unix_gc overload

README.md

unix_gc overload

The issue

Linux kernel has a garbage collection mechanism for inflight unix sockets passed to other unix sockets. This mechanism can be used to cause excessive load onto well behaved processes that are using regular unix sockets without any fd passing, because garbage collection is called in the socket write path.

The repro

The attached reproduction code in Go can be used to illustrate this:

$ go build -o /tmp/derp main.go && /tmp/derp

What it does:

1. Makes a unix connection to itself in a loop, writing some bytes and closing it every 50ms. This is the legitimate well behaving load.
2. Makes a unix connection to itself and puts 16.1k unix sockets into it. This is what forces unix_gc to run a lot more for the well behaved connection that has nothing to do with this.

With fewer than 16k file descriptors inflight, there's some gc, but not much:

$ sudo funclatency-bpfcc -uTi 1 unix_gc
Tracing 1 functions for "unix_gc"... Hit Ctrl-C to end.

22:08:34
     usecs               : count     distribution
         0 -> 1          : 0        |                                        |
         2 -> 3          : 0        |                                        |
         4 -> 7          : 0        |                                        |
         8 -> 15         : 0        |                                        |
        16 -> 31         : 0        |                                        |
        32 -> 63         : 0        |                                        |
        64 -> 127        : 0        |                                        |
       128 -> 255        : 0        |                                        |
       256 -> 511        : 0        |                                        |
       512 -> 1023       : 2        |*****                                   |
      1024 -> 2047       : 12       |**********************************      |
      2048 -> 4095       : 14       |****************************************|
      4096 -> 8191       : 6        |*****************                       |

avg = 2535 usecs, total: 86194 usecs, count: 34

This is triggered from unix_release_sock as long as there are any inflight sockets present at all (no matter how many):

• https://elixir.bootlin.com/linux/v6.1/source/net/unix/af_unix.c#L670

If you cross the threshold for the number of inflight sockets, it gets worse:

ivan@vm:~$ sudo funclatency-bpfcc -uTi 1 unix_gc
Tracing 1 functions for "unix_gc"... Hit Ctrl-C to end.

22:09:15
     usecs               : count     distribution
         0 -> 1          : 0        |                                        |
         2 -> 3          : 0        |                                        |
         4 -> 7          : 0        |                                        |
         8 -> 15         : 0        |                                        |
        16 -> 31         : 0        |                                        |
        32 -> 63         : 0        |                                        |
        64 -> 127        : 0        |                                        |
       128 -> 255        : 0        |                                        |
       256 -> 511        : 0        |                                        |
       512 -> 1023       : 456      |****************************************|
      1024 -> 2047       : 48       |****                                    |
      2048 -> 4095       : 2        |                                        |

avg = 979 usecs, total: 495498 usecs, count: 506

You can observe a lot more calls to unix_gc and a lot more work in each one. Most of the calls here are are from unix_stream_sendmsg:

• https://elixir.bootlin.com/linux/v6.1/source/net/unix/af_unix.c#L2158

flamegraph.svg

<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg version="1.1" width="1200" height="646" onload="init(evt)" viewBox="0 0 1200 646" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:fg="http://github.com/jonhoo/inferno"><!--Flame graph stack visualization. See https://github.com/brendangregg/FlameGraph for latest version, and http://www.brendangregg.com/flamegraphs.html for examples.--><!--NOTES: --><defs><linearGradient id="background" y1="0" y2="1" x1="0" x2="0"><stop stop-color="#eeeeee" offset="5%"/><stop stop-color="#eeeeb0" offset="95%"/></linearGradient></defs><style type="text/css">
text { font-family:monospace; font-size:12px }
#title { text-anchor:middle; font-size:17px; }
#matched { text-anchor:end; }
#search { text-anchor:end; opacity:0.1; cursor:pointer; }
#search:hover, #search.show { opacity:1; }
#subtitle { text-anchor:middle; font-color:rgb(160,160,160); }
#unzoom { cursor:pointer; }
#frames > *:hover { stroke:black; stroke-width:0.5; cursor:pointer; }
.hide { display:none; }
.parent { opacity:0.5; }
</style><script type="text/ecmascript"><![CDATA[
        var nametype = 'Function:';
        var fontsize = 12;
        var fontwidth = 0.59;
        var xpad = 10;
        var inverted = false;
        var searchcolor = 'rgb(230,0,230)';
        var fluiddrawing = true;
        var truncate_text_right = false;
    ]]><![CDATA["use strict";
var details, searchbtn, unzoombtn, matchedtxt, svg, searching, frames, known_font_width;
function init(evt) {
    details = document.getElementById("details").firstChild;
    searchbtn = document.getElementById("search");
    unzoombtn = document.getElementById("unzoom");
    matchedtxt = document.getElementById("matched");
    svg = document.getElementsByTagName("svg")[0];
    frames = document.getElementById("frames");
    known_font_width = get_monospace_width(frames);
    total_samples = parseInt(frames.attributes.total_samples.value);
    searching = 0;

    // Use GET parameters to restore a flamegraph's state.
    var restore_state = function() {
        var params = get_params();
        if (params.x && params.y)
            zoom(find_group(document.querySelector('[*|x="' + params.x + '"][y="' + params.y + '"]')));
        if (params.s)
            search(params.s);
    };

    if (fluiddrawing) {
        // Make width dynamic so the SVG fits its parent's width.
        svg.removeAttribute("width");
        // Edge requires us to have a viewBox that gets updated with size changes.
        var isEdge = /Edge\/\d./i.test(navigator.userAgent);
        if (!isEdge) {
            svg.removeAttribute("viewBox");
        }
        var update_for_width_change = function() {
            if (isEdge) {
                svg.attributes.viewBox.value = "0 0 " + svg.width.baseVal.value + " " + svg.height.baseVal.value;
            }

            // Keep consistent padding on left and right of frames container.
            frames.attributes.width.value = svg.width.baseVal.value - xpad * 2;

            // Text truncation needs to be adjusted for the current width.
            update_text_for_elements(frames.children);

            // Keep search elements at a fixed distance from right edge.
            var svgWidth = svg.width.baseVal.value;
            searchbtn.attributes.x.value = svgWidth - xpad;
            matchedtxt.attributes.x.value = svgWidth - xpad;
        };
        window.addEventListener('resize', function() {
            update_for_width_change();
        });
        // This needs to be done asynchronously for Safari to work.
        setTimeout(function() {
            unzoom();
            update_for_width_change();
            restore_state();
        }, 0);
    } else {
        restore_state();
    }
}
// event listeners
window.addEventListener("click", function(e) {
    var target = find_group(e.target);
    if (target) {
        if (target.nodeName == "a") {
            if (e.ctrlKey === false) return;
            e.preventDefault();
        }
        if (target.classList.contains("parent")) unzoom();
        zoom(target);

        // set parameters for zoom state
        var el = target.querySelector("rect");
        if (el && el.attributes && el.attributes.y && el.attributes["fg:x"]) {
            var params = get_params()
            params.x = el.attributes["fg:x"].value;
            params.y = el.attributes.y.value;
            history.replaceState(null, null, parse_params(params));
        }
    }
    else if (e.target.id == "unzoom") {
        unzoom();

        // remove zoom state
        var params = get_params();
        if (params.x) delete params.x;
        if (params.y) delete params.y;
        history.replaceState(null, null, parse_params(params));
    }
    else if (e.target.id == "search") search_prompt();
}, false)
// mouse-over for info
// show
window.addEventListener("mouseover", function(e) {
    var target = find_group(e.target);
    if (target) details.nodeValue = nametype + " " + g_to_text(target);
}, false)
// clear
window.addEventListener("mouseout", function(e) {
    var target = find_group(e.target);
    if (target) details.nodeValue = ' ';
}, false)
// ctrl-F for search
window.addEventListener("keydown",function (e) {
    if (e.keyCode === 114 || (e.ctrlKey && e.keyCode === 70)) {
        e.preventDefault();
        search_prompt();
    }
}, false)
// functions
function get_params() {
    var params = {};
    var paramsarr = window.location.search.substr(1).split('&');
    for (var i = 0; i < paramsarr.length; ++i) {
        var tmp = paramsarr[i].split("=");
        if (!tmp[0] || !tmp[1]) continue;
        params[tmp[0]]  = decodeURIComponent(tmp[1]);
    }
    return params;
}
function parse_params(params) {
    var uri = "?";
    for (var key in params) {
        uri += key + '=' + encodeURIComponent(params[key]) + '&';
    }
    if (uri.slice(-1) == "&")
        uri = uri.substring(0, uri.length - 1);
    if (uri == '?')
        uri = window.location.href.split('?')[0];
    return uri;
}
function find_child(node, selector) {
    var children = node.querySelectorAll(selector);
    if (children.length) return children[0];
    return;
}
function find_group(node) {
    var parent = node.parentElement;
    if (!parent) return;
    if (parent.id == "frames") return node;
    return find_group(parent);
}
function orig_save(e, attr, val) {
    if (e.attributes["fg:orig_" + attr] != undefined) return;
    if (e.attributes[attr] == undefined) return;
    if (val == undefined) val = e.attributes[attr].value;
    e.setAttribute("fg:orig_" + attr, val);
}
function orig_load(e, attr) {
    if (e.attributes["fg:orig_"+attr] == undefined) return;
    e.attributes[attr].value = e.attributes["fg:orig_" + attr].value;
    e.removeAttribute("fg:orig_" + attr);
}
function g_to_text(e) {
    var text = find_child(e, "title").firstChild.nodeValue;
    return (text)
}
function g_to_func(e) {
    var func = g_to_text(e);
    // if there's any manipulation we want to do to the function
    // name before it's searched, do it here before returning.
    return (func);
}
function get_monospace_width(frames) {
    // Given the id="frames" element, return the width of text characters if
    // this is a monospace font, otherwise return 0.
    text = find_child(frames.children[0], "text");
    originalContent = text.textContent;
    text.textContent = "!";
    bangWidth = text.getComputedTextLength();
    text.textContent = "W";
    wWidth = text.getComputedTextLength();
    text.textContent = originalContent;
    if (bangWidth === wWidth) {
        return bangWidth;
    } else {
        return 0;
    }
}
function update_text_for_elements(elements) {
    // In order to render quickly in the browser, you want to do one pass of
    // reading attributes, and one pass of mutating attributes. See
    // https://web.dev/avoid-large-complex-layouts-and-layout-thrashing/ for details.

    // Fall back to inefficient calculation, if we're variable-width font.
    // TODO This should be optimized somehow too.
    if (known_font_width === 0) {
        for (var i = 0; i < elements.length; i++) {
            update_text(elements[i]);
        }
        return;
    }

    var textElemNewAttributes = [];
    for (var i = 0; i < elements.length; i++) {
        var e = elements[i];
        var r = find_child(e, "rect");
        var t = find_child(e, "text");
        var w = parseFloat(r.attributes.width.value) * frames.attributes.width.value / 100 - 3;
        var txt = find_child(e, "title").textContent.replace(/\([^(]*\)$/,"");
        var newX = format_percent((parseFloat(r.attributes.x.value) + (100 * 3 / frames.attributes.width.value)));

        // Smaller than this size won't fit anything
        if (w < 2 * known_font_width) {
            textElemNewAttributes.push([newX, ""]);
            continue;
        }

        // Fit in full text width
        if (txt.length * known_font_width < w) {
            textElemNewAttributes.push([newX, txt]);
            continue;
        }

        var substringLength = Math.floor(w / known_font_width) - 2;
        if (truncate_text_right) {
            // Truncate the right side of the text.
            textElemNewAttributes.push([newX, txt.substring(0, substringLength) + ".."]);
            continue;
        } else {
            // Truncate the left side of the text.
            textElemNewAttributes.push([newX, ".." + txt.substring(txt.length - substringLength, txt.length)]);
            continue;
        }
    }

    console.assert(textElemNewAttributes.length === elements.length, "Resize failed, please file a bug at https://github.com/jonhoo/inferno/");

    // Now that we know new textContent, set it all in one go so we don't refresh a bazillion times.
    for (var i = 0; i < elements.length; i++) {
        var e = elements[i];
        var values = textElemNewAttributes[i];
        var t = find_child(e, "text");
        t.attributes.x.value = values[0];
        t.textContent = values[1];
    }
}

function update_text(e) {
    var r = find_child(e, "rect");
    var t = find_child(e, "text");
    var w = parseFloat(r.attributes.width.value) * frames.attributes.width.value / 100 - 3;
    var txt = find_child(e, "title").textContent.replace(/\([^(]*\)$/,"");
    t.attributes.x.value = format_percent((parseFloat(r.attributes.x.value) + (100 * 3 / frames.attributes.width.value)));

    // Smaller than this size won't fit anything
    if (w < 2 * fontsize * fontwidth) {
        t.textContent = "";
        return;
    }
    t.textContent = txt;
    // Fit in full text width
    if (t.getComputedTextLength() < w)
        return;
    if (truncate_text_right) {
        // Truncate the right side of the text.
        for (var x = txt.length - 2; x > 0; x--) {
            if (t.getSubStringLength(0, x + 2) <= w) {
                t.textContent = txt.substring(0, x) + "..";
                return;
            }
        }
    } else {
        // Truncate the left side of the text.
        for (var x = 2; x < txt.length; x++) {
            if (t.getSubStringLength(x - 2, txt.length) <= w) {
                t.textContent = ".." + txt.substring(x, txt.length);
                return;
            }
        }
    }
    t.textContent = "";
}
// zoom
function zoom_reset(e) {
    if (e.tagName == "rect") {
        e.attributes.x.value = format_percent(100 * parseInt(e.attributes["fg:x"].value) / total_samples);
        e.attributes.width.value = format_percent(100 * parseInt(e.attributes["fg:w"].value) / total_samples);
    }
    if (e.childNodes == undefined) return;
    for(var i = 0, c = e.childNodes; i < c.length; i++) {
        zoom_reset(c[i]);
    }
}
function zoom_child(e, x, zoomed_width_samples) {
    if (e.tagName == "text") {
        var parent_x = parseFloat(find_child(e.parentNode, "rect[x]").attributes.x.value);
        e.attributes.x.value = format_percent(parent_x + (100 * 3 / frames.attributes.width.value));
    } else if (e.tagName == "rect") {
        e.attributes.x.value = format_percent(100 * (parseInt(e.attributes["fg:x"].value) - x) / zoomed_width_samples);
        e.attributes.width.value = format_percent(100 * parseInt(e.attributes["fg:w"].value) / zoomed_width_samples);
    }
    if (e.childNodes == undefined) return;
    for(var i = 0, c = e.childNodes; i < c.length; i++) {
        zoom_child(c[i], x, zoomed_width_samples);
    }
}
function zoom_parent(e) {
    if (e.attributes) {
        if (e.attributes.x != undefined) {
            e.attributes.x.value = "0.0%";
        }
        if (e.attributes.width != undefined) {
            e.attributes.width.value = "100.0%";
        }
    }
    if (e.childNodes == undefined) return;
    for(var i = 0, c = e.childNodes; i < c.length; i++) {
        zoom_parent(c[i]);
    }
}
function zoom(node) {
    var attr = find_child(node, "rect").attributes;
    var width = parseInt(attr["fg:w"].value);
    var xmin = parseInt(attr["fg:x"].value);
    var xmax = xmin + width;
    var ymin = parseFloat(attr.y.value);
    unzoombtn.classList.remove("hide");
    var el = frames.children;
    var to_update_text = [];
    for (var i = 0; i < el.length; i++) {
        var e = el[i];
        var a = find_child(e, "rect").attributes;
        var ex = parseInt(a["fg:x"].value);
        var ew = parseInt(a["fg:w"].value);
        // Is it an ancestor
        if (!inverted) {
            var upstack = parseFloat(a.y.value) > ymin;
        } else {
            var upstack = parseFloat(a.y.value) < ymin;
        }
        if (upstack) {
            // Direct ancestor
            if (ex <= xmin && (ex+ew) >= xmax) {
                e.classList.add("parent");
                zoom_parent(e);
                to_update_text.push(e);
            }
            // not in current path
            else
                e.classList.add("hide");
        }
        // Children maybe
        else {
            // no common path
            if (ex < xmin || ex >= xmax) {
                e.classList.add("hide");
            }
            else {
                zoom_child(e, xmin, width);
                to_update_text.push(e);
            }
        }
    }
    update_text_for_elements(to_update_text);
}
function unzoom() {
    unzoombtn.classList.add("hide");
    var el = frames.children;
    for(var i = 0; i < el.length; i++) {
        el[i].classList.remove("parent");
        el[i].classList.remove("hide");
        zoom_reset(el[i]);
    }
    update_text_for_elements(el);
}
// search
function reset_search() {
    var el = document.querySelectorAll("#frames rect");
    for (var i = 0; i < el.length; i++) {
        orig_load(el[i], "fill")
    }
    var params = get_params();
    delete params.s;
    history.replaceState(null, null, parse_params(params));
}
function search_prompt() {
    if (!searching) {
        var term = prompt("Enter a search term (regexp " +
            "allowed, eg: ^ext4_)", "");
        if (term != null) {
            search(term)
        }
    } else {
        reset_search();
        searching = 0;
        searchbtn.classList.remove("show");
        searchbtn.firstChild.nodeValue = "Search"
        matchedtxt.classList.add("hide");
        matchedtxt.firstChild.nodeValue = ""
    }
}
function search(term) {
    var re = new RegExp(term);
    var el = frames.children;
    var matches = new Object();
    var maxwidth = 0;
    for (var i = 0; i < el.length; i++) {
        var e = el[i];
        // Skip over frames which are either not visible, or below the zoomed-to frame
        if (e.classList.contains("hide") || e.classList.contains("parent")) {
            continue;
        }
        var func = g_to_func(e);
        var rect = find_child(e, "rect");
        if (func == null || rect == null)
            continue;
        // Save max width. Only works as we have a root frame
        var w = parseInt(rect.attributes["fg:w"].value);
        if (w > maxwidth)
            maxwidth = w;
        if (func.match(re)) {
            // highlight
            var x = parseInt(rect.attributes["fg:x"].value);
            orig_save(rect, "fill");
            rect.attributes.fill.value = searchcolor;
            // remember matches
            if (matches[x] == undefined) {
                matches[x] = w;
            } else {
                if (w > matches[x]) {
                    // overwrite with parent
                    matches[x] = w;
                }
            }
            searching = 1;
        }
    }
    if (!searching)
        return;
    var params = get_params();
    params.s = term;
    history.replaceState(null, null, parse_params(params));

    searchbtn.classList.add("show");
    searchbtn.firstChild.nodeValue = "Reset Search";
    // calculate percent matched, excluding vertical overlap
    var count = 0;
    var lastx = -1;
    var lastw = 0;
    var keys = Array();
    for (k in matches) {
        if (matches.hasOwnProperty(k))
            keys.push(k);
    }
    // sort the matched frames by their x location
    // ascending, then width descending
    keys.sort(function(a, b){
        return a - b;
    });
    // Step through frames saving only the biggest bottom-up frames
    // thanks to the sort order. This relies on the tree property
    // where children are always smaller than their parents.
    for (var k in keys) {
        var x = parseInt(keys[k]);
        var w = matches[keys[k]];
        if (x >= lastx + lastw) {
            count += w;
            lastx = x;
            lastw = w;
        }
    }
    // display matched percent
    matchedtxt.classList.remove("hide");
    var pct = 100 * count / maxwidth;
    if (pct != 100) pct = pct.toFixed(1);
    matchedtxt.firstChild.nodeValue = "Matched: " + pct + "%";
}
function format_percent(n) {
    return n.toFixed(4) + "%";
}
]]></script><rect x="0" y="0" width="100%" height="646" fill="url(#background)"/><text id="title" fill="rgb(0,0,0)" x="50.0000%" y="24.00">Flame Graph</text><text id="details" fill="rgb(0,0,0)" x="10" y="629.00"> </text><text id="unzoom" class="hide" fill="rgb(0,0,0)" x="10" y="24.00">Reset Zoom</text><text id="search" fill="rgb(0,0,0)" x="1190" y="24.00">Search</text><text id="matched" fill="rgb(0,0,0)" x="1190" y="629.00"> </text><svg id="frames" x="10" width="1180" total_samples="2089"><g><title>[libc.so.6] (1 samples, 0.05%)</title><rect x="0.0000%" y="565" width="0.0479%" height="15" fill="rgb(227,0,7)" fg:x="0" fg:w="1"/><text x="0.2500%" y="575.50"></text></g><g><title>[libc.so.6] (1 samples, 0.05%)</title><rect x="0.0000%" y="549" width="0.0479%" height="15" fill="rgb(217,0,24)" fg:x="0" fg:w="1"/><text x="0.2500%" y="559.50"></text></g><g><title>runtime/cgo (1 samples, 0.05%)</title><rect x="0.0000%" y="533" width="0.0479%" height="15" fill="rgb(221,193,54)" fg:x="0" fg:w="1"/><text x="0.2500%" y="543.50"></text></g><g><title>crosscall1 (1 samples, 0.05%)</title><rect x="0.0000%" y="517" width="0.0479%" height="15" fill="rgb(248,212,6)" fg:x="0" fg:w="1"/><text x="0.2500%" y="527.50"></text></g><g><title>runtime.mstart.abi0 (1 samples, 0.05%)</title><rect x="0.0000%" y="501" width="0.0479%" height="15" fill="rgb(208,68,35)" fg:x="0" fg:w="1"/><text x="0.2500%" y="511.50"></text></g><g><title>runtime.mstart0 (1 samples, 0.05%)</title><rect x="0.0000%" y="485" width="0.0479%" height="15" fill="rgb(232,128,0)" fg:x="0" fg:w="1"/><text x="0.2500%" y="495.50"></text></g><g><title>runtime.mstart1 (1 samples, 0.05%)</title><rect x="0.0000%" y="469" width="0.0479%" height="15" fill="rgb(207,160,47)" fg:x="0" fg:w="1"/><text x="0.2500%" y="479.50"></text></g><g><title>runtime.sysmon (1 samples, 0.05%)</title><rect x="0.0000%" y="453" width="0.0479%" height="15" fill="rgb(228,23,34)" fg:x="0" fg:w="1"/><text x="0.2500%" y="463.50"></text></g><g><title>runtime.sysmon (1 samples, 0.05%)</title><rect x="0.0000%" y="437" width="0.0479%" height="15" fill="rgb(218,30,26)" fg:x="0" fg:w="1"/><text x="0.2500%" y="447.50"></text></g><g><title>runtime.usleep.abi0 (1 samples, 0.05%)</title><rect x="0.0000%" y="421" width="0.0479%" height="15" fill="rgb(220,122,19)" fg:x="0" fg:w="1"/><text x="0.2500%" y="431.50"></text></g><g><title>el0t_64_sync (1 samples, 0.05%)</title><rect x="0.0000%" y="405" width="0.0479%" height="15" fill="rgb(250,228,42)" fg:x="0" fg:w="1"/><text x="0.2500%" y="415.50"></text></g><g><title>el0t_64_sync_handler (1 samples, 0.05%)</title><rect x="0.0000%" y="389" width="0.0479%" height="15" fill="rgb(240,193,28)" fg:x="0" fg:w="1"/><text x="0.2500%" y="399.50"></text></g><g><title>el0_svc (1 samples, 0.05%)</title><rect x="0.0000%" y="373" width="0.0479%" height="15" fill="rgb(216,20,37)" fg:x="0" fg:w="1"/><text x="0.2500%" y="383.50"></text></g><g><title>do_el0_svc (1 samples, 0.05%)</title><rect x="0.0000%" y="357" width="0.0479%" height="15" fill="rgb(206,188,39)" fg:x="0" fg:w="1"/><text x="0.2500%" y="367.50"></text></g><g><title>invoke_syscall.constprop.0 (1 samples, 0.05%)</title><rect x="0.0000%" y="341" width="0.0479%" height="15" fill="rgb(217,207,13)" fg:x="0" fg:w="1"/><text x="0.2500%" y="351.50"></text></g><g><title>__arm64_sys_nanosleep (1 samples, 0.05%)</title><rect x="0.0000%" y="325" width="0.0479%" height="15" fill="rgb(231,73,38)" fg:x="0" fg:w="1"/><text x="0.2500%" y="335.50"></text></g><g><title>hrtimer_nanosleep (1 samples, 0.05%)</title><rect x="0.0000%" y="309" width="0.0479%" height="15" fill="rgb(225,20,46)" fg:x="0" fg:w="1"/><text x="0.2500%" y="319.50"></text></g><g><title>do_nanosleep (1 samples, 0.05%)</title><rect x="0.0000%" y="293" width="0.0479%" height="15" fill="rgb(210,31,41)" fg:x="0" fg:w="1"/><text x="0.2500%" y="303.50"></text></g><g><title>schedule (1 samples, 0.05%)</title><rect x="0.0000%" y="277" width="0.0479%" height="15" fill="rgb(221,200,47)" fg:x="0" fg:w="1"/><text x="0.2500%" y="287.50"></text></g><g><title>__schedule (1 samples, 0.05%)</title><rect x="0.0000%" y="261" width="0.0479%" height="15" fill="rgb(226,26,5)" fg:x="0" fg:w="1"/><text x="0.2500%" y="271.50"></text></g><g><title>finish_task_switch.isra.0 (1 samples, 0.05%)</title><rect x="0.0000%" y="245" width="0.0479%" height="15" fill="rgb(249,33,26)" fg:x="0" fg:w="1"/><text x="0.2500%" y="255.50"></text></g><g><title>log.Println (1 samples, 0.05%)</title><rect x="0.0479%" y="533" width="0.0479%" height="15" fill="rgb(235,183,28)" fg:x="1" fg:w="1"/><text x="0.2979%" y="543.50"></text></g><g><title>log.(*Logger).Output (1 samples, 0.05%)</title><rect x="0.0479%" y="517" width="0.0479%" height="15" fill="rgb(221,5,38)" fg:x="1" fg:w="1"/><text x="0.2979%" y="527.50"></text></g><g><title>os.(*File).Write (1 samples, 0.05%)</title><rect x="0.0479%" y="501" width="0.0479%" height="15" fill="rgb(247,18,42)" fg:x="1" fg:w="1"/><text x="0.2979%" y="511.50"></text></g><g><title>internal/poll.(*FD).Write (1 samples, 0.05%)</title><rect x="0.0479%" y="485" width="0.0479%" height="15" fill="rgb(241,131,45)" fg:x="1" fg:w="1"/><text x="0.2979%" y="495.50"></text></g><g><title>syscall.write (1 samples, 0.05%)</title><rect x="0.0479%" y="469" width="0.0479%" height="15" fill="rgb(249,31,29)" fg:x="1" fg:w="1"/><text x="0.2979%" y="479.50"></text></g><g><title>syscall.Syscall (1 samples, 0.05%)</title><rect x="0.0479%" y="453" width="0.0479%" height="15" fill="rgb(225,111,53)" fg:x="1" fg:w="1"/><text x="0.2979%" y="463.50"></text></g><g><title>syscall.RawSyscall6 (1 samples, 0.05%)</title><rect x="0.0479%" y="437" width="0.0479%" height="15" fill="rgb(238,160,17)" fg:x="1" fg:w="1"/><text x="0.2979%" y="447.50"></text></g><g><title>runtime/internal/syscall.Syscall6.abi0 (1 samples, 0.05%)</title><rect x="0.0479%" y="421" width="0.0479%" height="15" fill="rgb(214,148,48)" fg:x="1" fg:w="1"/><text x="0.2979%" y="431.50"></text></g><g><title>el0t_64_sync (1 samples, 0.05%)</title><rect x="0.0479%" y="405" width="0.0479%" height="15" fill="rgb(232,36,49)" fg:x="1" fg:w="1"/><text x="0.2979%" y="415.50"></text></g><g><title>el0t_64_sync_handler (1 samples, 0.05%)</title><rect x="0.0479%" y="389" width="0.0479%" height="15" fill="rgb(209,103,24)" fg:x="1" fg:w="1"/><text x="0.2979%" y="399.50"></text></g><g><title>el0_svc (1 samples, 0.05%)</title><rect x="0.0479%" y="373" width="0.0479%" height="15" fill="rgb(229,88,8)" fg:x="1" fg:w="1"/><text x="0.2979%" y="383.50"></text></g><g><title>do_el0_svc (1 samples, 0.05%)</title><rect x="0.0479%" y="357" width="0.0479%" height="15" fill="rgb(213,181,19)" fg:x="1" fg:w="1"/><text x="0.2979%" y="367.50"></text></g><g><title>invoke_syscall.constprop.0 (1 samples, 0.05%)</title><rect x="0.0479%" y="341" width="0.0479%" height="15" fill="rgb(254,191,54)" fg:x="1" fg:w="1"/><text x="0.2979%" y="351.50"></text></g><g><title>__arm64_sys_write (1 samples, 0.05%)</title><rect x="0.0479%" y="325" width="0.0479%" height="15" fill="rgb(241,83,37)" fg:x="1" fg:w="1"/><text x="0.2979%" y="335.50"></text></g><g><title>ksys_write (1 samples, 0.05%)</title><rect x="0.0479%" y="309" width="0.0479%" height="15" fill="rgb(233,36,39)" fg:x="1" fg:w="1"/><text x="0.2979%" y="319.50"></text></g><g><title>vfs_write (1 samples, 0.05%)</title><rect x="0.0479%" y="293" width="0.0479%" height="15" fill="rgb(226,3,54)" fg:x="1" fg:w="1"/><text x="0.2979%" y="303.50"></text></g><g><title>tty_write (1 samples, 0.05%)</title><rect x="0.0479%" y="277" width="0.0479%" height="15" fill="rgb(245,192,40)" fg:x="1" fg:w="1"/><text x="0.2979%" y="287.50"></text></g><g><title>file_tty_write.constprop.0 (1 samples, 0.05%)</title><rect x="0.0479%" y="261" width="0.0479%" height="15" fill="rgb(238,167,29)" fg:x="1" fg:w="1"/><text x="0.2979%" y="271.50"></text></g><g><title>n_tty_write (1 samples, 0.05%)</title><rect x="0.0479%" y="245" width="0.0479%" height="15" fill="rgb(232,182,51)" fg:x="1" fg:w="1"/><text x="0.2979%" y="255.50"></text></g><g><title>do_output_char (1 samples, 0.05%)</title><rect x="0.0479%" y="229" width="0.0479%" height="15" fill="rgb(231,60,39)" fg:x="1" fg:w="1"/><text x="0.2979%" y="239.50"></text></g><g><title>net.(*conn).Close (86 samples, 4.12%)</title><rect x="0.0957%" y="533" width="4.1168%" height="15" fill="rgb(208,69,12)" fg:x="2" fg:w="86"/><text x="0.3457%" y="543.50">net...</text></g><g><title>net.(*netFD).Close (86 samples, 4.12%)</title><rect x="0.0957%" y="517" width="4.1168%" height="15" fill="rgb(235,93,37)" fg:x="2" fg:w="86"/><text x="0.3457%" y="527.50">net...</text></g><g><title>internal/poll.(*FD).Close (86 samples, 4.12%)</title><rect x="0.0957%" y="501" width="4.1168%" height="15" fill="rgb(213,116,39)" fg:x="2" fg:w="86"/><text x="0.3457%" y="511.50">inte..</text></g><g><title>internal/poll.(*FD).decref (86 samples, 4.12%)</title><rect x="0.0957%" y="485" width="4.1168%" height="15" fill="rgb(222,207,29)" fg:x="2" fg:w="86"/><text x="0.3457%" y="495.50">inte..</text></g><g><title>internal/poll.(*FD).destroy (86 samples, 4.12%)</title><rect x="0.0957%" y="469" width="4.1168%" height="15" fill="rgb(206,96,30)" fg:x="2" fg:w="86"/><text x="0.3457%" y="479.50">inte..</text></g><g><title>syscall.Close (86 samples, 4.12%)</title><rect x="0.0957%" y="453" width="4.1168%" height="15" fill="rgb(218,138,4)" fg:x="2" fg:w="86"/><text x="0.3457%" y="463.50">sysc..</text></g><g><title>syscall.Syscall (86 samples, 4.12%)</title><rect x="0.0957%" y="437" width="4.1168%" height="15" fill="rgb(250,191,14)" fg:x="2" fg:w="86"/><text x="0.3457%" y="447.50">sysc..</text></g><g><title>syscall.RawSyscall6 (86 samples, 4.12%)</title><rect x="0.0957%" y="421" width="4.1168%" height="15" fill="rgb(239,60,40)" fg:x="2" fg:w="86"/><text x="0.3457%" y="431.50">sysc..</text></g><g><title>runtime/internal/syscall.Syscall6.abi0 (86 samples, 4.12%)</title><rect x="0.0957%" y="405" width="4.1168%" height="15" fill="rgb(206,27,48)" fg:x="2" fg:w="86"/><text x="0.3457%" y="415.50">runt..</text></g><g><title>el0t_64_sync (86 samples, 4.12%)</title><rect x="0.0957%" y="389" width="4.1168%" height="15" fill="rgb(225,35,8)" fg:x="2" fg:w="86"/><text x="0.3457%" y="399.50">el0t..</text></g><g><title>el0t_64_sync_handler (86 samples, 4.12%)</title><rect x="0.0957%" y="373" width="4.1168%" height="15" fill="rgb(250,213,24)" fg:x="2" fg:w="86"/><text x="0.3457%" y="383.50">el0t..</text></g><g><title>el0_svc (86 samples, 4.12%)</title><rect x="0.0957%" y="357" width="4.1168%" height="15" fill="rgb(247,123,22)" fg:x="2" fg:w="86"/><text x="0.3457%" y="367.50">el0_..</text></g><g><title>do_el0_svc (86 samples, 4.12%)</title><rect x="0.0957%" y="341" width="4.1168%" height="15" fill="rgb(231,138,38)" fg:x="2" fg:w="86"/><text x="0.3457%" y="351.50">do_e..</text></g><g><title>invoke_syscall.constprop.0 (86 samples, 4.12%)</title><rect x="0.0957%" y="325" width="4.1168%" height="15" fill="rgb(231,145,46)" fg:x="2" fg:w="86"/><text x="0.3457%" y="335.50">invo..</text></g><g><title>__arm64_sys_close (86 samples, 4.12%)</title><rect x="0.0957%" y="309" width="4.1168%" height="15" fill="rgb(251,118,11)" fg:x="2" fg:w="86"/><text x="0.3457%" y="319.50">__ar..</text></g><g><title>__fput_sync (86 samples, 4.12%)</title><rect x="0.0957%" y="293" width="4.1168%" height="15" fill="rgb(217,147,25)" fg:x="2" fg:w="86"/><text x="0.3457%" y="303.50">__fp..</text></g><g><title>__fput (86 samples, 4.12%)</title><rect x="0.0957%" y="277" width="4.1168%" height="15" fill="rgb(247,81,37)" fg:x="2" fg:w="86"/><text x="0.3457%" y="287.50">__fp..</text></g><g><title>sock_close (86 samples, 4.12%)</title><rect x="0.0957%" y="261" width="4.1168%" height="15" fill="rgb(209,12,38)" fg:x="2" fg:w="86"/><text x="0.3457%" y="271.50">sock..</text></g><g><title>__sock_release (86 samples, 4.12%)</title><rect x="0.0957%" y="245" width="4.1168%" height="15" fill="rgb(227,1,9)" fg:x="2" fg:w="86"/><text x="0.3457%" y="255.50">__so..</text></g><g><title>unix_release (86 samples, 4.12%)</title><rect x="0.0957%" y="229" width="4.1168%" height="15" fill="rgb(248,47,43)" fg:x="2" fg:w="86"/><text x="0.3457%" y="239.50">unix..</text></g><g><title>unix_release_sock (86 samples, 4.12%)</title><rect x="0.0957%" y="213" width="4.1168%" height="15" fill="rgb(221,10,30)" fg:x="2" fg:w="86"/><text x="0.3457%" y="223.50">unix..</text></g><g><title>unix_gc (86 samples, 4.12%)</title><rect x="0.0957%" y="197" width="4.1168%" height="15" fill="rgb(210,229,1)" fg:x="2" fg:w="86"/><text x="0.3457%" y="207.50">unix..</text></g><g><title>el1h_64_irq (6 samples, 0.29%)</title><rect x="3.9253%" y="181" width="0.2872%" height="15" fill="rgb(222,148,37)" fg:x="82" fg:w="6"/><text x="4.1753%" y="191.50"></text></g><g><title>el1h_64_irq_handler (6 samples, 0.29%)</title><rect x="3.9253%" y="165" width="0.2872%" height="15" fill="rgb(234,67,33)" fg:x="82" fg:w="6"/><text x="4.1753%" y="175.50"></text></g><g><title>el1_interrupt (6 samples, 0.29%)</title><rect x="3.9253%" y="149" width="0.2872%" height="15" fill="rgb(247,98,35)" fg:x="82" fg:w="6"/><text x="4.1753%" y="159.50"></text></g><g><title>irq_exit_rcu (6 samples, 0.29%)</title><rect x="3.9253%" y="133" width="0.2872%" height="15" fill="rgb(247,138,52)" fg:x="82" fg:w="6"/><text x="4.1753%" y="143.50"></text></g><g><title>do_softirq_own_stack (6 samples, 0.29%)</title><rect x="3.9253%" y="117" width="0.2872%" height="15" fill="rgb(213,79,30)" fg:x="82" fg:w="6"/><text x="4.1753%" y="127.50"></text></g><g><title>call_on_irq_stack (6 samples, 0.29%)</title><rect x="3.9253%" y="101" width="0.2872%" height="15" fill="rgb(246,177,23)" fg:x="82" fg:w="6"/><text x="4.1753%" y="111.50"></text></g><g><title>____do_softirq (6 samples, 0.29%)</title><rect x="3.9253%" y="85" width="0.2872%" height="15" fill="rgb(230,62,27)" fg:x="82" fg:w="6"/><text x="4.1753%" y="95.50"></text></g><g><title>__do_softirq (6 samples, 0.29%)</title><rect x="3.9253%" y="69" width="0.2872%" height="15" fill="rgb(216,154,8)" fg:x="82" fg:w="6"/><text x="4.1753%" y="79.50"></text></g><g><title>rcu_core_si (3 samples, 0.14%)</title><rect x="4.0689%" y="53" width="0.1436%" height="15" fill="rgb(244,35,45)" fg:x="85" fg:w="3"/><text x="4.3189%" y="63.50"></text></g><g><title>rcu_core (3 samples, 0.14%)</title><rect x="4.0689%" y="37" width="0.1436%" height="15" fill="rgb(251,115,12)" fg:x="85" fg:w="3"/><text x="4.3189%" y="47.50"></text></g><g><title>rw_verify_area (1 samples, 0.05%)</title><rect x="4.3562%" y="293" width="0.0479%" height="15" fill="rgb(240,54,50)" fg:x="91" fg:w="1"/><text x="4.6062%" y="303.50"></text></g><g><title>security_file_permission (1 samples, 0.05%)</title><rect x="4.3562%" y="277" width="0.0479%" height="15" fill="rgb(233,84,52)" fg:x="91" fg:w="1"/><text x="4.6062%" y="287.50"></text></g><g><title>selinux_file_permission (1 samples, 0.05%)</title><rect x="4.3562%" y="261" width="0.0479%" height="15" fill="rgb(207,117,47)" fg:x="91" fg:w="1"/><text x="4.6062%" y="271.50"></text></g><g><title>inode_security (1 samples, 0.05%)</title><rect x="4.3562%" y="245" width="0.0479%" height="15" fill="rgb(249,43,39)" fg:x="91" fg:w="1"/><text x="4.6062%" y="255.50"></text></g><g><title>__cond_resched (1 samples, 0.05%)</title><rect x="4.3562%" y="229" width="0.0479%" height="15" fill="rgb(209,38,44)" fg:x="91" fg:w="1"/><text x="4.6062%" y="239.50"></text></g><g><title>rcu_all_qs (1 samples, 0.05%)</title><rect x="4.3562%" y="213" width="0.0479%" height="15" fill="rgb(236,212,23)" fg:x="91" fg:w="1"/><text x="4.6062%" y="223.50"></text></g><g><title>skb_put (1 samples, 0.05%)</title><rect x="4.4040%" y="261" width="0.0479%" height="15" fill="rgb(242,79,21)" fg:x="92" fg:w="1"/><text x="4.6540%" y="271.50"></text></g><g><title>sock_def_readable (24 samples, 1.15%)</title><rect x="4.4519%" y="245" width="1.1489%" height="15" fill="rgb(211,96,35)" fg:x="93" fg:w="24"/><text x="4.7019%" y="255.50"></text></g><g><title>__wake_up_sync_key (24 samples, 1.15%)</title><rect x="4.4519%" y="229" width="1.1489%" height="15" fill="rgb(253,215,40)" fg:x="93" fg:w="24"/><text x="4.7019%" y="239.50"></text></g><g><title>__wake_up_common_lock (24 samples, 1.15%)</title><rect x="4.4519%" y="213" width="1.1489%" height="15" fill="rgb(211,81,21)" fg:x="93" fg:w="24"/><text x="4.7019%" y="223.50"></text></g><g><title>main.main (2,076 samples, 99.38%)</title><rect x="0.0479%" y="565" width="99.3777%" height="15" fill="rgb(208,190,38)" fg:x="1" fg:w="2076"/><text x="0.2979%" y="575.50">main.main</text></g><g><title>main.churnSockets (2,076 samples, 99.38%)</title><rect x="0.0479%" y="549" width="99.3777%" height="15" fill="rgb(235,213,38)" fg:x="1" fg:w="2076"/><text x="0.2979%" y="559.50">main.churnSockets</text></g><g><title>net.(*conn).Write (1,989 samples, 95.21%)</title><rect x="4.2125%" y="533" width="95.2130%" height="15" fill="rgb(237,122,38)" fg:x="88" fg:w="1989"/><text x="4.4625%" y="543.50">net.(*conn).Write</text></g><g><title>net.(*netFD).Write (1,989 samples, 95.21%)</title><rect x="4.2125%" y="517" width="95.2130%" height="15" fill="rgb(244,218,35)" fg:x="88" fg:w="1989"/><text x="4.4625%" y="527.50">net.(*netFD).Write</text></g><g><title>internal/poll.(*FD).Write (1,989 samples, 95.21%)</title><rect x="4.2125%" y="501" width="95.2130%" height="15" fill="rgb(240,68,47)" fg:x="88" fg:w="1989"/><text x="4.4625%" y="511.50">internal/poll.(*FD).Write</text></g><g><title>syscall.write (1,989 samples, 95.21%)</title><rect x="4.2125%" y="485" width="95.2130%" height="15" fill="rgb(210,16,53)" fg:x="88" fg:w="1989"/><text x="4.4625%" y="495.50">syscall.write</text></g><g><title>syscall.Syscall (1,989 samples, 95.21%)</title><rect x="4.2125%" y="469" width="95.2130%" height="15" fill="rgb(235,124,12)" fg:x="88" fg:w="1989"/><text x="4.4625%" y="479.50">syscall.Syscall</text></g><g><title>syscall.RawSyscall6 (1,989 samples, 95.21%)</title><rect x="4.2125%" y="453" width="95.2130%" height="15" fill="rgb(224,169,11)" fg:x="88" fg:w="1989"/><text x="4.4625%" y="463.50">syscall.RawSyscall6</text></g><g><title>runtime/internal/syscall.Syscall6.abi0 (1,989 samples, 95.21%)</title><rect x="4.2125%" y="437" width="95.2130%" height="15" fill="rgb(250,166,2)" fg:x="88" fg:w="1989"/><text x="4.4625%" y="447.50">runtime/internal/syscall.Syscall6.abi0</text></g><g><title>el0t_64_sync (1,986 samples, 95.07%)</title><rect x="4.3562%" y="421" width="95.0694%" height="15" fill="rgb(242,216,29)" fg:x="91" fg:w="1986"/><text x="4.6062%" y="431.50">el0t_64_sync</text></g><g><title>el0t_64_sync_handler (1,986 samples, 95.07%)</title><rect x="4.3562%" y="405" width="95.0694%" height="15" fill="rgb(230,116,27)" fg:x="91" fg:w="1986"/><text x="4.6062%" y="415.50">el0t_64_sync_handler</text></g><g><title>el0_svc (1,986 samples, 95.07%)</title><rect x="4.3562%" y="389" width="95.0694%" height="15" fill="rgb(228,99,48)" fg:x="91" fg:w="1986"/><text x="4.6062%" y="399.50">el0_svc</text></g><g><title>do_el0_svc (1,986 samples, 95.07%)</title><rect x="4.3562%" y="373" width="95.0694%" height="15" fill="rgb(253,11,6)" fg:x="91" fg:w="1986"/><text x="4.6062%" y="383.50">do_el0_svc</text></g><g><title>invoke_syscall.constprop.0 (1,986 samples, 95.07%)</title><rect x="4.3562%" y="357" width="95.0694%" height="15" fill="rgb(247,143,39)" fg:x="91" fg:w="1986"/><text x="4.6062%" y="367.50">invoke_syscall.constprop.0</text></g><g><title>__arm64_sys_write (1,986 samples, 95.07%)</title><rect x="4.3562%" y="341" width="95.0694%" height="15" fill="rgb(236,97,10)" fg:x="91" fg:w="1986"/><text x="4.6062%" y="351.50">__arm64_sys_write</text></g><g><title>ksys_write (1,986 samples, 95.07%)</title><rect x="4.3562%" y="325" width="95.0694%" height="15" fill="rgb(233,208,19)" fg:x="91" fg:w="1986"/><text x="4.6062%" y="335.50">ksys_write</text></g><g><title>vfs_write (1,986 samples, 95.07%)</title><rect x="4.3562%" y="309" width="95.0694%" height="15" fill="rgb(216,164,2)" fg:x="91" fg:w="1986"/><text x="4.6062%" y="319.50">vfs_write</text></g><g><title>sock_write_iter (1,985 samples, 95.02%)</title><rect x="4.4040%" y="293" width="95.0215%" height="15" fill="rgb(220,129,5)" fg:x="92" fg:w="1985"/><text x="4.6540%" y="303.50">sock_write_iter</text></g><g><title>sock_sendmsg (1,985 samples, 95.02%)</title><rect x="4.4040%" y="277" width="95.0215%" height="15" fill="rgb(242,17,10)" fg:x="92" fg:w="1985"/><text x="4.6540%" y="287.50">sock_sendmsg</text></g><g><title>unix_stream_sendmsg (1,984 samples, 94.97%)</title><rect x="4.4519%" y="261" width="94.9737%" height="15" fill="rgb(242,107,0)" fg:x="93" fg:w="1984"/><text x="4.7019%" y="271.50">unix_stream_sendmsg</text></g><g><title>wait_for_unix_gc (1,960 samples, 93.82%)</title><rect x="5.6008%" y="245" width="93.8248%" height="15" fill="rgb(251,28,31)" fg:x="117" fg:w="1960"/><text x="5.8508%" y="255.50">wait_for_unix_gc</text></g><g><title>unix_gc (1,960 samples, 93.82%)</title><rect x="5.6008%" y="229" width="93.8248%" height="15" fill="rgb(233,223,10)" fg:x="117" fg:w="1960"/><text x="5.8508%" y="239.50">unix_gc</text></g><g><title>runtime.mcall (2 samples, 0.10%)</title><rect x="99.4256%" y="565" width="0.0957%" height="15" fill="rgb(215,21,27)" fg:x="2077" fg:w="2"/><text x="99.6756%" y="575.50"></text></g><g><title>runtime.park_m (2 samples, 0.10%)</title><rect x="99.4256%" y="549" width="0.0957%" height="15" fill="rgb(232,23,21)" fg:x="2077" fg:w="2"/><text x="99.6756%" y="559.50"></text></g><g><title>runtime.schedule (2 samples, 0.10%)</title><rect x="99.4256%" y="533" width="0.0957%" height="15" fill="rgb(244,5,23)" fg:x="2077" fg:w="2"/><text x="99.6756%" y="543.50"></text></g><g><title>runtime.findRunnable (2 samples, 0.10%)</title><rect x="99.4256%" y="517" width="0.0957%" height="15" fill="rgb(226,81,46)" fg:x="2077" fg:w="2"/><text x="99.6756%" y="527.50"></text></g><g><title>runtime.netpoll (2 samples, 0.10%)</title><rect x="99.4256%" y="501" width="0.0957%" height="15" fill="rgb(247,70,30)" fg:x="2077" fg:w="2"/><text x="99.6756%" y="511.50"></text></g><g><title>runtime.epollwait.abi0 (2 samples, 0.10%)</title><rect x="99.4256%" y="485" width="0.0957%" height="15" fill="rgb(212,68,19)" fg:x="2077" fg:w="2"/><text x="99.6756%" y="495.50"></text></g><g><title>el0t_64_sync (1 samples, 0.05%)</title><rect x="99.4734%" y="469" width="0.0479%" height="15" fill="rgb(240,187,13)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="479.50"></text></g><g><title>el0t_64_sync_handler (1 samples, 0.05%)</title><rect x="99.4734%" y="453" width="0.0479%" height="15" fill="rgb(223,113,26)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="463.50"></text></g><g><title>el0_svc (1 samples, 0.05%)</title><rect x="99.4734%" y="437" width="0.0479%" height="15" fill="rgb(206,192,2)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="447.50"></text></g><g><title>do_el0_svc (1 samples, 0.05%)</title><rect x="99.4734%" y="421" width="0.0479%" height="15" fill="rgb(241,108,4)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="431.50"></text></g><g><title>invoke_syscall.constprop.0 (1 samples, 0.05%)</title><rect x="99.4734%" y="405" width="0.0479%" height="15" fill="rgb(247,173,49)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="415.50"></text></g><g><title>__arm64_sys_epoll_pwait (1 samples, 0.05%)</title><rect x="99.4734%" y="389" width="0.0479%" height="15" fill="rgb(224,114,35)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="399.50"></text></g><g><title>do_epoll_pwait.part.0 (1 samples, 0.05%)</title><rect x="99.4734%" y="373" width="0.0479%" height="15" fill="rgb(245,159,27)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="383.50"></text></g><g><title>do_epoll_wait (1 samples, 0.05%)</title><rect x="99.4734%" y="357" width="0.0479%" height="15" fill="rgb(245,172,44)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="367.50"></text></g><g><title>schedule_hrtimeout_range (1 samples, 0.05%)</title><rect x="99.4734%" y="341" width="0.0479%" height="15" fill="rgb(236,23,11)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="351.50"></text></g><g><title>schedule_hrtimeout_range_clock (1 samples, 0.05%)</title><rect x="99.4734%" y="325" width="0.0479%" height="15" fill="rgb(205,117,38)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="335.50"></text></g><g><title>schedule (1 samples, 0.05%)</title><rect x="99.4734%" y="309" width="0.0479%" height="15" fill="rgb(237,72,25)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="319.50"></text></g><g><title>__schedule (1 samples, 0.05%)</title><rect x="99.4734%" y="293" width="0.0479%" height="15" fill="rgb(244,70,9)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="303.50"></text></g><g><title>finish_task_switch.isra.0 (1 samples, 0.05%)</title><rect x="99.4734%" y="277" width="0.0479%" height="15" fill="rgb(217,125,39)" fg:x="2078" fg:w="1"/><text x="99.7234%" y="287.50"></text></g><g><title>runtime.acquirep (1 samples, 0.05%)</title><rect x="99.5213%" y="517" width="0.0479%" height="15" fill="rgb(235,36,10)" fg:x="2079" fg:w="1"/><text x="99.7713%" y="527.50"></text></g><g><title>runtime.acquirep (1 samples, 0.05%)</title><rect x="99.5213%" y="501" width="0.0479%" height="15" fill="rgb(251,123,47)" fg:x="2079" fg:w="1"/><text x="99.7713%" y="511.50"></text></g><g><title>runtime.(*mcache).prepareForSweep (1 samples, 0.05%)</title><rect x="99.5213%" y="485" width="0.0479%" height="15" fill="rgb(221,13,13)" fg:x="2079" fg:w="1"/><text x="99.7713%" y="495.50"></text></g><g><title>all (2,089 samples, 100%)</title><rect x="0.0000%" y="597" width="100.0000%" height="15" fill="rgb(238,131,9)" fg:x="0" fg:w="2089"/><text x="0.2500%" y="607.50"></text></g><g><title>derp (2,089 samples, 100.00%)</title><rect x="0.0000%" y="581" width="100.0000%" height="15" fill="rgb(211,50,8)" fg:x="0" fg:w="2089"/><text x="0.2500%" y="591.50">derp</text></g><g><title>runtime.systemstack.abi0 (10 samples, 0.48%)</title><rect x="99.5213%" y="565" width="0.4787%" height="15" fill="rgb(245,182,24)" fg:x="2079" fg:w="10"/><text x="99.7713%" y="575.50"></text></g><g><title>runtime.exitsyscallfast.func1 (10 samples, 0.48%)</title><rect x="99.5213%" y="549" width="0.4787%" height="15" fill="rgb(242,14,37)" fg:x="2079" fg:w="10"/><text x="99.7713%" y="559.50"></text></g><g><title>runtime.exitsyscallfast_pidle (10 samples, 0.48%)</title><rect x="99.5213%" y="533" width="0.4787%" height="15" fill="rgb(246,228,12)" fg:x="2079" fg:w="10"/><text x="99.7713%" y="543.50"></text></g><g><title>runtime.notewakeup (9 samples, 0.43%)</title><rect x="99.5692%" y="517" width="0.4308%" height="15" fill="rgb(213,55,15)" fg:x="2080" fg:w="9"/><text x="99.8192%" y="527.50"></text></g><g><title>runtime.futexwakeup (9 samples, 0.43%)</title><rect x="99.5692%" y="501" width="0.4308%" height="15" fill="rgb(209,9,3)" fg:x="2080" fg:w="9"/><text x="99.8192%" y="511.50"></text></g><g><title>runtime.futex.abi0 (9 samples, 0.43%)</title><rect x="99.5692%" y="485" width="0.4308%" height="15" fill="rgb(230,59,30)" fg:x="2080" fg:w="9"/><text x="99.8192%" y="495.50"></text></g><g><title>el0t_64_sync (9 samples, 0.43%)</title><rect x="99.5692%" y="469" width="0.4308%" height="15" fill="rgb(209,121,21)" fg:x="2080" fg:w="9"/><text x="99.8192%" y="479.50"></text></g><g><title>el0t_64_sync_handler (9 samples, 0.43%)</title><rect x="99.5692%" y="453" width="0.4308%" height="15" fill="rgb(220,109,13)" fg:x="2080" fg:w="9"/><text x="99.8192%" y="463.50"></text></g><g><title>el0_svc (9 samples, 0.43%)</title><rect x="99.5692%" y="437" width="0.4308%" height="15" fill="rgb(232,18,1)" fg:x="2080" fg:w="9"/><text x="99.8192%" y="447.50"></text></g><g><title>do_el0_svc (8 samples, 0.38%)</title><rect x="99.6170%" y="421" width="0.3830%" height="15" fill="rgb(215,41,42)" fg:x="2081" fg:w="8"/><text x="99.8670%" y="431.50"></text></g><g><title>invoke_syscall.constprop.0 (8 samples, 0.38%)</title><rect x="99.6170%" y="405" width="0.3830%" height="15" fill="rgb(224,123,36)" fg:x="2081" fg:w="8"/><text x="99.8670%" y="415.50"></text></g><g><title>__arm64_sys_futex (8 samples, 0.38%)</title><rect x="99.6170%" y="389" width="0.3830%" height="15" fill="rgb(240,125,3)" fg:x="2081" fg:w="8"/><text x="99.8670%" y="399.50"></text></g><g><title>do_futex (8 samples, 0.38%)</title><rect x="99.6170%" y="373" width="0.3830%" height="15" fill="rgb(205,98,50)" fg:x="2081" fg:w="8"/><text x="99.8670%" y="383.50"></text></g><g><title>futex_wake (8 samples, 0.38%)</title><rect x="99.6170%" y="357" width="0.3830%" height="15" fill="rgb(205,185,37)" fg:x="2081" fg:w="8"/><text x="99.8670%" y="367.50"></text></g><g><title>wake_up_q (8 samples, 0.38%)</title><rect x="99.6170%" y="341" width="0.3830%" height="15" fill="rgb(238,207,15)" fg:x="2081" fg:w="8"/><text x="99.8670%" y="351.50"></text></g><g><title>try_to_wake_up (8 samples, 0.38%)</title><rect x="99.6170%" y="325" width="0.3830%" height="15" fill="rgb(213,199,42)" fg:x="2081" fg:w="8"/><text x="99.8670%" y="335.50"></text></g></svg></svg>

main.go

package main

import (
	"log"
	"net"
	"os"
	"syscall"
	"time"
)

const okayAddr = "/tmp/unix-gc-storm.okay.sock"
const churnAddr = "/tmp/unix-gc-storm.churn.sock"
const scmAddr = "/tmp/unix-gc-storm.scm.sock"

func startOkayListener() net.Listener {
	os.Remove(okayAddr)
	listener, err := net.Listen("unix", okayAddr)
	if err != nil {
		panic(err)
	}

	return listener
}

func churnSockets() {
	os.Remove(churnAddr)
	listener, err := net.Listen("unix", churnAddr)
	if err != nil {
		panic(err)
	}

	for {
		client, err := net.Dial("unix", churnAddr)
		if err != nil {
			panic(err)
		}

		server, err := listener.Accept()
		if err != nil {
			panic(err)
		}

		for i := 0; i < 50; i++ {
			_, err = client.Write([]byte("hiiii"))
			if err != nil {
				panic(err)
			}
		}

		server.Close()
		client.Close()

		log.Println(".")

		time.Sleep(time.Millisecond * 50)
	}
}

func scm() net.Listener {
	os.Remove(scmAddr)
	listener, err := net.Listen("unix", scmAddr)
	if err != nil {
		panic(err)
	}

	scmConn, err := net.Dial("unix", scmAddr)
	if err != nil {
		panic(err)
	}

	scmConnFile, err := scmConn.(*net.UnixConn).File()
	if err != nil {
		panic(err)
	}

	// Use 159 for less load or 161 for more load.
	// The difference here is crossing the threshold:
	// * https://elixir.bootlin.com/linux/v6.1/source/net/unix/garbage.c#L198
	//
	// As long as there are more than 16k inflight sockets, unix_gc starts
	// to run in the unix socket write path, causing a lot more load:
	// * https://elixir.bootlin.com/linux/v6.1/source/net/unix/af_unix.c#L2158
	//
	// The load is applied to any socket, not just the offending one.
	for i := 0; i < 161; i++ {
		fds := []int{}

		for i := 0; i < 100; i++ {
			fd, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0)
			if err != nil {
				panic(err)
			}

			fds = append(fds, fd)
		}

		rights := syscall.UnixRights(fds...)

		err = syscall.Sendmsg(int(scmConnFile.Fd()), nil, rights, nil, 0)
		if err != nil {
			panic(err)
		}
	}

	return listener
}

func main() {
	okayListener := startOkayListener()

	scmListener := scm()

	churnSockets()

	okayListener.Close()
	scmListener.Close()

}