Use async iteration to iterate over large search results in Elasticsearch

scroll-search.ts

import {Client, SearchParams} from "elasticsearch";

/**
 * Iterate all search results one by one with async iteration without reading
 * it all to memory
 */
async function* scrollSearch<Document>(esClient: Client, params: SearchParams) {
    let res = await esClient.search<Document>(params);

    while (true) {
        const sourceHits = res.hits.hits;

        if (sourceHits.length === 0) {
            break;
        }

        for (const hit of sourceHits) {
            yield hit;
        }

        if (!res._scroll_id) {
            break;
        }

        res = await esClient.scroll<Document>({
            scrollId: res._scroll_id,
            scroll: params.scroll || "30s",
        });
    }
}

async function logAllDocuments(esClient: Client) {
    const body = {
        type: "foo",
        index: "bar",
        body: {
            match_all: {},
        },
    };

    for await (const hit of scrollSearch(esClient, {body})) {
        console.log(hit._source);
    }
}