how to insert large csv dataset in elastic using nodejs

insertLargeCsv.js

const { Client } = require('@elastic/elasticsearch');
const fs = require('fs');
const readline = require('readline');
const csv = require('csv-parser');

const client = new Client({ node: 'http://localhost:9200' });

const filePath = '/path/to/large/csv/file'; // Replace with the path to your CSV file
const chunkSize = 100000; // Replace with the desired chunk size

const rl = readline.createInterface({
  input: fs.createReadStream(filePath),
  crlfDelay: Infinity
});

let chunkIndex = 0;
let currentChunk = [];
rl.on('line', (line) => {
  currentChunk.push(line);
  if (currentChunk.length >= chunkSize) {
    insertChunk(currentChunk, chunkIndex);
    currentChunk = [];
    chunkIndex++;
  }
});

rl.on('close', () => {
  if (currentChunk.length > 0) {
    insertChunk(currentChunk, chunkIndex);
  }
});

function insertChunk(chunk, index) {
  const body = [];
  chunk.forEach((line) => {
    const fields = line.split(',');
    body.push({
      index: { _index: 'myindex', _id: index },
    });
    body.push({
      title: fields[0],
      // Add other fields here as needed
    });
    index++;
  });
  client.bulk({ body })
    .then(response => console.log(`Chunk ${index} inserted successfully`))
    .catch(error => console.log(`Error inserting chunk ${index}:`, error));
}

### for smaller files

const fs = require("fs");
const csv = require("csv-parser");
const { Client } = require("@elastic/elasticsearch");

const client = new Client({ node: "http://localhost:9200" });

fs.createReadStream("./hotflix.csv")
  .pipe(csv())
  .on("data", function (row) {
    client.index({
      index: "holamovies",
      body: row,
    });
  })
  .on("end", function () {
    console.log("csv inserted successfully");
  });