src/lib/db/utils.ts


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

export async function handleFile(
  filename: string,
  func: (line: string, idx: number) => void,
) {
  const file = Bun.file(filename);
  const s = file.stream();
  const reader = s.getReader();
  const decoder = new TextDecoder();
  let leftover = "";
  let lineCount = 0;
  while (true) {
    const { value, done } = await reader.read();
    if (done) break;
    const chunk = decoder.decode(value, { stream: true });
    const lines = (leftover + chunk).split("\n");

    // Process each line except the last (which might be incomplete)
    for (const line of lines.slice(0, -1)) {
      lineCount++;
      func(line, lineCount);
    }

    // Save the last incomplete line to process in the next iteration
    leftover = lines[lines.length - 1];
  }

  // Handle any remaining content after reading all chunks
  if (leftover) func(leftover, lineCount + 1);
}