vecfinder

commit 062d92b04571a4f93163cafab7756ced100ad282
parent 56f7e19439a079a67c5d5c873a7353945f0771a2
Author: finwo <finwo@pm.me>
Date:   Sat, 21 Mar 2026 04:39:51 +0100

Parallelism & spinner

Diffstat:
M
vecfinder.js
 | 
71
+++++++++++++++++++++++++++++++++++++++++++++++++++--------------------

1 file changed, 51 insertions(+), 20 deletions(-)
diff --git a/vecfinder.js b/vecfinder.js
@@ -137,6 +137,7 @@ class DocumentStore extends EventEmitter {
     super();
     this.documents = new Map();
     this.queryVec = null;
+    this.consumersActive = 0;
   }
 
   async addDocument(id, content, metadata = {}, vector) {
@@ -225,28 +226,46 @@ async function walkDir(dirPath) {
   return files;
 }
 
-async function indexDirectory(store, dirPath, ui) {
+async function indexDirectory(store, dirPath) {
   const files = await walkDir(dirPath);
+  const queue = [];
 
-  for (const filePath of files) {
-    try {
-      const stats = await fs.stat(filePath);
-      if (stats.size > 1024 * 1024) continue;
-      if (!(await isTextFile(filePath))) continue;
+  function consumer() {
+    if (queue.length === 0) return;
+    if (store.consumersActive >= config.parallelism) return;
 
-      const content = await fs.readFile(filePath, 'utf8');
-      const id = path.relative(dirPath, filePath);
+    store.consumersActive++;
+    const filePath = queue.shift();
 
-      let vector = await getCachedVector(filePath);
-      if (vector) {
-        await store.addDocument(id, content, { path: filePath }, vector);
-      } else {
-        const raw = await fetchEmbedding(content);
-        vector = normalize(raw);
-        await store.addDocument(id, content, { path: filePath }, vector);
-        await setCachedVector(filePath, vector);
-      }
-    } catch {}
+    (async () => {
+      try {
+        const content = await fs.readFile(filePath, 'utf8');
+        const id = path.relative(dirPath, filePath);
+
+        let vector = await getCachedVector(filePath);
+        if (vector) {
+          await store.addDocument(id, content, { path: filePath }, vector);
+        } else {
+          const raw = await fetchEmbedding(content);
+          vector = normalize(raw);
+          await store.addDocument(id, content, { path: filePath }, vector);
+          await setCachedVector(filePath, vector);
+        }
+      } catch {}
+
+      store.consumersActive--;
+      store.emit('documentAdded');
+      setImmediate(consumer);
+    })();
+  }
+
+  for (const filePath of files) {
+    const stats = await fs.stat(filePath).catch(() => null);
+    if (!stats || stats.size > 1024 * 1024) continue;
+    const text = await isTextFile(filePath);
+    if (!text) continue;
+    queue.push(filePath);
+    setImmediate(consumer);
   }
 }
 
@@ -261,6 +280,8 @@ class UI {
     this._previewWrapped = null;
     this._previewFileId = null;
     this._previewWidth = null;
+    this.spinnerFrame = 0;
+    this.spinnerInterval = null;
 
     this.store.on('documentAdded', () => this.render());
     this.store.on('queryUpdated', () => {
@@ -275,10 +296,15 @@ class UI {
     stdin.on('keypress', (str, key) => this.handleKeyPress(str, key));
     this.rl.on('line', () => this.handleEnter());
     this.rl.on('close', () => this.cleanup());
+    this.spinnerInterval = setInterval(() => {
+      this.spinnerFrame++;
+      this.render();
+    }, 80);
     this.render();
   }
 
   cleanup() {
+    clearInterval(this.spinnerInterval);
     stdin.removeAllListeners('keypress');
     if (this.rl) this.rl.close();
   }
@@ -359,7 +385,12 @@ class UI {
     const results = this.store.getResults();
 
     // Header
-    stderr.write('VecFinder\n');
+    const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+    if (this.store.consumersActive > 0) {
+      stderr.write(`${frames[this.spinnerFrame % frames.length]} VecFinder\n`);
+    } else {
+      stderr.write('\u2713 VecFinder\n');
+    }
     stderr.write('─'.repeat(leftWidth) + '┬' + '─'.repeat(rightWidth) + '\n');
 
     // Rows
@@ -432,7 +463,7 @@ async function main() {
 
   ui.init();
 
-  indexDirectory(store, process.cwd(), ui);
+  indexDirectory(store, process.cwd());
 }
 
 main().catch(err => {