[object Object]

← back to Butlr

upload-watcher: auto-import from connected folder every 30s

986f634a474ea6c859ed92cfc1e3472943f2443b · 2026-05-13 11:12:18 -0700 · SteveStudio2

Replaces manual "Sync now" click with a background scan that runs on
the server startup loop.

- lib/upload-watcher.js: scanOnce() walks meta.folder, dedups by
  (size+mtime) per source_path, copies new allowed-ext files into
  data/uploads/ with unguessable hex filenames. Logged with source:
  'auto_watcher' so manual + watcher imports are distinguishable.
  start({intervalMs:30000}) sets the recurring tick. HFM_NO_WATCHER=1
  disables (used by tests). First scan fires 1s after server boot.
- server.js: wires uploadWatcher.start() right after twilio.start().
- test/upload-watcher.test.js: 7/7 — imports allowed exts, skips bad
  ext + hidden, idempotent on second scan, picks up new file, meta
  source_* fields populated, missing folder gracefully reported,
  no-folder config is a noop.
- package.json: npm test now runs 41 assertions across 5 files.

Steve drops images in the configured folder; within 30s they appear
in /admin/uploads with copy-URL chips.

Files touched

Diff

commit 986f634a474ea6c859ed92cfc1e3472943f2443b
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Wed May 13 11:12:18 2026 -0700

    upload-watcher: auto-import from connected folder every 30s
    
    Replaces manual "Sync now" click with a background scan that runs on
    the server startup loop.
    
    - lib/upload-watcher.js: scanOnce() walks meta.folder, dedups by
      (size+mtime) per source_path, copies new allowed-ext files into
      data/uploads/ with unguessable hex filenames. Logged with source:
      'auto_watcher' so manual + watcher imports are distinguishable.
      start({intervalMs:30000}) sets the recurring tick. HFM_NO_WATCHER=1
      disables (used by tests). First scan fires 1s after server boot.
    - server.js: wires uploadWatcher.start() right after twilio.start().
    - test/upload-watcher.test.js: 7/7 — imports allowed exts, skips bad
      ext + hidden, idempotent on second scan, picks up new file, meta
      source_* fields populated, missing folder gracefully reported,
      no-folder config is a noop.
    - package.json: npm test now runs 41 assertions across 5 files.
    
    Steve drops images in the configured folder; within 30s they appear
    in /admin/uploads with copy-URL chips.
---
 data/dnc-blocks.jsonl       |   3 ++
 lib/upload-watcher.js       | 103 +++++++++++++++++++++++++++++++++++++++++
 package.json                |   2 +-
 server.js                   |   8 ++++
 test/upload-watcher.test.js | 109 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 224 insertions(+), 1 deletion(-)

diff --git a/data/dnc-blocks.jsonl b/data/dnc-blocks.jsonl
index d06a430..a6c3f2a 100644
--- a/data/dnc-blocks.jsonl
+++ b/data/dnc-blocks.jsonl
@@ -6,3 +6,6 @@
 {"at":"2026-05-13T16:31:41.679Z","phone_last4":"1212","phone_hash":"f84b6f1e6c516e8e33b58e707a9447030b9dbf8d0b22b1cb1c71fd942e7a54a7","channel":"call","reason":"federal_dnc","source":"ftc_donotcall"}
 {"at":"2026-05-13T16:31:41.682Z","phone_last4":"1212","phone_hash":"260c2ac371cd9dc5031bb35ee4604b8a86898bdf0050854abbd83a4816e392ce","channel":"sms","reason":"internal_suppression","source":"internal"}
 {"at":"2026-05-13T16:31:41.684Z","phone_last4":"1212","phone_hash":"f84b6f1e6c516e8e33b58e707a9447030b9dbf8d0b22b1cb1c71fd942e7a54a7","channel":"call","reason":"internal_suppression","source":"internal"}
+{"at":"2026-05-13T18:12:07.062Z","phone_last4":"1212","phone_hash":"f84b6f1e6c516e8e33b58e707a9447030b9dbf8d0b22b1cb1c71fd942e7a54a7","channel":"call","reason":"federal_dnc","source":"ftc_donotcall"}
+{"at":"2026-05-13T18:12:07.065Z","phone_last4":"1212","phone_hash":"260c2ac371cd9dc5031bb35ee4604b8a86898bdf0050854abbd83a4816e392ce","channel":"sms","reason":"internal_suppression","source":"internal"}
+{"at":"2026-05-13T18:12:07.067Z","phone_last4":"1212","phone_hash":"f84b6f1e6c516e8e33b58e707a9447030b9dbf8d0b22b1cb1c71fd942e7a54a7","channel":"call","reason":"internal_suppression","source":"internal"}
diff --git a/lib/upload-watcher.js b/lib/upload-watcher.js
new file mode 100644
index 0000000..9397c78
--- /dev/null
+++ b/lib/upload-watcher.js
@@ -0,0 +1,103 @@
+// Background folder watcher — polls data/uploads-meta.json.folder every
+// intervalMs and imports any new file into the /admin/uploads gallery.
+//
+// Sharing the same dedup logic (size+mtime per source path) as the
+// POST /admin/uploads/folder/sync endpoint. Best-effort — file errors are
+// logged + skipped; the next scan retries.
+//
+// Wired in server.js after attachListenBridge(). HFM_NO_WATCHER=1
+// disables (useful for tests).
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const UPLOADS_DIR = path.join(__dirname, '..', 'data', 'uploads');
+const META_PATH = path.join(__dirname, '..', 'data', 'uploads-meta.json');
+const ALLOWED_EXT = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.pdf', '.mp3', '.wav', '.mp4', '.txt', '.csv', '.json']);
+const MAX_SIZE = 10 * 1024 * 1024;
+
+function readMeta() {
+  try { return JSON.parse(fs.readFileSync(META_PATH, 'utf8')); } catch { return { files: [] }; }
+}
+function writeMeta(meta) {
+  const tmp = META_PATH + '.tmp';
+  fs.writeFileSync(tmp, JSON.stringify(meta, null, 2));
+  fs.renameSync(tmp, META_PATH);
+}
+
+function scanOnce() {
+  const meta = readMeta();
+  if (!meta.folder) return { skipped_no_folder: true };
+  if (!fs.existsSync(meta.folder)) return { skipped_missing: meta.folder };
+
+  fs.mkdirSync(UPLOADS_DIR, { recursive: true });
+  const existingKeys = new Set((meta.files || [])
+    .filter(f => f.source_path)
+    .map(f => `${f.source_size}:${f.source_mtime}`));
+
+  let entries;
+  try { entries = fs.readdirSync(meta.folder); }
+  catch (e) { return { error: 'readdir: ' + e.message }; }
+
+  const newEntries = [];
+  let imported = 0, skipped = 0;
+  for (const name of entries) {
+    if (name.startsWith('.')) continue;
+    const srcPath = path.join(meta.folder, name);
+    let st;
+    try { st = fs.statSync(srcPath); } catch { continue; }
+    if (!st.isFile()) continue;
+    if (st.size > MAX_SIZE) { continue; }
+    const ext = (path.extname(name) || '').toLowerCase();
+    if (!ALLOWED_EXT.has(ext)) { skipped++; continue; }
+    const key = `${st.size}:${st.mtime.getTime()}`;
+    if (existingKeys.has(key)) { skipped++; continue; }
+    try {
+      const newName = crypto.randomBytes(12).toString('hex') + ext;
+      fs.copyFileSync(srcPath, path.join(UPLOADS_DIR, newName));
+      newEntries.push({
+        filename: newName,
+        original_name: name,
+        size: st.size,
+        type: 'application/octet-stream',
+        uploaded_at: new Date().toISOString(),
+        source_path: srcPath,
+        source_size: st.size,
+        source_mtime: st.mtime.getTime(),
+        source: 'auto_watcher',
+      });
+      imported++;
+    } catch (e) {
+      console.error(`[upload-watcher] copy failed: ${name}: ${e.message}`);
+    }
+  }
+  if (newEntries.length) {
+    meta.files = (meta.files || []).concat(newEntries);
+    writeMeta(meta);
+  }
+  return { imported, skipped, folder: meta.folder };
+}
+
+let _interval = null;
+function start({ intervalMs = 30_000 } = {}) {
+  if (process.env.HFM_NO_WATCHER === '1') return;
+  if (_interval) return;
+  console.log(`[upload-watcher] start · interval=${intervalMs}ms`);
+  _interval = setInterval(() => {
+    try {
+      const r = scanOnce();
+      if (r && r.imported > 0) {
+        console.log(`[upload-watcher] imported ${r.imported} from ${r.folder} (skipped ${r.skipped})`);
+      }
+    } catch (e) {
+      console.error('[upload-watcher] tick error:', e.message);
+    }
+  }, intervalMs);
+  // immediate first scan
+  setTimeout(() => { try { scanOnce(); } catch {} }, 1000);
+}
+
+function stop() { if (_interval) { clearInterval(_interval); _interval = null; } }
+
+module.exports = { start, stop, scanOnce };
diff --git a/package.json b/package.json
index af05d1b..4f39a77 100644
--- a/package.json
+++ b/package.json
@@ -7,7 +7,7 @@
   "scripts": {
     "start": "node server.js",
     "dev": "node server.js",
-    "test": "node test/orphan-recordings.test.js && node test/admin-gate.test.js && node test/vapi-webhook.test.js && node test/dnc-check.test.js",
+    "test": "node test/orphan-recordings.test.js && node test/admin-gate.test.js && node test/vapi-webhook.test.js && node test/dnc-check.test.js && node test/upload-watcher.test.js",
     "report": "node scripts/call-quality-report.js --remote"
   },
   "dependencies": {
diff --git a/server.js b/server.js
index 295cb36..9f8e3f4 100644
--- a/server.js
+++ b/server.js
@@ -172,4 +172,12 @@ server.listen(PORT, () => {
       console.error('[twilio] failed to start worker:', e.message);
     }
   }
+  // Background folder watcher — auto-imports new files from the
+  // configured /admin/uploads folder every 30s. HFM_NO_WATCHER=1 disables.
+  try {
+    const uploadWatcher = require('./lib/upload-watcher');
+    uploadWatcher.start({ intervalMs: 30_000 });
+  } catch (e) {
+    console.error('[upload-watcher] failed to start:', e.message);
+  }
 });
diff --git a/test/upload-watcher.test.js b/test/upload-watcher.test.js
new file mode 100644
index 0000000..a0fbdb9
--- /dev/null
+++ b/test/upload-watcher.test.js
@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+// Unit test for lib/upload-watcher.scanOnce().
+// Builds a tmp meta + watched folder, asserts new files get imported,
+// existing files (by size+mtime) get skipped, unsupported ext gets skipped.
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const assert = require('assert');
+
+process.env.HFM_NO_WATCHER = '1';
+
+function ok(name, fn) {
+  try { fn(); console.log(`  ✓ ${name}`); return 1; }
+  catch (e) { console.error(`  ✗ ${name}`); console.error('     ', e.message); return 0; }
+}
+
+const ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'butlr-upload-watch-'));
+const WATCHED = path.join(ROOT, 'inbox');
+const UPLOADS = path.join(ROOT, 'uploads');
+const META = path.join(ROOT, 'uploads-meta.json');
+fs.mkdirSync(WATCHED, { recursive: true });
+fs.mkdirSync(UPLOADS, { recursive: true });
+
+// Seed input files
+fs.writeFileSync(path.join(WATCHED, 'a.png'), 'fakea');
+fs.writeFileSync(path.join(WATCHED, 'b.jpg'), 'fakeb');
+fs.writeFileSync(path.join(WATCHED, 'c.exe'), 'badext');  // should skip
+fs.writeFileSync(path.join(WATCHED, '.hidden'), 'hide');   // should skip
+
+// Patch __dirname-relative paths in the module by intercepting fs reads/writes
+// of the upload paths. Simpler: stub the require to substitute paths.
+// Cleanest approach: use the env-var pattern, but the module hardcodes paths.
+// Workaround: set up a fake project root in tmp with data/uploads/ structure
+// and chdir to it.
+const FAKE_ROOT = path.join(ROOT, 'project');
+fs.mkdirSync(path.join(FAKE_ROOT, 'data', 'uploads'), { recursive: true });
+fs.mkdirSync(path.join(FAKE_ROOT, 'lib'), { recursive: true });
+// Symlink the lib file
+fs.copyFileSync(path.join(__dirname, '..', 'lib', 'upload-watcher.js'),
+                path.join(FAKE_ROOT, 'lib', 'upload-watcher.js'));
+
+// Write meta pointing at WATCHED
+fs.writeFileSync(path.join(FAKE_ROOT, 'data', 'uploads-meta.json'),
+  JSON.stringify({ files: [], folder: WATCHED }, null, 2));
+
+const watcher = require(path.join(FAKE_ROOT, 'lib', 'upload-watcher.js'));
+
+let pass = 0, total = 0;
+
+// ── 1. Fresh folder — 2 files imported, 1 skipped (bad ext) ─────────
+total++; pass += ok('imports allowed ext, skips badext + hidden', () => {
+  const r = watcher.scanOnce();
+  assert.strictEqual(r.imported, 2, 'should import 2 files');
+  assert.strictEqual(r.skipped, 1, 'should skip 1 (bad ext)');
+});
+
+// ── 2. Idempotent — second scan skips everything ────────────────────
+total++; pass += ok('second scan is idempotent (same files skipped)', () => {
+  const r = watcher.scanOnce();
+  assert.strictEqual(r.imported, 0, 'second scan must not re-import');
+});
+
+// ── 3. New file added → imported ────────────────────────────────────
+total++; pass += ok('new file gets imported on next scan', () => {
+  fs.writeFileSync(path.join(WATCHED, 'd.webp'), 'newpic');
+  const r = watcher.scanOnce();
+  assert.strictEqual(r.imported, 1);
+});
+
+// ── 4. Meta updated with source_path/size/mtime ─────────────────────
+total++; pass += ok('meta entries have source_* fields for dedupe', () => {
+  const meta = JSON.parse(fs.readFileSync(path.join(FAKE_ROOT, 'data', 'uploads-meta.json'), 'utf8'));
+  assert.strictEqual(meta.files.length, 3);
+  for (const f of meta.files) {
+    assert.ok(f.source_path, 'source_path required');
+    assert.ok(typeof f.source_size === 'number');
+    assert.ok(typeof f.source_mtime === 'number');
+    assert.strictEqual(f.source, 'auto_watcher');
+  }
+});
+
+// ── 5. Filenames are 24-hex unguessable ─────────────────────────────
+total++; pass += ok('filenames are 24-hex + valid ext', () => {
+  const meta = JSON.parse(fs.readFileSync(path.join(FAKE_ROOT, 'data', 'uploads-meta.json'), 'utf8'));
+  for (const f of meta.files) {
+    assert.ok(/^[a-f0-9]{24}\.[a-z]+$/.test(f.filename), 'bad filename: ' + f.filename);
+  }
+});
+
+// ── 6. Missing folder gracefully reported ───────────────────────────
+total++; pass += ok('missing folder returns skipped_missing', () => {
+  const meta = JSON.parse(fs.readFileSync(path.join(FAKE_ROOT, 'data', 'uploads-meta.json'), 'utf8'));
+  meta.folder = '/no/such/path/anywhere';
+  fs.writeFileSync(path.join(FAKE_ROOT, 'data', 'uploads-meta.json'), JSON.stringify(meta));
+  const r = watcher.scanOnce();
+  assert.ok(r.skipped_missing);
+});
+
+// ── 7. No folder configured → graceful noop ─────────────────────────
+total++; pass += ok('no folder configured → skipped_no_folder', () => {
+  fs.writeFileSync(path.join(FAKE_ROOT, 'data', 'uploads-meta.json'), JSON.stringify({ files: [] }));
+  const r = watcher.scanOnce();
+  assert.strictEqual(r.skipped_no_folder, true);
+});
+
+fs.rmSync(ROOT, { recursive: true, force: true });
+console.log(`\n${pass}/${total} passed`);
+process.exit(pass === total ? 0 : 1);

← 81ccd55 uploads: folder-sync — one-shot scan + "Sync now" button  ·  back to Butlr  ·  snapshot: backup uncommitted work (4 files) 96ffa09 →