[object Object]

← back to Astek Landing

add monthly astek refresh: scrape->rebuild->deploy wrapper + launchd plist (1st of month 04:30)

7135c50a91ba63353cb351d1ff4096a5ea197e5e · 2026-07-12 23:59:45 -0700 · Steve

Files touched

Diff

commit 7135c50a91ba63353cb351d1ff4096a5ea197e5e
Author: Steve <steve@designerwallcoverings.com>
Date:   Sun Jul 12 23:59:45 2026 -0700

    add monthly astek refresh: scrape->rebuild->deploy wrapper + launchd plist (1st of month 04:30)
---
 launchd/com.steve.astek-monthly-refresh.plist | 23 +++++++++++++++
 scripts/monthly-refresh.sh                    | 41 +++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)

diff --git a/launchd/com.steve.astek-monthly-refresh.plist b/launchd/com.steve.astek-monthly-refresh.plist
new file mode 100644
index 0000000..284b4da
--- /dev/null
+++ b/launchd/com.steve.astek-monthly-refresh.plist
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0"><dict>
+  <key>Label</key><string>com.steve.astek-monthly-refresh</string>
+  <key>ProgramArguments</key>
+    <array>
+      <string>/bin/bash</string>
+      <string>/Users/macstudio3/Projects/astek-landing/scripts/monthly-refresh.sh</string>
+    </array>
+  <key>WorkingDirectory</key><string>/Users/macstudio3/Projects/astek-landing</string>
+  <!-- 1st of every month, 04:30 local -->
+  <key>StartCalendarInterval</key>
+    <dict>
+      <key>Day</key><integer>1</integer>
+      <key>Hour</key><integer>4</integer>
+      <key>Minute</key><integer>30</integer>
+    </dict>
+  <key>RunAtLoad</key><false/>
+  <key>StandardOutPath</key><string>/tmp/astek-monthly-refresh.log</string>
+  <key>StandardErrorPath</key><string>/tmp/astek-monthly-refresh.log</string>
+  <key>EnvironmentVariables</key>
+    <dict><key>PATH</key><string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin</string></dict>
+</dict></plist>
diff --git a/scripts/monthly-refresh.sh b/scripts/monthly-refresh.sh
new file mode 100755
index 0000000..97ef10b
--- /dev/null
+++ b/scripts/monthly-refresh.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Astek monthly full refresh: re-scrape astek.com -> astek_catalog -> rebuild
+# snapshot -> deploy the fresh data to the live (gated, internal) microsite.
+#
+# The existing com.steve.astek-data-refresh only REBUILDS the local snapshot
+# from the DB every 15 min; it never re-scrapes the vendor and never deploys.
+# This monthly job closes both gaps. Scrape+rebuild are $0 local; the deploy
+# targets an internal basic-auth microsite (no Shopify, no prices) so an
+# unattended monthly push is low-risk.
+set -uo pipefail
+
+# launchd hands us a minimal PATH — restore node + ssh/rsync (deploy.sh needs them).
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+
+PROJ="/Users/macstudio3/Projects/astek-landing"
+LOG="/tmp/astek-monthly-refresh.log"
+cd "$PROJ" || { echo "$(date '+%F %T') FATAL cd $PROJ failed" >>"$LOG"; exit 1; }
+
+echo "===== $(date '+%F %T') astek monthly refresh START =====" >>"$LOG"
+
+echo "[1/3] scrape astek.com -> astek_catalog" >>"$LOG"
+if ! node scripts/scrape-astek.js >>"$LOG" 2>&1; then
+  echo "$(date '+%F %T') SCRAPE FAILED — aborting (snapshot + live left untouched)" >>"$LOG"; exit 1
+fi
+
+echo "[2/3] rebuild data/products.json" >>"$LOG"
+if ! node scripts/build-products-json.js >>"$LOG" 2>&1; then
+  echo "$(date '+%F %T') REBUILD FAILED — aborting before deploy" >>"$LOG"; exit 1
+fi
+
+# Commit the refreshed data so the deploy ships a tracked snapshot.
+git add data/products.json data/astek-raw.json 2>/dev/null
+git -c user.email="steve@designerwallcoverings.com" \
+    commit -q -m "data: astek monthly refresh $(date '+%F')" 2>/dev/null || true
+
+echo "[3/3] deploy fresh snapshot to Kamatera" >>"$LOG"
+if ! bash /Users/macstudio3/Projects/_shared/scripts/deploy.sh >>"$LOG" 2>&1; then
+  echo "$(date '+%F %T') DEPLOY FAILED — local snapshot is fresh, live is stale" >>"$LOG"; exit 1
+fi
+
+echo "===== $(date '+%F %T') astek monthly refresh DONE =====" >>"$LOG"

← e5c3452 data: astek refresh 2026-07-13 (6187 variants re-scraped, sn  ·  back to Astek Landing  ·  auto-save: 2026-07-13T00:21:34 (1 files) — data/products.jso f3610c2 →