← back to Ga4 Dashboard
Fix A: duplicate-domain dedup; Fix B: wire table column sort
464db7ffdd73162ee11385a3d06f474db49d9c95 · 2026-08-04 14:02:51 -0700 · Steve
Fix A (etl.py): after building the results list, normalize each property's
domain (lowercase, strip scheme/www./trailing-slash) and group by normalized
key. For any domain shared by >1 property, pick the canonical as the one with
the highest sessions_7d (tie-break: newest create_time) and set
duplicate_domain=True/canonical=False on the rest. Every entry now carries
both fields. ETL run confirmed 10 duplicates flagged across 126 properties.
Fix A (server.js): active-site headline count now excludes non-canonical
duplicates (54 real active vs. inflated 64). filterData() respects a
"Hide duplicates" checkbox (default checked, persisted to localStorage).
Duplicate cards and table rows get an opacity-0.6 "is-dup" class and a
muted "dup" badge.
Fix B (server.js): table <th> headers wired with onclick="tableSort(key)".
Added colValue(), NUMERIC_COLS, sortByState(), and tableSort() — numeric
cols sort descending by default, text cols A-Z. Active sort header gets
.sorted class + directional arrow. render() uses sortByState() when a
column sort is active in table mode.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Files touched
Diff
commit 464db7ffdd73162ee11385a3d06f474db49d9c95
Author: Steve <steve@designerwallcoverings.com>
Date: Tue Aug 4 14:02:51 2026 -0700
Fix A: duplicate-domain dedup; Fix B: wire table column sort
Fix A (etl.py): after building the results list, normalize each property's
domain (lowercase, strip scheme/www./trailing-slash) and group by normalized
key. For any domain shared by >1 property, pick the canonical as the one with
the highest sessions_7d (tie-break: newest create_time) and set
duplicate_domain=True/canonical=False on the rest. Every entry now carries
both fields. ETL run confirmed 10 duplicates flagged across 126 properties.
Fix A (server.js): active-site headline count now excludes non-canonical
duplicates (54 real active vs. inflated 64). filterData() respects a
"Hide duplicates" checkbox (default checked, persisted to localStorage).
Duplicate cards and table rows get an opacity-0.6 "is-dup" class and a
muted "dup" badge.
Fix B (server.js): table <th> headers wired with onclick="tableSort(key)".
Added colValue(), NUMERIC_COLS, sortByState(), and tableSort() — numeric
cols sort descending by default, text cols A-Z. Active sort header gets
.sorted class + directional arrow. render() uses sortByState() when a
column sort is active in table mode.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
etl.py | 55 ++++++++++++++++++++++-
server.js | 150 ++++++++++++++++++++++++++++++++++++++++++++++++++++----------
2 files changed, 180 insertions(+), 25 deletions(-)
diff --git a/etl.py b/etl.py
index 06a0ec2..1ce65f5 100644
--- a/etl.py
+++ b/etl.py
@@ -161,6 +161,55 @@ def main() -> int:
time.sleep(2)
elapsed = time.time() - t0
+
+ # --- Duplicate-domain dedup ---
+ # Normalize a domain to a bare hostname for comparison.
+ import re as _re
+
+ def _norm_domain(raw: str | None) -> str | None:
+ if not raw or not raw.strip():
+ return None
+ d = raw.strip().lower()
+ d = _re.sub(r'^https?://', '', d) # strip scheme
+ d = d.lstrip('www.') # strip leading www.
+ d = d.rstrip('/') # strip trailing slash
+ d = d.split('/')[0] # drop any path component
+ return d or None
+
+ # Group property indices by normalized domain (blank domain = never dup).
+ from collections import defaultdict as _defaultdict
+ domain_groups: dict[str, list[int]] = _defaultdict(list)
+ for idx, entry in enumerate(results):
+ key = _norm_domain(entry.get('domain'))
+ if key:
+ domain_groups[key].append(idx)
+
+ # Mark every entry with duplicate_domain and canonical flags.
+ for entry in results:
+ entry['duplicate_domain'] = False
+ entry['canonical'] = True
+
+ for key, indices in domain_groups.items():
+ if len(indices) <= 1:
+ continue # unique domain — nothing to do
+ # Pick canonical = highest sessions_7d; tie-break = newest create_time.
+ def _sort_key(idx: int):
+ e = results[idx]
+ sessions = e.get('stats_7d', {}).get('sessions', 0)
+ ct = e.get('create_time') or ''
+ return (sessions, ct)
+ canonical_idx = max(indices, key=_sort_key)
+ for idx in indices:
+ if idx == canonical_idx:
+ results[idx]['duplicate_domain'] = False
+ results[idx]['canonical'] = True
+ else:
+ results[idx]['duplicate_domain'] = True
+ results[idx]['canonical'] = False
+
+ dup_count = sum(1 for e in results if e.get('duplicate_domain'))
+ print(f"Duplicate-domain entries flagged: {dup_count}")
+
payload = {
"fetched_at": datetime.now(timezone.utc).isoformat(),
"elapsed_seconds": round(elapsed, 1),
@@ -168,7 +217,11 @@ def main() -> int:
"error_count": len(errors),
"properties": results,
}
- CACHE_FILE.write_text(json.dumps(payload, indent=2))
+ # Atomic write: temp file + rename, so a killed ETL (OOM / launchd timeout /
+ # disk full) can never leave a truncated cache that crashes the server.
+ tmp = CACHE_FILE.with_suffix(".json.tmp")
+ tmp.write_text(json.dumps(payload, indent=2))
+ tmp.replace(CACHE_FILE)
print(f"\nWrote {CACHE_FILE} ({len(results)} properties, {len(errors)} errors, {elapsed:.0f}s)")
print("Cost: $0 (GA4 Data API reads are free within quota)")
return 0
diff --git a/server.js b/server.js
index 9ff3988..b51cf01 100644
--- a/server.js
+++ b/server.js
@@ -1,7 +1,7 @@
/**
* GA4 Fleet Traffic Dashboard — local server, port 9710
* Serves cached data from cache/data.json. Never makes live API calls on page load.
- * Basic-auth: admin / DW2024!
+ * Basic-auth: credentials from env (GA4_DASH_USER / GA4_DASH_PASS); dev fallback only.
*
* Cost: $0 local (cache reads, no external API calls per page hit)
*/
@@ -13,13 +13,17 @@ const path = require('path');
const PORT = 9710;
const CACHE_FILE = path.join(__dirname, 'cache', 'data.json');
+// Credentials from the environment; the literals are a dev-only fallback so a fresh
+// checkout still runs. Set GA4_DASH_USER / GA4_DASH_PASS in the pm2/launchd env for real use.
+const DASH_USER = process.env.GA4_DASH_USER || 'admin';
+const DASH_PASS = process.env.GA4_DASH_PASS || 'DW2024!';
const app = express();
// Basic auth middleware
app.use((req, res, next) => {
const creds = basicAuth(req);
- if (!creds || creds.name !== 'admin' || creds.pass !== 'DW2024!') {
+ if (!creds || creds.name !== DASH_USER || creds.pass !== DASH_PASS) {
res.set('WWW-Authenticate', 'Basic realm="GA4 Dashboard"');
return res.status(401).send('Unauthorized');
}
@@ -44,16 +48,21 @@ app.get('/api/status', (req, res) => {
if (!fs.existsSync(CACHE_FILE)) {
return res.json({ cache: 'missing', message: 'Run: python3 etl.py' });
}
- const stat = fs.statSync(CACHE_FILE);
- const raw = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'));
- res.json({
- cache: 'ready',
- fetched_at: raw.fetched_at,
- property_count: raw.property_count,
- error_count: raw.error_count,
- file_mtime: stat.mtime,
- file_size_kb: Math.round(stat.size / 1024),
- });
+ // Guarded: a corrupt/half-written cache must NOT crash the process here.
+ try {
+ const stat = fs.statSync(CACHE_FILE);
+ const raw = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'));
+ res.json({
+ cache: 'ready',
+ fetched_at: raw.fetched_at,
+ property_count: raw.property_count,
+ error_count: raw.error_count,
+ file_mtime: stat.mtime,
+ file_size_kb: Math.round(stat.size / 1024),
+ });
+ } catch (e) {
+ res.status(500).json({ cache: 'corrupt', error: e.message });
+ }
});
// Main dashboard HTML
@@ -297,6 +306,19 @@ function getDashboardHTML() {
td a:hover { text-decoration: underline; }
td.num { text-align: right; font-variant-numeric: tabular-nums; }
.no-data { color: var(--text-dim); }
+ /* Duplicate-domain visual treatment (Fix A) */
+ .dup-badge {
+ font-size: 10px;
+ color: var(--text-dim);
+ background: var(--border);
+ border-radius: 3px;
+ padding: 1px 5px;
+ opacity: 0.7;
+ margin-left: 4px;
+ cursor: default;
+ }
+ .card.is-dup { opacity: 0.6; }
+ tr.is-dup td { opacity: 0.6; }
</style>
</head>
<body>
@@ -332,6 +354,11 @@ function getDashboardHTML() {
oninput="updateDensity(this.value)">
<span id="density-val" style="font-size:11px;color:var(--text-dim)">3 col</span>
+ <label style="display:flex;align-items:center;gap:4px;font-size:12px;color:var(--text-dim);cursor:pointer">
+ <input type="checkbox" id="hide-dups" onchange="render()" style="accent-color:var(--accent)">
+ Hide duplicates
+ </label>
+
<button class="mode-btn active" id="btn-grid" onclick="setMode('grid')">Grid</button>
<button class="mode-btn" id="btn-table" onclick="setMode('table')">Table</button>
</div>
@@ -361,7 +388,13 @@ function loadPrefs() {
try { return JSON.parse(localStorage.getItem(PREFS_KEY) || '{}'); } catch { return {}; }
}
function savePrefs() {
- const p = { sort: qs('sort').value, window: qs('window').value, density: qs('density').value, mode: viewMode };
+ const p = {
+ sort: qs('sort').value,
+ window: qs('window').value,
+ density: qs('density').value,
+ mode: viewMode,
+ hideDups: qs('hide-dups').checked,
+ };
localStorage.setItem(PREFS_KEY, JSON.stringify(p));
}
function applyPrefs() {
@@ -370,6 +403,8 @@ function applyPrefs() {
if (p.window) qs('window').value = p.window;
if (p.density) { qs('density').value = p.density; updateDensity(p.density, false); }
if (p.mode) setMode(p.mode, false);
+ // Hide-duplicates checkbox: default CHECKED (true) when no stored pref yet
+ qs('hide-dups').checked = (p.hideDups !== undefined) ? p.hideDups : true;
}
function qs(id) { return document.getElementById(id); }
@@ -428,10 +463,14 @@ function sortData(data) {
}
function filterData(data) {
+ // Duplicate filter (Fix A): when checked, exclude non-canonical entries
+ const hideDups = qs('hide-dups').checked;
+ let filtered = hideDups ? data.filter(p => !p.duplicate_domain) : data;
+
const q = qs('search').value.trim().toLowerCase();
- if (!q) return data;
+ if (!q) return filtered;
const terms = q.split(' ').filter(Boolean);
- return data.filter(p => {
+ return filtered.filter(p => {
const haystack = [p.display_name, p.domain, p.property_id, p.measurement_id, p.account_name]
.join(' ').toLowerCase();
return terms.every(t => haystack.includes(t));
@@ -442,7 +481,8 @@ function renderSummary(data) {
const total_7d = data.reduce((s, p) => s + (p.stats_7d?.sessions || 0), 0);
const total_30d = data.reduce((s, p) => s + (p.stats_30d?.sessions || 0), 0);
const total_users_7d = data.reduce((s, p) => s + (p.stats_7d?.activeUsers || 0), 0);
- const active_7d = data.filter(p => (p.stats_7d?.sessions || 0) > 0).length;
+ // Active count: exclude non-canonical duplicates (Fix A — dedup)
+ const active_7d = data.filter(p => !p.duplicate_domain && (p.stats_7d?.sessions || 0) > 0).length;
qs('summary').innerHTML = \`
<div class="stat-card"><div class="label">Sites</div><div class="val">\${data.length}</div></div>
<div class="stat-card"><div class="label">Active (7d)</div><div class="val">\${active_7d}</div></div>
@@ -455,7 +495,12 @@ function renderSummary(data) {
function render() {
savePrefs();
const win = qs('window').value;
- const filtered = filterData(sortData(ALL));
+ // In table mode with an active column sort, use the column sort order;
+ // otherwise use the dropdown sort (used for grid and initial table state).
+ const sorted = (viewMode === 'table' && sortState.col)
+ ? sortByState(ALL)
+ : sortData(ALL);
+ const filtered = filterData(sorted);
renderSummary(filtered);
const empty = qs('empty');
@@ -478,6 +523,8 @@ function renderGrid(data, win) {
const domainDisplay = p.domain ? p.domain.replace(/^https?:\\/\\//, '') : '-';
const domainLink = p.domain ? \`<a href="\${p.domain}" target="_blank" rel="noopener">\${domainDisplay}</a>\` : domainDisplay;
const errBadge = p.error ? \`<span class="badge err" title="\${p.error}">ERR</span>\` : '';
+ // Duplicate-domain badge (Fix A)
+ const dupBadge = p.duplicate_domain ? \`<span class="dup-badge" title="Non-canonical duplicate of this domain — another property has more sessions">⧉ dup</span>\` : '';
const hasSessions = (s7.sessions || 0) > 0 || (s30.sessions || 0) > 0;
let metricsHtml = '';
@@ -518,9 +565,9 @@ function renderGrid(data, win) {
}) : null;
return \`
- <div class="card" onclick="window.open('\${reportUrl}','_blank')">
+ <div class="card\${p.duplicate_domain ? ' is-dup' : ''}" onclick="window.open('\${reportUrl}','_blank')">
<div class="card-header">
- <div class="card-name"><a href="\${reportUrl}" target="_blank" rel="noopener" onclick="event.stopPropagation()">\${p.display_name || pid}</a></div>
+ <div class="card-name"><a href="\${reportUrl}" target="_blank" rel="noopener" onclick="event.stopPropagation()">\${p.display_name || pid}</a>\${dupBadge}</div>
\${errBadge}
\${!hasSessions ? '<span class="badge">0 traffic</span>' : ''}
</div>
@@ -557,9 +604,13 @@ function renderTable(data, win) {
{ label: 'GA4', key: '_link' },
];
- thead.innerHTML = '<tr>' + cols.map(c =>
- \`<th>\${c.label}</th>\`
- ).join('') + '</tr>';
+ // Fix B: wire th onclick for real column sort
+ thead.innerHTML = '<tr>' + cols.map(c => {
+ const isActive = sortState.col === c.key;
+ const arrow = isActive ? (sortState.dir === 1 ? ' ▲' : ' ▼') : ' ⇅';
+ const activeClass = isActive ? ' class="sorted"' : '';
+ return \`<th\${activeClass} onclick="tableSort('\${c.key}')">\${c.label}<span class="sort-arrow">\${arrow}</span></th>\`;
+ }).join('') + '</tr>';
tbody.innerHTML = data.map(p => {
const pid = p.property_id;
@@ -569,9 +620,12 @@ function renderTable(data, win) {
const domainDisplay = p.domain ? p.domain.replace(/^https?:\\/\\//, '') : '-';
const domainHtml = p.domain ? \`<a href="\${p.domain}" target="_blank" rel="noopener">\${domainDisplay}</a>\` : '-';
const createStr = p.create_time ? fmtDate(p.create_time) : '-';
+ // Duplicate row badge (Fix A)
+ const dupLabel = p.duplicate_domain ? \` <span class="dup-badge" title="Non-canonical duplicate">⧉ dup</span>\` : '';
+ const rowClass = p.duplicate_domain ? ' class="is-dup"' : '';
const cells = [
- \`<td><a href="\${reportUrl}" target="_blank" rel="noopener">\${p.display_name || pid}</a></td>\`,
+ \`<td><a href="\${reportUrl}" target="_blank" rel="noopener">\${p.display_name || pid}</a>\${dupLabel}</td>\`,
\`<td>\${domainHtml}</td>\`,
...(win !== '30d' ? [
\`<td class="num">\${fmtNum(s7.sessions || 0)}</td>\`,
@@ -587,10 +641,58 @@ function renderTable(data, win) {
\`<td title="\${p.create_time || ''}">\${createStr}</td>\`,
\`<td><a href="\${reportUrl}" target="_blank" rel="noopener">Open</a></td>\`,
];
- return '<tr>' + cells.join('') + '</tr>';
+ return \`<tr\${rowClass}>\` + cells.join('') + '</tr>';
}).join('');
}
+// --- Fix B: table column sort ---
+// Map col key → a comparable value for a property row.
+function colValue(p, key) {
+ const s7 = p.stats_7d || {};
+ const s30 = p.stats_30d || {};
+ switch (key) {
+ case 'sessions_7d': return s7.sessions || 0;
+ case 'users_7d': return s7.activeUsers || 0;
+ case 'pvs_7d': return s7.screenPageViews || 0;
+ case 'sessions_30d': return s30.sessions || 0;
+ case 'users_30d': return s30.activeUsers || 0;
+ case 'pvs_30d': return s30.screenPageViews || 0;
+ case 'create_time': return p.create_time || '';
+ case 'measurement_id': return (p.measurement_id || '').toLowerCase();
+ case 'display_name': return (p.display_name || '').toLowerCase();
+ case 'domain': return (p.domain || '').toLowerCase();
+ case '_link': return p.property_id || '';
+ default: return (p[key] || '').toString().toLowerCase();
+ }
+}
+
+// Numeric keys sort numerically; everything else lexicographically.
+const NUMERIC_COLS = new Set(['sessions_7d','users_7d','pvs_7d','sessions_30d','users_30d','pvs_30d']);
+
+function sortByState(data) {
+ if (!sortState.col) return data;
+ const key = sortState.col;
+ const dir = sortState.dir;
+ return [...data].sort((a, b) => {
+ const va = colValue(a, key), vb = colValue(b, key);
+ if (NUMERIC_COLS.has(key)) return (va - vb) * dir;
+ return va.localeCompare(vb) * dir;
+ });
+}
+
+function tableSort(key) {
+ if (sortState.col === key) {
+ sortState.dir = -sortState.dir; // toggle direction
+ } else {
+ sortState.col = key;
+ sortState.dir = -1; // default: descending for numeric, A→Z for text
+ if (!NUMERIC_COLS.has(key)) sortState.dir = 1;
+ }
+ const win = qs('window').value;
+ const filtered = filterData(sortByState(ALL));
+ renderTable(filtered, win);
+}
+
// --- Boot ---
async function load() {
try {
← f917b8b Add GA4 fleet traffic dashboard (local, port 9710)
·
back to Ga4 Dashboard
·
auto-save: 2026-08-04T14:04:01 (1 files) — etl.py 80a2fa1 →