Parse compliance HTML before validating visibility

1f59efbfdb481d3e91665d2e5307a6fa216b18a7 · 2026-08-28 16:31:50 -0700 · Steve Abrams
Files touched

M lib/pre-send-gate.js
M test/pre-send-gate.test.js
M verification/e2e-proof.json
Diff

commit 1f59efbfdb481d3e91665d2e5307a6fa216b18a7
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Fri Aug 28 16:31:50 2026 -0700

    Parse compliance HTML before validating visibility
---
 lib/pre-send-gate.js        | 57 +++++++++++++++++++++++++++++++++------------
 test/pre-send-gate.test.js  |  9 +++++++
 verification/e2e-proof.json | 25 ++++++++++++++++----
 3 files changed, 71 insertions(+), 20 deletions(-)

diff --git a/lib/pre-send-gate.js b/lib/pre-send-gate.js
index 8027178..1545c87 100644
--- a/lib/pre-send-gate.js
+++ b/lib/pre-send-gate.js
@@ -1,9 +1,11 @@
 'use strict';
 
+const parse5 = require('parse5');
+
 const REQUIRED_FROM = 'info@designerwallcoverings.com';
 const ADDRESS_PATTERN = /15442\s+Ventura\s+Blvd(?:\.|\s)*(?:#|Suite)?\s*102[\s\S]*Sherman\s+Oaks,?\s*CA\s+91403/i;
 const MISLEADING_SUBJECT_PATTERN = /\b(?:re|fw|fwd)\s*:/i;
-const NON_RENDERED_ELEMENTS = 'script|style|template|noscript|head|svg|canvas';
+const NON_RENDERED_ELEMENTS = new Set(['script', 'style', 'template', 'noscript', 'head', 'svg', 'canvas']);
 
 function decodeHtmlEntities(value) {
   return String(value || '')
@@ -17,25 +19,50 @@ function decodeHtmlEntities(value) {
     .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)));
 }
 
-function sanitizeRenderedHtml(value) {
-  let html = String(value || '').replace(/<!--[\s\S]*?-->/g, '');
-  html = html.replace(new RegExp(`<(${NON_RENDERED_ELEMENTS})\\b[^>]*>[\\s\\S]*?<\\/\\1\\s*>`, 'gi'), '');
-  const hiddenBlock = /<([a-z][\w:-]*)\b(?=[^>]*(?:\bhidden(?:\s*=\s*(?:["'][^"']*["']|[^\s>]+))?|\baria-hidden\s*=\s*(["'])?true\2|\bstyle\s*=\s*(["'])[^"']*(?:display\s*:\s*none|visibility\s*:\s*hidden)[^"']*\3))[^>]*>[\s\S]*?<\/\1\s*>/gi;
-  let previous;
-  do { previous = html; html = html.replace(hiddenBlock, ''); } while (html !== previous);
-  return html;
+function attrs(node) {
+  return Object.fromEntries((node.attrs || []).map(({ name, value }) => [name.toLowerCase(), value]));
+}
+
+function isHiddenElement(node) {
+  if (!node.tagName) return false;
+  const tag = node.tagName.toLowerCase();
+  if (NON_RENDERED_ELEMENTS.has(tag)) return true;
+  const attributes = attrs(node);
+  if (Object.hasOwn(attributes, 'hidden')) return true;
+  if (String(attributes['aria-hidden'] || '').trim().toLowerCase() === 'true') return true;
+  if (tag === 'input' && String(attributes.type || '').trim().toLowerCase() === 'hidden') return true;
+  const style = String(attributes.style || '')
+    .replace(/\/\*[\s\S]*?\*\//g, '')
+    .replace(/\s+/g, '')
+    .toLowerCase();
+  return /(?:^|;)display:none(?:!important)?(?:;|$)/.test(style)
+    || /(?:^|;)visibility:hidden(?:!important)?(?:;|$)/.test(style);
+}
+
+function renderedContent(body) {
+  const fragment = parse5.parseFragment(String(body || ''));
+  const text = [];
+  const hrefs = [];
+  function visit(node, hidden = false) {
+    const nowHidden = hidden || isHiddenElement(node);
+    if (!nowHidden && node.nodeName === '#text') text.push(node.value || '');
+    if (!nowHidden && String(node.tagName || '').toLowerCase() === 'a') {
+      const href = attrs(node).href;
+      if (href) hrefs.push(href.trim());
+    }
+    for (const child of node.childNodes || []) visit(child, nowHidden);
+    if (node.content) visit(node.content, true);
+  }
+  visit(fragment);
+  return { visibleText: text.join(' ').replace(/\s+/g, ' ').trim(), hrefs };
 }
 
 function visibleTextFromHtml(body) {
-  const rendered = sanitizeRenderedHtml(body)
-    .replace(/<(?:br|\/p|\/div|\/li|\/tr|\/h[1-6])\b[^>]*>/gi, ' ')
-    .replace(/<[^>]*>/g, ' ');
-  return decodeHtmlEntities(rendered).replace(/\s+/g, ' ').trim();
+  return renderedContent(body).visibleText;
 }
 
 function hasWorkingOptOutLink(body) {
-  const rendered = sanitizeRenderedHtml(body);
-  const hrefs = [...rendered.matchAll(/<a\b[^>]*\bhref\s*=\s*(["'])(.*?)\1[^>]*>/gi)].map((match) => decodeHtmlEntities(match[2].trim()));
+  const { hrefs } = renderedContent(body);
   return hrefs.some((href) => {
     try {
       const url = new URL(href);
@@ -67,4 +94,4 @@ function assertPreSendCompliance({ from, subject, body }) {
   return true;
 }
 
-module.exports = { assertPreSendCompliance, hasWorkingOptOutLink, sanitizeRenderedHtml, visibleTextFromHtml, REQUIRED_FROM };
+module.exports = { assertPreSendCompliance, hasWorkingOptOutLink, renderedContent, visibleTextFromHtml, REQUIRED_FROM };
diff --git a/test/pre-send-gate.test.js b/test/pre-send-gate.test.js
index 16922bb..f20a060 100644
--- a/test/pre-send-gate.test.js
+++ b/test/pre-send-gate.test.js
@@ -54,6 +54,15 @@ assert.throws(() => assertPreSendCompliance({ from: REQUIRED_FROM, subject: comp
 const attackerOwned = bodyWithoutLink + '<a href="https://attacker.example.org/unsubscribe">unsubscribe</a>';
 assert.throws(() => assertPreSendCompliance({ from: REQUIRED_FROM, subject: compliant.subject, body: attackerOwned }), { code: 'PRE_SEND_COMPLIANCE_BLOCKED' });
 
+const validOptOut = compliant.html.match(/<a href="mailto:[\s\S]*?<\/a>/)[0];
+for (const parserBypass of [
+  '<div style="display&#58;none">15442 Ventura Blvd. #102, Sherman Oaks, CA 91403</div>' + validOptOut,
+  '<div style="display:/**/none">15442 Ventura Blvd. #102, Sherman Oaks, CA 91403</div>' + validOptOut,
+  '<script>15442 Ventura Blvd. #102, Sherman Oaks, CA 91403 <a href="mailto:info@designerwallcoverings.com?subject=Unsubscribe">unsubscribe</a>',
+]) {
+  assert.throws(() => assertPreSendCompliance({ from: REQUIRED_FROM, subject: compliant.subject, body: parserBypass }), { code: 'PRE_SEND_COMPLIANCE_BLOCKED' });
+}
+
 const reconciledFooter = `<div><p>Ship to:<br>Designer Wallcoverings<br>15442 Ventura Blvd. #102<br>Sherman Oaks, CA 91403<br>1-888-373-4564</p><p>Thank you!<br>Showroom Manager</p><p>Designer Wallcoverings · 15442 Ventura Blvd. #102 · Sherman Oaks, CA 91403<br>To stop receiving sample follow-up emails, <a href="mailto:info@designerwallcoverings.com?subject=Unsubscribe%20from%20sample%20follow-ups">unsubscribe here</a>.</p></div>`;
 assert.equal(assertPreSendCompliance({ from: REQUIRED_FROM, subject: 'Sample Follow-Up — Outstanding Memos — Designer Wallcoverings', body: reconciledFooter }), true);
 
diff --git a/verification/e2e-proof.json b/verification/e2e-proof.json
index 9087f53..0eeb231 100644
--- a/verification/e2e-proof.json
+++ b/verification/e2e-proof.json
@@ -2,20 +2,32 @@
   "intent": "Centralize fail-closed compliance enforcement and require visibly rendered, DW-owned compliance content",
   "risk_tier": "R3",
   "environment": "local repository only; injected HTTP spy; no George/Gmail/FileMaker/production network calls",
-  "build_identity": "Cycle 3 implementation commit 297d26a",
-  "timestamp": "2026-08-28T14:44:20-07:00",
+  "build_identity": "Cycle 4 working tree after a3c2bbb; final commit recorded after verification",
+  "timestamp": "2026-08-28T16:30:24-07:00",
   "ticket": "TK-10940-centralize-sample-follow-up-pre-send-com",
   "baseline": "Cycle 1 commit c74ebf7 had an ungated /api/drafts call, an ungated gmail_create_draft artifact, and accepted hidden/comment/plain-text/reserved-domain opt-out tokens",
   "commands": [
-    "for f in test/*.test.js; do node \"$f\"; done",
+    "npm test",
+    "npm audit --audit-level=high",
+    "isolated PORT=64846 node server.js plus localhost auth/state/send-one canary",
     "node --check on every changed JavaScript and MJS entry point",
     "git diff --check"
   ],
   "assertions": [
+    {
+      "boundary": "real local console API",
+      "verdict": "PASS",
+      "evidence": "isolated port 64846 returned 401 without auth, 200 for authenticated /api/state, and 422 PRE-SEND COMPLIANCE GATE BLOCKED for known stale dgd payload before George/Gmail"
+    },
     {
       "boundary": "compliance parser",
       "verdict": "PASS",
-      "evidence": "test/pre-send-gate.test.js rejects comments, script/template/hidden address text, script-only links, plain-text mailto tokens, invalid.example, and attacker.example.org while accepting lib/compose.js and reconciled-footer output"
+      "evidence": "parse5 HTML5 tree parsing plus test/pre-send-gate.test.js reject comments, unclosed script, entity-encoded/CSS-comment display:none, script/template/hidden address text, script-only links, plain-text mailto tokens, invalid.example, and attacker.example.org while accepting lib/compose.js and reconciled-footer output"
+    },
+    {
+      "boundary": "dependency integrity",
+      "verdict": "PASS",
+      "evidence": "parse5 is pinned at 7.3.0 in package-lock.json; npm audit reported 0 vulnerabilities"
     },
     {
       "boundary": "George /api/send and /api/drafts",
@@ -48,10 +60,13 @@
     "reserved invalid.example unsubscribe URL",
     "attacker-owned HTTPS unsubscribe URL",
     "postal address hidden with hidden/display:none",
+    "postal address hidden with entity-encoded display&#58;none",
+    "postal address hidden with CSS-comment display:/**/none",
     "postal address present only in script/template",
+    "postal address and opt-out present in malformed unclosed script",
     "opt-out link present only in script",
     "blocked transport makes zero requests"
   ],
-  "cleanup": "No external or persistent test state created; repository verification artifact intentionally retained",
+  "cleanup": "Isolated local server stopped with Ctrl-C; no external or persistent test state created; repository verification artifact intentionally retained",
   "verdict": "PASS"
 }
← 9c15a26 auto-data-snapshot: 2026-08-28T15:48:59 (2 data files) — REA · back to Sample Followup Sweep · Require visible opt-out link labels 900aa26 →