← back to Omega Watches 2
collectors/chrono24/collector.mjs
247 lines
/**
* Chrono24 Marketplace Collector
*
* Fetches sold/listed watch prices from Chrono24 for references
* in our database. Writes to the market_event ledger.
*
* Data flow:
* 1. Load watch references
* 2. For each reference, search Chrono24 listings
* 3. Parse sold/listed prices, conditions, dealer info
* 4. Insert into market_event (dedup: source_name + source_listing_id)
* 5. Quarantine suspicious prices
*
* Chrono24 public search endpoint:
* https://www.chrono24.com/search/index.htm?query={ref}&dosearch=true
*
* Note: This respects robots.txt and rate limits. Chrono24 allows
* crawling of search results per their robots.txt (as of 2025).
*/
import { BaseCollector, pool } from '../base-collector.mjs';
import https from 'https';
import crypto from 'crypto';
const CHRONO24_BASE = 'https://www.chrono24.com';
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
export default class Chrono24Collector extends BaseCollector {
constructor() {
super('chrono24');
this.references = [];
}
getJobType() { return 'incremental'; }
fetch(url) {
return new Promise((resolve, reject) => {
const parsedUrl = new URL(url);
const req = https.get({
hostname: parsedUrl.hostname,
path: parsedUrl.pathname + parsedUrl.search,
headers: {
'User-Agent': USER_AGENT,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'identity',
},
timeout: 30000,
}, (res) => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
const redirectUrl = res.headers.location.startsWith('http')
? res.headers.location
: `${CHRONO24_BASE}${res.headers.location}`;
return this.fetch(redirectUrl).then(resolve).catch(reject);
}
let body = '';
res.on('data', chunk => body += chunk);
res.on('end', () => resolve({ statusCode: res.statusCode, body }));
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });
});
}
/**
* Parse Chrono24 search results page for listing data
* Returns array of listing objects
*/
parseSearchResults(html, referenceNumber) {
const listings = [];
// Chrono24 embeds listing data in article elements
// Pattern: <article class="article-item" ... data-article-id="...">
const articlePattern = /<article[^>]*class="[^"]*article-item[^"]*"[^>]*data-article-id="(\d+)"[^>]*>([\s\S]*?)<\/article>/gi;
let match;
while ((match = articlePattern.exec(html)) !== null) {
const articleId = match[1];
const content = match[2];
const listing = {
source_listing_id: `c24-${articleId}`,
reference_number_raw: referenceNumber,
};
// Extract price
const priceMatch = content.match(/class="[^"]*price[^"]*"[^>]*>\s*(?:[\$€£])\s*([\d,]+)\s*</i)
|| content.match(/data-price="([\d.]+)"/i)
|| content.match(/>\s*(?:USD|US\$|\$)\s*([\d,]+)\s*</i);
if (priceMatch) {
listing.price_amount = parseFloat(priceMatch[1].replace(/,/g, ''));
}
// Extract currency
const currMatch = content.match(/(?:EUR|USD|GBP|CHF|JPY)/);
listing.currency = currMatch ? currMatch[0] : 'USD';
// Extract condition
const condMatch = content.match(/(?:New|Unworn|Very good|Good|Fair|Incomplete|Poor)/i);
if (condMatch) {
listing.condition_raw = condMatch[0];
listing.condition_normalized = this.normalizeCondition(condMatch[0]);
}
// Extract if it's a dealer or private seller
const dealerMatch = content.match(/class="[^"]*dealer-name[^"]*"/i);
listing.seller_type = dealerMatch ? 'dealer' : 'private';
// Determine event type
const soldMatch = content.match(/sold/i);
listing.event_type = soldMatch ? 'marketplace_sold' : 'dealer_ask';
// Extract listing URL
const hrefMatch = content.match(/href="(\/[^"]*omega[^"]*)"/i);
listing.source_url = hrefMatch ? `${CHRONO24_BASE}${hrefMatch[1]}` : `${CHRONO24_BASE}/omega/ref-${referenceNumber}.htm`;
if (listing.price_amount && listing.price_amount > 100) {
listings.push(listing);
}
}
// Fallback: try JSON-LD structured data
if (listings.length === 0) {
const ldBlocks = html.match(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi);
if (ldBlocks) {
for (const block of ldBlocks) {
try {
const jsonStr = block.replace(/<\/?script[^>]*>/gi, '');
const data = JSON.parse(jsonStr);
if (data['@type'] === 'ItemList' && data.itemListElement) {
for (const item of data.itemListElement) {
const product = item.item || item;
if (product.offers?.price) {
listings.push({
source_listing_id: `c24-ld-${product.sku || product.productID || crypto.randomBytes(4).toString('hex')}`,
reference_number_raw: referenceNumber,
price_amount: parseFloat(product.offers.price),
currency: product.offers.priceCurrency || 'USD',
event_type: 'dealer_ask',
condition_raw: product.itemCondition?.replace('https://schema.org/', '') || 'unknown',
condition_normalized: this.normalizeCondition(product.itemCondition || ''),
seller_type: 'dealer',
source_url: product.url || `${CHRONO24_BASE}/omega/ref-${referenceNumber}.htm`,
});
}
}
}
} catch {}
}
}
}
return listings;
}
normalizeCondition(raw) {
const lower = (raw || '').toLowerCase();
if (lower.includes('new') || lower.includes('unworn')) return 'new_unworn';
if (lower.includes('very good')) return 'very_good';
if (lower.includes('good')) return 'good';
if (lower.includes('fair')) return 'fair';
if (lower.includes('poor') || lower.includes('incomplete')) return 'poor';
return 'unknown';
}
async collect() {
const { rows } = await pool.query(
'SELECT id, reference_number, collection, model_name FROM watch_reference ORDER BY collection, reference_number'
);
this.references = rows;
console.log(`[chrono24] Searching Chrono24 for ${rows.length} references`);
let totalListings = 0;
for (const ref of rows) {
try {
await this.rateLimit();
const searchUrl = `${CHRONO24_BASE}/search/index.htm?query=${encodeURIComponent('Omega ' + ref.reference_number)}&dosearch=true&sortorder=5`;
let response;
try {
response = await this.fetch(searchUrl);
} catch (fetchErr) {
console.warn(`[chrono24] Fetch error for ${ref.reference_number}: ${fetchErr.message}`);
continue;
}
this.stats.fetched++;
if (response.statusCode !== 200) {
console.log(` ${ref.reference_number}: HTTP ${response.statusCode}`);
continue;
}
// Archive raw HTML
const pageHash = crypto.createHash('sha256').update(response.body).digest('hex');
await this.archiveSnapshot(searchUrl, response.body);
// Parse listings
const listings = this.parseSearchResults(response.body, ref.reference_number);
this.stats.parsed += listings.length;
let insertedForRef = 0;
for (const listing of listings) {
listing.source_name = 'chrono24';
listing.sale_date = new Date().toISOString().split('T')[0];
listing.page_hash = pageHash;
// For dealer_ask: price_amount is the ask, no hammer_price
if (listing.event_type === 'dealer_ask') {
listing.total_to_buyer = listing.price_amount;
listing.fee_semantics = 'unknown';
} else {
// marketplace_sold
listing.total_to_buyer = listing.price_amount;
listing.fee_semantics = 'total_inclusive';
}
const result = await this.insertMarketEvent(listing);
if (result.action === 'inserted') insertedForRef++;
}
totalListings += listings.length;
if (listings.length > 0) {
console.log(` ${ref.reference_number}: ${listings.length} listings found, ${insertedForRef} new`);
}
} catch (err) {
console.error(` ${ref.reference_number}: Error — ${err.message}`);
}
}
console.log(`[chrono24] Complete: ${totalListings} total listings processed, ${this.stats.inserted} inserted, ${this.stats.deduplicated} deduped`);
}
validate(event) {
// Chrono24-specific validation
const base = super.validate(event);
if (base) return base;
// Omega watches: reasonable price range $500 - $500k
if (event.price_amount < 500) return 'price_below_minimum';
if (event.price_amount > 500000) return 'price_above_maximum';
return null;
}
}