← back to Watches
scripts/bulk-wayback-crawler.py
471 lines
#!/usr/bin/env python3
"""
Bulk Wayback Machine Crawler for Historical Omega Watch Prices
Crawls archived Chrono24, eBay, and WatchCharts pages from 2010-2025.
Outputs to data/historical-prices.json with resume capability.
"""
import json
import sys
import os
import argparse
import time
import re
from datetime import datetime, timedelta
from pathlib import Path
try:
import requests
except ImportError:
print("Installing requests...", file=sys.stderr)
import subprocess
subprocess.check_call([sys.executable, "-m", "pip", "install", "requests", "-q"])
import requests
# Configuration
SCRIPT_DIR = Path(__file__).parent.absolute()
PROJECT_DIR = SCRIPT_DIR.parent
DATA_FILE = PROJECT_DIR / "data" / "historical-prices.json"
STATUS_FILE = PROJECT_DIR / "data" / "crawl-status.json"
# Rate limiting
REQUEST_DELAY = 2.0 # seconds between requests
BATCH_DELAY = 300 # 5 minute pause every 100 requests
BATCH_SIZE = 100
# Watch models to crawl
WATCH_MODELS = {
"speedmaster-moonwatch": {
"chrono24": "chrono24.com/omega/speedmaster-moonwatch",
"ebay": "ebay.com/sch/i.html?_nkw=omega+speedmaster+moonwatch",
"keywords": ["speedmaster", "moonwatch", "professional"]
},
"speedmaster-alaska": {
"chrono24": "chrono24.com/omega/speedmaster-alaska",
"keywords": ["speedmaster", "alaska"]
},
"seamaster-300": {
"chrono24": "chrono24.com/omega/seamaster-300",
"ebay": "ebay.com/sch/i.html?_nkw=omega+seamaster+300",
"keywords": ["seamaster", "300"]
},
"seamaster-ploprof": {
"chrono24": "chrono24.com/omega/seamaster-ploprof",
"keywords": ["seamaster", "ploprof", "1200"]
},
"seamaster-bond": {
"chrono24": "chrono24.com/omega/seamaster-diver-300m",
"ebay": "ebay.com/sch/i.html?_nkw=omega+seamaster+diver+300m",
"keywords": ["seamaster", "diver", "300m", "bond"]
},
"constellation-piepan": {
"chrono24": "chrono24.com/omega/constellation-pie-pan",
"keywords": ["constellation", "pie", "pan"]
},
"constellation-manhattan": {
"chrono24": "chrono24.com/omega/constellation-manhattan",
"keywords": ["constellation", "manhattan"]
},
"deville-tresor": {
"chrono24": "chrono24.com/omega/de-ville-tresor",
"keywords": ["de", "ville", "tresor"]
},
"deville-coaxial": {
"chrono24": "chrono24.com/omega/de-ville-co-axial",
"keywords": ["de", "ville", "coaxial", "co-axial"]
},
"railmaster-original": {
"chrono24": "chrono24.com/omega/railmaster",
"keywords": ["railmaster"]
},
"aqua-terra": {
"chrono24": "chrono24.com/omega/seamaster-aqua-terra",
"ebay": "ebay.com/sch/i.html?_nkw=omega+aqua+terra",
"keywords": ["aqua", "terra"]
},
"planet-ocean": {
"chrono24": "chrono24.com/omega/seamaster-planet-ocean",
"ebay": "ebay.com/sch/i.html?_nkw=omega+planet+ocean",
"keywords": ["planet", "ocean"]
},
"speedmaster-reduced": {
"chrono24": "chrono24.com/omega/speedmaster-reduced",
"keywords": ["speedmaster", "reduced"]
},
"speedmaster-mark2": {
"chrono24": "chrono24.com/omega/speedmaster-mark-ii",
"keywords": ["speedmaster", "mark", "ii", "2"]
},
"flightmaster": {
"chrono24": "chrono24.com/omega/flightmaster",
"keywords": ["flightmaster"]
}
}
# HTTP headers
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}
def load_existing_data():
"""Load existing historical prices data."""
if DATA_FILE.exists():
with open(DATA_FILE) as f:
return json.load(f)
return {
"metadata": {
"version": "1.0",
"created": datetime.now().isoformat(),
"lastUpdated": None,
"totalRecords": 0,
"dateRange": {"from": "2010-01-01", "to": "2025-12-31"},
"sources": ["chrono24", "ebay", "watchcharts"],
"watches": len(WATCH_MODELS)
},
"crawlStatus": {
"lastCrawl": None,
"status": "not_started",
"progress": {"completed": 0, "total": len(WATCH_MODELS), "currentWatch": None}
},
"prices": {}
}
def save_data(data):
"""Save data to file."""
data["metadata"]["lastUpdated"] = datetime.now().isoformat()
data["metadata"]["totalRecords"] = sum(
len(records) for records in data["prices"].values()
)
with open(DATA_FILE, 'w') as f:
json.dump(data, f, indent=2)
print(f"Saved {data['metadata']['totalRecords']} records to {DATA_FILE}", file=sys.stderr)
def load_status():
"""Load crawl status."""
if STATUS_FILE.exists():
with open(STATUS_FILE) as f:
return json.load(f)
return {
"watchesCompleted": [],
"currentWatch": None,
"yearsCompleted": {},
"totalRequests": 0,
"lastRequest": None,
"errors": []
}
def save_status(status):
"""Save crawl status."""
status["lastRequest"] = datetime.now().isoformat()
with open(STATUS_FILE, 'w') as f:
json.dump(status, f, indent=2)
def fetch_cdx_snapshots(url, from_year, to_year, limit=100):
"""Fetch CDX index data for a URL from Wayback Machine."""
all_snapshots = []
# Query year by year to avoid timeouts
for year in range(from_year, to_year + 1):
cdx_url = "https://web.archive.org/cdx/search/cdx"
params = {
"url": url,
"matchType": "prefix",
"from": str(year),
"to": str(year),
"output": "json",
"filter": "statuscode:200",
"collapse": "timestamp:6", # One per month
"limit": str(limit)
}
for attempt in range(3): # Retry up to 3 times
try:
resp = requests.get(cdx_url, params=params, headers=HEADERS, timeout=90)
resp.raise_for_status()
data = resp.json()
if len(data) > 1:
all_snapshots.extend(data[1:]) # Skip header row
print(f" {year}: {len(data)-1 if len(data) > 1 else 0} snapshots", file=sys.stderr)
time.sleep(1) # Small delay between year queries
break
except Exception as e:
if attempt < 2:
print(f" {year}: Retry {attempt+1} after error: {e}", file=sys.stderr)
time.sleep(5)
else:
print(f" {year}: Failed after 3 attempts: {e}", file=sys.stderr)
return all_snapshots
def extract_prices_from_html(html_content, source):
"""Extract prices from archived HTML page."""
prices = []
if not html_content:
return prices
# Different patterns for different sources
patterns = [
# Chrono24 patterns
r'\$\s*([\d,]+(?:\.\d{2})?)',
r'USD\s*([\d,]+(?:\.\d{2})?)',
r'([\d,]+(?:\.\d{2})?)\s*USD',
r'"price"\s*:\s*"?\$?([\d,]+)"?',
r'data-price="([\d,]+)"',
# eBay patterns
r'itemprop="price"\s+content="([\d.]+)"',
r'class="s-item__price"[^>]*>\$?([\d,]+)',
# Generic price patterns
r'class="[^"]*price[^"]*"[^>]*>\s*\$?([\d,]+)',
]
for pattern in patterns:
matches = re.findall(pattern, html_content, re.IGNORECASE)
for match in matches:
try:
price_str = str(match).replace(',', '').replace('$', '')
price = int(float(price_str))
# Filter reasonable watch prices ($500 - $100,000)
if 500 <= price <= 100000:
prices.append(price)
except (ValueError, TypeError):
continue
# Deduplicate and return
return list(set(prices))
def fetch_archived_page(archive_url):
"""Fetch content from an archived page."""
try:
resp = requests.get(archive_url, headers=HEADERS, timeout=60)
resp.raise_for_status()
return resp.text
except Exception as e:
print(f" Page fetch error: {e}", file=sys.stderr)
return None
def crawl_watch(watch_id, watch_config, from_year, to_year, data, status):
"""Crawl all historical data for a single watch."""
print(f"\n{'='*60}", file=sys.stderr)
print(f"Crawling: {watch_id}", file=sys.stderr)
print(f"Date range: {from_year} - {to_year}", file=sys.stderr)
print(f"{'='*60}", file=sys.stderr)
if watch_id not in data["prices"]:
data["prices"][watch_id] = []
existing_dates = {r["date"] for r in data["prices"][watch_id]}
request_count = 0
new_records = 0
# Crawl each source
sources_to_crawl = []
if "chrono24" in watch_config:
sources_to_crawl.append(("chrono24", watch_config["chrono24"]))
if "ebay" in watch_config:
sources_to_crawl.append(("ebay", watch_config["ebay"]))
for source_name, source_url in sources_to_crawl:
print(f"\n Source: {source_name}", file=sys.stderr)
# Fetch CDX snapshots
snapshots = fetch_cdx_snapshots(source_url, from_year, to_year)
print(f" Found {len(snapshots)} snapshots", file=sys.stderr)
for i, snapshot in enumerate(snapshots):
try:
# CDX format: urlkey, timestamp, original, mimetype, statuscode, digest, length
timestamp = snapshot[1]
original_url = snapshot[2]
# Parse date
date_str = datetime.strptime(timestamp[:8], '%Y%m%d').strftime('%Y-%m-%d')
# Skip if we already have data for this date
if date_str in existing_dates:
continue
# Build archive URL
archive_url = f"https://web.archive.org/web/{timestamp}/{original_url}"
print(f" [{i+1}/{len(snapshots)}] {date_str}...", file=sys.stderr, end=" ")
# Fetch and parse page
html = fetch_archived_page(archive_url)
if html:
prices = extract_prices_from_html(html, source_name)
if prices:
# Add record for each price found
for price in prices[:5]: # Max 5 prices per snapshot
record = {
"date": date_str,
"price": price,
"source": source_name,
"archiveUrl": archive_url,
"crawledAt": datetime.now().isoformat()
}
data["prices"][watch_id].append(record)
new_records += 1
print(f"Found {len(prices)} prices (${min(prices):,} - ${max(prices):,})", file=sys.stderr)
existing_dates.add(date_str)
else:
print("No prices found", file=sys.stderr)
else:
print("Failed to fetch", file=sys.stderr)
request_count += 1
status["totalRequests"] += 1
# Rate limiting
time.sleep(REQUEST_DELAY)
# Batch pause
if request_count % BATCH_SIZE == 0:
print(f"\n Pausing {BATCH_DELAY}s after {request_count} requests...", file=sys.stderr)
save_data(data)
save_status(status)
time.sleep(BATCH_DELAY)
except Exception as e:
print(f"Error processing snapshot: {e}", file=sys.stderr)
status["errors"].append({
"watch": watch_id,
"error": str(e),
"timestamp": datetime.now().isoformat()
})
continue
# Sort by date
data["prices"][watch_id].sort(key=lambda x: x["date"])
print(f"\n Completed: {new_records} new records, {len(data['prices'][watch_id])} total", file=sys.stderr)
return new_records
def crawl_all(from_year, to_year, resume=True):
"""Crawl all watches."""
print(f"\n{'#'*60}", file=sys.stderr)
print(f"# BULK HISTORICAL PRICE CRAWL", file=sys.stderr)
print(f"# Date range: {from_year} - {to_year}", file=sys.stderr)
print(f"# Watches: {len(WATCH_MODELS)}", file=sys.stderr)
print(f"{'#'*60}\n", file=sys.stderr)
data = load_existing_data()
status = load_status()
# Determine which watches to crawl
watches_to_crawl = list(WATCH_MODELS.keys())
if resume and status["watchesCompleted"]:
watches_to_crawl = [w for w in watches_to_crawl if w not in status["watchesCompleted"]]
print(f"Resuming: {len(status['watchesCompleted'])} already completed", file=sys.stderr)
start_time = datetime.now()
total_new_records = 0
for i, watch_id in enumerate(watches_to_crawl):
watch_config = WATCH_MODELS[watch_id]
status["currentWatch"] = watch_id
data["crawlStatus"]["progress"]["currentWatch"] = watch_id
data["crawlStatus"]["progress"]["completed"] = len(status["watchesCompleted"])
data["crawlStatus"]["status"] = "in_progress"
save_status(status)
new_records = crawl_watch(watch_id, watch_config, from_year, to_year, data, status)
total_new_records += new_records
status["watchesCompleted"].append(watch_id)
save_data(data)
save_status(status)
# Progress report
elapsed = (datetime.now() - start_time).total_seconds()
remaining = len(watches_to_crawl) - (i + 1)
eta = (elapsed / (i + 1)) * remaining if i > 0 else 0
print(f"\nProgress: {i+1}/{len(watches_to_crawl)} watches", file=sys.stderr)
print(f"Total new records: {total_new_records}", file=sys.stderr)
print(f"ETA: {int(eta/60)} minutes", file=sys.stderr)
# Final status
data["crawlStatus"]["status"] = "completed"
data["crawlStatus"]["lastCrawl"] = datetime.now().isoformat()
save_data(data)
elapsed = (datetime.now() - start_time).total_seconds()
print(f"\n{'#'*60}", file=sys.stderr)
print(f"# CRAWL COMPLETE", file=sys.stderr)
print(f"# Total records: {data['metadata']['totalRecords']}", file=sys.stderr)
print(f"# New records: {total_new_records}", file=sys.stderr)
print(f"# Total time: {int(elapsed/60)} minutes", file=sys.stderr)
print(f"# Requests made: {status['totalRequests']}", file=sys.stderr)
print(f"{'#'*60}\n", file=sys.stderr)
return data
def main():
parser = argparse.ArgumentParser(description='Bulk crawl historical Omega prices from Wayback Machine')
parser.add_argument('--from-year', type=int, default=2010, help='Start year (default: 2010)')
parser.add_argument('--to-year', type=int, default=2025, help='End year (default: 2025)')
parser.add_argument('--watch', help='Crawl single watch ID only')
parser.add_argument('--no-resume', action='store_true', help='Start fresh (ignore previous progress)')
parser.add_argument('--dry-run', action='store_true', help='Show what would be crawled without fetching')
parser.add_argument('--status', action='store_true', help='Show crawl status and exit')
args = parser.parse_args()
if args.status:
status = load_status()
data = load_existing_data()
print(json.dumps({
"status": data["crawlStatus"],
"watchesCompleted": status["watchesCompleted"],
"totalRecords": data["metadata"]["totalRecords"],
"totalRequests": status["totalRequests"],
"errors": len(status["errors"])
}, indent=2))
return
if args.dry_run:
print(f"Would crawl {len(WATCH_MODELS)} watches from {args.from_year} to {args.to_year}")
for watch_id in WATCH_MODELS:
print(f" - {watch_id}")
return
if args.watch:
if args.watch not in WATCH_MODELS:
print(f"Unknown watch: {args.watch}", file=sys.stderr)
print(f"Available: {', '.join(WATCH_MODELS.keys())}", file=sys.stderr)
sys.exit(1)
data = load_existing_data()
status = load_status()
crawl_watch(args.watch, WATCH_MODELS[args.watch], args.from_year, args.to_year, data, status)
save_data(data)
save_status(status)
else:
data = crawl_all(args.from_year, args.to_year, resume=not args.no_resume)
# Output final stats
print(json.dumps({
"success": True,
"totalRecords": data["metadata"]["totalRecords"],
"watches": len(data["prices"]),
"dateRange": data["metadata"]["dateRange"]
}, indent=2))
if __name__ == '__main__':
main()