← back to Handbag Authentication
scripts/mega_handbag_scraper.js
623 lines
#!/usr/bin/env node
/**
* MEGA HANDBAG DATA SCRAPER
* Comprehensive data collection from European and Japanese institutions
*
* FOCUS: Handbags, purses, bags, clutches, maroquinerie only
* SOURCE TRACKING: Every item includes complete source attribution
*
* Target Sources:
* - Europeana Fashion API (EU-wide, 50M+ items)
* - Rijksmuseum API (Netherlands, 500K+ items)
* - Paris Musées GraphQL (France, 280K+ items)
* - V&A Museum API (UK)
* - Harvard Art Museums API (US, but includes European pieces)
* - KCI Kyoto (Japan)
* - Metropolitan Museum API (US, but includes European/Japanese pieces)
* - Cooper Hewitt API
* - More institutions as discovered
*/
const https = require('https');
const http = require('http');
const fs = require('fs');
const path = require('path');
const BASE_DIR = path.join(__dirname, '..');
const DATA_DIR = path.join(BASE_DIR, 'handbag_data');
const OUTPUT_DIR = path.join(DATA_DIR, 'mega_collection');
const LOG_DIR = path.join(OUTPUT_DIR, 'logs');
// Ensure directories exist
[OUTPUT_DIR, LOG_DIR].forEach(dir => {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
});
// Search terms in multiple languages (HANDBAG-SPECIFIC ONLY)
const SEARCH_TERMS = {
en: ['handbag', 'purse', 'bag', 'clutch', 'evening bag', 'tote bag', 'shoulder bag', 'crossbody bag'],
fr: ['sac à main', 'sac', 'pochette', 'bourse', 'porte-monnaie', 'maroquinerie', 'cabas', 'sac bandoulière'],
it: ['borsetta', 'borsa', 'borsellino', 'pochette', 'pelletteria'],
nl: ['handtas', 'tas', 'beurs', 'clutch'],
de: ['Handtasche', 'Tasche', 'Geldbörse', 'Clutch'],
es: ['bolso', 'bolso de mano', 'cartera', 'monedero'],
ja: ['ハンドバッグ', 'バッグ', '財布', 'かばん', '鞄']
};
// Luxury brands to specifically search for (handbag-focused)
const LUXURY_BRANDS = [
'Hermès', 'Hermes', 'Chanel', 'Louis Vuitton', 'Dior', 'Celine',
'Balenciaga', 'YSL', 'Yves Saint Laurent', 'Givenchy', 'Longchamp',
'Gucci', 'Prada', 'Fendi', 'Versace', 'Bottega Veneta', 'Ferragamo',
'Valentino', 'Dolce & Gabbana', 'Burberry', 'Mulberry', 'Chloe'
];
/**
* Configuration for all museum/institution APIs
*/
const INSTITUTIONS = {
europeana: {
name: 'Europeana Fashion (Pan-European)',
country: 'EU',
baseUrl: 'https://api.europeana.eu/record/v2',
searchUrl: 'https://api.europeana.eu/record/v2/search.json',
needsApiKey: true,
apiKeyEnv: 'EUROPEANA_API_KEY',
apiKeyParam: 'wskey',
maxResults: 100,
rateLimit: 1000, // ms between requests
enabled: true,
searchStrategy: 'multilingual',
notes: 'Register at https://pro.europeana.eu/page/get-api'
},
rijksmuseum: {
name: 'Rijksmuseum Amsterdam',
country: 'Netherlands',
baseUrl: 'https://www.rijksmuseum.nl/api',
searchUrl: 'https://www.rijksmuseum.nl/api/nl/collection',
needsApiKey: true,
apiKeyEnv: 'RIJKSMUSEUM_API_KEY',
apiKeyParam: 'key',
maxResults: 100,
rateLimit: 1000,
enabled: true,
searchStrategy: 'dutch',
notes: 'Register at https://data.rijksmuseum.nl/'
},
vam: {
name: 'Victoria & Albert Museum',
country: 'UK',
baseUrl: 'https://api.vam.ac.uk/v2',
searchUrl: 'https://api.vam.ac.uk/v2/objects/search',
needsApiKey: false,
maxResults: 100,
rateLimit: 500,
enabled: true,
searchStrategy: 'english',
notes: 'Open API, no key required'
},
met: {
name: 'Metropolitan Museum of Art',
country: 'US',
baseUrl: 'https://collectionapi.metmuseum.org/public/collection/v1',
searchUrl: 'https://collectionapi.metmuseum.org/public/collection/v1/search',
needsApiKey: false,
maxResults: 1000, // Returns IDs, then fetch individually
rateLimit: 200,
enabled: true,
searchStrategy: 'english',
notes: 'Open API, includes European and Japanese historical pieces'
},
harvard: {
name: 'Harvard Art Museums',
country: 'US',
baseUrl: 'https://api.harvardartmuseums.org/v1',
searchUrl: 'https://api.harvardartmuseums.org/v1/object',
needsApiKey: true,
apiKeyEnv: 'HARVARD_API_KEY',
apiKeyParam: 'apikey',
maxResults: 100,
rateLimit: 1000,
enabled: true,
searchStrategy: 'english',
notes: 'Register at https://harvardartmuseums.org/collections/api'
},
cooperhewitt: {
name: 'Cooper Hewitt Smithsonian Design Museum',
country: 'US',
baseUrl: 'https://api.collection.cooperhewitt.org/rest',
searchUrl: 'https://api.collection.cooperhewitt.org/rest',
needsApiKey: true,
apiKeyEnv: 'COOPERHEWITT_API_KEY',
apiKeyParam: 'access_token',
maxResults: 100,
rateLimit: 1000,
enabled: true,
searchStrategy: 'english',
notes: 'Register at https://collection.cooperhewitt.org/api/'
}
};
/**
* Logging utility
*/
function log(message, level = 'INFO') {
const timestamp = new Date().toISOString();
const logMessage = `[${timestamp}] [${level}] ${message}`;
console.log(logMessage);
// Also write to log file
const logFile = path.join(LOG_DIR, `scraper_${new Date().toISOString().split('T')[0]}.log`);
fs.appendFileSync(logFile, logMessage + '\n');
}
/**
* Make HTTPS/HTTP request with error handling
*/
function makeRequest(url, protocol = 'https') {
return new Promise((resolve, reject) => {
const client = protocol === 'https' ? https : http;
client.get(url, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => {
if (res.statusCode !== 200) {
reject(new Error(`HTTP ${res.statusCode}: ${data}`));
return;
}
try {
resolve(JSON.parse(data));
} catch (e) {
reject(new Error(`Failed to parse JSON from ${url}: ${e.message}`));
}
});
}).on('error', reject);
});
}
/**
* Rate limiter
*/
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Normalize item data with SOURCE TRACKING
*/
function normalizeItem(item, source, institution, searchTerm, timestamp) {
const normalized = {
// SOURCE ATTRIBUTION (REQUIRED)
source_institution: institution.name,
source_country: institution.country,
source_api: source,
source_url: institution.baseUrl,
source_search_term: searchTerm,
collected_at: timestamp,
// ITEM DATA
item_id: null,
title: null,
description: null,
date: null,
century: null,
maker: null,
culture: null,
classification: null,
medium: null,
materials: null,
dimensions: null,
item_url: null,
image_url: null,
image_urls: [],
// HANDBAG SPECIFIC
type: null, // handbag, purse, clutch, etc.
brand: null,
// RAW DATA (for reference)
raw_data: item
};
return normalized;
}
/**
* Fetch from Europeana Fashion API
*/
async function fetchEuropeana(apiKey) {
const institutionKey = 'europeana';
const institution = INSTITUTIONS[institutionKey];
log(`\n🇪🇺 Starting ${institution.name}...`, 'INFO');
if (!apiKey) {
log('No API key provided. Skipping.', 'WARN');
return { institution: institution.name, items: [], error: 'No API key' };
}
const allItems = [];
const timestamp = new Date().toISOString();
// Search for handbag terms in multiple languages
const termsToSearch = [
...SEARCH_TERMS.en,
...SEARCH_TERMS.fr,
...SEARCH_TERMS.it,
...SEARCH_TERMS.nl,
...SEARCH_TERMS.de
];
for (const term of termsToSearch) {
try {
// Search with Fashion provider filter
const url = `${institution.searchUrl}?${institution.apiKeyParam}=${apiKey}&query=${encodeURIComponent(term)}&qf=PROVIDER:"Europeana Fashion"&rows=${institution.maxResults}`;
log(` Searching: "${term}"...`);
const data = await makeRequest(url);
if (data.items && data.items.length > 0) {
log(` ✓ Found ${data.items.length} items`);
// Normalize and add source tracking
data.items.forEach(item => {
const normalized = normalizeItem(item, institutionKey, institution, term, timestamp);
// Extract Europeana-specific fields
normalized.item_id = item.id;
normalized.title = item.title ? item.title[0] : null;
normalized.description = item.dcDescription ? item.dcDescription[0] : null;
normalized.date = item.year ? item.year[0] : null;
normalized.maker = item.dcCreator ? item.dcCreator[0] : null;
normalized.item_url = item.guid;
normalized.image_url = item.edmPreview ? item.edmPreview[0] : null;
normalized.type = item.type;
allItems.push(normalized);
});
} else {
log(` - No items found for "${term}"`);
}
await sleep(institution.rateLimit);
} catch (error) {
log(` ✗ Error searching "${term}": ${error.message}`, 'ERROR');
}
}
// Also search for luxury brands
log(` Searching luxury brands...`);
for (const brand of LUXURY_BRANDS.slice(0, 10)) { // Limit to top 10 brands
try {
const url = `${institution.searchUrl}?${institution.apiKeyParam}=${apiKey}&query=${encodeURIComponent(brand + ' bag')}&qf=PROVIDER:"Europeana Fashion"&rows=${institution.maxResults}`;
log(` Brand: ${brand}...`);
const data = await makeRequest(url);
if (data.items && data.items.length > 0) {
log(` ✓ Found ${data.items.length} items`);
data.items.forEach(item => {
const normalized = normalizeItem(item, institutionKey, institution, `${brand} bag`, timestamp);
normalized.brand = brand;
normalized.item_id = item.id;
normalized.title = item.title ? item.title[0] : null;
normalized.item_url = item.guid;
normalized.image_url = item.edmPreview ? item.edmPreview[0] : null;
allItems.push(normalized);
});
}
await sleep(institution.rateLimit);
} catch (error) {
log(` ✗ Error: ${error.message}`, 'ERROR');
}
}
// Save results
const outputFile = path.join(OUTPUT_DIR, `europeana_handbags_${Date.now()}.json`);
fs.writeFileSync(outputFile, JSON.stringify(allItems, null, 2));
log(`💾 Saved ${allItems.length} items to ${outputFile}`, 'SUCCESS');
return { institution: institution.name, items: allItems, count: allItems.length };
}
/**
* Fetch from Rijksmuseum API
*/
async function fetchRijksmuseum(apiKey) {
const institutionKey = 'rijksmuseum';
const institution = INSTITUTIONS[institutionKey];
log(`\n🇳🇱 Starting ${institution.name}...`, 'INFO');
if (!apiKey) {
log('No API key provided. Skipping.', 'WARN');
return { institution: institution.name, items: [], error: 'No API key' };
}
const allItems = [];
const timestamp = new Date().toISOString();
// Search Dutch terms + English fallbacks
const termsToSearch = [...SEARCH_TERMS.nl, ...SEARCH_TERMS.en.slice(0, 3)];
for (const term of termsToSearch) {
try {
const url = `${institution.searchUrl}?${institution.apiKeyParam}=${apiKey}&q=${encodeURIComponent(term)}&imgonly=true&ps=${institution.maxResults}`;
log(` Searching: "${term}"...`);
const data = await makeRequest(url);
if (data.artObjects && data.artObjects.length > 0) {
log(` ✓ Found ${data.artObjects.length} items`);
data.artObjects.forEach(item => {
const normalized = normalizeItem(item, institutionKey, institution, term, timestamp);
normalized.item_id = item.objectNumber;
normalized.title = item.title;
normalized.maker = item.principalOrFirstMaker;
normalized.item_url = item.links?.web || `https://www.rijksmuseum.nl/nl/collectie/${item.objectNumber}`;
normalized.image_url = item.webImage?.url || item.headerImage?.url;
allItems.push(normalized);
});
}
await sleep(institution.rateLimit);
} catch (error) {
log(` ✗ Error: ${error.message}`, 'ERROR');
}
}
const outputFile = path.join(OUTPUT_DIR, `rijksmuseum_handbags_${Date.now()}.json`);
fs.writeFileSync(outputFile, JSON.stringify(allItems, null, 2));
log(`💾 Saved ${allItems.length} items to ${outputFile}`, 'SUCCESS');
return { institution: institution.name, items: allItems, count: allItems.length };
}
/**
* Fetch from V&A Museum API
*/
async function fetchVAM() {
const institutionKey = 'vam';
const institution = INSTITUTIONS[institutionKey];
log(`\n🇬🇧 Starting ${institution.name}...`, 'INFO');
const allItems = [];
const timestamp = new Date().toISOString();
for (const term of SEARCH_TERMS.en) {
try {
const url = `${institution.searchUrl}?q=${encodeURIComponent(term)}&page_size=${institution.maxResults}`;
log(` Searching: "${term}"...`);
const data = await makeRequest(url);
if (data.records && data.records.length > 0) {
log(` ✓ Found ${data.records.length} items`);
data.records.forEach(item => {
const normalized = normalizeItem(item, institutionKey, institution, term, timestamp);
normalized.item_id = item.systemNumber;
normalized.title = item._primaryTitle;
normalized.date = item._primaryDate;
normalized.maker = item._primaryMaker?.name;
normalized.description = item._primaryDescription;
normalized.item_url = `https://collections.vam.ac.uk/item/${item.systemNumber}/`;
normalized.image_url = item._primaryImageId ?
`https://framemark.vam.ac.uk/collections/${item._primaryImageId}/full/735,/0/default.jpg` : null;
allItems.push(normalized);
});
}
await sleep(institution.rateLimit);
} catch (error) {
log(` ✗ Error: ${error.message}`, 'ERROR');
}
}
const outputFile = path.join(OUTPUT_DIR, `vam_handbags_${Date.now()}.json`);
fs.writeFileSync(outputFile, JSON.stringify(allItems, null, 2));
log(`💾 Saved ${allItems.length} items to ${outputFile}`, 'SUCCESS');
return { institution: institution.name, items: allItems, count: allItems.length };
}
/**
* Fetch from Met Museum API
*/
async function fetchMet() {
const institutionKey = 'met';
const institution = INSTITUTIONS[institutionKey];
log(`\n🏛️ Starting ${institution.name}...`, 'INFO');
const allItems = [];
const timestamp = new Date().toISOString();
for (const term of SEARCH_TERMS.en) {
try {
// Step 1: Search for object IDs
const searchUrl = `${institution.searchUrl}?q=${encodeURIComponent(term)}`;
log(` Searching: "${term}"...`);
const searchData = await makeRequest(searchUrl);
if (searchData.objectIDs && searchData.objectIDs.length > 0) {
const objectIds = searchData.objectIDs.slice(0, 50); // Limit to 50 per term
log(` ✓ Found ${searchData.total} items (fetching first ${objectIds.length})...`);
// Step 2: Fetch individual objects
for (const id of objectIds) {
try {
const objectUrl = `${institution.baseUrl}/objects/${id}`;
const item = await makeRequest(objectUrl);
const normalized = normalizeItem(item, institutionKey, institution, term, timestamp);
normalized.item_id = item.objectID;
normalized.title = item.title;
normalized.date = item.objectDate;
normalized.century = item.period;
normalized.maker = item.artistDisplayName;
normalized.culture = item.culture;
normalized.classification = item.classification;
normalized.medium = item.medium;
normalized.dimensions = item.dimensions;
normalized.item_url = item.objectURL;
normalized.image_url = item.primaryImage;
normalized.image_urls = item.additionalImages || [];
allItems.push(normalized);
await sleep(institution.rateLimit);
} catch (error) {
log(` ✗ Error fetching object ${id}: ${error.message}`, 'ERROR');
}
}
}
} catch (error) {
log(` ✗ Error: ${error.message}`, 'ERROR');
}
}
const outputFile = path.join(OUTPUT_DIR, `met_handbags_${Date.now()}.json`);
fs.writeFileSync(outputFile, JSON.stringify(allItems, null, 2));
log(`💾 Saved ${allItems.length} items to ${outputFile}`, 'SUCCESS');
return { institution: institution.name, items: allItems, count: allItems.length };
}
/**
* Main execution
*/
async function main() {
log('='.repeat(80), 'INFO');
log('🚀 MEGA HANDBAG DATA SCRAPER - Starting...', 'INFO');
log('='.repeat(80), 'INFO');
log('Focus: Handbags, purses, bags, clutches only', 'INFO');
log('Source tracking: Every item includes complete attribution', 'INFO');
log('='.repeat(80), 'INFO');
const startTime = Date.now();
const results = [];
// Load API keys from environment
const apiKeys = {
europeana: process.env.EUROPEANA_API_KEY,
rijksmuseum: process.env.RIJKSMUSEUM_API_KEY,
harvard: process.env.HARVARD_API_KEY,
cooperhewitt: process.env.COOPERHEWITT_API_KEY
};
// Fetch from all enabled institutions
if (INSTITUTIONS.europeana.enabled) {
try {
const result = await fetchEuropeana(apiKeys.europeana);
results.push(result);
} catch (error) {
log(`Failed to fetch from Europeana: ${error.message}`, 'ERROR');
}
}
if (INSTITUTIONS.rijksmuseum.enabled) {
try {
const result = await fetchRijksmuseum(apiKeys.rijksmuseum);
results.push(result);
} catch (error) {
log(`Failed to fetch from Rijksmuseum: ${error.message}`, 'ERROR');
}
}
if (INSTITUTIONS.vam.enabled) {
try {
const result = await fetchVAM();
results.push(result);
} catch (error) {
log(`Failed to fetch from V&A: ${error.message}`, 'ERROR');
}
}
if (INSTITUTIONS.met.enabled) {
try {
const result = await fetchMet();
results.push(result);
} catch (error) {
log(`Failed to fetch from Met: ${error.message}`, 'ERROR');
}
}
// Generate summary report
const endTime = Date.now();
const duration = ((endTime - startTime) / 1000 / 60).toFixed(2);
const summary = {
scraper_version: '1.0.0',
execution_date: new Date().toISOString(),
duration_minutes: duration,
total_institutions_queried: results.length,
total_items_collected: results.reduce((sum, r) => sum + r.count, 0),
results_by_institution: results.map(r => ({
institution: r.institution,
items_collected: r.count,
error: r.error || null
})),
search_terms_used: SEARCH_TERMS,
luxury_brands_searched: LUXURY_BRANDS,
output_directory: OUTPUT_DIR,
notes: [
'All items include complete source attribution',
'Focus: Handbags, purses, bags, clutches only',
'Multilingual searches performed',
'Rate limiting respected for all APIs'
]
};
const summaryFile = path.join(OUTPUT_DIR, `scraper_summary_${Date.now()}.json`);
fs.writeFileSync(summaryFile, JSON.stringify(summary, null, 2));
log('='.repeat(80), 'INFO');
log('✅ SCRAPING COMPLETE!', 'SUCCESS');
log(`⏱️ Duration: ${duration} minutes`, 'INFO');
log(`📊 Total items collected: ${summary.total_items_collected}`, 'INFO');
log(`📁 Output directory: ${OUTPUT_DIR}`, 'INFO');
log(`📄 Summary: ${summaryFile}`, 'INFO');
log('='.repeat(80), 'INFO');
// Print institution breakdown
results.forEach(r => {
log(` ${r.institution}: ${r.count} items`, 'INFO');
});
log('='.repeat(80), 'INFO');
return summary;
}
// Run if called directly
if (require.main === module) {
main().catch(error => {
log(`FATAL ERROR: ${error.message}`, 'ERROR');
log(error.stack, 'ERROR');
process.exit(1);
});
}
module.exports = { main, INSTITUTIONS, SEARCH_TERMS, LUXURY_BRANDS };