← back to Handbag Authentication
scripts/japanese_handbag_scraper.js
515 lines
#!/usr/bin/env node
/**
* JAPANESE HANDBAG INSTITUTIONS SCRAPER
* Specialized scraper for Japanese fashion museums and universities
*
* FOCUS: Handbags only
* SOURCE TRACKING: Complete attribution for every item
*
* Target Institutions:
* 1. Kyoto Costume Institute (KCI) - 300 online items
* 2. Bunka Gakuen Costume Museum - Web scraping
* 3. Kobe Fashion Museum - Web scraping
* 4. Tokyo National Museum - API if available
*/
const https = require('https');
const fs = require('fs');
const path = require('path');
const BASE_DIR = path.join(__dirname, '..');
const DATA_DIR = path.join(BASE_DIR, 'handbag_data');
const OUTPUT_DIR = path.join(DATA_DIR, 'japanese_collections');
const LOG_DIR = path.join(OUTPUT_DIR, 'logs');
// Ensure directories exist
[OUTPUT_DIR, LOG_DIR].forEach(dir => {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
});
// Japanese handbag search terms
const JAPANESE_TERMS = {
handbag: ['ハンドバッグ', 'handobaggu'],
bag: ['バッグ', 'baggu', 'かばん', '鞄'],
purse: ['財布', 'saifu', 'がま口', 'gamaguchi'],
clutch: ['クラッチバッグ', 'kuracchi baggu'],
leather_goods: ['革製品', 'kawa seihin'],
luxury_brand: ['高級ブランド', 'kōkyū burando']
};
// Japanese institutions configuration
const JAPANESE_INSTITUTIONS = {
kci: {
name: 'Kyoto Costume Institute',
name_ja: '京都服飾文化研究財団',
location: 'Kyoto, Japan',
url: 'https://www.kci.or.jp',
digital_archive: 'https://www.kci.or.jp/en/archives/digital_archives/',
collection_size: '13,000+ items (300 online)',
type: 'museum',
focus: 'Western fashion history, haute couture',
handbag_holdings: 'French and Japanese designer bags',
access_method: 'web_scraping',
status: 'active'
},
bunka: {
name: 'Bunka Gakuen Costume Museum',
name_ja: '文化学園服飾博物館',
location: 'Tokyo, Japan',
url: 'https://museum.bunka.ac.jp',
collection_size: '30,000+ items',
type: 'university_museum',
affiliation: 'Bunka Fashion College',
focus: 'Historical costume, ethnic dress, modern fashion',
handbag_holdings: 'Japanese and Western bags',
access_method: 'web_scraping',
status: 'research_needed'
},
kobe: {
name: 'Kobe Fashion Museum',
name_ja: '神戸ファッション美術館',
location: 'Kobe, Japan',
url: 'https://www.fashionmuseum.jp',
collection_size: '9,000+ items',
type: 'museum',
focus: '20th century Western fashion',
handbag_holdings: '20th century handbags',
access_method: 'web_scraping',
status: 'active'
},
tnm: {
name: 'Tokyo National Museum',
name_ja: '東京国立博物館',
location: 'Tokyo, Japan',
url: 'https://www.tnm.jp',
api: 'https://www.tnm.jp/modules/r_free_page/index.php?id=1941',
collection_size: '120,000+ objects',
type: 'national_museum',
focus: 'Japanese art and cultural artifacts',
handbag_holdings: 'Traditional Japanese bags (kinchaku, etc.)',
access_method: 'api_or_web',
status: 'research_needed'
}
};
/**
* Logging utility
*/
function log(message, level = 'INFO') {
const timestamp = new Date().toISOString();
const logMessage = `[${timestamp}] [${level}] ${message}`;
console.log(logMessage);
const logFile = path.join(LOG_DIR, `japanese_scraper_${new Date().toISOString().split('T')[0]}.log`);
fs.appendFileSync(logFile, logMessage + '\n');
}
/**
* Make HTTPS request
*/
function makeRequest(url) {
return new Promise((resolve, reject) => {
https.get(url, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => {
if (res.statusCode !== 200) {
reject(new Error(`HTTP ${res.statusCode}: ${url}`));
return;
}
resolve(data); // Return raw HTML/text
});
}).on('error', reject);
});
}
/**
* Sleep/rate limiter
*/
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Normalize Japanese item data with source tracking
*/
function normalizeJapaneseItem(item, institutionKey, searchTerm, timestamp) {
const institution = JAPANESE_INSTITUTIONS[institutionKey];
return {
// SOURCE ATTRIBUTION (REQUIRED)
source_institution: institution.name,
source_institution_ja: institution.name_ja,
source_country: 'Japan',
source_location: institution.location,
source_url: institution.url,
source_search_term: searchTerm,
source_collection_type: institution.type,
collected_at: timestamp,
// ITEM DATA
item_id: item.id || null,
title: item.title || null,
title_ja: item.title_ja || null,
description: item.description || null,
description_ja: item.description_ja || null,
date: item.date || null,
era: item.era || null, // Japanese era (Meiji, Taisho, Showa, etc.)
maker: item.maker || null,
designer: item.designer || null,
brand: item.brand || null,
culture: 'Japanese',
classification: item.classification || 'handbag',
materials: item.materials || null,
dimensions: item.dimensions || null,
item_url: item.url || null,
image_url: item.image_url || null,
image_urls: item.image_urls || [],
// HANDBAG SPECIFIC
type: item.type || 'handbag',
style: item.style || null,
// RAW DATA
raw_data: item
};
}
/**
* Scrape KCI Kyoto Digital Archives
* NOTE: This is a basic implementation. Full scraping would require puppeteer.
*/
async function scrapeKCIKyoto() {
const institutionKey = 'kci';
const institution = JAPANESE_INSTITUTIONS[institutionKey];
log(`\n🇯🇵 Starting ${institution.name} (KCI Kyoto)...`, 'INFO');
log(`URL: ${institution.digital_archive}`, 'INFO');
const items = [];
const timestamp = new Date().toISOString();
// NOTE: KCI digital archives requires JavaScript rendering
// This is a placeholder structure showing how data should be collected
log('⚠️ KCI Kyoto requires browser automation (Puppeteer) for full scraping', 'WARN');
log('Manual collection recommended for 300 online items', 'INFO');
// Create manual collection template
const template = {
institution: institution.name,
url: institution.digital_archive,
collection_method: 'MANUAL',
instructions: [
'1. Visit: https://www.kci.or.jp/en/archives/digital_archives/',
'2. Browse chronologically or by category',
'3. Look for handbag, bag, purse items',
'4. For each item, collect:',
' - Image URL',
' - Title (English + Japanese if available)',
' - Date/Era',
' - Designer/Maker',
' - Description',
' - Item page URL',
'5. Save data in JSON format using normalizeJapaneseItem structure'
],
search_keywords: [
'bag', 'handbag', 'purse', 'pochette',
'Hermès', 'Chanel', 'Louis Vuitton'
],
expected_items: '50-300 handbag items',
notes: [
'KCI has extensive French designer bag collection',
'High-quality museum photography',
'Bilingual metadata (Japanese/English)',
'Focus on 17th-21st century pieces'
]
};
const templateFile = path.join(OUTPUT_DIR, 'kci_manual_collection_template.json');
fs.writeFileSync(templateFile, JSON.stringify(template, null, 2));
log(`📄 Created manual collection template: ${templateFile}`, 'INFO');
// Example item structure (to be filled manually or with Puppeteer)
const exampleItem = normalizeJapaneseItem({
id: 'KCI-AC-XXXX',
title: 'Handbag',
title_ja: 'ハンドバッグ',
date: '1960s',
designer: 'Hermès',
brand: 'Hermès',
type: 'handbag',
url: 'https://www.kci.or.jp/en/archives/digital_archives/[item_id]',
image_url: 'https://www.kci.or.jp/images/[image_id].jpg'
}, institutionKey, 'handbag', timestamp);
const exampleFile = path.join(OUTPUT_DIR, 'kci_example_item_structure.json');
fs.writeFileSync(exampleFile, JSON.stringify(exampleItem, null, 2));
log(`📄 Created example item structure: ${exampleFile}`, 'INFO');
return { institution: institution.name, items: items, count: 0, status: 'manual_collection_required' };
}
/**
* Research Bunka Gakuen Museum
*/
async function researchBunkaGakuen() {
const institutionKey = 'bunka';
const institution = JAPANESE_INSTITUTIONS[institutionKey];
log(`\n🇯🇵 Researching ${institution.name}...`, 'INFO');
log(`URL: ${institution.url}`, 'INFO');
const researchData = {
institution: institution.name,
name_ja: institution.name_ja,
url: institution.url,
collection_size: institution.collection_size,
research_notes: [
'Affiliated with Bunka Fashion College (top Japanese fashion school)',
'Collection includes 30,000+ fashion items',
'Focus: Historical costume, ethnic dress, modern fashion',
'Likely has Japanese and Western handbag collection',
'May have online database - needs research'
],
next_steps: [
'1. Visit website and search for online collections',
'2. Look for digital archive or database',
'3. Search for: バッグ, ハンドバッグ, 財布',
'4. Contact museum for research access if needed',
'5. Check if collection data available via Japanese museum aggregators'
],
contact_info: {
address: '3-22-7 Yoyogi, Shibuya-ku, Tokyo',
phone: '+81-3-3299-2387',
note: 'Contact for research access to collections'
},
potential_data_sources: [
'Museum website',
'Japanese museum databases (J-GLOBAL, CiNii)',
'Academic publications from Bunka Fashion College',
'Fashion research archives'
]
};
const outputFile = path.join(OUTPUT_DIR, 'bunka_gakuen_research.json');
fs.writeFileSync(outputFile, JSON.stringify(researchData, null, 2));
log(`📄 Saved research data: ${outputFile}`, 'INFO');
return { institution: institution.name, status: 'research_complete' };
}
/**
* Research Kobe Fashion Museum
*/
async function researchKobeFashion() {
const institutionKey = 'kobe';
const institution = JAPANESE_INSTITUTIONS[institutionKey];
log(`\n🇯🇵 Researching ${institution.name}...`, 'INFO');
log(`URL: ${institution.url}`, 'INFO');
const researchData = {
institution: institution.name,
name_ja: institution.name_ja,
url: institution.url,
collection_size: institution.collection_size,
focus: '20th century Western fashion',
handbag_potential: 'High - 9,000+ Western fashion items likely include handbags',
research_notes: [
'First Japanese museum dedicated to fashion',
'Opened 1997',
'Collection: 9,000+ Western fashion items',
'Focus: 20th century fashion',
'Should have handbag collection from major designers',
'Website may have online collection database'
],
next_steps: [
'1. Explore website for online collection',
'2. Search for: handbag, bag, purse (English/Japanese)',
'3. Look for exhibition archives with handbag photos',
'4. Check for collection API or data export',
'5. Contact museum for digital collection access'
],
contact_info: {
address: '2-9-1 Koyocho-naka, Higashinada-ku, Kobe',
website: 'https://www.fashionmuseum.jp',
note: 'Check website for current exhibitions and collections'
}
};
const outputFile = path.join(OUTPUT_DIR, 'kobe_fashion_museum_research.json');
fs.writeFileSync(outputFile, JSON.stringify(researchData, null, 2));
log(`📄 Saved research data: ${outputFile}`, 'INFO');
return { institution: institution.name, status: 'research_complete' };
}
/**
* Create comprehensive Japanese institution guide
*/
function createJapaneseInstitutionGuide() {
const guide = {
title: 'Japanese Handbag Collections - Data Collection Guide',
generated: new Date().toISOString(),
focus: 'Handbags, purses, bags only',
institutions: JAPANESE_INSTITUTIONS,
search_terms: JAPANESE_TERMS,
collection_strategy: {
phase_1: {
name: 'Manual Collection - KCI Kyoto',
priority: 'HIGH',
items: '50-300 handbag items',
effort: 'Medium (4-8 hours)',
method: 'Manual browsing + data entry',
url: 'https://www.kci.or.jp/en/archives/digital_archives/'
},
phase_2: {
name: 'Research - Bunka Gakuen',
priority: 'HIGH',
potential: '500-2,000 items',
effort: 'Research access required',
method: 'Contact museum, check online database'
},
phase_3: {
name: 'Research - Kobe Fashion Museum',
priority: 'MEDIUM',
potential: '200-500 items',
effort: 'Website exploration + scraping',
method: 'Check online collections, possible API'
},
phase_4: {
name: 'Automated Scraping',
priority: 'LONG-TERM',
tools: 'Puppeteer, Playwright',
effort: 'High (development required)',
method: 'Browser automation for dynamic sites'
}
},
data_requirements: {
mandatory_fields: [
'source_institution',
'source_country',
'collected_at',
'item_url or source_url',
'title or description'
],
recommended_fields: [
'designer/maker',
'date/era',
'brand',
'image_url',
'materials',
'type (handbag/purse/clutch/etc)'
]
},
japanese_luxury_brands: [
'Hermès (French, but in Japanese collections)',
'Chanel (French, but in Japanese collections)',
'Louis Vuitton (French, but in Japanese collections)',
'Traditional Japanese: kinchaku, gamaguchi purses'
],
notes: [
'Japanese museum sites often require JavaScript rendering',
'Many collections have limited online access',
'On-site research access may be required for comprehensive data',
'Bilingual data (Japanese/English) when available',
'High-quality museum photography typical',
'Strong focus on French designer pieces in major collections'
]
};
const guideFile = path.join(OUTPUT_DIR, 'JAPANESE_COLLECTION_GUIDE.json');
fs.writeFileSync(guideFile, JSON.stringify(guide, null, 2));
log(`📄 Created comprehensive guide: ${guideFile}`, 'SUCCESS');
return guide;
}
/**
* Main execution
*/
async function main() {
log('='.repeat(80), 'INFO');
log('🇯🇵 JAPANESE HANDBAG INSTITUTIONS SCRAPER', 'INFO');
log('='.repeat(80), 'INFO');
const results = [];
try {
// KCI Kyoto
const kciResult = await scrapeKCIKyoto();
results.push(kciResult);
await sleep(1000);
// Bunka Gakuen Research
const bunkaResult = await researchBunkaGakuen();
results.push(bunkaResult);
await sleep(1000);
// Kobe Fashion Museum Research
const kobeResult = await researchKobeFashion();
results.push(kobeResult);
// Create comprehensive guide
const guide = createJapaneseInstitutionGuide();
// Summary
const summary = {
execution_date: new Date().toISOString(),
institutions_processed: results.length,
results: results,
output_directory: OUTPUT_DIR,
guide_file: 'JAPANESE_COLLECTION_GUIDE.json',
next_actions: [
'1. Manual collection from KCI Kyoto (priority)',
'2. Contact Bunka Gakuen for research access',
'3. Explore Kobe Fashion Museum website',
'4. Consider Puppeteer development for automation',
'5. Research Japanese museum aggregators'
],
notes: [
'Japanese collections require more manual effort',
'High-quality data with strong provenance',
'Extensive French designer bag holdings in major collections',
'Traditional Japanese bags (kinchaku) also available'
]
};
const summaryFile = path.join(OUTPUT_DIR, `japanese_scraper_summary_${Date.now()}.json`);
fs.writeFileSync(summaryFile, JSON.stringify(summary, null, 2));
log('='.repeat(80), 'INFO');
log('✅ Japanese institutions research complete!', 'SUCCESS');
log(`📁 Output directory: ${OUTPUT_DIR}`, 'INFO');
log(`📄 Summary: ${summaryFile}`, 'INFO');
log('='.repeat(80), 'INFO');
return summary;
} catch (error) {
log(`ERROR: ${error.message}`, 'ERROR');
log(error.stack, 'ERROR');
throw error;
}
}
// Run if called directly
if (require.main === module) {
main().catch(error => {
log(`FATAL ERROR: ${error.message}`, 'ERROR');
process.exit(1);
});
}
module.exports = { main, JAPANESE_INSTITUTIONS, JAPANESE_TERMS };