← back to Handbag Authentication

scripts/fetch_international_museum_data.js

393 lines

#!/usr/bin/env node

/**
 * International Museum Handbag Data Fetcher
 * Fetches handbag and purse data from European, Asian, and global museum APIs
 *
 * Sources:
 * 1. Europeana (EU) - 50M+ items including French fashion
 * 2. Rijksmuseum (Netherlands) - 500K+ items
 * 3. Palais Galliera (France) via Europeana
 * 4. Gallica BNF (French National Library)
 * 5. KCI Kyoto Costume Institute (Japan) - 13K items
 */

const https = require('https');
const fs = require('fs');
const path = require('path');

const BASE_DIR = path.join(__dirname, '..');
const DATA_DIR = path.join(BASE_DIR, 'handbag_data');
const INTL_DIR = path.join(DATA_DIR, 'international_collections');

// API configurations
const APIS = {
    europeana: {
        name: 'Europeana Fashion',
        baseUrl: 'https://api.europeana.eu/record/v2',
        searchUrl: 'https://api.europeana.eu/record/v2/search.json',
        needsApiKey: true,
        apiKeyEnv: 'EUROPEANA_API_KEY',
        description: 'Register at https://pro.europeana.eu/page/get-api',
        languages: ['fr', 'it', 'nl', 'de', 'es']
    },
    rijksmuseum: {
        name: 'Rijksmuseum Amsterdam',
        baseUrl: 'https://www.rijksmuseum.nl/api',
        searchUrl: 'https://www.rijksmuseum.nl/api/nl/collection',
        needsApiKey: true,
        apiKeyEnv: 'RIJKSMUSEUM_API_KEY',
        description: 'Register at https://data.rijksmuseum.nl/',
        language: 'nl'
    },
    parisMusees: {
        name: 'Paris Musées (includes Palais Galliera)',
        baseUrl: 'https://apicollections.parismusees.paris.fr',
        searchUrl: 'https://apicollections.parismusees.paris.fr/graphql',
        needsApiKey: false,
        description: 'Open API - no key required',
        language: 'fr'
    }
};

/**
 * Make HTTPS request
 */
function httpsRequest(url) {
    return new Promise((resolve, reject) => {
        https.get(url, (res) => {
            let data = '';
            res.on('data', chunk => data += chunk);
            res.on('end', () => {
                try {
                    resolve(JSON.parse(data));
                } catch (e) {
                    reject(new Error(`Failed to parse JSON from ${url}: ${e.message}`));
                }
            });
        }).on('error', reject);
    });
}

/**
 * Fetch from Europeana Fashion
 */
async function fetchEuropeanaFashion(apiKey) {
    console.log('\n🇪🇺 Fetching from Europeana Fashion...');

    if (!apiKey) {
        console.log('  ⚠️  No API key provided. Skipping Europeana.');
        console.log('  ℹ️  Get a free API key at: https://pro.europeana.eu/page/get-api');
        return [];
    }

    const searchTerms = [
        { term: 'handbag', lang: 'en' },
        { term: 'sac à main', lang: 'fr' },  // French: handbag
        { term: 'borsetta', lang: 'it' },    // Italian: handbag
        { term: 'handtas', lang: 'nl' },     // Dutch: handbag
        { term: 'Hermès', lang: 'fr' },
        { term: 'Chanel', lang: 'fr' },
        { term: 'Louis Vuitton', lang: 'fr' }
    ];

    const allResults = [];

    for (const { term, lang } of searchTerms) {
        try {
            // Query Europeana Fashion provider
            const url = `https://api.europeana.eu/record/v2/search.json?wskey=${apiKey}&query=${encodeURIComponent(term)}&qf=PROVIDER:"Europeana Fashion"&rows=100`;
            console.log(`  Searching for: "${term}" (${lang})...`);

            const data = await httpsRequest(url);

            if (data.items) {
                console.log(`    ✓ Found ${data.items.length} items`);
                allResults.push(...data.items.map(item => ({
                    ...item,
                    search_term: term,
                    search_language: lang
                })));
            }

            // Be nice to the API
            await new Promise(resolve => setTimeout(resolve, 1000));
        } catch (error) {
            console.error(`    ✗ Error fetching "${term}":`, error.message);
        }
    }

    // Save raw data
    const outputFile = path.join(INTL_DIR, 'european_union', 'europeana_fashion_handbags.json');
    fs.mkdirSync(path.dirname(outputFile), { recursive: true });
    fs.writeFileSync(outputFile, JSON.stringify(allResults, null, 2));
    console.log(`  ✓ Saved ${allResults.length} items to: ${outputFile}`);

    return allResults;
}

/**
 * Fetch from Rijksmuseum
 */
async function fetchRijksmuseum(apiKey) {
    console.log('\n🇳🇱 Fetching from Rijksmuseum Amsterdam...');

    if (!apiKey) {
        console.log('  ⚠️  No API key provided. Skipping Rijksmuseum.');
        console.log('  ℹ️  Get a free API key at: https://data.rijksmuseum.nl/');
        return [];
    }

    const searchTerms = [
        'tas',      // Dutch: bag
        'handtas',  // Dutch: handbag
        'beurs',    // Dutch: purse
    ];

    const allResults = [];

    for (const term of searchTerms) {
        try {
            const url = `https://www.rijksmuseum.nl/api/nl/collection?key=${apiKey}&q=${encodeURIComponent(term)}&ps=100&imgonly=true`;
            console.log(`  Searching for: "${term}"...`);

            const data = await httpsRequest(url);

            if (data.artObjects) {
                console.log(`    ✓ Found ${data.artObjects.length} items`);
                allResults.push(...data.artObjects.map(item => ({
                    ...item,
                    search_term: term
                })));
            }

            await new Promise(resolve => setTimeout(resolve, 1000));
        } catch (error) {
            console.error(`    ✗ Error fetching "${term}":`, error.message);
        }
    }

    // Save raw data
    const outputFile = path.join(INTL_DIR, 'netherlands', 'rijksmuseum_handbags.json');
    fs.mkdirSync(path.dirname(outputFile), { recursive: true });
    fs.writeFileSync(outputFile, JSON.stringify(allResults, null, 2));
    console.log(`  ✓ Saved ${allResults.length} items to: ${outputFile}`);

    return allResults;
}

/**
 * Generate instructions for manual data access
 */
function generateAccessInstructions() {
    const instructions = {
        generated_at: new Date().toISOString(),
        api_access: {
            europeana: {
                name: 'Europeana Fashion',
                registration: 'https://pro.europeana.eu/page/get-api',
                documentation: 'https://pro.europeana.eu/page/search',
                data_coverage: '50 million+ cultural items including extensive French fashion',
                search_examples: [
                    'sac à main (French handbag)',
                    'Hermès, Chanel, Louis Vuitton',
                    'Palais Galliera collections'
                ],
                note: 'Provides access to Palais Galliera (Paris Fashion Museum) collections'
            },
            rijksmuseum: {
                name: 'Rijksmuseum Amsterdam',
                registration: 'https://data.rijksmuseum.nl/',
                documentation: 'https://data.rijksmuseum.nl/object-metadata/api/',
                data_coverage: '500,000+ objects including fashion and textiles',
                search_terms: ['handtas', 'tas', 'beurs'],
                note: 'Dutch museum with extensive fashion collection'
            },
            gallica_bnf: {
                name: 'Gallica - French National Library',
                url: 'https://gallica.bnf.fr/',
                api: 'https://api.bnf.fr/api-gallica-de-recherche',
                documentation: 'https://api.bnf.fr/en/',
                data_coverage: 'Millions of digitized documents including fashion illustrations',
                search_examples: [
                    'sac à main',
                    'maroquinerie',
                    'mode française'
                ],
                note: 'Historical French fashion documents, requires French language searches'
            },
            kci_kyoto: {
                name: 'Kyoto Costume Institute',
                url: 'https://www.kci.or.jp/en/archives/digital_archives/',
                data_coverage: '13,000 items from 17th century to present',
                access: 'Online: 300 items publicly accessible; Full access: reservation required',
                brands_included: ['Chanel', 'Balenciaga', 'Hermès', 'Japanese designers'],
                note: 'Includes both Western and Japanese fashion, particularly strong in haute couture'
            },
            paris_musees: {
                name: 'Paris Musées Collections API',
                url: 'https://apicollections.parismusees.paris.fr/',
                documentation: 'https://apicollections.parismusees.paris.fr/en',
                data_coverage: '280,000+ works including Palais Galliera items',
                access: 'Open API, no key required',
                note: 'Includes Palais Galliera Fashion Museum collections'
            }
        },
        datasets_on_github: {
            fancy_dataset: {
                name: 'FANCY - Fashion Show Dataset',
                repo: 'https://github.com/youngseungjeon/FANCY',
                size: '302,772 images',
                coverage: 'Vogue fashion shows 2010-2019',
                brands: '987 brands including Chanel, Hermès, Louis Vuitton, Gucci, Prada, Fendi',
                attributes: '146 fashion attributes, 25 style groups',
                note: 'Comprehensive French and Italian luxury brand coverage'
            },
            louis_vuitton_data: {
                name: 'Louis Vuitton Web Scraped Data',
                repo: 'https://github.com/databoutique/Louis-Vuitton-web-scraped-data',
                status: 'Commercial dataset',
                note: 'Product catalog snapshot'
            },
            gucci_data: {
                name: 'Gucci Web Scraped Data',
                repo: 'https://github.com/databoutique/Gucci-web-scraped-data',
                status: 'Commercial dataset',
                note: 'Product catalog snapshot'
            }
        },
        french_brands_coverage: {
            major_brands: [
                'Hermès',
                'Louis Vuitton',
                'Chanel',
                'Dior',
                'Celine',
                'Goyard',
                'Longchamp',
                'Balenciaga (Spanish-French)',
                'Yves Saint Laurent'
            ],
            data_sources: [
                'Europeana Fashion (Palais Galliera collections)',
                'FANCY dataset (fashion shows)',
                'Gallica BNF (historical documents)',
                'KCI Kyoto (haute couture pieces)'
            ]
        },
        italian_brands_coverage: {
            major_brands: [
                'Gucci',
                'Prada',
                'Fendi',
                'Versace',
                'Bottega Veneta',
                'Salvatore Ferragamo',
                'Valentino'
            ],
            data_sources: [
                'FANCY dataset (fashion shows)',
                'Europeana Fashion',
                'DeepFashion2 dataset'
            ]
        },
        translation_requirements: {
            french_terms: {
                'handbag': 'sac à main',
                'purse': 'bourse / porte-monnaie',
                'bag': 'sac',
                'leather goods': 'maroquinerie',
                'fashion': 'mode',
                'luxury': 'luxe',
                'vintage': 'vintage'
            },
            italian_terms: {
                'handbag': 'borsetta',
                'purse': 'borsellino',
                'bag': 'borsa',
                'leather goods': 'pelletteria',
                'fashion': 'moda',
                'luxury': 'lusso'
            },
            dutch_terms: {
                'handbag': 'handtas',
                'purse': 'beurs',
                'bag': 'tas',
                'fashion': 'mode'
            },
            japanese_terms: {
                'handbag': 'ハンドバッグ (handobaggu)',
                'purse': '財布 (saifu)',
                'bag': 'バッグ (baggu)',
                'fashion': 'ファッション (fasshon)'
            }
        }
    };

    return instructions;
}

/**
 * Main execution
 */
async function main() {
    console.log('🌍 International Museum Handbag Data Fetcher');
    console.log('='.repeat(60));

    try {
        // Get API keys from environment
        const europeanaKey = process.env.EUROPEANA_API_KEY || '';
        const rijksmuseumKey = process.env.RIJKSMUSEUM_API_KEY || '';

        // Fetch data
        const europeanaData = await fetchEuropeanaFashion(europeanaKey);
        const rijksmuseumData = await fetchRijksmuseum(rijksmuseumKey);

        // Generate access instructions
        const instructions = generateAccessInstructions();
        const instructionsFile = path.join(INTL_DIR, 'INTERNATIONAL_ACCESS_GUIDE.json');
        fs.writeFileSync(instructionsFile, JSON.stringify(instructions, null, 2));
        console.log(`\n📚 Saved access instructions to: ${instructionsFile}`);

        // Summary
        console.log('\n📊 International Data Summary:');
        console.log(`  Europeana Fashion: ${europeanaData.length} items`);
        console.log(`  Rijksmuseum: ${rijksmuseumData.length} items`);
        console.log(`  Total international records: ${europeanaData.length + rijksmuseumData.length}`);

        console.log('\n✅ International data fetch complete!');

        // Print next steps
        console.log('\n📝 Additional Data Sources Available:');
        console.log('  1. FANCY Dataset: 302,772 fashion show images (cloned)');
        console.log('     - Includes French brands: Hermès, Chanel, Louis Vuitton');
        console.log('     - Includes Italian brands: Gucci, Prada, Fendi, Versace');
        console.log('  2. Gallica BNF: French National Library (manual access)');
        console.log('     - Search: https://gallica.bnf.fr/');
        console.log('  3. KCI Kyoto: 300 items online, 13,000 total');
        console.log('     - Access: https://www.kci.or.jp/en/archives/digital_archives/');
        console.log('  4. Paris Musées API: Palais Galliera collections');
        console.log('     - GraphQL API (advanced usage)');

        console.log('\n🔑 To Get More Data:');
        console.log('  1. Register for Europeana API key:');
        console.log('     https://pro.europeana.eu/page/get-api');
        console.log('     Then: export EUROPEANA_API_KEY=your_key');
        console.log('\n  2. Register for Rijksmuseum API key:');
        console.log('     https://data.rijksmuseum.nl/');
        console.log('     Then: export RIJKSMUSEUM_API_KEY=your_key');

    } catch (error) {
        console.error('\n❌ Error during fetch:', error.message);
        console.error(error.stack);
        process.exit(1);
    }
}

// Run if called directly
if (require.main === module) {
    main();
}

module.exports = { main, fetchEuropeanaFashion, fetchRijksmuseum };