`scripts/fetch_museum_data.js`

295 lines
#!/usr/bin/env node

/**
 * Museum Handbag Data Fetcher
 * Fetches handbag and purse data from museum APIs
 *
 * Sources:
 * 1. Harvard Art Museums API
 * 2. Victoria & Albert Museum API
 * 3. Met Museum (CSV-based, already cloned)
 */

const https = require('https');
const fs = require('fs');
const path = require('path');

const BASE_DIR = path.join(__dirname, '..');
const DATA_DIR = path.join(BASE_DIR, 'handbag_data');
const MUSEUM_DIR = path.join(DATA_DIR, 'museum_collections');
const OUTPUT_DIR = path.join(DATA_DIR, 'processed');

// Museum API configurations
const MUSEUMS = {
    harvard: {
        name: 'Harvard Art Museums',
        baseUrl: 'https://api.harvardartmuseums.org/v1',
        needsApiKey: true,
        apiKeyEnv: 'HARVARD_API_KEY',
        description: 'Requires free API key from https://harvardartmuseums.org/collections/api'
    },
    vam: {
        name: 'Victoria & Albert Museum',
        baseUrl: 'https://api.vam.ac.uk/v2',
        needsApiKey: false,
        description: 'Open API, no key required'
    }
};

/**
 * Make HTTPS request
 */
function httpsRequest(url) {
    return new Promise((resolve, reject) => {
        https.get(url, (res) => {
            let data = '';
            res.on('data', chunk => data += chunk);
            res.on('end', () => {
                try {
                    resolve(JSON.parse(data));
                } catch (e) {
                    reject(new Error(`Failed to parse JSON from ${url}: ${e.message}`));
                }
            });
        }).on('error', reject);
    });
}

/**
 * Fetch from Harvard Art Museums
 */
async function fetchHarvardData(apiKey) {
    console.log('\n🏛️  Fetching from Harvard Art Museums...');

    if (!apiKey) {
        console.log('  ⚠️  No API key provided. Skipping Harvard.');
        console.log('  ℹ️  Get a free API key at: https://harvardartmuseums.org/collections/api');
        return [];
    }

    const searchTerms = ['handbag', 'purse', 'bag', 'clutch'];
    const allResults = [];

    for (const term of searchTerms) {
        try {
            const url = `https://api.harvardartmuseums.org/v1/object?apikey=${apiKey}&keyword=${term}&size=100&classification=Accessories`;
            console.log(`  Searching for: "${term}"...`);

            const data = await httpsRequest(url);

            if (data.records) {
                console.log(`    ✓ Found ${data.records.length} items`);
                allResults.push(...data.records);
            }
        } catch (error) {
            console.error(`    ✗ Error fetching "${term}":`, error.message);
        }
    }

    // Save raw data
    const outputFile = path.join(MUSEUM_DIR, 'harvard_handbags.json');
    fs.writeFileSync(outputFile, JSON.stringify(allResults, null, 2));
    console.log(`  ✓ Saved ${allResults.length} items to: ${outputFile}`);

    return allResults;
}

/**
 * Fetch from Victoria & Albert Museum
 */
async function fetchVAMData() {
    console.log('\n🏛️  Fetching from Victoria & Albert Museum...');

    const searchTerms = ['handbag', 'purse', 'bag'];
    const allResults = [];

    for (const term of searchTerms) {
        try {
            const url = `https://api.vam.ac.uk/v2/objects/search?q=${term}&page_size=100`;
            console.log(`  Searching for: "${term}"...`);

            const data = await httpsRequest(url);

            if (data.records) {
                console.log(`    ✓ Found ${data.records.length} items`);
                allResults.push(...data.records);
            } else if (data.info && data.info.record_count) {
                console.log(`    ℹ️  Total available: ${data.info.record_count} items`);
            }
        } catch (error) {
            console.error(`    ✗ Error fetching "${term}":`, error.message);
        }
    }

    // Save raw data
    const outputFile = path.join(MUSEUM_DIR, 'vam_handbags.json');
    fs.writeFileSync(outputFile, JSON.stringify(allResults, null, 2));
    console.log(`  ✓ Saved ${allResults.length} items to: ${outputFile}`);

    return allResults;
}

/**
 * Process and normalize museum data
 */
function normalizeMuseumData(harvardData, vamData) {
    console.log('\n📊 Normalizing museum data...');

    const normalized = [];

    // Process Harvard data
    harvardData.forEach(item => {
        normalized.push({
            source: 'harvard_art_museums',
            museum_id: item.id,
            title: item.title,
            dated: item.dated,
            period: item.period,
            culture: item.culture,
            classification: item.classification,
            medium: item.medium,
            dimensions: item.dimensions,
            description: item.description,
            url: item.url,
            image_url: item.primaryimageurl,
            century: item.century,
            division: item.division,
            department: item.department
        });
    });

    // Process V&A data
    vamData.forEach(item => {
        normalized.push({
            source: 'victoria_albert_museum',
            museum_id: item.systemNumber || item.id,
            title: item._primaryTitle || item.title,
            dated: item._primaryDate || item.productionDates,
            culture: item._primaryPlace || '',
            classification: item.objectType,
            medium: item._primaryMaterialTechnique || '',
            dimensions: '',
            description: item._primaryDescription || '',
            url: `https://collections.vam.ac.uk/item/${item.systemNumber}/`,
            image_url: item._primaryImageId ? `https://framemark.vam.ac.uk/collections/${item._primaryImageId}/full/735,/0/default.jpg` : '',
            artist_maker: item._primaryMaker?.name || ''
        });
    });

    console.log(`  ✓ Normalized ${normalized.length} museum records`);

    return normalized;
}

/**
 * Generate museum data report
 */
function generateReport(harvardData, vamData, normalizedData) {
    const report = {
        generated_at: new Date().toISOString(),
        sources: {
            harvard_art_museums: {
                total_items: harvardData.length,
                api_url: MUSEUMS.harvard.baseUrl,
                documentation: 'https://harvardartmuseums.org/collections/api'
            },
            victoria_albert_museum: {
                total_items: vamData.length,
                api_url: MUSEUMS.vam.baseUrl,
                documentation: 'https://developers.vam.ac.uk/'
            },
            met_museum: {
                note: 'CSV data requires Git LFS. Download from: https://github.com/metmuseum/openaccess',
                total_objects: '492,000+',
                costume_institute: '33,000+ fashion items'
            }
        },
        summary: {
            total_museum_records: normalizedData.length,
            harvard_items: harvardData.length,
            vam_items: vamData.length
        },
        output_files: {
            harvard_raw: 'handbag_data/museum_collections/harvard_handbags.json',
            vam_raw: 'handbag_data/museum_collections/vam_handbags.json',
            normalized: 'handbag_data/processed/museum_handbags_normalized.json'
        },
        notes: {
            met_museum: 'Met Museum CSV (317MB) requires Git LFS installation to download',
            harvard_api_key: 'Get free API key at https://harvardartmuseums.org/collections/api',
            vam_pagination: 'V&A API returns max 100 items per request. Use pagination for more.',
            historical_data: 'Museum collections include handbags from multiple centuries'
        }
    };

    return report;
}

/**
 * Main execution
 */
async function main() {
    console.log('🏛️  Museum Handbag Data Fetcher');
    console.log('='.repeat(60));

    // Ensure directories exist
    if (!fs.existsSync(MUSEUM_DIR)) {
        fs.mkdirSync(MUSEUM_DIR, { recursive: true });
    }
    if (!fs.existsSync(OUTPUT_DIR)) {
        fs.mkdirSync(OUTPUT_DIR, { recursive: true });
    }

    try {
        // Get API keys from environment
        const harvardApiKey = process.env.HARVARD_API_KEY || '';

        // Fetch data
        const harvardData = await fetchHarvardData(harvardApiKey);
        const vamData = await fetchVAMData();

        // Normalize data
        const normalizedData = normalizeMuseumData(harvardData, vamData);

        // Save normalized data
        const normalizedFile = path.join(OUTPUT_DIR, 'museum_handbags_normalized.json');
        fs.writeFileSync(normalizedFile, JSON.stringify(normalizedData, null, 2));
        console.log(`\n💾 Saved normalized data to: ${normalizedFile}`);

        // Generate and save report
        const report = generateReport(harvardData, vamData, normalizedData);
        const reportFile = path.join(OUTPUT_DIR, 'museum_data_report.json');
        fs.writeFileSync(reportFile, JSON.stringify(report, null, 2));
        console.log(`💾 Saved report to: ${reportFile}`);

        // Print summary
        console.log('\n📊 Museum Data Summary:');
        console.log(`  Harvard Art Museums: ${harvardData.length} items`);
        console.log(`  Victoria & Albert: ${vamData.length} items`);
        console.log(`  Total normalized records: ${normalizedData.length}`);

        console.log('\n✅ Museum data fetch complete!');

        // Print instructions
        console.log('\n📝 Next Steps:');
        console.log('  1. For Met Museum data, install Git LFS:');
        console.log('     apt-get install git-lfs');
        console.log('     cd handbag_data/museum_collections/openaccess && git lfs pull');
        console.log('\n  2. For more Harvard data, get API key:');
        console.log('     https://harvardartmuseums.org/collections/api');
        console.log('     Then: export HARVARD_API_KEY=your_key');

    } catch (error) {
        console.error('\n❌ Error during fetch:', error.message);
        console.error(error.stack);
        process.exit(1);
    }
}

// Run if called directly
if (require.main === module) {
    main();
}

module.exports = { main, fetchHarvardData, fetchVAMData };