← back to Handbag Authentication
scripts/us_government_bigquery_scraper.js
660 lines
#!/usr/bin/env node
/**
* US GOVERNMENT & BIGQUERY HANDBAG DATA SCRAPER
* Access US government cultural data + BigQuery public datasets
*
* FOCUS: Handbags only
* SOURCE TRACKING: Complete attribution
*
* Target Sources:
* 1. Smithsonian Institution Open Access API (20M+ items)
* 2. Library of Congress API (Fashion, textiles, historical)
* 3. DPLA (Digital Public Library of America) - 40M+ items
* 4. Internet Archive (Fashion catalogs, vintage magazines)
* 5. Google BigQuery Public Datasets (Fashion/product data)
*/
const https = require('https');
const http = require('http');
const fs = require('fs');
const path = require('path');
const BASE_DIR = path.join(__dirname, '..');
const DATA_DIR = path.join(BASE_DIR, 'handbag_data');
const OUTPUT_DIR = path.join(DATA_DIR, 'us_government_bigquery');
const LOG_DIR = path.join(OUTPUT_DIR, 'logs');
// Ensure directories exist
[OUTPUT_DIR, LOG_DIR].forEach(dir => {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
});
// US Government & BigQuery sources
const US_GOV_SOURCES = {
smithsonian: {
name: 'Smithsonian Institution Open Access',
country: 'US',
url: 'https://api.si.edu/openaccess/api/v1.0',
searchUrl: 'https://api.si.edu/openaccess/api/v1.0/search',
needsApiKey: true,
apiKeyEnv: 'SMITHSONIAN_API_KEY',
apiKeyParam: 'api_key',
collection_size: '20+ million objects',
maxResults: 1000,
rateLimit: 500,
enabled: true,
notes: 'Free API key at https://api.data.gov/signup/',
handbag_holdings: 'Extensive fashion and textile collection'
},
loc: {
name: 'Library of Congress',
country: 'US',
url: 'https://www.loc.gov/collections/',
searchUrl: 'https://www.loc.gov/collections/',
apiUrl: 'https://www.loc.gov/collections/?fo=json',
needsApiKey: false,
collection_size: '170+ million items',
rateLimit: 1000,
enabled: true,
notes: 'Open API, no key required',
handbag_holdings: 'Fashion catalogs, vintage advertisements, historical textiles'
},
dpla: {
name: 'Digital Public Library of America',
country: 'US',
url: 'https://dp.la/',
searchUrl: 'https://api.dp.la/v2/items',
needsApiKey: true,
apiKeyEnv: 'DPLA_API_KEY',
apiKeyParam: 'api_key',
collection_size: '40+ million items',
maxResults: 500,
rateLimit: 1000,
enabled: true,
notes: 'Free API key at https://pro.dp.la/developers/api-key-request',
handbag_holdings: 'Aggregates cultural institutions nationwide'
},
internetarchive: {
name: 'Internet Archive',
country: 'US',
url: 'https://archive.org/',
searchUrl: 'https://archive.org/advancedsearch.php',
apiUrl: 'https://archive.org/metadata/',
needsApiKey: false,
collection_size: '700+ billion pages',
rateLimit: 500,
enabled: true,
notes: 'Open API, scrape fashion magazines/catalogs',
handbag_holdings: 'Vintage fashion magazines, designer catalogs, historical documents'
}
};
// BigQuery public datasets for fashion/handbags
const BIGQUERY_DATASETS = {
info: 'Google BigQuery public datasets require Google Cloud setup',
datasets: [
{
name: 'Google Open Images',
description: '9M+ images with annotations',
handbag_relevance: 'Product images, fashion items',
access: 'bigquery-public-data.open_images',
query_example: 'SELECT * FROM `bigquery-public-data.open_images.annotations` WHERE label_name LIKE "%bag%" OR label_name LIKE "%handbag%" OR label_name LIKE "%purse%"'
},
{
name: 'Fashion MNIST',
description: '70,000 fashion product images',
handbag_relevance: 'Fashion classification dataset',
access: 'Via Kaggle/TensorFlow',
notes: 'Already downloaded in many datasets'
},
{
name: 'Common Crawl',
description: 'Web crawl data',
handbag_relevance: 'E-commerce product data, fashion sites',
access: 'bigquery-public-data.common_crawl',
notes: 'Requires processing for handbag-specific data'
}
],
setup_required: true,
documentation: 'https://cloud.google.com/bigquery/public-data'
};
/**
* Logging utility
*/
function log(message, level = 'INFO') {
const timestamp = new Date().toISOString();
const logMessage = `[${timestamp}] [${level}] ${message}`;
console.log(logMessage);
const logFile = path.join(LOG_DIR, `us_gov_bigquery_scraper_${new Date().toISOString().split('T')[0]}.log`);
fs.appendFileSync(logFile, logMessage + '\n');
}
/**
* Make HTTPS request
*/
function makeRequest(url) {
return new Promise((resolve, reject) => {
const client = url.startsWith('https') ? https : http;
client.get(url, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => {
if (res.statusCode !== 200) {
reject(new Error(`HTTP ${res.statusCode}: ${url}`));
return;
}
try {
resolve(JSON.parse(data));
} catch (e) {
// Some APIs return HTML or plain text
resolve(data);
}
});
}).on('error', reject);
});
}
/**
* Sleep/rate limiter
*/
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Normalize US Gov/BigQuery data with source tracking
*/
function normalizeItem(item, sourceKey, institution, searchTerm, timestamp) {
return {
// SOURCE ATTRIBUTION (REQUIRED)
source_institution: institution.name,
source_country: institution.country,
source_api: sourceKey,
source_url: institution.url,
source_search_term: searchTerm,
source_type: 'US_Government_Public_Data',
collected_at: timestamp,
// ITEM DATA
item_id: item.id || item.objectID || item.identifier || null,
title: item.title || item.name || null,
description: item.description || null,
date: item.date || item.year || null,
maker: item.maker || item.creator || item.artist || null,
culture: item.culture || 'American',
classification: item.classification || 'handbag',
materials: item.materials || item.medium || null,
item_url: item.url || item.link || null,
image_url: item.image_url || item.thumbnail || null,
// HANDBAG SPECIFIC
type: 'handbag',
brand: item.brand || null,
// RAW DATA
raw_data: item
};
}
/**
* Fetch from Smithsonian Open Access API
*/
async function fetchSmithsonian(apiKey) {
const sourceKey = 'smithsonian';
const institution = US_GOV_SOURCES[sourceKey];
log(`\n🏛️ Starting ${institution.name}...`, 'INFO');
if (!apiKey) {
log('⚠️ No API key provided. Skipping Smithsonian.', 'WARN');
log(`ℹ️ Get free API key at: https://api.data.gov/signup/`, 'INFO');
return { institution: institution.name, items: [], error: 'No API key' };
}
const allItems = [];
const timestamp = new Date().toISOString();
const searchTerms = ['handbag', 'purse', 'bag', 'clutch', 'evening bag'];
for (const term of searchTerms) {
try {
// Smithsonian Open Access search
const url = `${institution.searchUrl}?${institution.apiKeyParam}=${apiKey}&q=${encodeURIComponent(term)}&rows=100`;
log(` Searching: "${term}"...`);
const data = await makeRequest(url);
if (data.response && data.response.rows) {
log(` ✓ Found ${data.response.rows.length} items`);
data.response.rows.forEach(item => {
const normalized = normalizeItem(item, sourceKey, institution, term, timestamp);
// Extract Smithsonian-specific fields
if (item.content) {
normalized.title = item.content.descriptiveNonRepeating?.title?.content || item.title;
normalized.item_id = item.id;
normalized.item_url = `https://collections.si.edu/search/detail/${item.id}`;
// Extract images
if (item.content.descriptiveNonRepeating?.online_media?.media) {
const media = item.content.descriptiveNonRepeating.online_media.media;
if (media.length > 0) {
normalized.image_url = media[0].thumbnail || media[0].idsId;
}
}
}
allItems.push(normalized);
});
} else {
log(` - No items found for "${term}"`);
}
await sleep(institution.rateLimit);
} catch (error) {
log(` ✗ Error searching "${term}": ${error.message}`, 'ERROR');
}
}
const outputFile = path.join(OUTPUT_DIR, `smithsonian_handbags_${Date.now()}.json`);
fs.writeFileSync(outputFile, JSON.stringify(allItems, null, 2));
log(`💾 Saved ${allItems.length} items to ${outputFile}`, 'SUCCESS');
return { institution: institution.name, items: allItems, count: allItems.length };
}
/**
* Fetch from DPLA (Digital Public Library of America)
*/
async function fetchDPLA(apiKey) {
const sourceKey = 'dpla';
const institution = US_GOV_SOURCES[sourceKey];
log(`\n📚 Starting ${institution.name}...`, 'INFO');
if (!apiKey) {
log('⚠️ No API key provided. Skipping DPLA.', 'WARN');
log(`ℹ️ Get free API key at: https://pro.dp.la/developers/api-key-request`, 'INFO');
return { institution: institution.name, items: [], error: 'No API key' };
}
const allItems = [];
const timestamp = new Date().toISOString();
const searchTerms = ['handbag', 'purse', 'bag', 'clutch'];
for (const term of searchTerms) {
try {
const url = `${institution.searchUrl}?${institution.apiKeyParam}=${apiKey}&q=${encodeURIComponent(term)}&page_size=100`;
log(` Searching: "${term}"...`);
const data = await makeRequest(url);
if (data.docs && data.docs.length > 0) {
log(` ✓ Found ${data.docs.length} items`);
data.docs.forEach(item => {
const normalized = normalizeItem(item, sourceKey, institution, term, timestamp);
normalized.item_id = item.id;
normalized.title = item.sourceResource?.title || item.title;
normalized.description = item.sourceResource?.description;
normalized.date = item.sourceResource?.date?.displayDate;
normalized.maker = item.sourceResource?.creator;
normalized.item_url = item.isShownAt;
normalized.image_url = item.object;
allItems.push(normalized);
});
}
await sleep(institution.rateLimit);
} catch (error) {
log(` ✗ Error: ${error.message}`, 'ERROR');
}
}
const outputFile = path.join(OUTPUT_DIR, `dpla_handbags_${Date.now()}.json`);
fs.writeFileSync(outputFile, JSON.stringify(allItems, null, 2));
log(`💾 Saved ${allItems.length} items to ${outputFile}`, 'SUCCESS');
return { institution: institution.name, items: allItems, count: allItems.length };
}
/**
* Research Library of Congress collections
*/
async function researchLibraryOfCongress() {
const sourceKey = 'loc';
const institution = US_GOV_SOURCES[sourceKey];
log(`\n📖 Researching ${institution.name}...`, 'INFO');
const researchData = {
institution: institution.name,
url: institution.url,
api_documentation: 'https://www.loc.gov/apis/',
collection_size: institution.collection_size,
relevant_collections: [
{
name: 'American Fashion Archives',
url: 'https://www.loc.gov/collections/american-fashion/',
handbag_potential: 'High - fashion history'
},
{
name: 'Printed Ephemera Collection',
url: 'https://www.loc.gov/collections/printed-ephemera/',
handbag_potential: 'Medium - vintage advertisements'
},
{
name: 'Magazine and Journal Collections',
url: 'https://www.loc.gov/collections/magazine-and-journal-collections/',
handbag_potential: 'High - fashion magazines'
},
{
name: 'Business Americana',
url: 'https://www.loc.gov/collections/business-americana/',
handbag_potential: 'Medium - retail catalogs'
}
],
search_strategy: [
'1. Use JSON API: https://www.loc.gov/collections/?fo=json',
'2. Search terms: handbag, purse, bag, leather goods',
'3. Filter by: photographs, prints, manuscripts',
'4. Focus on: 1900-1970 fashion materials'
],
access_method: 'Open API, no key required',
notes: [
'LoC has extensive fashion catalog and magazine collections',
'Historical handbag advertisements and fashion plates',
'Designer catalogs from major American and European brands',
'Requires custom scraping for image extraction'
]
};
const outputFile = path.join(OUTPUT_DIR, 'library_of_congress_research.json');
fs.writeFileSync(outputFile, JSON.stringify(researchData, null, 2));
log(`📄 Saved research data: ${outputFile}`, 'INFO');
return { institution: institution.name, status: 'research_complete' };
}
/**
* Research Internet Archive fashion collections
*/
async function researchInternetArchive() {
const sourceKey = 'internetarchive';
const institution = US_GOV_SOURCES[sourceKey];
log(`\n📦 Researching ${institution.name}...`, 'INFO');
const researchData = {
institution: institution.name,
url: institution.url,
api_documentation: 'https://archive.org/services/docs/api/',
collection_size: institution.collection_size,
relevant_collections: [
{
name: 'Fashion Magazines',
search_url: 'https://archive.org/search.php?query=fashion%20magazine&and[]=mediatype%3A%22texts%22',
handbag_potential: 'Very High - Vogue, Harper\'s Bazaar, etc.'
},
{
name: 'Department Store Catalogs',
search_url: 'https://archive.org/search.php?query=department%20store%20catalog',
handbag_potential: 'Very High - Sears, Montgomery Ward catalogs'
},
{
name: 'Designer Catalogs',
search_url: 'https://archive.org/search.php?query=designer%20catalog%20fashion',
handbag_potential: 'High - Hermès, Chanel materials'
},
{
name: 'Fashion History Books',
search_url: 'https://archive.org/search.php?query=fashion%20history%20accessories',
handbag_potential: 'Medium - academic resources'
}
],
search_strategy: [
'1. Use Advanced Search API',
'2. Search queries:',
' - "handbag fashion"',
' - "purse vintage"',
' - "Vogue handbag"',
' - "designer bag catalog"',
'3. Filter by: texts, images, mediatype',
'4. Download PDFs/images for extraction'
],
api_examples: [
'https://archive.org/metadata/ITEM_ID',
'https://archive.org/advancedsearch.php?q=handbag&fl[]=identifier&output=json'
],
access_method: 'Open API, no key required',
extraction_method: 'Download PDFs/magazines → OCR → Extract handbag images/text',
notes: [
'Massive collection of vintage fashion magazines',
'Full-text searchable after OCR',
'High-quality scans of designer catalogs',
'Requires PDF processing and image extraction',
'Best for historical handbag data (pre-2000)'
]
};
const outputFile = path.join(OUTPUT_DIR, 'internet_archive_research.json');
fs.writeFileSync(outputFile, JSON.stringify(researchData, null, 2));
log(`📄 Saved research data: ${outputFile}`, 'INFO');
return { institution: institution.name, status: 'research_complete' };
}
/**
* Create BigQuery setup guide
*/
function createBigQueryGuide() {
log(`\n☁️ Creating BigQuery setup guide...`, 'INFO');
const guide = {
title: 'Google BigQuery Public Datasets for Handbag Data',
description: 'Access fashion/product datasets via BigQuery',
datasets: BIGQUERY_DATASETS.datasets,
setup_steps: [
'1. Create Google Cloud Project:',
' https://console.cloud.google.com/',
'',
'2. Enable BigQuery API',
'',
'3. Install Google Cloud SDK:',
' curl https://sdk.cloud.google.com | bash',
'',
'4. Authenticate:',
' gcloud auth login',
' gcloud config set project YOUR_PROJECT_ID',
'',
'5. Query public datasets (free up to 1TB/month):',
' bq query --use_legacy_sql=false \'',
' SELECT * FROM `bigquery-public-data.open_images.annotations`',
' WHERE label_name LIKE "%bag%" LIMIT 1000',
' \''
],
handbag_relevant_queries: [
{
dataset: 'Open Images',
query: `
SELECT
image_id,
label_name,
confidence,
xmin, ymin, xmax, ymax
FROM \`bigquery-public-data.open_images.annotations\`
WHERE
(label_name LIKE '%bag%'
OR label_name LIKE '%handbag%'
OR label_name LIKE '%purse%'
OR label_name LIKE '%clutch%')
AND confidence > 0.8
LIMIT 10000
`.trim()
}
],
cost_estimate: {
query_cost: 'Free for first 1TB/month',
storage_cost: 'Results stored in your project (minimal)',
download_cost: 'Egress charges may apply for large downloads'
},
alternative_access: [
'Kaggle Datasets (pre-downloaded): https://www.kaggle.com/datasets',
'TensorFlow Datasets: https://www.tensorflow.org/datasets',
'Hugging Face Datasets: https://huggingface.co/datasets'
],
notes: [
'BigQuery best for large-scale image datasets',
'Open Images has 9M+ product/fashion images',
'Requires Google Cloud setup (15-30 minutes)',
'Free tier sufficient for this project',
'Consider Kaggle/TF Datasets for easier access'
]
};
const guideFile = path.join(OUTPUT_DIR, 'BIGQUERY_SETUP_GUIDE.json');
fs.writeFileSync(guideFile, JSON.stringify(guide, null, 2));
log(`📄 Created BigQuery guide: ${guideFile}`, 'SUCCESS');
return guide;
}
/**
* Main execution
*/
async function main() {
log('='.repeat(80), 'INFO');
log('🇺🇸 US GOVERNMENT & BIGQUERY HANDBAG DATA SCRAPER', 'INFO');
log('='.repeat(80), 'INFO');
const results = [];
// Load API keys
const apiKeys = {
smithsonian: process.env.SMITHSONIAN_API_KEY,
dpla: process.env.DPLA_API_KEY
};
try {
// Smithsonian
if (US_GOV_SOURCES.smithsonian.enabled) {
const result = await fetchSmithsonian(apiKeys.smithsonian);
results.push(result);
}
// DPLA
if (US_GOV_SOURCES.dpla.enabled) {
const result = await fetchDPLA(apiKeys.dpla);
results.push(result);
}
// Library of Congress Research
const locResult = await researchLibraryOfCongress();
results.push(locResult);
// Internet Archive Research
const iaResult = await researchInternetArchive();
results.push(iaResult);
// BigQuery Guide
const bigqueryGuide = createBigQueryGuide();
// Summary
const summary = {
execution_date: new Date().toISOString(),
sources_processed: results.length,
results: results,
api_registration_needed: [
{
source: 'Smithsonian Open Access',
url: 'https://api.data.gov/signup/',
priority: 'HIGH',
expected_data: '1,000-10,000 handbag items'
},
{
source: 'DPLA',
url: 'https://pro.dp.la/developers/api-key-request',
priority: 'HIGH',
expected_data: '500-5,000 handbag items'
},
{
source: 'Google Cloud (for BigQuery)',
url: 'https://console.cloud.google.com/',
priority: 'MEDIUM',
expected_data: '10,000+ product images'
}
],
manual_collection_sources: [
'Library of Congress Collections',
'Internet Archive Fashion Magazines',
'Internet Archive Department Store Catalogs'
],
output_directory: OUTPUT_DIR,
notes: [
'US Government data is public domain',
'All sources include complete attribution',
'BigQuery requires Google Cloud setup',
'Internet Archive best for historical data'
]
};
const summaryFile = path.join(OUTPUT_DIR, `us_gov_bigquery_summary_${Date.now()}.json`);
fs.writeFileSync(summaryFile, JSON.stringify(summary, null, 2));
log('='.repeat(80), 'INFO');
log('✅ US Government & BigQuery research complete!', 'SUCCESS');
log(`📁 Output directory: ${OUTPUT_DIR}`, 'INFO');
log(`📄 Summary: ${summaryFile}`, 'INFO');
log('='.repeat(80), 'INFO');
return summary;
} catch (error) {
log(`ERROR: ${error.message}`, 'ERROR');
log(error.stack, 'ERROR');
throw error;
}
}
// Run if called directly
if (require.main === module) {
main().catch(error => {
log(`FATAL ERROR: ${error.message}`, 'ERROR');
process.exit(1);
});
}
module.exports = { main, US_GOV_SOURCES, BIGQUERY_DATASETS };