← back to Handbag Authentication
scripts/aggregate_handbag_data.js
296 lines
#!/usr/bin/env node
/**
* Handbag Data Aggregation Script
* Aggregates handbag data from multiple GitHub sources
*
* Sources:
* 1. Pursearch - Retailer handbag data (2,345 bags)
* 2. Luxury Handbag Price Prediction - Expert-curated pricing (143 luxury bags)
* 3. Met Museum - Historical fashion accessories
* 4. DeepFashion2 - Image dataset with annotations
*/
const fs = require('fs');
const path = require('path');
const csv = require('csv-parser');
const { createObjectCsvWriter } = require('csv-writer');
const BASE_DIR = path.join(__dirname, '..');
const DATA_DIR = path.join(BASE_DIR, 'handbag_data');
const OUTPUT_DIR = path.join(DATA_DIR, 'processed');
// Ensure output directory exists
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
}
/**
* Read CSV file and return array of objects
*/
function readCSV(filePath) {
return new Promise((resolve, reject) => {
const results = [];
fs.createReadStream(filePath)
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => resolve(results))
.on('error', (error) => reject(error));
});
}
/**
* Process Pursearch retailer data
*/
async function processPursearchData() {
console.log('\n📦 Processing Pursearch retailer data...');
const saksBags = path.join(DATA_DIR, 'github_datasets/Pursearch/data/shop/file/shop_bag_index_final.csv');
const sheinBags = path.join(DATA_DIR, 'github_datasets/Pursearch/data/shop/file/shop_shein_index_final.csv');
const saksData = await readCSV(saksBags);
const sheinData = await readCSV(sheinBags);
console.log(` ✓ Saks Fifth Avenue: ${saksData.length} handbags`);
console.log(` ✓ Shein: ${sheinData.length} handbags`);
// Normalize data structure
const normalizedData = [...saksData, ...sheinData].map(bag => ({
source: 'pursearch',
brand: bag.brand || '',
product_name: bag.product || '',
price: bag.price ? parseFloat(bag.price.replace(/[$,]/g, '')) : null,
price_original: bag.price || '',
price_level: bag.price_level || '',
type: bag.type || '',
image_url: bag.image || '',
page_url: bag.page || '',
retailer: bag.page?.includes('shein') ? 'Shein' : 'Saks Fifth Avenue'
}));
return normalizedData;
}
/**
* Process Luxury Handbag Price Prediction data
*/
async function processLuxuryPriceData() {
console.log('\n💎 Processing luxury handbag price data...');
const cleanDataPath = path.join(DATA_DIR, 'github_datasets/Luxury-Handbag-Price-Prediction/Data/clean_data.csv');
const data = await readCSV(cleanDataPath);
console.log(` ✓ Luxury handbags: ${data.length} items`);
// Normalize data structure
const normalizedData = data.map(bag => ({
source: 'luxury_price_prediction',
brand: bag.brand || '',
bag_style: bag['bag style'] || '',
skin_type: bag['skin type'] || '',
hardware_metal: bag['hardware - metal type'] || '',
hardware_zips: parseInt(bag['hardware - num zips']) || 0,
hardware_strap_type: bag['hardware - strap type'] || '',
strap_length: parseFloat(bag['strap length']) || null,
num_compartments: parseInt(bag['num compartments']) || 0,
num_components: parseInt(bag['num components']) || 0,
major_color: bag['major color'] || '',
num_colors: parseInt(bag['num colors']) || 0,
volume: parseFloat(bag.volume) || null,
num_functionality: parseInt(bag['num functionality']) || 0,
inner_material: bag['inner material'] || '',
price: parseFloat(bag.price) || null,
accessories: bag.accessories || ''
}));
return normalizedData;
}
/**
* Analyze data and generate statistics
*/
function analyzeData(pursearchData, luxuryData) {
console.log('\n📊 Data Analysis Summary\n');
console.log('=' .repeat(60));
// Total counts
console.log(`Total handbags collected: ${pursearchData.length + luxuryData.length}`);
console.log(` - Retailer data (Pursearch): ${pursearchData.length}`);
console.log(` - Luxury expert data: ${luxuryData.length}`);
// Brand analysis from Pursearch
const brandCounts = {};
pursearchData.forEach(bag => {
if (bag.brand) {
brandCounts[bag.brand] = (brandCounts[bag.brand] || 0) + 1;
}
});
const topBrands = Object.entries(brandCounts)
.sort((a, b) => b[1] - a[1])
.slice(0, 10);
console.log('\n🏷️ Top 10 Brands (by count):');
topBrands.forEach(([brand, count], idx) => {
console.log(` ${idx + 1}. ${brand}: ${count} items`);
});
// Price analysis
const prices = pursearchData
.map(bag => bag.price)
.filter(price => price !== null && price > 0);
if (prices.length > 0) {
const avgPrice = prices.reduce((sum, p) => sum + p, 0) / prices.length;
const minPrice = Math.min(...prices);
const maxPrice = Math.max(...prices);
console.log('\n💰 Price Analysis (Retailer Data):');
console.log(` Average: $${avgPrice.toFixed(2)}`);
console.log(` Min: $${minPrice.toFixed(2)}`);
console.log(` Max: $${maxPrice.toFixed(2)}`);
}
// Luxury price analysis
const luxuryPrices = luxuryData
.map(bag => bag.price)
.filter(price => price !== null && price > 0);
if (luxuryPrices.length > 0) {
const avgLuxury = luxuryPrices.reduce((sum, p) => sum + p, 0) / luxuryPrices.length;
const minLuxury = Math.min(...luxuryPrices);
const maxLuxury = Math.max(...luxuryPrices);
console.log('\n💎 Price Analysis (Luxury Expert Data):');
console.log(` Average: $${avgLuxury.toFixed(2)}`);
console.log(` Min: $${minLuxury.toFixed(2)}`);
console.log(` Max: $${maxLuxury.toFixed(2)}`);
}
// Type analysis
const types = {};
pursearchData.forEach(bag => {
if (bag.type) {
types[bag.type] = (types[bag.type] || 0) + 1;
}
});
console.log('\n👜 Handbag Types:');
Object.entries(types)
.sort((a, b) => b[1] - a[1])
.forEach(([type, count]) => {
console.log(` ${type}: ${count} items`);
});
console.log('\n' + '='.repeat(60));
}
/**
* Save aggregated data
*/
async function saveAggregatedData(pursearchData, luxuryData) {
console.log('\n💾 Saving aggregated data...');
// Save Pursearch data
const pursearchOutput = path.join(OUTPUT_DIR, 'retailer_handbags.csv');
const pursearchWriter = createObjectCsvWriter({
path: pursearchOutput,
header: [
{ id: 'source', title: 'source' },
{ id: 'brand', title: 'brand' },
{ id: 'product_name', title: 'product_name' },
{ id: 'price', title: 'price_numeric' },
{ id: 'price_original', title: 'price_original' },
{ id: 'price_level', title: 'price_level' },
{ id: 'type', title: 'type' },
{ id: 'retailer', title: 'retailer' },
{ id: 'image_url', title: 'image_url' },
{ id: 'page_url', title: 'page_url' }
]
});
await pursearchWriter.writeRecords(pursearchData);
console.log(` ✓ Saved: ${pursearchOutput}`);
// Save luxury data
const luxuryOutput = path.join(OUTPUT_DIR, 'luxury_handbags.csv');
const luxuryWriter = createObjectCsvWriter({
path: luxuryOutput,
header: [
{ id: 'source', title: 'source' },
{ id: 'brand', title: 'brand' },
{ id: 'bag_style', title: 'bag_style' },
{ id: 'skin_type', title: 'skin_type' },
{ id: 'hardware_metal', title: 'hardware_metal' },
{ id: 'hardware_zips', title: 'hardware_zips' },
{ id: 'hardware_strap_type', title: 'hardware_strap_type' },
{ id: 'strap_length', title: 'strap_length' },
{ id: 'num_compartments', title: 'num_compartments' },
{ id: 'num_components', title: 'num_components' },
{ id: 'major_color', title: 'major_color' },
{ id: 'num_colors', title: 'num_colors' },
{ id: 'volume', title: 'volume' },
{ id: 'num_functionality', title: 'num_functionality' },
{ id: 'inner_material', title: 'inner_material' },
{ id: 'price', title: 'price' },
{ id: 'accessories', title: 'accessories' }
]
});
await luxuryWriter.writeRecords(luxuryData);
console.log(` ✓ Saved: ${luxuryOutput}`);
// Save combined summary
const allData = [...pursearchData, ...luxuryData];
const summaryOutput = path.join(OUTPUT_DIR, 'all_handbags_summary.json');
fs.writeFileSync(summaryOutput, JSON.stringify({
metadata: {
generated_at: new Date().toISOString(),
total_records: allData.length,
sources: {
pursearch: pursearchData.length,
luxury_price_prediction: luxuryData.length
}
},
summary: {
total_handbags: allData.length,
retailer_items: pursearchData.length,
luxury_items: luxuryData.length
}
}, null, 2));
console.log(` ✓ Saved: ${summaryOutput}`);
}
/**
* Main execution
*/
async function main() {
console.log('🎒 Handbag Data Aggregation Tool');
console.log('='.repeat(60));
try {
// Process all data sources
const pursearchData = await processPursearchData();
const luxuryData = await processLuxuryPriceData();
// Analyze
analyzeData(pursearchData, luxuryData);
// Save aggregated data
await saveAggregatedData(pursearchData, luxuryData);
console.log('\n✅ Data aggregation complete!');
console.log(`\nOutput files saved to: ${OUTPUT_DIR}`);
} catch (error) {
console.error('\n❌ Error during aggregation:', error.message);
console.error(error.stack);
process.exit(1);
}
}
// Run if called directly
if (require.main === module) {
main();
}
module.exports = { main, processPursearchData, processLuxuryPriceData };