`scripts/setup-cron.js`

61 lines
const cron = require('node-cron');
const { exec } = require('child_process');
const path = require('path');

// Setup cron to run every 6 hours
// Runs at: 00:00, 06:00, 12:00, 18:00

const cronExpression = '0 */6 * * *'; // Every 6 hours

console.log('Setting up cron job for crawling every 6 hours...');
console.log(`Cron schedule: ${cronExpression}`);

// Create a persistent cron job using node-cron
const task = cron.schedule(cronExpression, () => {
  const timestamp = new Date().toISOString();
  console.log(`\n[${timestamp}] Starting scheduled crawl...`);

  const crawlerPath = path.join(__dirname, '../crawler/index.js');

  exec(`node ${crawlerPath}`, (error, stdout, stderr) => {
    if (error) {
      console.error(`Crawl error: ${error.message}`);
      return;
    }
    if (stderr) {
      console.error(`Crawl stderr: ${stderr}`);
    }
    console.log(stdout);
    console.log(`[${new Date().toISOString()}] Crawl completed`);

    // After crawl, run AI analysis
    const analyzerPath = path.join(__dirname, '../ai/analyzer.js');
    exec(`node ${analyzerPath}`, (error, stdout, stderr) => {
      if (error) {
        console.error(`Analysis error: ${error.message}`);
        return;
      }
      console.log(stdout);
      console.log(`[${new Date().toISOString()}] Analysis completed`);
    });
  });
}, {
  scheduled: true,
  timezone: "UTC"
});

console.log('✓ Cron job configured successfully!');
console.log('The crawler will run automatically every 6 hours.');
console.log('Keep this process running to maintain the schedule.\n');

// Keep the process alive
process.on('SIGINT', () => {
  console.log('\nStopping cron job...');
  task.stop();
  process.exit(0);
});

// Optional: Run immediately on startup
console.log('Run initial crawl now? (The cron job will start regardless)');
console.log('To run manually: npm run crawler\n');