← back to Animate Museum Posts

src/animateWithReplicate.js

133 lines

import Replicate from 'replicate';
import axios from 'axios';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';
import sharp from 'sharp';

dotenv.config();

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

export async function createRealAIVideo(imagePath, metadata) {
  try {
    const REPLICATE_API_TOKEN = process.env.REPLICATE_API_TOKEN;

    if (!REPLICATE_API_TOKEN) {
      throw new Error('REPLICATE_API_TOKEN not configured - get it from https://replicate.com/account/api-tokens');
    }

    console.log('Creating REAL AI video with Stable Video Diffusion...');

    const replicate = new Replicate({
      auth: REPLICATE_API_TOKEN,
    });

    // Prepare the image - SVD works best with 1024x576 resolution
    const optimizedImagePath = path.join(path.dirname(imagePath), `optimized_${path.basename(imagePath)}`);
    await sharp(imagePath)
      .resize(1024, 576, {
        fit: 'cover',
        position: 'center'
      })
      .jpeg({ quality: 95 })
      .toFile(optimizedImagePath);

    // Convert image to base64 data URI
    const imageBuffer = fs.readFileSync(optimizedImagePath);
    const base64Image = imageBuffer.toString('base64');
    const dataUri = `data:image/jpeg;base64,${base64Image}`;

    console.log('Sending to Stable Video Diffusion...');
    console.log('Cost estimate: ~$0.02 per video');

    // Run Stable Video Diffusion - allow up to 3 minutes
    console.log('Please wait 60-180 seconds for AI video generation...');
    console.log('Creating SURREAL METAMORPHOSIS - objects transform, morph, come alive!');

    const output = await Promise.race([
      replicate.run(
        "stability-ai/stable-video-diffusion:3f0457e4619daac51203dedb472816fd4af51f3149fa7a9e0b5ffcf1b8172438",
        {
          input: {
            input_image: dataUri,
            video_length: "25_frames_with_svd_xt",
            sizing_strategy: "maintain_aspect_ratio",
            frames_per_second: 8,  // Slightly slower for dramatic effect
            motion_bucket_id: 255,  // MAXIMUM motion for dramatic metamorphosis
            cond_aug: 0.1,  // Higher variation for surreal transformations
            decoding_t: 14,
            seed: Math.floor(Math.random() * 1000000)
          }
        }
      ),
      new Promise((_, reject) => setTimeout(() => reject(new Error('Replicate timeout after 180s')), 180000))
    ]);

    console.log('Video generation complete!');

    if (!output) {
      throw new Error('No video generated');
    }

    // Download the generated video
    console.log('Downloading animated video...');
    const videoUrl = output;
    const videoResponse = await axios.get(videoUrl, {
      responseType: 'arraybuffer'
    });

    const outputDir = path.join(__dirname, '..', 'output');
    const timestamp = new Date().getTime();
    const videoPath = path.join(outputDir, `ai_video_${timestamp}.mp4`);

    fs.writeFileSync(videoPath, videoResponse.data);

    // Clean up optimized image
    if (fs.existsSync(optimizedImagePath)) {
      fs.unlinkSync(optimizedImagePath);
    }

    const stats = fs.statSync(videoPath);
    console.log(`AI video created: ${(stats.size / 1024 / 1024).toFixed(2)} MB`);

    // Check if we need to convert to GIF for Twitter
    // Twitter supports MP4 but has size limits
    if (stats.size > 15 * 1024 * 1024) {
      console.log('Video too large for Twitter, converting to optimized version...');

      // For now, return the video as is
      // In production, you'd use ffmpeg to compress it
    }

    return {
      originalImage: imagePath,
      animatedVideo: videoPath,
      artworkMetadata: metadata,
      isAnimated: true,
      animationType: 'stable-video-diffusion',
      videoFormat: 'mp4'
    };

  } catch (error) {
    console.error('Error creating AI video:', error.message);

    if (error.message.includes('REPLICATE_API_TOKEN')) {
      console.error('\n⚠️  To use real AI video animation:');
      console.error('1. Go to https://replicate.com');
      console.error('2. Sign up for free account');
      console.error('3. Get API token from https://replicate.com/account/api-tokens');
      console.error('4. Add to .env: REPLICATE_API_TOKEN=your_token_here');
      console.error('5. Cost: ~$0.02 per video\n');
    }

    return {
      originalImage: imagePath,
      isStatic: true,
      artworkMetadata: metadata,
      error: error.message
    };
  }
}