← back to Animate Museum Posts

src/animateWithRunway.js

113 lines

import RunwayML from '@runwayml/sdk';
import axios from 'axios';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';

dotenv.config();

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

export async function createRunwayAnimation(imagePath, metadata) {
  try {
    const RUNWAY_API_TOKEN = process.env.RUNWAY_API_TOKEN;

    if (!RUNWAY_API_TOKEN) {
      throw new Error('RUNWAY_API_TOKEN not configured - get it from https://app.runwayml.com/settings/apikeys');
    }

    console.log('Creating REAL object animation with RunwayML Gen-2...');

    const client = new RunwayML({
      apiKey: RUNWAY_API_TOKEN,
    });

    // Read and prepare the image
    const imageBuffer = fs.readFileSync(imagePath);
    const base64Image = imageBuffer.toString('base64');

    // Create animation prompt based on the painting
    let animationPrompt = '';

    if (metadata.title.toLowerCase().includes('sunflower')) {
      animationPrompt = 'Make the sunflowers gently sway in a breeze, petals moving individually, stems bending slightly, natural flower movement';
    } else if (metadata.title.toLowerCase().includes('water') || metadata.title.toLowerCase().includes('wave')) {
      animationPrompt = 'Animate the water with realistic waves, ripples, and flow. Make the water move naturally';
    } else if (metadata.title.toLowerCase().includes('night')) {
      animationPrompt = 'Add twinkling stars, moving clouds, flickering lights, create a living night scene';
    } else {
      animationPrompt = `Bring the painting "${metadata.title}" to life. Make elements move naturally - fabric swaying, leaves rustling, light shifting, everything should have subtle natural movement. No camera panning or zooming.`;
    }

    console.log('Animation prompt:', animationPrompt);

    // Create the generation task
    const task = await client.imageToVideo.create({
      model: 'gen-3-turbo',  // Or 'gen-2' for older model
      image: `data:image/jpeg;base64,${base64Image}`,
      prompt: animationPrompt,
      duration: 4,  // 4 seconds
      aspectRatio: '16:9'
    });

    console.log('Generation started, task ID:', task.id);
    console.log('Waiting for video generation (this takes 1-2 minutes)...');

    // Poll for completion
    let result = await task.poll({
      interval: 5000,  // Check every 5 seconds
      timeout: 180000  // 3 minute timeout
    });

    if (result.status === 'succeeded' && result.output) {
      console.log('Video generation complete!');

      // Download the video
      const videoResponse = await axios.get(result.output, {
        responseType: 'arraybuffer'
      });

      const outputDir = path.join(__dirname, '..', 'output');
      const timestamp = new Date().getTime();
      const videoPath = path.join(outputDir, `runway_animated_${timestamp}.mp4`);

      fs.writeFileSync(videoPath, videoResponse.data);

      const stats = fs.statSync(videoPath);
      console.log(`RunwayML video created: ${(stats.size / 1024 / 1024).toFixed(2)} MB`);

      return {
        originalImage: imagePath,
        animatedVideo: videoPath,
        artworkMetadata: metadata,
        isAnimated: true,
        animationType: 'runway-gen2',
        videoFormat: 'mp4'
      };
    } else {
      throw new Error('Video generation failed: ' + result.status);
    }

  } catch (error) {
    console.error('Error creating RunwayML animation:', error.message);

    if (error.message.includes('RUNWAY_API_TOKEN')) {
      console.error('\n⚠️  To use RunwayML Gen-2:');
      console.error('1. Go to https://app.runwayml.com');
      console.error('2. Sign up (they have free credits to start)');
      console.error('3. Go to Settings > API Keys');
      console.error('4. Create an API key');
      console.error('5. Add to .env: RUNWAY_API_TOKEN=your_token_here');
      console.error('6. Cost: ~$0.05 per 4-second video\n');
    }

    return {
      originalImage: imagePath,
      isStatic: true,
      artworkMetadata: metadata,
      error: error.message
    };
  }
}