← back to Animate Museum Posts

src/animateWithReplicateAlt.js

142 lines

import Replicate from 'replicate';
import axios from 'axios';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';
import sharp from 'sharp';

dotenv.config();

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

export async function createRealAnimation(imagePath, metadata) {
  try {
    const REPLICATE_API_TOKEN = process.env.REPLICATE_API_TOKEN;

    if (!REPLICATE_API_TOKEN) {
      throw new Error('REPLICATE_API_TOKEN not configured');
    }

    console.log('Creating REAL animation with AI...');

    const replicate = new Replicate({
      auth: REPLICATE_API_TOKEN,
    });

    // Convert image to base64
    const imageBuffer = fs.readFileSync(imagePath);
    const base64Image = imageBuffer.toString('base64');
    const dataUri = `data:image/jpeg;base64,${base64Image}`;

    // Try different animation models
    console.log('Attempting AnimateDiff for artistic animation...');

    try {
      // AnimateDiff - better for artistic animation
      const output = await replicate.run(
        "lucataco/animate-diff:1531004ee4c98894ab11f8a4ce6206099e732c1da15121987a8eef54828f0663",
        {
          input: {
            path: dataUri,
            prompt: `animate ${metadata.title}, make the elements move naturally, flowers swaying, leaves rustling, light changing, everything should move and come alive, not camera movement but actual animation of objects`,
            negative_prompt: "panning, zooming, camera movement, static",
            num_frames: 16,
            num_inference_steps: 25,
            guidance_scale: 7.5,
            seed: Math.floor(Math.random() * 1000000)
          }
        }
      );

      if (output) {
        console.log('AnimateDiff animation created!');
        return await downloadVideo(output, metadata);
      }
    } catch (e) {
      console.log('AnimateDiff not available, trying alternative...');
    }

    // Try I2VGen-XL for image to video
    try {
      const output = await replicate.run(
        "ali-vilab/i2vgen-xl:5821a338d00033abaaba89080a17eb8783d9a17ed710a6b4246a18e0900ccad4",
        {
          input: {
            image: dataUri,
            prompt: `Make the ${metadata.title} painting come alive with natural movement. Animate the flowers, make petals move, leaves sway, create life and motion in the scene. No camera movement.`,
            max_frames: 16,
            guidance_scale: 9,
            seed: Math.floor(Math.random() * 1000000)
          }
        }
      );

      if (output) {
        console.log('I2VGen-XL animation created!');
        return await downloadVideo(output, metadata);
      }
    } catch (e) {
      console.log('I2VGen-XL not available, trying next...');
    }

    // Last resort - try SVD with different prompt
    console.log('Using modified Stable Video Diffusion...');

    const output = await replicate.run(
      "stability-ai/stable-video-diffusion:3f0457e4619daac51203dedb472816fd4af51f3149fa7a9e0b5ffcf1b8172438",
      {
        input: {
          input_image: dataUri,
          video_length: "25_frames_with_svd_xt",
          sizing_strategy: "crop_to_16_9",  // Try different strategy
          frames_per_second: 6,
          motion_bucket_id: 200,  // High but not max
          cond_aug: 0.3,
          decoding_t: 14,
          seed: Math.floor(Math.random() * 1000000)
        }
      }
    );

    if (output) {
      return await downloadVideo(output, metadata);
    }

    throw new Error('No animation model succeeded');

  } catch (error) {
    console.error('Error creating animation:', error.message);
    return {
      originalImage: imagePath,
      isStatic: true,
      artworkMetadata: metadata,
      error: error.message
    };
  }
}

async function downloadVideo(videoUrl, metadata) {
  console.log('Downloading animated video...');
  const videoResponse = await axios.get(videoUrl, {
    responseType: 'arraybuffer'
  });

  const outputDir = path.join(__dirname, '..', 'output');
  const timestamp = new Date().getTime();
  const videoPath = path.join(outputDir, `ai_animated_${timestamp}.mp4`);

  fs.writeFileSync(videoPath, videoResponse.data);

  const stats = fs.statSync(videoPath);
  console.log(`AI video created: ${(stats.size / 1024 / 1024).toFixed(2)} MB`);

  return {
    animatedVideo: videoPath,
    artworkMetadata: metadata,
    isAnimated: true,
    animationType: 'ai-animation',
    videoFormat: 'mp4'
  };
}