← back to Animate Museum Posts

src/animateWithMorphing.js

171 lines

import Replicate from 'replicate';
import axios from 'axios';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import sharp from 'sharp';
import { execSync } from 'child_process';
import dotenv from 'dotenv';

dotenv.config();

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const OUTPUT_DIR = path.join(__dirname, '..', 'output');

/**
 * Generate creative morphing prompts based on artwork
 */
function getMorphingPrompt(metadata) {
  const title = (metadata.title || '').toLowerCase();

  const morphEffects = [
    "A playful pitbull dog runs into frame, sniffs the artwork, then magically morphs into a fluffy cat that walks away",
    "Butterflies emerge from the painting and transform into colorful birds flying away",
    "The scene comes alive as a dog appears, chases its tail, then transforms into a sleeping cat",
    "Flowers in the image bloom and morph into flying hummingbirds",
    "A curious puppy enters, looks at viewer, then shapeshifts into a majestic lion",
    "Water ripples form into fish that leap out and transform into dolphins",
    "Clouds drift and morph into galloping horses running across the sky",
    "A pitbull trots through the scene, stops, and metamorphoses into a cat stretching lazily",
    "Elements swirl and transform - dogs become cats, birds become butterflies",
    "Surreal transformation: everyday objects morph into animals that come alive"
  ];

  // Pick random morphing effect
  const effect = morphEffects[Math.floor(Math.random() * morphEffects.length)];

  return `Cinematic video of famous artwork "${metadata.title}". ${effect}. Dramatic lighting, smooth animation, 4K quality, dreamlike atmosphere.`;
}

/**
 * Create morphing video using prompt-based models
 */
export async function createMorphingVideo(imagePath, metadata) {
  console.log('Creating MORPHING video with cats & dogs...');

  const replicate = new Replicate({ auth: process.env.REPLICATE_API_TOKEN });

  // Prepare image
  const optimizedPath = path.join(OUTPUT_DIR, `morph_opt_${Date.now()}.jpg`);
  await sharp(imagePath)
    .resize(1280, 720, { fit: 'cover', position: 'center' })
    .jpeg({ quality: 95 })
    .toFile(optimizedPath);

  const imageBuffer = fs.readFileSync(optimizedPath);
  const base64Image = imageBuffer.toString('base64');
  const dataUri = `data:image/jpeg;base64,${base64Image}`;

  const prompt = getMorphingPrompt(metadata);
  console.log(`Prompt: "${prompt.substring(0, 100)}..."`);

  let output;
  let modelUsed;

  // Try Minimax first (reliable for image-to-video with prompts)
  try {
    console.log('Trying Minimax video-01...');
    output = await Promise.race([
      replicate.run("minimax/video-01", {
        input: {
          prompt: prompt,
          first_frame_image: dataUri
        }
      }),
      new Promise((_, reject) => setTimeout(() => reject(new Error('Minimax timeout')), 300000))
    ]);
    modelUsed = 'minimax';
  } catch (e) {
    console.log('Minimax failed:', e.message);

    // Try Luma Photon
    try {
      console.log('Trying Luma Photon...');
      output = await Promise.race([
        replicate.run("luma/photon", {
          input: {
            prompt: prompt,
            image: dataUri,
            aspect_ratio: "16:9"
          }
        }),
        new Promise((_, reject) => setTimeout(() => reject(new Error('Luma timeout')), 300000))
      ]);
      modelUsed = 'luma-photon';
    } catch (e2) {
      console.log('Luma failed:', e2.message);

      // Final fallback to SVD with max motion
      try {
        console.log('Fallback to SVD max motion...');
        output = await Promise.race([
          replicate.run("stability-ai/stable-video-diffusion:3f0457e4619daac51203dedb472816fd4af51f3149fa7a9e0b5ffcf1b8172438", {
            input: {
              input_image: dataUri,
              video_length: "25_frames_with_svd_xt",
              frames_per_second: 8,
              motion_bucket_id: 255,
              cond_aug: 0.1,
              seed: Math.floor(Math.random() * 1000000)
            }
          }),
          new Promise((_, reject) => setTimeout(() => reject(new Error('SVD timeout')), 200000))
        ]);
        modelUsed = 'svd-max';
      } catch (e3) {
        console.log('All models failed');
        fs.unlinkSync(optimizedPath);
        throw new Error('No animation model available');
      }
    }
  }

  console.log(`Video generated with ${modelUsed}!`);

  // Get video URL from output
  let videoUrl;
  if (typeof output === 'string') {
    videoUrl = output;
  } else if (output?.video) {
    videoUrl = output.video;
  } else if (Array.isArray(output)) {
    videoUrl = output[0];
  } else {
    videoUrl = output;
  }

  // Download video
  const videoResponse = await axios.get(videoUrl, { responseType: 'arraybuffer' });
  const timestamp = Date.now();
  const videoPath = path.join(OUTPUT_DIR, `morphing_${timestamp}.mp4`);
  fs.writeFileSync(videoPath, videoResponse.data);

  // Convert to H.264 for Twitter
  const h264Path = videoPath.replace('.mp4', '_h264.mp4');
  execSync(`ffmpeg -y -i "${videoPath}" -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p -movflags +faststart -an "${h264Path}"`, { stdio: 'pipe' });
  fs.unlinkSync(videoPath);
  fs.renameSync(h264Path, videoPath);

  // Cleanup
  fs.unlinkSync(optimizedPath);

  const stats = fs.statSync(videoPath);
  console.log(`Morphing video created: ${(stats.size / 1024 / 1024).toFixed(2)} MB`);

  return {
    videoPath,
    modelUsed,
    prompt,
    motionSettings: { model: modelUsed, prompt }
  };
}

// Test if run directly
if (process.argv[1] === fileURLToPath(import.meta.url)) {
  const testImage = process.argv[2] || './output/great_wave_source.jpg';
  createMorphingVideo(testImage, { title: 'The Great Wave', artist: 'Hokusai' })
    .then(r => console.log('Result:', r))
    .catch(e => console.error('Error:', e));
}