← back to Animate Museum Posts

src/animateWithLuma.js

261 lines

import Replicate from 'replicate';
import axios from 'axios';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { execSync } from 'child_process';
import sharp from 'sharp';
import dotenv from 'dotenv';

dotenv.config();

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

/**
 * Generate a creative animation prompt based on artwork content
 */
function generateAnimationPrompt(metadata) {
  const title = (metadata.title || '').toLowerCase();
  const description = (metadata.description || '').toLowerCase();
  const department = (metadata.department || '').toLowerCase();

  // Analyze content and generate appropriate animation prompts
  const prompts = [];

  // Portrait/figure detection
  if (title.includes('portrait') || title.includes('head') || title.includes('bust') ||
      title.includes('figure') || title.includes('man') || title.includes('woman') ||
      title.includes('child') || department.includes('portrait')) {
    prompts.push(
      "the subject's eyes slowly blink and look around, subtle breathing movement, slight head turn",
      "gentle eye movement, subtle smile forming, natural breathing animation",
      "the figure comes to life with blinking eyes and subtle facial expressions"
    );
  }

  // Landscape/nature detection
  if (title.includes('landscape') || title.includes('garden') || title.includes('forest') ||
      title.includes('river') || title.includes('lake') || title.includes('mountain') ||
      title.includes('tree') || title.includes('flower')) {
    prompts.push(
      "wind gently moves through the trees, clouds drift slowly, water ripples",
      "leaves rustle in the breeze, sunlight shifts through clouds, nature comes alive",
      "gentle wind animation, flowing water, moving clouds, swaying plants"
    );
  }

  // Animal detection
  if (title.includes('horse') || title.includes('dog') || title.includes('cat') ||
      title.includes('bird') || title.includes('lion') || title.includes('stork') ||
      title.includes('animal') || department.includes('animal')) {
    prompts.push(
      "the animal breathes, blinks, and makes subtle natural movements",
      "gentle breathing, tail or ear movement, the creature comes to life",
      "realistic animal animation with breathing and natural movement"
    );
  }

  // Water/maritime detection
  if (title.includes('sea') || title.includes('ocean') || title.includes('ship') ||
      title.includes('boat') || title.includes('water') || title.includes('wave')) {
    prompts.push(
      "waves gently roll, water sparkles, boats rock gently on the water",
      "ocean waves in motion, reflections shimmer, maritime scene comes alive"
    );
  }

  // Still life detection
  if (title.includes('still life') || title.includes('vase') || title.includes('flower') ||
      title.includes('fruit') || title.includes('table')) {
    prompts.push(
      "subtle lighting shifts, flower petals gently sway, dust motes float in light",
      "candlelight flickers, shadows shift, delicate movement in the scene"
    );
  }

  // Mask/sculpture detection
  if (title.includes('mask') || title.includes('sculpture') || title.includes('statue') ||
      title.includes('memorial') || title.includes('head')) {
    prompts.push(
      "the mask suddenly comes alive with glowing eyes and speaking mouth",
      "mystical animation where the sculpture begins to breathe and look around",
      "dramatic awakening - the figure opens its eyes and starts to speak"
    );
  }

  // Default fallback
  if (prompts.length === 0) {
    prompts.push(
      "subtle movement brings the artwork to life, gentle animation",
      "cinematic camera movement, lighting shifts, scene comes alive",
      "dramatic animation with natural movement and atmospheric effects"
    );
  }

  // Return random prompt from generated options
  return prompts[Math.floor(Math.random() * prompts.length)];
}

export async function createLumaAnimation(imagePath, metadata) {
  try {
    const REPLICATE_API_TOKEN = process.env.REPLICATE_API_TOKEN;

    if (!REPLICATE_API_TOKEN) {
      throw new Error('REPLICATE_API_TOKEN not configured');
    }

    console.log('Creating animation with Luma Dream Machine...');
    console.log('This will make the artwork come ALIVE!');

    const replicate = new Replicate({
      auth: REPLICATE_API_TOKEN,
    });

    // Prepare image - Luma works well with various resolutions
    const optimizedImagePath = path.join(path.dirname(imagePath), `luma_${path.basename(imagePath)}`);
    await sharp(imagePath)
      .resize(1280, 720, {
        fit: 'inside',
        withoutEnlargement: false
      })
      .jpeg({ quality: 95 })
      .toFile(optimizedImagePath);

    // Convert to base64 data URI
    const imageBuffer = fs.readFileSync(optimizedImagePath);
    const base64Image = imageBuffer.toString('base64');
    const dataUri = `data:image/jpeg;base64,${base64Image}`;

    // Generate creative animation prompt
    const animationPrompt = generateAnimationPrompt(metadata);
    console.log(`Animation prompt: "${animationPrompt}"`);

    console.log('Sending to Luma Dream Machine...');
    console.log('Please wait 60-120 seconds for dramatic animation...');

    // Try Luma models on Replicate
    let output;
    try {
      // Try luma-photon first (faster)
      output = await Promise.race([
        replicate.run(
          "luma/photon",
          {
            input: {
              prompt: animationPrompt,
              image: dataUri,
              aspect_ratio: "16:9"
            }
          }
        ),
        new Promise((_, reject) => setTimeout(() => reject(new Error('Luma timeout')), 180000))
      ]);
    } catch (e1) {
      console.log('Luma Photon failed, trying Ray2...');
      try {
        output = await Promise.race([
          replicate.run(
            "luma/ray-2",
            {
              input: {
                prompt: animationPrompt,
                image: dataUri
              }
            }
          ),
          new Promise((_, reject) => setTimeout(() => reject(new Error('Ray2 timeout')), 180000))
        ]);
      } catch (e2) {
        console.log('Ray2 failed, trying Dream Machine...');
        output = await Promise.race([
          replicate.run(
            "luma/dream-machine",
            {
              input: {
                prompt: animationPrompt,
                image_url: dataUri
              }
            }
          ),
          new Promise((_, reject) => setTimeout(() => reject(new Error('Dream Machine timeout')), 180000))
        ]);
      }
    }

    console.log('Luma animation complete!');

    if (!output) {
      throw new Error('No video generated from Luma');
    }

    // Download the generated video
    console.log('Downloading animated video...');
    const videoUrl = typeof output === 'string' ? output : output.video || output;
    const videoResponse = await axios.get(videoUrl, {
      responseType: 'arraybuffer'
    });

    const outputDir = path.join(__dirname, '..', 'output');
    const timestamp = Date.now();
    const videoPath = path.join(outputDir, `luma_video_${timestamp}.mp4`);

    fs.writeFileSync(videoPath, videoResponse.data);

    // Convert to Twitter-compatible H.264 format
    console.log('Converting to Twitter-compatible H.264 format...');
    const h264VideoPath = videoPath.replace('.mp4', '_h264.mp4');
    try {
      execSync(`ffmpeg -y -i "${videoPath}" -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p -movflags +faststart -an "${h264VideoPath}"`, {
        stdio: 'pipe',
        timeout: 60000
      });
      // Replace original with converted
      fs.unlinkSync(videoPath);
      fs.renameSync(h264VideoPath, videoPath);
      console.log('Video converted to H.264 successfully');
    } catch (ffmpegError) {
      console.log('FFmpeg conversion failed, using original:', ffmpegError.message);
    }

    // Clean up temp file
    if (fs.existsSync(optimizedImagePath)) {
      fs.unlinkSync(optimizedImagePath);
    }

    const stats = fs.statSync(videoPath);
    console.log(`Luma video created: ${(stats.size / 1024 / 1024).toFixed(2)} MB`);

    return {
      originalImage: imagePath,
      animatedVideo: videoPath,
      artworkMetadata: metadata,
      isAnimated: true,
      animationType: 'luma-dream-machine',
      animationPrompt: animationPrompt,
      videoFormat: 'mp4'
    };

  } catch (error) {
    console.error('Luma animation failed:', error.message);
    return {
      originalImage: imagePath,
      isStatic: true,
      artworkMetadata: metadata,
      error: error.message
    };
  }
}

// Test if run directly
if (process.argv[1] === fileURLToPath(import.meta.url)) {
  const testImage = process.argv[2];
  if (testImage) {
    createLumaAnimation(testImage, { title: 'Test', artist: 'Test' })
      .then(r => console.log('Result:', r))
      .catch(e => console.error('Error:', e));
  } else {
    console.log('Usage: node animateWithLuma.js <image-path>');
  }
}