← back to Animate Museum Posts

src/animateWithAI.js

154 lines

import axios from 'axios';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';
import sharp from 'sharp';

dotenv.config();

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

// Free AI animation options
const ANIMATION_SERVICES = {
  // LeiaPix - Free depth-based animation (no API key needed)
  LEIAPIX: {
    url: 'https://www.immersity.ai/api/v1/animation',
    free: true
  },
  // MyHeritage AI - Free for limited use
  MYHERITAGE: {
    url: 'https://www.myheritage.com/deep-nostalgia/api',
    free: true
  },
  // Replicate - Pay per use but very cheap ($0.02 per generation)
  REPLICATE: {
    url: 'https://api.replicate.com/v1/predictions',
    model: 'stability-ai/stable-video-diffusion',
    free: false
  }
};

export async function animateWithFreeAI(imagePath, metadata) {
  try {
    console.log('Attempting free AI animation...');

    // Option 1: Try Immersity AI (LeiaPix) - Creates 3D depth animation
    try {
      console.log('Trying Immersity AI for 3D depth animation...');

      // This service creates a 3D depth map and animates it
      // Note: This is a simplified example - the actual API might require registration
      const imageBuffer = fs.readFileSync(imagePath);
      const base64Image = imageBuffer.toString('base64');

      // For Immersity AI, we'd need to:
      // 1. Upload the image
      // 2. Wait for processing
      // 3. Download the animated result

      // Since this requires web scraping or unofficial API, let's try another approach

    } catch (error) {
      console.log('Immersity AI not available');
    }

    // Option 2: Use Stable Video Diffusion via Replicate (very cheap but not free)
    if (process.env.REPLICATE_API_TOKEN) {
      console.log('Using Replicate Stable Video Diffusion...');

      const response = await axios.post(
        'https://api.replicate.com/v1/predictions',
        {
          version: 'stability-ai/stable-video-diffusion:3f0457e4619daac51203dedb472816fd4af51f3149fa7a9e0b5ffcf1b8172438',
          input: {
            input_image: `data:image/jpeg;base64,${fs.readFileSync(imagePath).toString('base64')}`,
            frames_per_second: 7,
            motion_bucket_id: 127,
            cond_aug: 0.02,
            decoding_t: 7,
            seed: Math.floor(Math.random() * 1000000)
          }
        },
        {
          headers: {
            'Authorization': `Token ${process.env.REPLICATE_API_TOKEN}`,
            'Content-Type': 'application/json'
          }
        }
      );

      // Poll for completion
      let prediction = response.data;
      while (prediction.status !== 'succeeded' && prediction.status !== 'failed') {
        await new Promise(resolve => setTimeout(resolve, 1000));
        const checkResponse = await axios.get(
          `https://api.replicate.com/v1/predictions/${prediction.id}`,
          {
            headers: {
              'Authorization': `Token ${process.env.REPLICATE_API_TOKEN}`
            }
          }
        );
        prediction = checkResponse.data;
      }

      if (prediction.status === 'succeeded' && prediction.output) {
        // Download the video
        const videoResponse = await axios.get(prediction.output, {
          responseType: 'arraybuffer'
        });

        const outputDir = path.join(__dirname, '..', 'output');
        const timestamp = new Date().getTime();
        const videoPath = path.join(outputDir, `animated_${timestamp}.mp4`);

        fs.writeFileSync(videoPath, videoResponse.data);

        console.log('AI-animated video created successfully!');
        return {
          originalImage: imagePath,
          animatedVideo: videoPath,
          artworkMetadata: metadata,
          isAnimated: true,
          animationType: 'ai-video'
        };
      }
    }

    // Option 3: Create a cinemagraph-style animation with depth estimation
    console.log('Creating depth-based animation effect...');

    // This would use depth estimation to create parallax effect
    // For now, returning original as we need proper AI service

    return {
      originalImage: imagePath,
      isStatic: true,
      artworkMetadata: metadata,
      needsAIService: true
    };

  } catch (error) {
    console.error('Error in AI animation:', error.message);
    return {
      originalImage: imagePath,
      isStatic: true,
      artworkMetadata: metadata,
      error: error.message
    };
  }
}

// Helper function to create depth-based animation locally
export async function createDepthAnimation(imagePath) {
  // This would:
  // 1. Use MiDaS or DPT for depth estimation
  // 2. Create layers based on depth
  // 3. Animate layers with parallax
  // But requires Python integration or pre-trained models

  console.log('Depth animation requires additional setup');
  return null;
}