← back to Animate Museum Posts

src/animateWithHuggingFace.js

172 lines

import { HfInference } from '@huggingface/inference';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';
import axios from 'axios';

dotenv.config();

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

export async function createHuggingFaceAnimation(imagePath, metadata) {
  try {
    const HF_TOKEN = process.env.HUGGINGFACE_TOKEN;

    if (!HF_TOKEN) {
      throw new Error('HUGGINGFACE_TOKEN not configured - get it from https://huggingface.co/settings/tokens');
    }

    console.log('Creating animation with Hugging Face models...');

    const hf = new HfInference(HF_TOKEN);
    const TIMEOUT = 60000; // 60 second timeout

    // Read the image
    const imageBuffer = fs.readFileSync(imagePath);

    // Try different video generation models on Hugging Face
    console.log('Attempting video generation...');

    try {
      // Option 1: Try Stable Video Diffusion on HuggingFace (PRO required)
      console.log('Trying Stable Video Diffusion on HuggingFace PRO...');
      const response = await hf.imageToVideo({
        model: 'stabilityai/stable-video-diffusion-img2vid-xt',
        inputs: imageBuffer,
        parameters: {
          num_frames: 25,
          motion_bucket_id: 180,  // Balanced motion (not too chaotic)
          fps: 7,
          decode_chunk_size: 8
        },
        options: {
          use_gpu: true,
          wait_for_model: true
        }
      });

      if (response) {
        const outputDir = path.join(__dirname, '..', 'output');
        const timestamp = new Date().getTime();
        const videoPath = path.join(outputDir, `hf_animated_${timestamp}.mp4`);

        // Save the video
        const videoBuffer = await response.arrayBuffer();
        fs.writeFileSync(videoPath, Buffer.from(videoBuffer));

        console.log('Hugging Face video created!');
        return {
          originalImage: imagePath,
          animatedVideo: videoPath,
          artworkMetadata: metadata,
          isAnimated: true,
          animationType: 'huggingface-svd'
        };
      }
    } catch (e) {
      console.log('SVD error:', e.message);
      if (e.message.includes('quota')) {
        console.log('GPU quota exceeded - try again later');
      } else if (e.message.includes('Pro subscription')) {
        console.log('Model requires PRO subscription');
      }
    }

    // Option 2: Try AnimateDiff
    try {
      const response = await hf.textToVideo({
        model: 'ByteDance/AnimateDiff-Lightning',
        inputs: `animate the painting ${metadata.title}, make elements move naturally, flowers swaying, no camera movement`,
        parameters: {
          num_frames: 16,
          guidance_scale: 7.5
        }
      });

      if (response) {
        const outputDir = path.join(__dirname, '..', 'output');
        const timestamp = new Date().getTime();
        const videoPath = path.join(outputDir, `hf_animated_${timestamp}.mp4`);

        const videoBuffer = await response.arrayBuffer();
        fs.writeFileSync(videoPath, Buffer.from(videoBuffer));

        console.log('AnimateDiff video created!');
        return {
          originalImage: imagePath,
          animatedVideo: videoPath,
          artworkMetadata: metadata,
          isAnimated: true,
          animationType: 'animatediff'
        };
      }
    } catch (e) {
      console.log('AnimateDiff not available');
    }

    // Option 3: Use image-to-image to create animation frames
    console.log('Creating animation frames with img2img...');

    const frames = [];
    const frameCount = 10;

    for (let i = 0; i < frameCount; i++) {
      const prompt = `${metadata.title} painting with subtle movement, frame ${i + 1} of animation sequence, slight variation`;

      try {
        const response = await hf.imageToImage({
          model: 'stabilityai/stable-diffusion-xl-refiner-1.0',
          inputs: imageBuffer,
          parameters: {
            prompt: prompt,
            strength: 0.3,  // Low strength to keep it similar
            guidance_scale: 7
          }
        });

        if (response) {
          frames.push(await response.arrayBuffer());
        }
      } catch (e) {
        console.log(`Frame ${i} failed`);
      }
    }

    if (frames.length > 5) {
      // Convert frames to video (would need ffmpeg or similar)
      console.log(`Generated ${frames.length} frames for animation`);

      // For now, return the original
      return {
        originalImage: imagePath,
        artworkMetadata: metadata,
        isAnimated: false,
        frames: frames.length,
        needsVideoCompilation: true
      };
    }

    throw new Error('No animation method succeeded');

  } catch (error) {
    console.error('Error with Hugging Face animation:', error.message);

    if (error.message.includes('HUGGINGFACE_TOKEN')) {
      console.error('\n⚠️  To use Hugging Face:');
      console.error('1. Go to https://huggingface.co/join');
      console.error('2. Sign up (free)');
      console.error('3. Go to https://huggingface.co/settings/tokens');
      console.error('4. Create new token');
      console.error('5. Add to .env: HUGGINGFACE_TOKEN=hf_xxxxx\n');
    }

    return {
      originalImage: imagePath,
      isStatic: true,
      artworkMetadata: metadata,
      error: error.message
    };
  }
}