`series2/audio.py`

65 lines
#!/usr/bin/env python3
"""Local Kokoro narrator, measured caption times, speech-driven mouth envelope, restrained score."""
import json, math, subprocess
from pathlib import Path
import numpy as np
import soundfile as sf
from kokoro_onnx import Kokoro

ROOT=Path(__file__).resolve().parent
CACHE=Path.home()/'.cache/hyperframes/tts'
engine=Kokoro(str(CACHE/'models/kokoro-v1.0.onnx'),str(CACHE/'voices/voices-v1.0.bin'))
episodes=json.loads((ROOT/'episodes.json').read_text())
rate=24000
metadata=[]
for ep_index,ep in enumerate(episodes):
    voice=np.zeros(rate*20,dtype=np.float32)
    segments=[]
    starts=[0.6,5.1,9.6,14.1]
    for i,line in enumerate(ep['lines']):
        output=ROOT/'assets/voice'/f'{ep["slug"]}-{i}.wav'
        if output.exists(): samples,sr=sf.read(output,dtype='float32')
        else:
            samples,sr=engine.create(line,voice='am_michael',speed=1.02,lang='en-us')
            sf.write(output,samples,sr)
        duration=len(samples)/sr
        max_duration=4.22 if i<3 else 4.8
        if duration>max_duration:
            tempo=duration/max_duration
            assert tempo<1.4, (ep['slug'],i,duration)
            fitted=output.with_name(output.stem+'-fit.wav')
            subprocess.run(['ffmpeg','-v','error','-y','-i',str(output),'-af',f'atempo={tempo}','-ar',str(rate),str(fitted)],check=True)
            samples,sr=sf.read(fitted,dtype='float32')
        duration=len(samples)/sr
        assert sr==rate and starts[i]+duration<19.5
        samples=samples*.65/max(.65,float(np.max(np.abs(samples))))
        pos=round(starts[i]*rate);voice[pos:pos+len(samples)]+=samples
        segments.append({'index':i,'start':starts[i],'end':round(starts[i]+duration,3),'text':line,'caption':ep['captions'][i],'duration':round(duration,3)})
        print(ep['slug'],i,round(duration,2),'s',flush=True)
    # A sparse original warm keyboard score, kept well below the spoken voice.
    score=np.zeros(rate*20,dtype=np.float32)
    chords=[[130.813,164.814,195.998,246.942],[110,146.832,174.614,220],[130.813,164.814,195.998,261.626],[97.999,146.832,195.998,246.942]]
    for beat in range(24):
        start=beat*.8
        note=chords[(beat//6+ep_index)%4][beat%4]
        length=min(2.5,20-start)
        if length<=0:continue
        ts=np.arange(int(length*rate))/rate
        tone=(np.sin(2*np.pi*note*ts)+.23*np.sin(2*np.pi*note*2*ts)+.07*np.sin(2*np.pi*note*3*ts))
        env=np.minimum(1,ts/.012)*np.exp(-ts*2.1)*.019
        at=int(start*rate);score[at:at+len(ts)]+=tone*env
    # Measured speech energy also controls mouth openness and dynamic bed ducking.
    energy=[]
    for i in range(1000):
        frame=voice[i*480:(i+1)*480]
        energy.append(float(min(1,np.sqrt(np.mean(frame**2))/.13)))
    duck=np.repeat(1-np.minimum(1,np.array(energy)*3)*.60,480)
    score*=duck
    fade=np.minimum(1,np.arange(rate*20)/(rate*.25))*np.minimum(1,(rate*20-np.arange(rate*20))/(rate*.7))
    mix=(voice+score)*fade
    stereo=np.column_stack((mix,mix))
    sf.write(ROOT/'assets'/f'mix-{ep_index}.wav',stereo,rate,subtype='PCM_16')
    metadata.append({'slug':ep['slug'],'provider':'Kokoro local','voice':'am_michael','duration':20,'segments':segments,'energy':[round(v,4) for v in energy],'peak':float(np.max(np.abs(mix)))})
(ROOT/'audio.json').write_text(json.dumps(metadata,indent=2)+'\n')
print('Five narrated 20-second mixes complete.',flush=True)