feat: auto-captions

apps/transcription/README.md

@@ -0,0 +1,86 @@
+Before you follow anything in this guide, please make sure you've followed the steps in the [README](../../README.md) (under "Optional: Auto-captions (Transcription) Setup").
+
+Open your terminal and make sure you're in the `apps/transcription` directory.
+
+1. Create virtual environment
+
+```bash
+python -m venv env
+```
+
+2. Activate it
+
+**Windows:**
+
+```bash
+env\Scripts\activate
+```
+
+**macOS/Linux:**
+
+```bash
+source env/bin/activate
+```
+
+> Note: if you're using VS Code/Cursor and you're seeing errors with the imports about the modules not being found,
+> You might have to press CTRL + Shift + P -> Python: Select Interpreter -> Enter interpreter path -> Find -> env -> scripts -> python.exe
+
+3. Install libraries/packages/whatever you wanna call them
+
+```bash
+pip install -r requirements.txt
+```
+
+4. Make sure you have a Modal account. If you don't: [create one](https://modal.com/)
+
+> If you don't know what Modal is: it allows us to process the actual audio and transcribe with Whisper by providing the infra to run Python code with a lot of RAM, generally affordable.
+
+5. Once you've got a Modal accoumt, run this:
+
+```bash
+python -m modal setup
+```
+
+It's gonna open a browser so you can authenticate.
+
+6. Test it if you want to make sure it actually works:
+
+```bash
+modal run transcription.py
+```
+
+6. Deploy the function!
+
+```bash
+modal deploy transcription.py
+```
+
+7. Set the required secrets in Modal
+
+So the script we just deployed interacts with Cloudflare to do two things:
+
+- Download the audio (so it can be transcribed with Whisper)
+- Delete the file after processing (privacy)
+
+To do those things, the script needs access to these environment variables:
+```bash
+CLOUDFLARE_ACCOUNT_ID=your-account-id
+R2_ACCESS_KEY_ID=your-access-key-id
+R2_SECRET_ACCESS_KEY=your-secret-access-key
+R2_BUCKET_NAME=opencut-transcription
+```
+
+Remember, we set these earlier in `.env.local`.
+
+So let's do it:
+
+   - Go to [Modal Secrets](https://modal.com/secrets/mazewinther/main)
+   - Click "Custom" and enter "opencut-r2-secrets" for the name.
+   - Now you can just click "Import .env" and copy/paste the 4 variables from your `.env.local` file. Copy and paste these only:
+      ```bash
+      CLOUDFLARE_ACCOUNT_ID=your-account-id
+      R2_ACCESS_KEY_ID=your-access-key-id
+      R2_SECRET_ACCESS_KEY=your-secret-access-key
+      R2_BUCKET_NAME=opencut-transcription
+      ```
+    - Click "Done" and you should see some cool particles!

apps/transcription/requirements.txt

@@ -0,0 +1,5 @@
+modal
+openai-whisper
+boto3
+pydantic
+cryptography

apps/transcription/transcription.py

@@ -0,0 +1,143 @@
+import modal
+from pydantic import BaseModel
+
+app = modal.App("opencut-transcription")
+
+class TranscribeRequest(BaseModel):
+    filename: str
+    language: str = "auto"
+    decryptionKey: str = None
+    iv: str = None
+
+@app.function(
+    image=modal.Image.debian_slim()
+        .apt_install(["ffmpeg"])
+        .pip_install(["openai-whisper", "boto3", "fastapi[standard]", "pydantic", "cryptography"]),
+    gpu="A10G",
+    timeout=300, # 5m
+    secrets=[modal.Secret.from_name("opencut-r2-secrets")]
+)
+@modal.fastapi_endpoint(method="POST")
+def transcribe_audio(request: TranscribeRequest):
+    import whisper
+    import boto3
+    import tempfile
+    import os
+    import json
+    
+    try:
+        filename = request.filename
+        language = request.language
+        decryption_key = request.decryptionKey
+        iv = request.iv
+        
+        if not filename:
+            return {
+                "error": "Missing filename parameter"
+            }
+        
+        # Initialize R2 client
+        s3_client = boto3.client(
+            's3',
+            endpoint_url=f'https://{os.environ["CLOUDFLARE_ACCOUNT_ID"]}.r2.cloudflarestorage.com',
+            aws_access_key_id=os.environ["R2_ACCESS_KEY_ID"],
+            aws_secret_access_key=os.environ["R2_SECRET_ACCESS_KEY"],
+            region_name='auto'
+        )
+        
+        # Create temporary file for audio
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as temp_file:
+            temp_path = temp_file.name
+            
+            try:
+                # Download audio from R2
+                s3_client.download_file(
+                    os.environ["R2_BUCKET_NAME"], 
+                    filename, 
+                    temp_path
+                )
+                
+                # If decryption key provided, decrypt the file directly (zero-knowledge)
+                if decryption_key and iv:
+                    import base64
+                    from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+                    from cryptography.hazmat.backends import default_backend
+                    
+                    # Read the encrypted file
+                    with open(temp_path, 'rb') as f:
+                        encrypted_data = f.read()
+                    
+                    # Decode the key and IV from base64
+                    key_bytes = base64.b64decode(decryption_key)
+                    iv_bytes = base64.b64decode(iv)
+                    
+                    # Decrypt the data using AES-GCM
+                    # Extract the tag (last 16 bytes) and ciphertext
+                    tag = encrypted_data[-16:]
+                    ciphertext = encrypted_data[:-16]
+                    
+                    cipher = Cipher(
+                        algorithms.AES(key_bytes),
+                        modes.GCM(iv_bytes, tag),
+                        backend=default_backend()
+                    )
+                    decryptor = cipher.decryptor()
+                    decrypted_data = decryptor.update(ciphertext) + decryptor.finalize()
+                    
+                    # Write decrypted audio back to temp file
+                    with open(temp_path, 'wb') as f:
+                        f.write(decrypted_data)
+                
+                # Load Whisper model
+                model = whisper.load_model("base")
+                
+                # Transcribe audio
+                if language == "auto":
+                    result = model.transcribe(temp_path)
+                else:
+                    result = model.transcribe(temp_path, language=language.lower())
+                
+                # Delete audio file from R2 (cleanup)
+                s3_client.delete_object(
+                    Bucket=os.environ["R2_BUCKET_NAME"],
+                    Key=filename
+                )
+                
+                # Adjust segment timing - Whisper is consistently late by ~500ms
+                adjusted_segments = []
+                for segment in result["segments"]:
+                    adjusted_segment = segment.copy()
+                    # Shift start/end times earlier by 500ms, don't go below 0
+                    adjusted_segment["start"] = max(0, segment["start"] - 0.5)
+                    adjusted_segment["end"] = max(0.5, segment["end"] - 0.5)  # Ensure duration is at least 0.5s
+                    adjusted_segments.append(adjusted_segment)
+                
+                return {
+                    "text": result["text"],
+                    "segments": adjusted_segments,
+                    "language": result["language"]
+                }
+                
+            finally:
+                # Clean up temporary file
+                if os.path.exists(temp_path):
+                    os.unlink(temp_path)
+                    
+    except Exception as e:
+        import traceback
+        print(f"Transcription error: {str(e)}")
+        print(f"Traceback: {traceback.format_exc()}")
+        
+        # Return error response that matches expected format
+        return {
+            "error": str(e),
+            "text": "",
+            "segments": [],
+            "language": "unknown"
+        }
+
+@app.local_entrypoint()
+def main():
+    # Test function - you can call this with modal run transcription.py
+    print("Transcription service is ready to deploy!")
+    print("Deploy with: modal deploy transcription.py")