mirror of
https://github.com/OpenCut-app/OpenCut.git
synced 2026-07-13 21:52:53 +02:00
lots of stuff
This commit is contained in:
Maze Winther
@@ -4,215 +4,223 @@ import type { TimelineTrack } from "@/types/timeline";
|
||||
import type { MediaAsset } from "@/types/assets";
|
||||
|
||||
export async function getVideoInfo({
|
||||
videoFile,
|
||||
videoFile,
|
||||
}: {
|
||||
videoFile: File;
|
||||
videoFile: File;
|
||||
}): Promise<{
|
||||
duration: number;
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
duration: number;
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
}> {
|
||||
const input = new Input({
|
||||
source: new BlobSource(videoFile),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
const input = new Input({
|
||||
source: new BlobSource(videoFile),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const duration = await input.computeDuration();
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
const duration = await input.computeDuration();
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
}
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
}
|
||||
|
||||
const packetStats = await videoTrack.computePacketStats(100);
|
||||
const fps = packetStats.averagePacketRate;
|
||||
const packetStats = await videoTrack.computePacketStats(100);
|
||||
const fps = packetStats.averagePacketRate;
|
||||
|
||||
return {
|
||||
duration,
|
||||
width: videoTrack.displayWidth,
|
||||
height: videoTrack.displayHeight,
|
||||
fps,
|
||||
};
|
||||
return {
|
||||
duration,
|
||||
width: videoTrack.displayWidth,
|
||||
height: videoTrack.displayHeight,
|
||||
fps,
|
||||
};
|
||||
}
|
||||
|
||||
const SAMPLE_RATE = 44100;
|
||||
const NUM_CHANNELS = 2;
|
||||
|
||||
export const extractTimelineAudio = async ({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
totalDuration,
|
||||
onProgress,
|
||||
tracks,
|
||||
mediaAssets,
|
||||
totalDuration,
|
||||
onProgress,
|
||||
}: {
|
||||
tracks: TimelineTrack[];
|
||||
mediaAssets: MediaAsset[];
|
||||
totalDuration: number;
|
||||
onProgress?: (progress: number) => void;
|
||||
tracks: TimelineTrack[];
|
||||
mediaAssets: MediaAsset[];
|
||||
totalDuration: number;
|
||||
onProgress?: (progress: number) => void;
|
||||
}): Promise<Blob> => {
|
||||
if (totalDuration === 0) {
|
||||
return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });
|
||||
}
|
||||
if (totalDuration === 0) {
|
||||
return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });
|
||||
}
|
||||
|
||||
const audioMixSources = await collectAudioMixSources({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
});
|
||||
const audioMixSources = await collectAudioMixSources({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
});
|
||||
|
||||
if (audioMixSources.length === 0) {
|
||||
const silentDuration = Math.max(1, totalDuration);
|
||||
const silentSamples = new Float32Array(
|
||||
Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,
|
||||
);
|
||||
return createWavBlob({ samples: silentSamples });
|
||||
}
|
||||
if (audioMixSources.length === 0) {
|
||||
const silentDuration = Math.max(1, totalDuration);
|
||||
const silentSamples = new Float32Array(
|
||||
Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,
|
||||
);
|
||||
return createWavBlob({ samples: silentSamples });
|
||||
}
|
||||
|
||||
const totalSamples = Math.ceil(totalDuration * SAMPLE_RATE);
|
||||
const mixBuffers = [
|
||||
new Float32Array(totalSamples),
|
||||
new Float32Array(totalSamples),
|
||||
];
|
||||
const totalSamples = Math.ceil(totalDuration * SAMPLE_RATE);
|
||||
const mixBuffers = [
|
||||
new Float32Array(totalSamples),
|
||||
new Float32Array(totalSamples),
|
||||
];
|
||||
|
||||
for (let i = 0; i < audioMixSources.length; i++) {
|
||||
const source = audioMixSources[i];
|
||||
for (let i = 0; i < audioMixSources.length; i++) {
|
||||
const source = audioMixSources[i];
|
||||
|
||||
if (onProgress) {
|
||||
onProgress((i / audioMixSources.length) * 90);
|
||||
}
|
||||
if (onProgress) {
|
||||
onProgress((i / audioMixSources.length) * 90);
|
||||
}
|
||||
|
||||
try {
|
||||
await decodeAndMixAudioSource({
|
||||
source,
|
||||
mixBuffers,
|
||||
totalSamples,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(`Failed to process audio source ${source.file.name}:`, error);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await decodeAndMixAudioSource({
|
||||
source,
|
||||
mixBuffers,
|
||||
totalSamples,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`Failed to process audio source ${source.file.name}:`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// clamp to prevent clipping
|
||||
for (const channel of mixBuffers) {
|
||||
for (let i = 0; i < channel.length; i++) {
|
||||
channel[i] = Math.max(-1, Math.min(1, channel[i]));
|
||||
}
|
||||
}
|
||||
// clamp to prevent clipping
|
||||
for (const channel of mixBuffers) {
|
||||
for (let i = 0; i < channel.length; i++) {
|
||||
channel[i] = Math.max(-1, Math.min(1, channel[i]));
|
||||
}
|
||||
}
|
||||
|
||||
// interleave channels for wav output
|
||||
const interleavedSamples = new Float32Array(totalSamples * NUM_CHANNELS);
|
||||
for (let i = 0; i < totalSamples; i++) {
|
||||
interleavedSamples[i * 2] = mixBuffers[0][i];
|
||||
interleavedSamples[i * 2 + 1] = mixBuffers[1][i];
|
||||
}
|
||||
// interleave channels for wav output
|
||||
const interleavedSamples = new Float32Array(totalSamples * NUM_CHANNELS);
|
||||
for (let i = 0; i < totalSamples; i++) {
|
||||
interleavedSamples[i * 2] = mixBuffers[0][i];
|
||||
interleavedSamples[i * 2 + 1] = mixBuffers[1][i];
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(100);
|
||||
}
|
||||
if (onProgress) {
|
||||
onProgress(100);
|
||||
}
|
||||
|
||||
return createWavBlob({ samples: interleavedSamples });
|
||||
return createWavBlob({ samples: interleavedSamples });
|
||||
};
|
||||
|
||||
async function decodeAndMixAudioSource({
|
||||
source,
|
||||
mixBuffers,
|
||||
totalSamples,
|
||||
source,
|
||||
mixBuffers,
|
||||
totalSamples,
|
||||
}: {
|
||||
source: { file: File; startTime: number; duration: number; trimStart: number };
|
||||
mixBuffers: Float32Array[];
|
||||
totalSamples: number;
|
||||
source: {
|
||||
file: File;
|
||||
startTime: number;
|
||||
duration: number;
|
||||
trimStart: number;
|
||||
};
|
||||
mixBuffers: Float32Array[];
|
||||
totalSamples: number;
|
||||
}): Promise<void> {
|
||||
const input = new Input({
|
||||
source: new BlobSource(source.file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
const input = new Input({
|
||||
source: new BlobSource(source.file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
if (!audioTrack) return;
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
if (!audioTrack) return;
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
const trimEnd = source.trimStart + source.duration;
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
const trimEnd = source.trimStart + source.duration;
|
||||
|
||||
for await (const { buffer, timestamp } of sink.buffers(
|
||||
source.trimStart,
|
||||
trimEnd,
|
||||
)) {
|
||||
const relativeTime = timestamp - source.trimStart;
|
||||
const outputStartSample = Math.floor(
|
||||
(source.startTime + relativeTime) * SAMPLE_RATE,
|
||||
);
|
||||
for await (const { buffer, timestamp } of sink.buffers(
|
||||
source.trimStart,
|
||||
trimEnd,
|
||||
)) {
|
||||
const relativeTime = timestamp - source.trimStart;
|
||||
const outputStartSample = Math.floor(
|
||||
(source.startTime + relativeTime) * SAMPLE_RATE,
|
||||
);
|
||||
|
||||
// resample if needed
|
||||
const resampleRatio = SAMPLE_RATE / buffer.sampleRate;
|
||||
// resample if needed
|
||||
const resampleRatio = SAMPLE_RATE / buffer.sampleRate;
|
||||
|
||||
for (let ch = 0; ch < NUM_CHANNELS; ch++) {
|
||||
const sourceChannel = Math.min(ch, buffer.numberOfChannels - 1);
|
||||
const channelData = buffer.getChannelData(sourceChannel);
|
||||
const outputChannel = mixBuffers[ch];
|
||||
for (let ch = 0; ch < NUM_CHANNELS; ch++) {
|
||||
const sourceChannel = Math.min(ch, buffer.numberOfChannels - 1);
|
||||
const channelData = buffer.getChannelData(sourceChannel);
|
||||
const outputChannel = mixBuffers[ch];
|
||||
|
||||
const resampledLength = Math.floor(channelData.length * resampleRatio);
|
||||
for (let i = 0; i < resampledLength; i++) {
|
||||
const outputIdx = outputStartSample + i;
|
||||
if (outputIdx < 0 || outputIdx >= totalSamples) continue;
|
||||
const resampledLength = Math.floor(channelData.length * resampleRatio);
|
||||
for (let i = 0; i < resampledLength; i++) {
|
||||
const outputIdx = outputStartSample + i;
|
||||
if (outputIdx < 0 || outputIdx >= totalSamples) continue;
|
||||
|
||||
const sourceIdx = Math.floor(i / resampleRatio);
|
||||
if (sourceIdx < channelData.length) {
|
||||
outputChannel[outputIdx] += channelData[sourceIdx];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const sourceIdx = Math.floor(i / resampleRatio);
|
||||
if (sourceIdx < channelData.length) {
|
||||
outputChannel[outputIdx] += channelData[sourceIdx];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createWavBlob({ samples }: { samples: Float32Array }): Blob {
|
||||
const numChannels = NUM_CHANNELS;
|
||||
const bitsPerSample = 16;
|
||||
const bytesPerSample = bitsPerSample / 8;
|
||||
const numSamples = samples.length / numChannels;
|
||||
const dataSize = numSamples * numChannels * bytesPerSample;
|
||||
const buffer = new ArrayBuffer(44 + dataSize);
|
||||
const view = new DataView(buffer);
|
||||
const numChannels = NUM_CHANNELS;
|
||||
const bitsPerSample = 16;
|
||||
const bytesPerSample = bitsPerSample / 8;
|
||||
const numSamples = samples.length / numChannels;
|
||||
const dataSize = numSamples * numChannels * bytesPerSample;
|
||||
const buffer = new ArrayBuffer(44 + dataSize);
|
||||
const view = new DataView(buffer);
|
||||
|
||||
// riff header
|
||||
writeString({ view, offset: 0, str: "RIFF" });
|
||||
view.setUint32(4, 36 + dataSize, true);
|
||||
writeString({ view, offset: 8, str: "WAVE" });
|
||||
// riff header
|
||||
writeString({ view, offset: 0, str: "RIFF" });
|
||||
view.setUint32(4, 36 + dataSize, true);
|
||||
writeString({ view, offset: 8, str: "WAVE" });
|
||||
|
||||
// fmt chunk
|
||||
writeString({ view, offset: 12, str: "fmt " });
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, numChannels, true);
|
||||
view.setUint32(24, SAMPLE_RATE, true);
|
||||
view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
|
||||
view.setUint16(32, numChannels * bytesPerSample, true);
|
||||
view.setUint16(34, bitsPerSample, true);
|
||||
// fmt chunk
|
||||
writeString({ view, offset: 12, str: "fmt " });
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, numChannels, true);
|
||||
view.setUint32(24, SAMPLE_RATE, true);
|
||||
view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
|
||||
view.setUint16(32, numChannels * bytesPerSample, true);
|
||||
view.setUint16(34, bitsPerSample, true);
|
||||
|
||||
// data chunk
|
||||
writeString({ view, offset: 36, str: "data" });
|
||||
view.setUint32(40, dataSize, true);
|
||||
// data chunk
|
||||
writeString({ view, offset: 36, str: "data" });
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
// convert float32 to int16 and write
|
||||
let offset = 44;
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const sample = Math.max(-1, Math.min(1, samples[i]));
|
||||
const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
|
||||
view.setInt16(offset, int16, true);
|
||||
offset += 2;
|
||||
}
|
||||
// convert float32 to int16 and write
|
||||
let offset = 44;
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const sample = Math.max(-1, Math.min(1, samples[i]));
|
||||
const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
|
||||
view.setInt16(offset, int16, true);
|
||||
offset += 2;
|
||||
}
|
||||
|
||||
return new Blob([buffer], { type: "audio/wav" });
|
||||
return new Blob([buffer], { type: "audio/wav" });
|
||||
}
|
||||
|
||||
function writeString({
|
||||
view,
|
||||
offset,
|
||||
str,
|
||||
view,
|
||||
offset,
|
||||
str,
|
||||
}: {
|
||||
view: DataView;
|
||||
offset: number;
|
||||
str: string;
|
||||
view: DataView;
|
||||
offset: number;
|
||||
str: string;
|
||||
}): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user