TSM.ID
285 lines
14 KiB
TypeScript
285 lines
14 KiB
TypeScript
/* eslint-disable */
|
|
// @ts-nocheck
|
|
import fs from 'fs';
|
|
|
|
const decoderPath = 'C:/X/workspace/jumpa.id/vc/lib/xcu-quantum-decoder.ts';
|
|
let decoderContent = fs.readFileSync(decoderPath, 'utf-8');
|
|
|
|
// 1. Add VP8
|
|
decoderContent = decoderContent.replace(
|
|
`"vp09.00.10.08", // VP9\n\t\t\t"avc1.42E01F" // H.264`,
|
|
`"vp09.00.10.08", // VP9\n\t\t\t"vp8", // VP8\n\t\t\t"avc1.42E01F" // H.264`
|
|
);
|
|
|
|
// 2. Add Audio Engine Mode & Encoders
|
|
if (!decoderContent.includes("public audioEngineMode")) {
|
|
decoderContent = decoderContent.replace(
|
|
`public videoEngineMode: "canvas" | "webcodecs" = "canvas";`,
|
|
`public videoEngineMode: "canvas" | "webcodecs" = "canvas";\n\tpublic audioEngineMode: "pcm" | "xcu-neural" = "pcm";\n\tprivate audioEncoder: unknown = null;\n\tprivate audioDecoders: Map<number, unknown> = new Map();`
|
|
);
|
|
}
|
|
|
|
// 3. Audio Encoder Logic inside ScriptProcessor
|
|
const uplinkAudioOld = ` const float32 = e.inputBuffer.getChannelData(0);
|
|
const int16 = new Int16Array(float32.length);
|
|
let sum = 0;
|
|
for (let i = 0; i < float32.length; i++) {
|
|
const s = Math.max(-1, Math.min(1, float32[i]));
|
|
int16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
if (i % 10 === 0) sum += Math.abs(s);
|
|
}
|
|
|
|
if (this.onAudioLevel) {
|
|
this.onAudioLevel(Math.min(100, (sum / (float32.length / 10)) * 500));
|
|
}`;
|
|
|
|
const uplinkAudioNew = ` const float32 = e.inputBuffer.getChannelData(0);
|
|
|
|
// XCU VAD (Voice Activity Detection) - Neural Gate
|
|
let sum = 0;
|
|
for (let i = 0; i < float32.length; i++) {
|
|
if (i % 10 === 0) sum += Math.abs(float32[i]);
|
|
}
|
|
const rms = sum / (float32.length / 10);
|
|
|
|
if (this.onAudioLevel) {
|
|
this.onAudioLevel(Math.min(100, rms * 500));
|
|
}
|
|
|
|
// Zero-Bandwidth Silence Drop
|
|
if (this.audioEngineMode === "xcu-neural" && rms < 0.005) {
|
|
return; // DROP PACKET: 0 Byte Bandwidth!
|
|
}
|
|
|
|
if (this.audioEngineMode === "xcu-neural" && this.audioEncoder) {
|
|
try {
|
|
const ad = new (window as any).AudioData({
|
|
format: 'f32-planar',
|
|
sampleRate: audioContext.sampleRate,
|
|
numberOfFrames: float32.length,
|
|
numberOfChannels: 1,
|
|
timestamp: performance.now() * 1000,
|
|
data: float32
|
|
});
|
|
(this.audioEncoder as any).encode(ad);
|
|
ad.close();
|
|
} catch (err) {}
|
|
return;
|
|
}
|
|
|
|
const int16 = new Int16Array(float32.length);
|
|
for (let i = 0; i < float32.length; i++) {
|
|
const s = Math.max(-1, Math.min(1, float32[i]));
|
|
int16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
}`;
|
|
|
|
decoderContent = decoderContent.replace(uplinkAudioOld, uplinkAudioNew);
|
|
|
|
// Initialize AudioEncoder in activateUplink
|
|
const initAudioOld = `const processor = audioContext.createScriptProcessor(4096, 1, 1);`;
|
|
const initAudioNew = `const processor = audioContext.createScriptProcessor(4096, 1, 1);
|
|
if (this.audioEngineMode === "xcu-neural") {
|
|
console.log("[QUANTUM AUDIO] XCU Neural Audio (Opus + VAD) Diaktifkan!");
|
|
this.audioEncoder = new (window as any).AudioEncoder({
|
|
output: (chunk: any) => {
|
|
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
|
const chunkData = new Uint8Array(chunk.byteLength);
|
|
chunk.copyTo(chunkData);
|
|
|
|
// Payload: [FRAME_AUDIO] [Quality] [SenderID x 2] [Length x 4] | [CodecID=1] [Opus Data]
|
|
const packetLen = 8 + 1 + chunkData.length;
|
|
const packet = new Uint8Array(packetLen);
|
|
packet[0] = 2; // FRAME_AUDIO
|
|
packet[1] = 1;
|
|
const view = new DataView(packet.buffer);
|
|
view.setUint16(2, this.participantId, true);
|
|
view.setUint32(4, packetLen - 8, true);
|
|
packet[8] = 1; // XCU Opus Codec ID
|
|
packet.set(chunkData, 9);
|
|
this.ws!.send(packet);
|
|
},
|
|
error: (e: any) => console.error("AudioEncoder Error", e)
|
|
});
|
|
(this.audioEncoder as any).configure({
|
|
codec: 'opus',
|
|
sampleRate: audioContext.sampleRate,
|
|
numberOfChannels: 1,
|
|
bitrate: 32000 // Ultra-efficient
|
|
});
|
|
}`;
|
|
decoderContent = decoderContent.replace(initAudioOld, initAudioNew);
|
|
|
|
|
|
// 4. Audio Decoder Logic
|
|
const rxAudioOld = ` if (frameType === 2) { // FRAME_AUDIO
|
|
// Format: [4 bytes uint32 sample rate] [n bytes Int16Array PCM]
|
|
if (quality === 2 && this.e2eeKey && payloadData.length > 12) {
|
|
const iv = payloadData.slice(0, 12);
|
|
const cipher = payloadData.slice(12);
|
|
try {
|
|
const plainBuf = await window.crypto.subtle.decrypt(
|
|
{ name: "AES-GCM", iv: iv },
|
|
this.e2eeKey,
|
|
cipher,
|
|
);
|
|
payloadData = new Uint8Array(plainBuf);
|
|
} catch (_e) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
let ctx = this.audioContexts.get(senderId);`;
|
|
|
|
const rxAudioNew = ` if (frameType === 2) { // FRAME_AUDIO
|
|
if (quality === 2 && this.e2eeKey && payloadData.length > 12) {
|
|
const iv = payloadData.slice(0, 12);
|
|
const cipher = payloadData.slice(12);
|
|
try {
|
|
const plainBuf = await window.crypto.subtle.decrypt({ name: "AES-GCM", iv: iv }, this.e2eeKey, cipher);
|
|
payloadData = new Uint8Array(plainBuf);
|
|
} catch (_e) { continue; }
|
|
}
|
|
|
|
let ctx = this.audioContexts.get(senderId);
|
|
if (!ctx) {
|
|
ctx = new AudioContext();
|
|
this.audioContexts.set(senderId, ctx as any);
|
|
ctx.nextPlayTime = ctx.currentTime + 0.05;
|
|
}
|
|
|
|
// Cek XCU Codec ID
|
|
if (payloadData[0] === 1) { // Opus Chunk
|
|
const opusData = payloadData.slice(1);
|
|
let dec = this.audioDecoders.get(senderId) as any;
|
|
if (!dec) {
|
|
dec = new (window as any).AudioDecoder({
|
|
output: (audioData: any) => {
|
|
const float32 = new Float32Array(audioData.numberOfFrames);
|
|
audioData.copyTo(float32, { planeIndex: 0 });
|
|
const audioBuffer = ctx.createBuffer(1, audioData.numberOfFrames, audioData.sampleRate);
|
|
audioBuffer.getChannelData(0).set(float32);
|
|
const source = ctx.createBufferSource();
|
|
source.buffer = audioBuffer;
|
|
source.connect(ctx.destination);
|
|
|
|
if (ctx.nextPlayTime < ctx.currentTime) ctx.nextPlayTime = ctx.currentTime + 0.03;
|
|
else if (ctx.nextPlayTime > ctx.currentTime + 0.5) ctx.nextPlayTime = ctx.currentTime + 0.03;
|
|
|
|
source.start(ctx.nextPlayTime);
|
|
ctx.nextPlayTime += audioBuffer.duration;
|
|
audioData.close();
|
|
},
|
|
error: (e:any) => console.error("AudioDecoder", e)
|
|
});
|
|
dec.configure({ codec: 'opus', sampleRate: 48000, numberOfChannels: 1 });
|
|
this.audioDecoders.set(senderId, dec);
|
|
}
|
|
try {
|
|
dec.decode(new (window as any).EncodedAudioChunk({
|
|
type: 'key', timestamp: performance.now() * 1000, data: opusData
|
|
}));
|
|
} catch(e) {}
|
|
continue;
|
|
}
|
|
|
|
// Fallback PCM`;
|
|
|
|
decoderContent = decoderContent.replace(rxAudioOld, rxAudioNew);
|
|
|
|
// Add Hot-Swap Support
|
|
const hotswap = ` public async hotSwapVideoEngine(mode: "canvas" | "webcodecs") {
|
|
if (this.videoEngineMode === mode) return;
|
|
console.log(\`[QUANTUM HOT-SWAP] Mengalihkan Video Engine ke: \${mode}\`);
|
|
this.videoEngineMode = mode;
|
|
if (this.mediaStream) {
|
|
await this.deactivateUplink();
|
|
await this.activateUplink('camera');
|
|
}
|
|
}
|
|
|
|
public async hotSwapAudioEngine(mode: "pcm" | "xcu-neural") {
|
|
if (this.audioEngineMode === mode) return;
|
|
console.log(\`[QUANTUM HOT-SWAP] Mengalihkan Audio Engine ke: \${mode}\`);
|
|
this.audioEngineMode = mode;
|
|
if (this.mediaStream) {
|
|
await this.deactivateUplink();
|
|
await this.activateUplink('camera');
|
|
}
|
|
}`;
|
|
|
|
decoderContent = decoderContent.replace(
|
|
/public async hotSwapVideoEngine[\s\S]*?activateUplink\('camera'\);\n\t\t\}\n\t\}/,
|
|
hotswap
|
|
);
|
|
|
|
fs.writeFileSync(decoderPath, decoderContent, 'utf-8');
|
|
|
|
|
|
// Now UI xcuRoom.tsx
|
|
const roomPath = 'C:/X/workspace/jumpa.id/vc/components/xcuRoom.tsx';
|
|
let roomContent = fs.readFileSync(roomPath, 'utf-8');
|
|
|
|
if (!roomContent.includes("audioEngineMode")) {
|
|
roomContent = roomContent.replace(
|
|
`const [videoEngineMode, setVideoEngineMode] = useState<'canvas' | 'webcodecs'>('webcodecs');`,
|
|
`const [videoEngineMode, setVideoEngineMode] = useState<'canvas' | 'webcodecs'>('webcodecs');\n const [audioEngineMode, setAudioEngineMode] = useState<'pcm' | 'xcu-neural'>('xcu-neural');`
|
|
);
|
|
|
|
roomContent = roomContent.replace(
|
|
`matrix.videoEngineMode = videoEngineMode;`,
|
|
`matrix.videoEngineMode = videoEngineMode;\n matrix.audioEngineMode = audioEngineMode;`
|
|
);
|
|
|
|
const toggleAudio = ` const handleToggleAudioEngine = () => {
|
|
const newMode = audioEngineMode === 'xcu-neural' ? 'pcm' : 'xcu-neural';
|
|
setAudioEngineMode(newMode);
|
|
if (matrixRef.current) {
|
|
matrixRef.current.hotSwapAudioEngine(newMode);
|
|
alert(newMode === 'xcu-neural' ? '???? XCU Neural Audio AKTIF! (VAD & Zero Bandwidth).' : '?????? Raw PCM Audio Aktif. Peringatan: Konsumsi Bandwidth Tinggi.');
|
|
}
|
|
};`;
|
|
|
|
roomContent = roomContent.replace(`const handleToggleEngine = () => {`, toggleAudio + `\n const handleToggleEngine = () => {`);
|
|
|
|
const oldButton = `<button onClick={handleToggleEngine} className={\`flex flex-col items-center justify-center w-14 h-14 rounded-lg transition-all duration-300 \${videoEngineMode === 'webcodecs' ? 'bg-cyan-900/40 text-cyan-400 border border-cyan-500/50 shadow-[0_0_15px_rgba(34,211,238,0.4)]' : 'hover:bg-slate-800 text-slate-400'}\`}>
|
|
<div className="relative">
|
|
<svg className={\`w-6 h-6 mb-1 \${videoEngineMode === 'webcodecs' ? 'animate-pulse' : ''}\`} fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
|
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M19.428 15.428a2 2 0 00-1.022-.547l-2.387-.477a6 6 0 00-3.86.517l-.318.158a6 6 0 01-3.86.517L6.05 15.21a2 2 0 00-1.806.547M8 4h8l-1 1v5.172a2 2 0 00.586 1.414l5 5c1.26 1.26.367 3.414-1.415 3.414H4.828c-1.782 0-2.674-2.154-1.414-3.414l5-5A2 2 0 009 10.172V5L8 4z"></path>
|
|
</svg>
|
|
{videoEngineMode === 'webcodecs' && <div className="absolute inset-0 bg-cyan-400 blur-md rounded-full opacity-40 animate-ping"></div>}
|
|
</div>
|
|
<span className="text-[9px] leading-tight text-center font-bold">{videoEngineMode === 'webcodecs' ? 'XCU WebCodecs' : 'Canvas Mode'}</span>
|
|
</button>`;
|
|
|
|
const capsuleUI = `<!-- Quantum Engine Capsule -->
|
|
<div className="flex bg-slate-800/50 p-1 rounded-xl border border-slate-700/50 shadow-inner">
|
|
{/* Video Segment */}
|
|
<button onClick={handleToggleEngine} className={\`flex flex-col items-center justify-center w-16 h-14 rounded-lg transition-all duration-300 \${videoEngineMode === 'webcodecs' ? 'bg-cyan-900/60 text-cyan-400 shadow-[0_0_15px_rgba(34,211,238,0.5)] z-10' : 'text-slate-400 hover:bg-slate-700/50'}\`}>
|
|
<div className="relative">
|
|
<svg className={\`w-5 h-5 mb-1 \${videoEngineMode === 'webcodecs' ? 'animate-pulse' : ''}\`} fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
|
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M15 10l4.553-2.276A1 1 0 0121 8.618v6.764a1 1 0 01-1.447.894L15 14M5 18h8a2 2 0 002-2V8a2 2 0 00-2-2H5a2 2 0 00-2 2v8a2 2 0 002 2z"></path>
|
|
</svg>
|
|
</div>
|
|
<span className="text-[8px] leading-tight text-center font-bold">{videoEngineMode === 'webcodecs' ? 'GPU VIDEO' : 'CPU VIDEO'}</span>
|
|
</button>
|
|
|
|
{/* Divider */}
|
|
<div className="w-[1px] bg-slate-700 mx-1 self-center h-8"></div>
|
|
|
|
{/* Audio Segment */}
|
|
<button onClick={handleToggleAudioEngine} className={\`flex flex-col items-center justify-center w-16 h-14 rounded-lg transition-all duration-300 \${audioEngineMode === 'xcu-neural' ? 'bg-fuchsia-900/60 text-fuchsia-400 shadow-[0_0_15px_rgba(217,70,239,0.5)] z-10' : 'text-slate-400 hover:bg-slate-700/50'}\`}>
|
|
<div className="relative">
|
|
<svg className={\`w-5 h-5 mb-1 \${audioEngineMode === 'xcu-neural' ? 'animate-bounce' : ''}\`} fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
|
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z"></path>
|
|
</svg>
|
|
</div>
|
|
<span className="text-[8px] leading-tight text-center font-bold">{audioEngineMode === 'xcu-neural' ? 'XCU NEURAL' : 'RAW PCM'}</span>
|
|
</button>
|
|
</div>`;
|
|
|
|
roomContent = roomContent.replace(oldButton, capsuleUI);
|
|
fs.writeFileSync(roomPath, roomContent, 'utf-8');
|
|
}
|
|
|
|
console.log("Audio Codec update script complete");
|