// [TSM.ID].[11031972] β€” All Rights Reserved. Proprietary & Confidential. /* eslint-disable */ // @ts-nocheck import { XCUPulsarCodec } from './xcu-pulsar-codec'; import { XCUResonanceCodec } from './xcu-resonance-codec'; // Frame type constants for XCU Binary Protocol v3 const FRAME_VIDEO_DELTA = 0; const FRAME_VIDEO_KEY = 1; const FRAME_AUDIO = 2; const FRAME_CONTROL = 3; const FRAME_HEARTBEAT = 5; const FRAME_LEDGER = 6; const FRAME_PULSAR = 10; class XCUWebGLFilter { public canvas: HTMLCanvasElement; private gl: WebGLRenderingContext; private program: WebGLProgram; private positionLocation: number; private texCoordLocation: number; private texture: WebGLTexture; private resolutionLocation: WebGLUniformLocation; private timeLocation: WebGLUniformLocation; private modeLocation: WebGLUniformLocation; constructor(width: number, height: number) { this.canvas = document.createElement('canvas'); this.canvas.width = width; this.canvas.height = height; const gl = this.canvas.getContext('webgl')!; this.gl = gl; const vs = ` attribute vec2 a_position; attribute vec2 a_texCoord; varying vec2 v_texCoord; void main() { gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0); v_texCoord = a_texCoord; } `; const fs = ` precision mediump float; uniform sampler2D u_image; uniform vec2 u_resolution; uniform float u_time; uniform int u_mode; varying vec2 v_texCoord; float rand(vec2 co){ return fract(sin(dot(co.xy ,vec2(12.9898,78.233))) * 43758.5453); } void main() { vec2 uv = v_texCoord; vec4 color = texture2D(u_image, uv); vec2 px = 1.0 / u_resolution; // Mode 1, 3, 5: Beauty Filter (Fast 5-tap Bilateral) if (u_mode == 1 || u_mode == 3 || u_mode == 5) { vec4 s1 = texture2D(u_image, uv + vec2(px.x, 0.0)); vec4 s2 = texture2D(u_image, uv + vec2(-px.x, 0.0)); vec4 s3 = texture2D(u_image, uv + vec2(0.0, px.y)); vec4 s4 = texture2D(u_image, uv + vec2(0.0, -px.y)); vec4 sum = color; float w = 1.0; float d1 = exp(-pow(length(s1.rgb - color.rgb), 2.0) * 50.0); sum += s1 * d1; w += d1; float d2 = exp(-pow(length(s2.rgb - color.rgb), 2.0) * 50.0); sum += s2 * d2; w += d2; float d3 = exp(-pow(length(s3.rgb - color.rgb), 2.0) * 50.0); sum += s3 * d3; w += d3; float d4 = exp(-pow(length(s4.rgb - color.rgb), 2.0) * 50.0); sum += s4 * d4; w += d4; color = sum / w; color.rgb = mix(color.rgb, color.rgb * 1.05 + 0.02, 0.5); } // Naive Edge Segmentation float s00 = length(texture2D(u_image, uv + vec2(-px.x, -px.y)).rgb); float s22 = length(texture2D(u_image, uv + vec2(px.x, px.y)).rgb); float edge = abs(s00 - s22); float distCenter = distance(uv, vec2(0.5, 0.5)); bool isBg = (distCenter > 0.35 && edge < 0.15); // Mode 2, 3: Hologram BG if ((u_mode == 2 || u_mode == 3) && isBg) { float m = fract(uv.y * 10.0 - u_time * 2.0 + rand(vec2(uv.x, 0.0))); vec3 bg = vec3(0.0, m * 0.5, m * 0.2); color.rgb = mix(color.rgb, bg, 0.85); } // Mode 4, 5: Bokeh Blur BG (Fast 5-tap box blur) if ((u_mode == 4 || u_mode == 5) && isBg) { vec4 b1 = texture2D(u_image, uv + vec2(px.x*4.0, 0.0)); vec4 b2 = texture2D(u_image, uv + vec2(-px.x*4.0, 0.0)); vec4 b3 = texture2D(u_image, uv + vec2(0.0, px.y*4.0)); vec4 b4 = texture2D(u_image, uv + vec2(0.0, -px.y*4.0)); vec4 blurSum = color + b1 + b2 + b3 + b4; color.rgb = mix(color.rgb, (blurSum / 5.0).rgb, 0.95); } gl_FragColor = color; } `; const createShader = (type: number, source: string) => { const shader = gl.createShader(type)!; gl.shaderSource(shader, source); gl.compileShader(shader); return shader; }; const vShader = createShader(gl.VERTEX_SHADER, vs); const fShader = createShader(gl.FRAGMENT_SHADER, fs); this.program = gl.createProgram()!; gl.attachShader(this.program, vShader); gl.attachShader(this.program, fShader); gl.linkProgram(this.program); this.positionLocation = gl.getAttribLocation(this.program, 'a_position'); this.texCoordLocation = gl.getAttribLocation(this.program, 'a_texCoord'); this.resolutionLocation = gl.getUniformLocation(this.program, 'u_resolution')!; this.timeLocation = gl.getUniformLocation(this.program, 'u_time')!; this.modeLocation = gl.getUniformLocation(this.program, 'u_mode')!; const positionBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([ -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0 ]), gl.STATIC_DRAW); const texCoordBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([ 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0 ]), gl.STATIC_DRAW); this.texture = gl.createTexture()!; gl.bindTexture(gl.TEXTURE_2D, this.texture); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.useProgram(this.program); gl.enableVertexAttribArray(this.positionLocation); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); gl.vertexAttribPointer(this.positionLocation, 2, gl.FLOAT, false, 0, 0); gl.enableVertexAttribArray(this.texCoordLocation); gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); gl.vertexAttribPointer(this.texCoordLocation, 2, gl.FLOAT, false, 0, 0); } public render(video: HTMLVideoElement, mode: number) { const gl = this.gl; gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); gl.bindTexture(gl.TEXTURE_2D, this.texture); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video); gl.useProgram(this.program); gl.uniform2f(this.resolutionLocation, gl.canvas.width, gl.canvas.height); gl.uniform1f(this.timeLocation, performance.now() / 1000.0); gl.uniform1i(this.modeLocation, mode); gl.drawArrays(gl.TRIANGLES, 0, 6); } } export class XCUQuantumMatrix { private transport: WebTransport | null = null; private stream: WebTransportBidirectionalStream | null = null; private streamWriter: WritableStreamDefaultWriter | null = null; private streamReader: ReadableStreamDefaultReader | null = null; private videoDecoders: Map = new Map(); private canvasCtxMap: Map = new Map(); private firstKeyFrameReceived: Map = new Map(); private videoEncoder: VideoEncoder | null = null; private isRunning: boolean = false; private natPingInterval: any = null; private mediaStream: MediaStream | null = null; private ws: WebSocket | null = null; private _frameCount: number = 0; public onAudioLevel: ((level: number) => void) | null = null; private vadAudioCtx: AudioContext | null = null; private vadAnalyser: AnalyserNode | null = null; private vadInterval: any = null; private heartbeatTimer: ReturnType | null = null; public participantRole: "PANELIST" | "AUDIENCE" = "PANELIST"; public videoEngineMode: "auto" | "canvas" | "webcodecs" = "auto"; public audioEngineMode: "auto" | "pcm" | "xcu-neural" | "xcu-resonance" = "auto"; public activeVideoCodec: string = "STANDBY"; public activeAudioCodec: string = "STANDBY"; public targetFps: 15 | 30 | 60 = 30; // Default 30fps β€” user can set via Matrix UI public fpsConfirmed60: boolean = false; // AutoPilot: true setelah konfirmasi device support 60fps private fpsProbeCount: number = 0; // Counter untuk probe 60fps public currentBandwidth: number = 0; public displayName: string = ''; // Display name for this participant (email or custom) private autoPilotInterval: any = null; private isSwappingEncoder: boolean = false; private audioEncoder: unknown = null; private audioDecoders: Map = new Map(); private trackProcessor: unknown = null; private trackGenerator: unknown = null; private activeCodecStr: string = "avc1.42E01F"; public useVirtualBg: number = 0; // 0=off, 2=Hologram, 4=Bokeh public useBeautyFilter: boolean = false; private pulsarEncoder: XCUPulsarCodec | null = null; private pulsarDecoders: Map = new Map(); private pulsarCanvas: HTMLCanvasElement | null = null; private pulsarCtx: CanvasRenderingContext2D | null = null; private usePulsarCodec: boolean = false; // Primary codec flag private participantLastSeen: Map = new Map(); private ghostCleanupInterval: ReturnType | null = null; private isMicMuted: boolean = false; private isEncodingCanvas: boolean = false; private reconnectAttempt: number = 0; // Exponential backoff counter private maxReconnectDelay: number = 30000; // Max 30s between reconnects private adaptiveQuality: number = 0.92; // Smart JPEG quality (0.5-0.95) private adaptiveResScale: number = 1.0; // Dynamic resolution scale (0.5-1.0) private framesDropped: number = 0; // Frame drop counter for telemetry private lastBandwidthEstimate: number = 10; // Mbps from real TX rate private audioJitterMap: Map = new Map(); private videoRenderQueue: Map = new Map(); private videoSyncLoopActive: boolean = false; private startVideoSyncLoop() { if (this.videoSyncLoopActive) return; this.videoSyncLoopActive = true; console.log("[QUANTUM MATRIX] A/V Sync Render Loop Started [TSM.ID].[11031972]"); const loop = () => { if (!this.videoSyncLoopActive) return; requestAnimationFrame(loop); const now = performance.now(); this.videoRenderQueue.forEach((queue, senderId) => { if (queue.length === 0) return; queue.sort((a, b) => a.targetTime - b.targetTime); const ctx = this.canvasCtxMap.get(senderId); if (!ctx) { queue.forEach(item => { if (item.isWebCodec && item.frame.close) { try { item.frame.close(); } catch(e) {} } }); queue.length = 0; return; } while (queue.length > 0) { const item = queue[0]; if (now >= item.targetTime) { queue.shift(); if (now - item.targetTime > 150) { if (item.isWebCodec && item.frame.close) { try { item.frame.close(); } catch(e) {} } continue; } if (item.isWebCodec) { const vf = item.frame as VideoFrame; ctx.canvas.width = vf.displayWidth; ctx.canvas.height = vf.displayHeight; ctx.drawImage(vf, 0, 0); vf.close(); } else { const img = item.frame as HTMLImageElement; ctx.canvas.width = img.width; ctx.canvas.height = img.height; ctx.drawImage(img, 0, 0); } } else { break; } } }); }; requestAnimationFrame(loop); } private stopVideoSyncLoop() { this.videoSyncLoopActive = false; this.videoRenderQueue.forEach((queue) => { queue.forEach(item => { if (item.isWebCodec && item.frame.close) { try { item.frame.close(); } catch(e) {} } }); queue.length = 0; }); this.videoRenderQueue.clear(); } public onRemoteAudio?: (participantId: number, stream: MediaStream) => void; public onModuleUnlocked?: (moduleId: number) => void; public onQuantumResonance?: (senderId: number, reactionType: string) => void; public onSovereignSignal?: (type: string, payload?: any) => void; public isDesktop: boolean = false; public pulsarCodec: any = null; public resonanceCodec: XCUResonanceCodec = new XCUResonanceCodec(); // REAL Traffic Counters β€” NO DUMMY (counts actual ws.send/ws.onmessage bytes) public trafficStats = { tx: { video: 0, audio: 0, control: 0, total: 0 }, rx: { video: 0, audio: 0, control: 0, total: 0 }, rates: { txVideo: 0, txAudio: 0, txTotal: 0, rxVideo: 0, rxAudio: 0, rxTotal: 0, }, _lastSnapshot: { ts: 0, txVideo: 0, txAudio: 0, txTotal: 0, rxVideo: 0, rxAudio: 0, rxTotal: 0, }, startTime: 0, wsState: 'CLOSED' as string, }; private trafficInterval: ReturnType | null = null; public pulsarWasmMemory: WebAssembly.Memory | null = null; public downlinkAudioCtx: AudioContext | null = null; public downlinkAudioDest: MediaStreamAudioDestinationNode | null = null; private downlinkAudioEl: HTMLAudioElement | null = null; private initDownlinkAudio() { if (!this.downlinkAudioCtx) { const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext; this.downlinkAudioCtx = new AudioContextClass(); (this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx!.currentTime; this.downlinkAudioDest = this.downlinkAudioCtx!.createMediaStreamDestination(); this.downlinkAudioEl = document.createElement('audio'); this.downlinkAudioEl.autoplay = true; this.downlinkAudioEl.srcObject = this.downlinkAudioDest.stream; document.body.appendChild(this.downlinkAudioEl); this.downlinkAudioEl.play().catch(e => console.warn("[AEC] Audio auto-play dicekal browser", e)); console.log("[AEC MATRIX] Web Audio to DOM Pipeline Active"); } } private e2eeKey: CryptoKey | null = null; private e2eeKeyStr: string = ""; public async setE2EEKey(keyStr: string) { if (!keyStr || keyStr === "NO_KEY" || keyStr === "none") { this.e2eeKey = null; this.e2eeKeyStr = ""; console.log("[QUANTUM MATRIX] E2EE Kriptografi DINONAKTIFKAN."); return; } try { // Derivasi Password ke CryptoKey AES-GCM (PBKDF2) - True Hardware Acceleration const enc = new TextEncoder(); const keyMaterial = await crypto.subtle.importKey( "raw", enc.encode(keyStr), "PBKDF2", false, ["deriveBits", "deriveKey"] ); // Salt tetap agar sinkron antar perangkat const salt = enc.encode("XCOM_ULTRA_QUANTUM_SALT_2026"); this.e2eeKey = await crypto.subtle.deriveKey( { name: "PBKDF2", salt: salt, iterations: 100000, hash: "SHA-256" }, keyMaterial, { name: "AES-GCM", length: 256 }, false, ["encrypt", "decrypt"] ); this.e2eeKeyStr = keyStr; console.log("[QUANTUM MATRIX] πŸ›‘οΈ E2EE Kriptografi AKTIF (AES-GCM 256-bit)"); } catch (_e) { console.error("[QUANTUM MATRIX] Gagal merakit E2EE Key:", _e); } } constructor( private roomName: string, public participantId: number, private jwtToken: string // Entitlement Token ) { this.isDesktop = !/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test( navigator.userAgent ); console.log(`[QUANTUM MATRIX] Node ${this.participantId} Initialized.`); // ANTI 3-SECOND AUDIO BUG: Eagerly create AudioContext so it can be resumed by global click events try { this.initDownlinkAudio(); (this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx.currentTime; } catch (_e) { console.warn("AudioContext failed to initialize early", _e); } } // Callbacks public onParticipantJoined: ((id: number) => void) | null = null; public onParticipantLeft: ((id: number) => void) | null = null; public onParticipantNameReceived: ((id: number, name: string) => void) | null = null; public onQuantumDataReceived: | ((senderId: number, payload: string) => void) | null = null; public onLocalStream: ((stream: MediaStream) => void) | null = null; public onActiveSpeakerChanged: ((speakerId: number) => void) | null = null; public onAudioLevel: ((level: number) => void) | null = null; private vadLocalAudioCtx: AudioContext | null = null; private uplinkAudioCtx: AudioContext | null = null; private uplinkScriptNode: ScriptProcessorNode | null = null; private uplinkAudioSource: MediaStreamAudioSourceNode | null = null; public registerCanvas(participantId: number, canvasId: string) { const canvas = document.getElementById(canvasId) as HTMLCanvasElement; if (canvas) { const ctx = canvas.getContext("2d"); if (ctx) this.canvasCtxMap.set(participantId, ctx); console.log( `[QUANTUM MATRIX] Canvas didaftarkan untuk Partisipan ${participantId}`, ); } } /** Try to find and register canvas by participant ID */ private tryAutoRegisterCanvas(participantId: number): boolean { if (this.canvasCtxMap.has(participantId)) return true; const canvasId = `quantum-matrix-${participantId}`; const canvas = document.getElementById(canvasId) as HTMLCanvasElement; if (canvas) { const ctx = canvas.getContext("2d"); if (ctx) { this.canvasCtxMap.set(participantId, ctx); return true; } } return false; } public async ignite(roomName: string, serverUrl: string = "/xcu-engine", quantumHash?: string) { this.isRunning = true; this.startVideoSyncLoop(); console.log("[QUANTUM MATRIX] Ignite v3.0 β€” WebTransport QUIC Primary + WebSocket Fallback"); // Attempt WebTransport QUIC first, then fall back to WebSocket const host = window.location.hostname; // mesh.ultramodul.xyz const wtUrl = `https://${host}:8443`; let useWebTransport = false; // === WebTransport QUIC (PRIMARY) === if (typeof WebTransport !== 'undefined') { try { console.log(`[QUANTUM MATRIX] Attempting WebTransport QUIC: ${wtUrl}`); // Fetch server cert hash for serverCertificateHashes (needed for self-signed or pinning) let certHashes: any[] | undefined; try { const certRes = await fetch('/api/v1/system/cert'); const certData = await certRes.json(); if (certData.hash) { // Convert hex hash to Uint8Array const hashBytes = new Uint8Array(certData.hash.match(/.{1,2}/g).map((b: string) => parseInt(b, 16))); certHashes = [{ algorithm: 'sha-256', value: hashBytes.buffer }]; console.log('[QUANTUM MATRIX] Cert hash loaded for WebTransport pinning'); } } catch(e) { console.log('[QUANTUM MATRIX] No cert hash, using system trust'); } const wtOptions: any = {}; // Only use serverCertificateHashes if available and needed // For Let's Encrypt certs, browser trusts them natively const wt = new WebTransport(wtUrl, wtOptions); await Promise.race([ wt.ready, new Promise((_, reject) => setTimeout(() => reject(new Error('WebTransport timeout')), 5000)) ]); console.log("[QUANTUM MATRIX] βœ… WebTransport QUIC CONNECTED! Zero Head-of-Line Blocking ACTIVE"); this.transport = wt; this.trafficStats.wsState = 'QUIC'; useWebTransport = true; this.reconnectAttempt = 0; // Reset backoff on success // Setup datagram reader const reader = wt.datagrams.readable.getReader(); const writer = wt.datagrams.writable.getWriter(); this.streamWriter = writer; // Send registration datagram: [type=99, flags=0, pId_lo, pId_hi, room_len, ...room_bytes] const roomBytes = new TextEncoder().encode(roomName); const regPayload = new Uint8Array(5 + roomBytes.length); regPayload[0] = 99; // CONTROL regPayload[1] = 0; regPayload[2] = this.participantId & 0xFF; regPayload[3] = (this.participantId >> 8) & 0xFF; regPayload[4] = roomBytes.length; regPayload.set(roomBytes, 5); await writer.write(regPayload); this.trafficStats.tx.control += regPayload.length; this.trafficStats.tx.total += regPayload.length; // Start NAT Keep-Alive Ping this.natPingInterval = setInterval(() => { if (this.streamWriter) { const ping = new Uint8Array([99, 1]); this.streamWriter.write(ping).catch(() => {}); } }, 3000); // Read datagrams in loop (async () => { try { while (true) { const { value, done } = await reader.read(); if (done) break; if (!value || value.byteLength < 4) continue; // REAL RX counter const rxLen = value.byteLength; this.trafficStats.rx.total += rxLen; const frameType = value[0]; if (frameType === 1 || frameType === 10) this.trafficStats.rx.video += rxLen; else if (frameType === 2) this.trafficStats.rx.audio += rxLen; else this.trafficStats.rx.control += rxLen; this.handleIncomingFrame(value); } } catch(e) { console.warn('[QUANTUM MATRIX] WebTransport datagram read ended:', e); } })(); // Handle connection close β€” with exponential backoff wt.closed.then(() => { console.log('[QUANTUM MATRIX] WebTransport closed'); this.trafficStats.wsState = 'CLOSED'; if (this.isRunning) { const delay = Math.min(1000 * Math.pow(1.5, this.reconnectAttempt), this.maxReconnectDelay); this.reconnectAttempt++; console.log(`[QUANTUM MATRIX] Reconnecting WebTransport in ${(delay/1000).toFixed(1)}s (attempt ${this.reconnectAttempt})...`); setTimeout(() => this.ignite(roomName, serverUrl, quantumHash), delay); } }).catch(() => { if (this.natPingInterval) clearInterval(this.natPingInterval); this.trafficStats.wsState = 'CLOSED'; if (this.isRunning) { const delay = Math.min(1000 * Math.pow(1.5, this.reconnectAttempt), this.maxReconnectDelay); this.reconnectAttempt++; console.log(`[QUANTUM MATRIX] Reconnecting WebTransport in ${(delay/1000).toFixed(1)}s (attempt ${this.reconnectAttempt})...`); setTimeout(() => this.ignite(roomName, serverUrl, quantumHash), delay); } }); wt.closed.then(() => { if (this.natPingInterval) clearInterval(this.natPingInterval); }).catch(() => {}); this.startQuantumAutoPilot(); this.startTrafficMonitor(); this.startGhostCleanup(); // BUG FIX: Send heartbeat IMMEDIATELY on connect for instant visibility this.sendPresenceHeartbeat(); this.startHeartbeatLoop(); } catch(e) { console.warn(`[QUANTUM MATRIX] WebTransport failed: ${e}. Falling back to WebSocket...`); useWebTransport = false; } } // === WebSocket Fallback (SECONDARY) === if (!useWebTransport) { try { const wsHost = window.location.host; const wsProto = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; const wsUrl = `${wsProto}//${wsHost}/ws/${roomName}`; console.log("[QUANTUM MATRIX] WebSocket Fallback URL:", wsUrl); const ws = new WebSocket(wsUrl); ws.binaryType = "arraybuffer"; this.ws = ws; this.trafficStats.wsState = 'WS-CONNECTING'; let resolveReady: () => void; let rejectReady: (e: unknown) => void; const readyPromise = new Promise((res, rej) => { resolveReady = res; rejectReady = rej; }); ws.onopen = () => { console.log("[QUANTUM MATRIX] WebSocket Fallback CONNECTED (TCP)"); this.trafficStats.wsState = 'WS-OPEN'; this.reconnectAttempt = 0; // Reset backoff on success ws.send(JSON.stringify({ type: "qcg_handshake", token: this.jwtToken, displayName: this.displayName || '' })); resolveReady!(); }; ws.onerror = (e) => { console.error("[QUANTUM MATRIX] WebSocket Error:", e); this.trafficStats.wsState = 'WS-ERROR'; rejectReady!(e); }; ws.onclose = () => { console.log("[QUANTUM MATRIX] WebSocket Closed"); this.trafficStats.wsState = 'WS-CLOSED'; if (this.isRunning) { const delay = Math.min(1000 * Math.pow(1.5, this.reconnectAttempt), this.maxReconnectDelay); this.reconnectAttempt++; console.log(`[QUANTUM MATRIX] Reconnecting WebSocket in ${(delay/1000).toFixed(1)}s (attempt ${this.reconnectAttempt})...`); setTimeout(() => this.ignite(roomName, serverUrl, quantumHash), delay); } }; ws.onmessage = (event) => { if (typeof event.data === "string") { try { const msg = JSON.parse(event.data); if (msg.type === 'participant_list') { console.log('[QUANTUM MATRIX] Participant list:', msg.participants); } } catch(e) {} return; } const data = new Uint8Array(event.data); if (data.byteLength < 4) return; // REAL RX counter const rxLen = data.byteLength; this.trafficStats.rx.total += rxLen; const frameType = data[0]; if (frameType === 1 || frameType === 10) this.trafficStats.rx.video += rxLen; else if (frameType === 2) this.trafficStats.rx.audio += rxLen; else this.trafficStats.rx.control += rxLen; this.handleIncomingFrame(data); }; await readyPromise; this.startQuantumAutoPilot(); this.startTrafficMonitor(); this.startGhostCleanup(); // BUG FIX: Send heartbeat IMMEDIATELY on connect for instant visibility this.sendPresenceHeartbeat(); this.startHeartbeatLoop(); } catch(e) { console.error("[QUANTUM MATRIX] Both WebTransport and WebSocket failed:", e); this.trafficStats.wsState = 'FAILED'; } } // 3. Start video capture pipeline if (this.mediaStream) { this.setupVideoCapturePipeline(); } } /** Unified incoming frame handler for both WebTransport and WebSocket */ private async handleIncomingFrame(data: Uint8Array) { if (data.byteLength < 8) return; const dataView = new DataView(data.buffer, data.byteOffset, data.byteLength); const frameType = dataView.getUint8(0); const quality = dataView.getUint8(1); const senderId = dataView.getUint16(2, true); const frameLength = dataView.getUint32(4, true); if (senderId === this.participantId) return; // Skip own frames this.participantLastSeen.set(senderId, Date.now()); if (data.byteLength < 8 + frameLength) return; const payloadData = data.slice(8, 8 + frameLength); if (frameType === 5) { // FRAME_HEARTBEAT if (!this.canvasCtxMap.has(senderId)) { if (this.onParticipantJoined) this.onParticipantJoined(senderId); } // Parse display name from heartbeat payload (if present) if (payloadData.length > 0) { try { const nameStr = new TextDecoder().decode(payloadData); if (nameStr && nameStr.length > 0 && nameStr.length < 256) { const parsed = JSON.parse(nameStr); if (parsed.type === 'NAME_ANNOUNCE' && parsed.name) { if (this.onParticipantNameReceived) this.onParticipantNameReceived(senderId, parsed.name); } } } catch(_) { /* Not JSON, ignore */ } } return; } if (frameType === 10 || frameType === 1) { // FRAME_PULSAR let jpegPayload = payloadData; let isDecryptionFailed = false; if (quality === 2) { if (this.e2eeKey && payloadData.length > 12) { const iv = payloadData.slice(0, 12); const cipher = payloadData.slice(12); try { const plainBuf = await window.crypto.subtle.decrypt( { name: "AES-GCM", iv: iv }, this.e2eeKey, cipher ); jpegPayload = new Uint8Array(plainBuf); } catch (_e) { isDecryptionFailed = true; } } else { isDecryptionFailed = true; } } if (isDecryptionFailed) { this.tryAutoRegisterCanvas(senderId); const ctx = this.canvasCtxMap.get(senderId); if (ctx) { const w = ctx.canvas.width || 640; const h = ctx.canvas.height || 360; const idata = ctx.createImageData(w, h); const d32 = new Uint32Array(idata.data.buffer); for(let i=0; i { const jitter = this.audioJitterMap.get(senderId) || 50; let queue = this.videoRenderQueue.get(senderId); if (!queue) { queue = []; this.videoRenderQueue.set(senderId, queue); } queue.push({ frame: img, targetTime: performance.now() + jitter, isWebCodec: false }); URL.revokeObjectURL(bmpUrl); }; img.onerror = () => URL.revokeObjectURL(bmpUrl); img.src = bmpUrl; } else if (frameType === 2) { // AUDIO frame let audioPayload = payloadData; let isDecryptionFailed = false; if (quality === 2) { if (this.e2eeKey && payloadData.length > 12) { const iv = payloadData.slice(0, 12); const cipher = payloadData.slice(12); try { const plainBuf = await window.crypto.subtle.decrypt( { name: "AES-GCM", iv: iv }, this.e2eeKey, cipher ); audioPayload = new Uint8Array(plainBuf); } catch (_e) { isDecryptionFailed = true; } } else { isDecryptionFailed = true; } } if (isDecryptionFailed) return; // Play audio if method exists this.playRemoteAudio(senderId, audioPayload); } else if (frameType === 6) { // FRAME_LEDGER β€” JSON data (Chat, Resonance, Name Announce, etc.) try { const jsonStr = new TextDecoder().decode(payloadData); const parsed = JSON.parse(jsonStr); if (parsed.pkepxType === 'NAME_ANNOUNCE' && parsed.name) { if (this.onParticipantNameReceived) this.onParticipantNameReceived(senderId, parsed.name); } else if (parsed.pkepxType === 'RESONANCE' && parsed.payload) { if (this.onQuantumResonance) this.onQuantumResonance(senderId, parsed.payload); } else if (parsed.pkepxType === 'SOVEREIGN_SIGNAL') { if (this.onSovereignSignal) this.onSovereignSignal(parsed.signalType, parsed.payload); } // Also forward to generic data handler if (this.onQuantumDataReceived) this.onQuantumDataReceived(senderId, jsonStr); } catch(_) { /* Not valid JSON */ } } } /** Send data via WebTransport (QUIC datagram) or WebSocket (TCP) */ public async sendFrame(payload: Uint8Array) { // REAL TX counter this.trafficStats.tx.total += payload.byteLength; const frameType = payload[0]; if (frameType === 1 || frameType === 10) this.trafficStats.tx.video += payload.byteLength; else if (frameType === 2) this.trafficStats.tx.audio += payload.byteLength; else this.trafficStats.tx.control += payload.byteLength; // Send via WebTransport QUIC datagram (primary) or WebSocket (fallback) if (this.streamWriter) { try { await this.streamWriter.write(payload); return; } catch(e) { // WebTransport write failed, try WebSocket } } if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(payload); } } public async hotSwapVideoEngine(mode: "auto" | "canvas" | "webcodecs") { if (this.videoEngineMode === mode) return; console.log(`[QUANTUM HOT-SWAP] Mengalihkan Video Engine ke: ${mode}`); this.videoEngineMode = mode; if (this.mediaStream) { await this.deactivateUplink(); await this.activateUplink('camera'); } } public async hotSwapAudioEngine(mode: "auto" | "pcm" | "xcu-neural" | "xcu-resonance") { if (this.audioEngineMode === mode) return; console.log(`[QUANTUM HOT-SWAP] Mengalihkan Audio Engine ke: ${mode}`); this.audioEngineMode = mode; if (this.mediaStream) { await this.deactivateUplink(); await this.activateUplink('camera'); } } public unlockAudio() { if (!this.downlinkAudioCtx) { this.initDownlinkAudio(); (this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx.currentTime; } if (this.downlinkAudioCtx.state === "suspended") { this.downlinkAudioCtx.resume().catch(() => {}); } if (!this.uplinkAudioCtx) { this.uplinkAudioCtx = new AudioContext(); } if (this.uplinkAudioCtx.state === "suspended") { this.uplinkAudioCtx.resume().catch(() => {}); } console.log("[QUANTUM MATRIX] Audio Contexts Pre-Created and Unlocked via User Gesture"); } public toggleMic(enabled: boolean) { this.isMicMuted = !enabled; if (this.mediaStream) { this.mediaStream.getAudioTracks().forEach((track) => { track.enabled = enabled; }); } console.log(`[QUANTUM UPLINK] Mikrofon ${enabled ? "AKTIF" : "MATI"} β€” pipeline terjaga untuk mencegah bug Mac Safari/Chrome.`); } public resumeAudioContext() { if (this.downlinkAudioCtx && this.downlinkAudioCtx.state === "suspended") { this.downlinkAudioCtx.resume().then(() => { console.log("[QUANTUM MATRIX] AudioContext berhasil dibangunkan secara manual!"); }).catch(e => { console.warn("[QUANTUM MATRIX] Gagal membangunkan AudioContext:", e); }); } if (this.uplinkAudioCtx && this.uplinkAudioCtx.state === "suspended") { this.uplinkAudioCtx.resume().catch(() => {}); } if (this.vadAudioCtx && this.vadAudioCtx.state === "suspended") { this.vadAudioCtx.resume().catch(() => {}); } } public setEffects(virtualBgMode: number, beautyFilter: boolean) { this.useVirtualBg = virtualBgMode; this.useBeautyFilter = beautyFilter; console.log(`[QUANTUM EFFECTS] Virtual BG Mode: ${virtualBgMode}, Beauty: ${beautyFilter}`); } private async detectBestCodec(): Promise { const codecs = [ "av01.0.04M.08", // AV1 "vp09.00.10.08", // VP9 "vp8", // VP8 "avc1.42E01F" // H.264 ]; for (const c of codecs) { try { const support = await VideoEncoder.isConfigSupported({ codec: c, width: 1920, height: 1080, bitrate: 8_000_000, framerate: 30 }); if (support.supported) { console.log(`[QUANTUM WEBCODECS] Hardware GPU Codec Terdeteksi: ${c}`); return c; } } catch (_e) {} } return "avc1.42E01F"; // Fallback H.264 } private createDecoderForParticipant(senderId: number, codecStr: string = "avc1.42E01F") { // Jika decoder sudah ada dan codec-nya berbeda, tutup dulu if (this.videoDecoders.has(senderId)) { try { this.videoDecoders.get(senderId)!.close(); } catch(e){} this.videoDecoders.delete(senderId); } const decoder = new VideoDecoder({ output: (frame: unknown) => { this.tryAutoRegisterCanvas(senderId); const jitter = this.audioJitterMap.get(senderId) || 50; let queue = this.videoRenderQueue.get(senderId); if (!queue) { queue = []; this.videoRenderQueue.set(senderId, queue); } queue.push({ frame: frame, targetTime: performance.now() + jitter, isWebCodec: true }); }, error: (e: unknown) => console.error(`Decoder Error untuk ${senderId}:`, e), }); try { decoder.configure({ codec: codecStr, codedWidth: 1280, codedHeight: 720 }); this.videoDecoders.set(senderId, decoder); this.firstKeyFrameReceived.set(senderId, false); console.log(`[QUANTUM WEBCODECS] Hardware Decoder (${codecStr}) siap untuk Partisipan ${senderId}`); } catch (_e) { console.error("[QUANTUM WEBCODECS] Gagal konfigurasi decoder:", e); } // Notifikasi React untuk merender tile if (this.onParticipantJoined) { this.onParticipantJoined(senderId); } } public async activateUplink(source: "camera" | "screen" = "camera", facingMode: "user" | "environment" = "user") { if (this.mediaStream) { console.warn( "[QUANTUM UPLINK] Kamera/Layar sudah aktif! Mengabaikan perintah ganda untuk mencegah kebocoran memori.", ); return; } console.log( `[QUANTUM UPLINK] Memulai Injeksi ${source === "screen" ? "Layar" : "Kamera"} ke WebCodecs...`, ); try { if (source === "screen") { this.mediaStream = await navigator.mediaDevices.getDisplayMedia({ video: { width: { ideal: 1920 }, height: { ideal: 1080 }, frameRate: { ideal: this.targetFps } }, audio: false, }); } else { this.mediaStream = await navigator.mediaDevices.getUserMedia({ video: { width: { ideal: 1920 }, height: { ideal: 1080 }, frameRate: { ideal: this.targetFps }, facingMode: facingMode }, audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }); } if (this.onLocalStream) { this.onLocalStream(this.mediaStream); } this.startVAD(); // AUDIENCE mode: skip encoding if (this.participantRole === "AUDIENCE") { console.log("[UPLINK] AUDIENCE mode - no video upload."); await this.sendRoleSignal(); return; } const videoTrack = this.mediaStream.getVideoTracks()[0]; // === Canvas JPEG Pipeline: Works on ALL browsers === const captureVideo = document.createElement("video"); captureVideo.srcObject = new MediaStream([videoTrack]); captureVideo.muted = true; captureVideo.playsInline = true; captureVideo.autoplay = true; try { await captureVideo.play(); } catch { /* autoplay blocked */ } // Adaptive canvas resolution based on bandwidth const conn = (navigator as any).connection; const dl = conn ? conn.downlink : 10; let captureW = 1280, captureH = 720; // Default HD if (dl > 5) { captureW = 1920; captureH = 1080; } // FHD for good bandwidth else if (dl < 1.5) { captureW = 854; captureH = 480; } // SD for low bandwidth const captureCvs = document.createElement("canvas"); captureCvs.width = captureW; captureCvs.height = captureH; const captureCtx = captureCvs.getContext("2d")!; console.log(`[UPLINK] Canvas JPEG pipeline active (${captureW}x${captureH} @ ${this.targetFps}fps) [TSM.ID].[11031972]`); let lastVideoFrame = 0; const captureLoop = (now: number) => { if (!this.isRunning) return; requestAnimationFrame(captureLoop); const fpsInterval = 1000 / this.targetFps; if (now - lastVideoFrame < fpsInterval) return; lastVideoFrame = now; if (this.videoEngineMode === "webcodecs") return; if (!this.streamWriter && (!this.ws || this.ws.readyState !== WebSocket.OPEN)) return; // === ULTRA BACKPRESSURE CONTROL === // If WebSocket send buffer > 64KB, skip this frame (prevent 18KB backlog!) if (this.ws && this.ws.bufferedAmount > 65536) { this.framesDropped++; // Smart downgrade: reduce quality when buffer is full if (this.adaptiveQuality > 0.5) { this.adaptiveQuality = Math.max(0.5, this.adaptiveQuality - 0.05); console.log(`[AUTOPILOT] ⚑ Backpressure! Quality β†’ ${(this.adaptiveQuality*100).toFixed(0)}% (dropped: ${this.framesDropped})`); } if (this.adaptiveResScale > 0.5) { this.adaptiveResScale = Math.max(0.5, this.adaptiveResScale - 0.1); } return; } // === SMART RESOLUTION SCALING === const effectiveW = Math.round(captureW * this.adaptiveResScale); const effectiveH = Math.round(captureH * this.adaptiveResScale); if (captureCvs.width !== effectiveW || captureCvs.height !== effectiveH) { captureCvs.width = effectiveW; captureCvs.height = effectiveH; } try { let mode = 0; if (this.useBeautyFilter && this.useVirtualBg === 2) mode = 3; else if (this.useBeautyFilter && this.useVirtualBg === 4) mode = 5; else if (this.useVirtualBg > 0) mode = this.useVirtualBg; else if (this.useBeautyFilter) mode = 1; if (mode > 0) { if (!(this as any).webglFilter) (this as any).webglFilter = new XCUWebGLFilter(effectiveW, effectiveH); (this as any).webglFilter.render(captureVideo, mode); captureCtx.drawImage((this as any).webglFilter.canvas, 0, 0, effectiveW, effectiveH); } else { if (captureVideo.readyState >= 2) { captureCtx.drawImage(captureVideo, 0, 0, effectiveW, effectiveH); } } if (this.isEncodingCanvas) return; // Prevent OOM stacking this.isEncodingCanvas = true; captureCvs.toBlob((blob) => { this.isEncodingCanvas = false; if (!blob) return; // Check both WS and WT if (!this.streamWriter && (!this.ws || this.ws.readyState !== WebSocket.OPEN)) return; blob.arrayBuffer().then(async (buf) => { let jpegData = new Uint8Array(buf); let packetLen = 8 + jpegData.length; let isEncrypted = 0; const iv = new Uint8Array(12); if (this.e2eeKey) { window.crypto.getRandomValues(iv); const cipherBuffer = await window.crypto.subtle.encrypt( { name: "AES-GCM", iv: iv }, this.e2eeKey, jpegData ); jpegData = new Uint8Array(cipherBuffer); packetLen = 8 + 12 + jpegData.length; isEncrypted = 2; } const packet = new Uint8Array(packetLen); packet[0] = 10; // FRAME_PULSAR packet[1] = isEncrypted ? 2 : 1; const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); if (isEncrypted) { view.setUint32(4, 12 + jpegData.length, true); packet.set(iv, 8); packet.set(jpegData, 20); } else { view.setUint32(4, jpegData.length, true); packet.set(jpegData, 8); } this.sendFrame(packet); this._frameCount++; }); }, "image/jpeg", this.adaptiveQuality); } catch (err) { console.warn("[TX] capture error", err); } }; requestAnimationFrame(captureLoop); // === Audio PCM Pipeline β€” ULTRA TUNE-UP === const audioTracks = this.mediaStream.getAudioTracks(); if (audioTracks.length > 0) { if (!this.uplinkAudioCtx) this.uplinkAudioCtx = new AudioContext(); this.uplinkAudioSource = this.uplinkAudioCtx.createMediaStreamSource(new MediaStream([audioTracks[0]])); // Smaller buffer = lower latency (2048 samples = ~42ms @ 48kHz) this.uplinkScriptNode = this.uplinkAudioCtx.createScriptProcessor(2048, 1, 1); this.uplinkScriptNode.onaudioprocess = (ev: AudioProcessingEvent) => { if (this.isMicMuted) { const out = ev.outputBuffer.getChannelData(0); for (let i = 0; i < out.length; i++) out[i] = 0; return; } const float32 = ev.inputBuffer.getChannelData(0); // SILENCE DETECTION: Skip sending if audio is nearly silent (saves bandwidth) let rms = 0; for (let i = 0; i < float32.length; i += 16) rms += float32[i] * float32[i]; rms = Math.sqrt(rms / (float32.length / 16)); if (rms < 0.005) { const out = ev.outputBuffer.getChannelData(0); for (let i = 0; i < out.length; i++) out[i] = 0; return; // Skip silent frames } // DOWNSAMPLE 48kHz β†’ 16kHz (3x bandwidth reduction) const srcRate = this.uplinkAudioCtx!.sampleRate; const targetRate = 16000; const ratio = srcRate / targetRate; const downLen = Math.floor(float32.length / ratio); const int16 = new Int16Array(downLen); for (let i = 0; i < downLen; i++) { // Linear interpolation for cleaner downsampling const srcIdx = i * ratio; const idx0 = Math.floor(srcIdx); const idx1 = Math.min(idx0 + 1, float32.length - 1); const frac = srcIdx - idx0; const sample = float32[idx0] * (1 - frac) + float32[idx1] * frac; int16[i] = Math.max(-32768, Math.min(32767, Math.round(sample * 32767))); } const out = ev.outputBuffer.getChannelData(0); for (let i = 0; i < out.length; i++) out[i] = 0; const pcmBytes = new Uint8Array(int16.buffer); if (this.e2eeKey) { const iv = window.crypto.getRandomValues(new Uint8Array(12)); window.crypto.subtle.encrypt({ name: "AES-GCM", iv: iv }, this.e2eeKey, pcmBytes) .then(cipherBuffer => { const cipherBytes = new Uint8Array(cipherBuffer); const packet = new Uint8Array(8 + 4 + 12 + cipherBytes.length); packet[0] = 2; packet[1] = 2; const av = new DataView(packet.buffer); av.setUint16(2, this.participantId, true); av.setUint32(4, 4 + 12 + cipherBytes.length, true); av.setUint32(8, targetRate, true); packet.set(iv, 12); packet.set(cipherBytes, 24); this.sendFrame(packet); }).catch(e => console.error("Audio Encrypt Error", e)); } else { const packet = new Uint8Array(8 + 4 + pcmBytes.length); packet[0] = 2; packet[1] = 0; const av = new DataView(packet.buffer); av.setUint16(2, this.participantId, true); av.setUint32(4, 4 + pcmBytes.length, true); av.setUint32(8, targetRate, true); packet.set(pcmBytes, 12); this.sendFrame(packet); } }; this.uplinkAudioSource.connect(this.uplinkScriptNode); this.uplinkScriptNode.connect(this.uplinkAudioCtx.destination); (window as unknown as { [key: string]: unknown }).uplinkScriptNode = this.uplinkScriptNode; console.log("[UPLINK] Audio pipeline active (16kHz downsampled, silence gated) [TSM.ID].[11031972]"); this.startVadLoop(); } } catch (e: unknown) { console.error("[UPLINK] Camera error:", e); } } /** Calculate real bytes/sec rates from cumulative counters */ // Ghost Participant Cleanup β€” auto-remove stale participants after 8s of no data private startGhostCleanup() { if (this.ghostCleanupInterval) clearInterval(this.ghostCleanupInterval); this.ghostCleanupInterval = setInterval(() => { const now = Date.now(); const GHOST_TIMEOUT = 5000; // 5s β€” faster cleanup for accurate participant list this.participantLastSeen.forEach((lastSeen, senderId) => { if (now - lastSeen > GHOST_TIMEOUT) { console.log(`[GHOST CLEANUP] Removing stale participant ${senderId} (${Math.round((now - lastSeen)/1000)}s stale)`); this.participantLastSeen.delete(senderId); this.canvasCtxMap.delete(senderId); if (this.videoDecoders.has(senderId)) { try { this.videoDecoders.get(senderId)!.close(); } catch(e) {} this.videoDecoders.delete(senderId); } if (this.onParticipantLeft) { this.onParticipantLeft(senderId); } } }); }, 5000); } public startTrafficMonitor() { this.trafficStats.startTime = performance.now(); this.trafficStats._lastSnapshot.ts = performance.now(); if (this.trafficInterval) clearInterval(this.trafficInterval); this.trafficInterval = setInterval(() => { const now = performance.now(); const elapsed = (now - this.trafficStats._lastSnapshot.ts) / 1000; if (elapsed < 0.5) return; const snap = this.trafficStats._lastSnapshot; this.trafficStats.rates.txVideo = Math.round((this.trafficStats.tx.video - snap.txVideo) / elapsed); this.trafficStats.rates.txAudio = Math.round((this.trafficStats.tx.audio - snap.txAudio) / elapsed); this.trafficStats.rates.txTotal = Math.round((this.trafficStats.tx.total - snap.txTotal) / elapsed); this.trafficStats.rates.rxVideo = Math.round((this.trafficStats.rx.video - snap.rxVideo) / elapsed); this.trafficStats.rates.rxAudio = Math.round((this.trafficStats.rx.audio - snap.rxAudio) / elapsed); this.trafficStats.rates.rxTotal = Math.round((this.trafficStats.rx.total - snap.rxTotal) / elapsed); snap.txVideo = this.trafficStats.tx.video; snap.txAudio = this.trafficStats.tx.audio; snap.txTotal = this.trafficStats.tx.total; snap.rxVideo = this.trafficStats.rx.video; snap.rxAudio = this.trafficStats.rx.audio; snap.rxTotal = this.trafficStats.rx.total; snap.ts = now; this.trafficStats.wsState = this.ws?.readyState === WebSocket.OPEN ? 'OPEN' : this.ws?.readyState === WebSocket.CONNECTING ? 'CONNECTING' : 'CLOSED'; }, 1000); } public stopTrafficMonitor() { if (this.trafficInterval) { clearInterval(this.trafficInterval); this.trafficInterval = null; } } public startQuantumAutoPilot() { if (this.autoPilotInterval) clearInterval(this.autoPilotInterval); this.autoPilotInterval = setInterval(async () => { const conn = (navigator as any).connection; const downlink = conn ? conn.downlink : 10; // Mbps this.currentBandwidth = downlink; // === BANDWIDTH ESTIMATION (OPTION 1 & 2) === // Option 1: Standard Network API (Downlink) + Traffic Jitter let bwScore = 3; // Default FHD if (downlink < 0.5) bwScore = 0; // Base/Audio Only else if (downlink < 1.5) bwScore = 1; // SD else if (downlink < 4) bwScore = 2; // HD // Option 2: Target Buffer Throttle (Dynamic RX Drop simulation) // Jika RX video macet (Buffer tersendat), paksa turunkan skor! if (this.trafficStats.rates.rxVideo < 5000 && this.participantRole === 'AUDIENCE') { // Terindikasi video membeku walau downlink besar bwScore = Math.max(0, bwScore - 1); } // Kirim Telemetry Datagram ke XCU Router (Ring-0 Simulator) if (this.streamWriter) { const bwTelemetry = new Uint8Array(4); bwTelemetry[0] = 10; // Tipe: Bandwidth Telemetry bwTelemetry[1] = bwScore; // Skor 0-3 bwTelemetry[2] = this.participantId & 0xFF; bwTelemetry[3] = (this.participantId >> 8) & 0xFF; this.streamWriter.write(bwTelemetry).catch(() => {}); } if (this.isSwappingEncoder) return; // --- AUTO VIDEO (XCU Pulsar = INSTANT BOOT, WebCodecs = PERFORMANCE UPGRADE) --- // PKX Konstitusi 5 Pasal 3: XCU Pulsar STARTS FIRST, kemudian upgrade jika hardware tersedia if (this.videoEngineMode === 'auto') { if (!this.usePulsarCodec && !this.videoEncoder) { // BOOT PERTAMA: XCU Pulsar langsung aktif tanpa delay this.usePulsarCodec = true; this.activeVideoCodec = 'XCU PULSAR (Delta)'; console.log(`[AUTO-PILOT] XCU Pulsar BOOT PERTAMA β€” instant start`); } else if (this.usePulsarCodec && this.videoEncoder && downlink > 1) { // UPGRADE: WebCodecs hardware tersedia + bandwidth cukup β†’ upgrade untuk performa this.isSwappingEncoder = true; let targetCodec = "avc1.42E01F"; // H.264 baseline (paling kompatibel) let targetBitrate = 800_000; if (downlink > 10) { targetCodec = "vp09.00.10.08"; // VP9 targetBitrate = 2_500_000; } else if (downlink > 5) { targetCodec = "vp8"; targetBitrate = 1_500_000; } else if (downlink > 2) { targetCodec = "avc1.42E01F"; // H.264 targetBitrate = 1_000_000; } console.log(`[AUTO-PILOT] UPGRADE dari Pulsar ke WebCodecs ${targetCodec} (${downlink} Mbps)`); try { try { (this.videoEncoder as any).close(); } catch(_e) {} this.activeCodecStr = targetCodec; this.activeVideoCodec = `WebCodecs ${targetCodec.split('.')[0].toUpperCase()}`; this.videoEncoder = new (window as any).VideoEncoder({ output: (chunk: any, metadata: any) => this.handleVideoChunk(chunk, metadata), error: (e: any) => { console.error("[AUTO-PILOT] WebCodecs error, fallback ke Pulsar", e); this.usePulsarCodec = true; this.activeVideoCodec = 'XCU PULSAR (Delta)'; } }); (this.videoEncoder as any).configure({ codec: this.activeCodecStr, width: 1920, height: 1080, bitrate: targetBitrate, framerate: this.targetFps }); this.usePulsarCodec = false; // Switch berhasil } catch(e) { // WebCodecs tidak support: tetap Pulsar (PKX fallback) this.usePulsarCodec = true; this.activeVideoCodec = 'XCU PULSAR (Delta)'; console.log('[AUTO-PILOT] WebCodecs tidak tersedia, Pulsar tetap aktif'); } this.isSwappingEncoder = false; } else if (!this.usePulsarCodec && this.videoEncoder && downlink < 0.5) { // DOWNGRADE: Bandwidth sangat rendah β†’ kembali ke Pulsar this.usePulsarCodec = true; this.activeVideoCodec = 'XCU PULSAR (Delta)'; console.log(`[AUTO-PILOT] Bandwidth rendah (${downlink} Mbps), kembali ke Pulsar`); } } // --- AUTO AUDIO (XCU Resonance = INSTANT BOOT, Opus = PERFORMANCE UPGRADE) --- // PKX: Resonance STARTS FIRST, upgrade ke Opus jika bandwidth stabil if (this.audioEngineMode === 'auto') { if (!this.audioEncoder && downlink > 2) { // UPGRADE: Bandwidth cukup β†’ Opus untuk kualitas audio lebih baik this.isSwappingEncoder = true; try { this.audioEncoder = new (window as any).AudioEncoder({ output: (chunk: any) => this.handleAudioChunk(chunk), error: (e: any) => { console.error("[AUTO-PILOT] Opus gagal, Resonance tetap aktif", e); this.audioEncoder = null; this.activeAudioCodec = 'XCU RESONANCE (300bps)'; } }); (this.audioEncoder as any).configure({ codec: 'opus', sampleRate: 48000, numberOfChannels: 1, bitrate: 32000 }); this.activeAudioCodec = 'XCU NEURAL (Opus 32k)'; console.log(`[AUTO-PILOT] UPGRADE Audio ke Opus (${downlink} Mbps)`); } catch(e) { this.audioEncoder = null; this.activeAudioCodec = 'XCU RESONANCE (300bps)'; } this.isSwappingEncoder = false; } else if (this.audioEncoder && downlink < 0.5) { // DOWNGRADE: Bandwidth rendah β†’ kembali ke Resonance try { (this.audioEncoder as any).close(); } catch(_e) {} this.audioEncoder = null; this.activeAudioCodec = 'XCU RESONANCE (300bps)'; console.log(`[AUTO-PILOT] Bandwidth rendah, kembali ke Resonance`); } else if (!this.audioEncoder) { this.activeAudioCodec = 'XCU RESONANCE (300bps)'; } } // --- AUTO FPS (30fps β†’ 60fps probe β€” PKX Extreme Smooth) --- if (this.targetFps === 30 && !this.fpsConfirmed60 && downlink > 3) { this.fpsProbeCount++; if (this.fpsProbeCount >= 4) { // 4 cycles Γ— 3 sec = 12 detik stabil this.fpsConfirmed60 = true; this.targetFps = 60; console.log(`[AUTO-PILOT] 60fps CONFIRMED! Bandwidth stabil ${downlink} Mbps selama 12 detik. Upgrade ke 60fps.`); // Re-configure video track jika memungkinkan try { const videoTrack = this.mediaStream?.getVideoTracks()[0]; if (videoTrack) { videoTrack.applyConstraints({ frameRate: { ideal: 60 } }).catch(() => {}); } } catch(_e) {} } } else if (this.targetFps === 30 && downlink <= 3) { this.fpsProbeCount = 0; // Reset probe jika bandwidth tidak stabil } // --- DOWNGRADE FPS jika bandwidth sangat rendah --- if (this.targetFps === 60 && downlink < 1) { this.targetFps = 30; this.fpsConfirmed60 = false; this.fpsProbeCount = 0; console.log(`[AUTO-PILOT] Bandwidth drop (${downlink} Mbps). Downgrade ke 30fps.`); } }, 3000); // === ULTRA AUTOPILOT: Smart Quality Recovery === // Gradually restore quality when bandwidth improves setInterval(() => { const wsBuffered = this.ws ? this.ws.bufferedAmount : 0; const txRate = this.trafficStats.rates.txTotal; // bytes/sec // Estimate real bandwidth from TX rate this.lastBandwidthEstimate = txRate > 0 ? (txRate * 8 / 1_000_000) : this.currentBandwidth; // Buffer is clear + bandwidth ok β†’ gradually RESTORE quality if (wsBuffered < 8192 && this.adaptiveQuality < 0.92) { this.adaptiveQuality = Math.min(0.92, this.adaptiveQuality + 0.02); console.log(`[AUTOPILOT] πŸ“ˆ Quality recovering β†’ ${(this.adaptiveQuality*100).toFixed(0)}%`); } if (wsBuffered < 4096 && this.adaptiveResScale < 1.0) { this.adaptiveResScale = Math.min(1.0, this.adaptiveResScale + 0.05); console.log(`[AUTOPILOT] πŸ“ˆ Resolution recovering β†’ ${(this.adaptiveResScale*100).toFixed(0)}%`); } // Severe congestion β†’ aggressive downgrade if (wsBuffered > 131072) { // >128KB stuck this.adaptiveQuality = 0.5; this.adaptiveResScale = 0.5; if (this.targetFps > 15) { this.targetFps = 15; console.log(`[AUTOPILOT] 🚨 SEVERE CONGESTION! Qualityβ†’50%, Resβ†’50%, FPSβ†’15`); } } }, 2000); } /** * Set FPS manual dari Matrix UI * @param fps - 15, 30, atau 60 * Setelah di-set manual, AutoPilot FPS probe di-nonaktifkan */ public setFps(fps: 15 | 30 | 60): void { this.targetFps = fps; this.fpsConfirmed60 = fps === 60; // Jika manual 60, tandai confirmed this.fpsProbeCount = 999; // Disable auto probe console.log(`[MATRIX] FPS manual di-set ke ${fps}fps oleh user`); // Re-configure video track try { const videoTrack = this.mediaStream?.getVideoTracks()[0]; if (videoTrack) { videoTrack.applyConstraints({ frameRate: { ideal: fps } }).catch(() => {}); } } catch(_e) {} } private handleVideoChunk(chunk: any, metadata: any) { if ((this.ws && this.ws.readyState === WebSocket.OPEN) || this.streamWriter) { if (this.ws && this.ws.readyState === WebSocket.OPEN && this.ws.bufferedAmount > 65536) { // Jaringan kewalahan (backlog TCP menumpuk). Buang frame delta! if (chunk.type !== "key") return; } const chunkData = new Uint8Array(chunk.byteLength); chunk.copyTo(chunkData); const packetLen = 8 + chunkData.length; const packet = new Uint8Array(packetLen); packet[0] = chunk.type === "key" ? 3 : 4; packet[1] = 0; const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); view.setUint32(4, packetLen - 8, true); packet.set(chunkData, 8); this.sendFrame(packet); } } private handleAudioChunk(chunk: any) { if ((this.ws && this.ws.readyState === WebSocket.OPEN) || this.streamWriter) { if (this.ws && this.ws.readyState === WebSocket.OPEN && this.ws.bufferedAmount > 131072) { // Jika antrean > 128KB, buang audio juga untuk reset TCP return; } const chunkData = new Uint8Array(chunk.byteLength); chunk.copyTo(chunkData); const packetLen = 8 + 1 + chunkData.length; const packet = new Uint8Array(packetLen); packet[0] = 2; // FRAME_AUDIO packet[1] = 1; const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); view.setUint32(4, packetLen - 8, true); packet[8] = 1; // Opus Codec ID packet.set(chunkData, 9); this.sendFrame(packet); } } public async deactivateUplink() { console.log("[QUANTUM UPLINK] Mematikan Kamera (Downlink tetap aktif)..."); if (this.mediaStream) { this.mediaStream.getTracks().forEach((t) => { t.stop(); }); this.mediaStream = null; } if (this.videoEncoder && this.videoEncoder.state !== "closed") { this.videoEncoder.close(); this.videoEncoder = null; } if (this.trackProcessor) { try { /* Let GC handle trackProcessor */ } catch(e){} this.trackProcessor = null; } } private async encodeLoop(reader: ReadableStreamDefaultReader) { while (this.isRunning) { try { const { done, value: frame } = await reader.read(); if (done || !frame) break; if (this.videoEncoder && this.videoEncoder.state === "configured") { if (this.videoEncoder.encodeQueueSize > 2) { // Drop frame before encoding to prevent queue buildup (CPU bottleneck) console.warn("[QUANTUM ENCODER] Skipping frame to prevent latency"); } else { this.videoEncoder.encode(frame, { keyFrame: this._frameCount % 30 === 0 }); } this._frameCount++; } if (frame) frame.close(); } catch { // Stream terputus atau kamera dimatikan break; } } } private async startReceiver() { if (!this.transport) return; try { console.log("[QUANTUM MATRIX] Menciptakan QUIC Bidirectional Stream..."); // Klien yang berinisiatif membuka Bi-Directional Stream pertama kali this.stream = await this.transport.createBidirectionalStream(); this.streamWriter = this.stream.writable.getWriter(); this.streamReader = this.stream.readable.getReader(); let buffer = new Uint8Array(0); while (this.isRunning) { const { value, done } = await this.streamReader.read(); if (done) { console.log("QUIC Stream Ditutup oleh Server."); break; } if (value) { // Gabungkan chunk yang baru datang ke dalam buffer const newBuffer = new Uint8Array(buffer.length + value.length); newBuffer.set(buffer); newBuffer.set(value, buffer.length); buffer = newBuffer; // Ekstrak Frame berdasarkan 8-byte header while (buffer.length >= 8) { const frameType = buffer[0]; const quality = buffer[1]; const view = new DataView( buffer.buffer, buffer.byteOffset, buffer.byteLength, ); const senderId = view.getUint16(2, true); const frameLength = view.getUint32(4, true); if (buffer.length >= 8 + frameLength) { const payloadData = buffer.slice(8, 8 + frameLength); buffer = buffer.slice(8 + frameLength); try { if (senderId === this.participantId) continue; this.participantLastSeen.set(senderId, Date.now()); if (frameType === FRAME_LEDGER) { const textDecoder = new TextDecoder(); const payloadStr = textDecoder.decode(payloadData); // Intersepsi PKEPX Signals try { const sig = JSON.parse(payloadStr); if (sig.pkepxType === 'SOVEREIGN') { this.handleSovereignSignal(sig, senderId); continue; } else if (sig.pkepxType === 'RESONANCE') { if (this.onQuantumResonance) this.onQuantumResonance(senderId, sig.payload); continue; } } catch(e) {} if (this.onQuantumDataReceived) { this.onQuantumDataReceived(senderId, payloadStr); } continue; } if (frameType === FRAME_CONTROL && quality === 2) { if (this.onActiveSpeakerChanged) { this.onActiveSpeakerChanged(senderId); } continue; } if (frameType === FRAME_HEARTBEAT) { if (!this.pulsarDecoders.has(senderId) && !this.videoDecoders.has(senderId)) { console.log("[PRESENCE] Remote participant detected via heartbeat:", senderId); if (this.onParticipantJoined) { this.onParticipantJoined(senderId); } } continue; } if (frameType === FRAME_PULSAR) { // JPEG image frame - decode with Image API (works everywhere) let jpegPayload = payloadData; let isDecryptionFailed = false; if (quality === 2) { if (this.e2eeKey && payloadData.length > 12) { const iv = payloadData.slice(0, 12); const cipher = payloadData.slice(12); try { const plainBuf = await window.crypto.subtle.decrypt( { name: "AES-GCM", iv: iv }, this.e2eeKey, cipher ); jpegPayload = new Uint8Array(plainBuf); } catch (_e) { isDecryptionFailed = true; } } else { isDecryptionFailed = true; // No key to decrypt } } if (isDecryptionFailed) { // Render Quantum Noise (TV Semut) this.tryAutoRegisterCanvas(senderId); const ctx = this.canvasCtxMap.get(senderId); if (ctx) { const w = ctx.canvas.width || 640; const h = ctx.canvas.height || 360; const idata = ctx.createImageData(w, h); const d32 = new Uint32Array(idata.data.buffer); for(let i=0; i { this.tryAutoRegisterCanvas(senderId); const jitter = this.audioJitterMap.get(senderId) || 50; let queue = this.videoRenderQueue.get(senderId); if (!queue) { queue = []; this.videoRenderQueue.set(senderId, queue); } queue.push({ frame: img, targetTime: performance.now() + jitter, isWebCodec: false }); URL.revokeObjectURL(bmpUrl); }; img.src = bmpUrl; continue; } if (frameType === 2) { // FRAME_AUDIO let audioPayload = payloadData; let isDecryptionFailed = false; if (quality === 2) { if (this.e2eeKey && payloadData.length > 12) { const iv = payloadData.slice(0, 12); const cipher = payloadData.slice(12); try { const plainBuf = await window.crypto.subtle.decrypt( { name: "AES-GCM", iv: iv }, this.e2eeKey, cipher ); audioPayload = new Uint8Array(plainBuf); } catch (_e) { isDecryptionFailed = true; } } else { isDecryptionFailed = true; } } if (isDecryptionFailed) { // Quantum Noise Audio (White Noise Intelijen) if (audioPayload.length > 4) { const fakePayload = new Uint8Array(audioPayload.length); fakePayload.set(audioPayload.slice(0, 4), 0); // Keep header for (let i = 4; i < fakePayload.length; i++) { fakePayload[i] = Math.floor(Math.random() * 255); } audioPayload = fakePayload; } else { continue; } } if (!this.downlinkAudioCtx) { this.initDownlinkAudio(); (this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx.currentTime; } // Jangan panggil resume di sini karena bukan berasal dari user gesture! const ctx = this.downlinkAudioCtx as any; if (audioPayload.length <= 4) continue; // SAFE PARSING via DataView const dv = new DataView(audioPayload.buffer, audioPayload.byteOffset, audioPayload.byteLength); const senderSampleRate = dv.getUint32(0, true) || 48000; const numSamples = Math.floor((audioPayload.length - 4) / 2); const float32 = new Float32Array(numSamples); let sum = 0; for (let i = 0; i < numSamples; i++) { const intSample = dv.getInt16(4 + i * 2, true); float32[i] = intSample / 32768.0; if (i % 10 === 0) sum += Math.abs(float32[i]); } // The buffer uses the exact sender sample rate, the hardware context will auto-resample const audioBuffer = ctx.createBuffer(1, numSamples, senderSampleRate); audioBuffer.getChannelData(0).set(float32); const source = ctx.createBufferSource(); source.buffer = audioBuffer; if (this.downlinkAudioDest) { source.connect(this.downlinkAudioDest); } else { source.connect(ctx.destination); } const delay = ctx.nextPlayTime - ctx.currentTime; this.audioJitterMap.set(senderId, Math.max(0, delay * 1000)); if (delay > 0.15) { // EXTREME TUNE-UP: Jika delay > 150ms, BUANG paket ini agar tetap real-time & cegah GEMURUH! ctx.nextPlayTime = ctx.currentTime + 0.05; continue; // Drop packet } if (ctx.nextPlayTime < ctx.currentTime) { ctx.nextPlayTime = ctx.currentTime + 0.05; // 50ms initial buffer } source.start(ctx.nextPlayTime); ctx.nextPlayTime += audioBuffer.duration; if (this.onAudioLevel) { this.onAudioLevel(Math.min(100, (sum / (numSamples/10)) * 500)); } if (this.onActiveSpeakerChanged && sum > 5) { this.onActiveSpeakerChanged(senderId); } continue; } if (frameType === FRAME_VIDEO_DELTA || frameType === FRAME_VIDEO_KEY) { if (!this.videoDecoders.has(senderId)) { this.createDecoderForParticipant(senderId); } if (frameType === FRAME_VIDEO_DELTA && !this.firstKeyFrameReceived.get(senderId)) { continue; } if (frameType === FRAME_VIDEO_KEY) { this.firstKeyFrameReceived.set(senderId, true); } this.tryAutoRegisterCanvas(senderId); const decoder = this.videoDecoders.get(senderId); if (decoder && decoder.state === "configured") { if (decoder.decodeQueueSize > 5) { if (frameType === FRAME_VIDEO_DELTA) { // DROP delta frame if decoder is backed up to catch up to real-time! this.firstKeyFrameReceived.set(senderId, false); continue; } } const chunk = new EncodedVideoChunk({ type: frameType === FRAME_VIDEO_KEY ? "key" : "delta", timestamp: performance.now() * 1000, data: payloadData, }); decoder.decode(chunk); } continue; } if (frameType === 99) { // QUANTUM CIPHER GATE: WASM BYTECODE INJECTION // Asymmetric Bytecode Streaming dari XCU Core const dv2 = new DataView(payloadData.buffer, payloadData.byteOffset, payloadData.byteLength); const moduleId = dv2.getUint16(2, true); const wasmBytes = payloadData.slice(4); // Payload murni WebAssembly console.log(`[QUANTUM MATRIX] QCG: Menerima Injeksi Bytecode Modul ${moduleId} (${wasmBytes.byteLength} bytes). Merakit...`); try { const _module = await WebAssembly.instantiate(wasmBytes, {}); console.log(`[QUANTUM MATRIX] QCG: Modul ${moduleId} Berhasil Dirakit di Memori!`); if (moduleId === 43) { this.pulsarCodec = module.instance.exports; this.pulsarWasmMemory = this.pulsarCodec.memory; } if (this.onModuleUnlocked) { this.onModuleUnlocked(moduleId); } } catch (_e) { console.error(`[QUANTUM MATRIX] QCG: Gagal merakit modul ${moduleId} (Hack Attempt?)`, e); } continue; } } catch (frameErr) { console.error("[QUANTUM DECODER] Frame parse error (ignored):", frameErr); } } else { // Butuh lebih banyak byte untuk membentuk frame utuh break; } } } } } catch (e: unknown) { console.error("[QUANTUM MATRIX] Kegagalan Aliran (Stream Failure):", e); } } // Fase 37: Voice Activity Detection (VAD) Loop - QUANTUM WASM EDITION private startVadLoop() { try { if (!this.uplinkAudioCtx || !this.uplinkAudioSource) return; // Jika WASM Modul 43 belum turun, gunakan Analyser JS biasa sebagai fallback const useWasm = this.pulsarCodec && this.pulsarCodec.pulsar_vad && this.pulsarWasmMemory; const analyser = this.uplinkAudioCtx.createAnalyser(); analyser.fftSize = 1024; // Gunakan 1024 frame agar cukup untuk WASM this.uplinkAudioSource.connect(analyser); const jsDataArray = new Uint8Array(analyser.frequencyBinCount); const f32DataArray = new Float32Array(analyser.fftSize); let speakingFrames = 0; setInterval(() => { let energyScore = 0; if (useWasm) { // 1. Eksekusi Asimetris: Dapatkan Pointer Memori WASM analyser.getFloatTimeDomainData(f32DataArray); const bufferPtr = this.pulsarCodec.get_buffer_ptr(); const capacity = this.pulsarCodec.get_buffer_capacity(); // 2. Tembus Batas RAM: Tulis gelombang suara murni (PCM f32) ke memori WebAssembly const wasmMemArray = new Float32Array(this.pulsarWasmMemory!.buffer, bufferPtr, capacity); const copyLen = Math.min(f32DataArray.length, capacity); wasmMemArray.set(f32DataArray.subarray(0, copyLen)); // 3. Kalkulasi Matematis oleh Sang Algojo Rust energyScore = this.pulsarCodec.pulsar_vad(copyLen); } else { // Fallback Mode JS analyser.getByteFrequencyData(jsDataArray); const sum = jsDataArray.reduce((a, b) => a + b, 0); energyScore = sum / jsDataArray.length; } // Tampilkan ke UI if (this.onAudioLevel) { // Wasm score: 0.0 to ~10.0+. JS score: 0 to 255. const normalized = useWasm ? Math.min(100, energyScore * 10) : energyScore; this.onAudioLevel(normalized); } // Ambang Deteksi const threshold = useWasm ? 0.5 : 5; if (energyScore > threshold) { speakingFrames++; if (speakingFrames === 3) { if (this.streamWriter && this.isRunning) { this.sendDirectorSignal(2); } } } else { speakingFrames = 0; } }, 100); } catch (e: unknown) { console.warn( "[VAD] Audio Context tidak didukung atau dicekal browser.", e, ); } } private async sendDirectorSignal(actionQuality: number) { if (!this.streamWriter) return; const packet = new Uint8Array(8); packet[0] = 3; // Control packet[1] = actionQuality; const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); view.setUint32(4, 0, true); if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); } } // Fase 34: Zero-CPU Omniscient Recording public async triggerQuantumRecording() { if (!this.streamWriter) return; const packet = new Uint8Array(8); packet[0] = 3; // Control packet[1] = 6; // Set Action: Record const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); view.setUint32(4, 0, true); if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); } console.log( "[VAULT MATRIX] Sinyal Perekaman Kuantum Langsung Ke SSD Server Dikirim.", ); } // Fase 35: SVC Downgrade (Anti-Lag Manual/Otomatis) public async activateAntiLagDowngrade() { if (this.streamWriter) { console.log("[SVC MATRIX] Mengirim sinyal Downgrade ke Server Rust..."); const controlPacket = new Uint8Array(8); controlPacket[0] = 3; // Type: Control controlPacket[1] = 0; // Quality: Low (Drop paket 1080p) const view = new DataView(controlPacket.buffer); view.setUint16(2, this.participantId, true); view.setUint32(4, 0, true); // Payload Length: 0 await this.streamWriter.write(controlPacket); } } // Fase 39: Webinar Asimetris (Audience) public async joinAsAudience() { this.participantRole = "AUDIENCE"; await this.sendRoleSignal(); console.log( "[WEBINAR MATRIX] Beralih ke Mode AUDIENCE (Uplink Dimatikan).", ); } private async sendRoleSignal() { if (!this.streamWriter) return; const packet = new Uint8Array(8); packet[0] = 3; // Control packet[1] = 4; // Set Role const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); packet[4] = this.participantRole === "AUDIENCE" ? 1 : 0; // 1=Audience, 0=Panelist if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); } } // Fase 39: Podcast Mode (Matikan Video di Routing Level) public async setPodcastMode(isPodcast: boolean) { if (!this.streamWriter) return; const packet = new Uint8Array(8); packet[0] = 3; // Control packet[1] = 5; // Set Mode const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); packet[4] = isPodcast ? 1 : 0; // 1=Podcast, 0=Webinar if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); } console.log( `[PODCAST MATRIX] Sinyal ${isPodcast ? "PODCAST" : "WEBINAR"} dikirim ke Core.`, ); } // Fase 40: The Quantum Ledger (Pengganti WebSocket Node.js) // Menembakkan Chat, JSON, Emoji, Koordinat langsung via QUIC public async sendQuantumData(payload: string) { if (!this.streamWriter) return; const textEncoder = new TextEncoder(); const payloadBytes = textEncoder.encode(payload); const packet = new Uint8Array(8 + payloadBytes.byteLength); packet[0] = 6; // Type: Ledger Data packet[1] = 0; // Quality: N/A const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); view.setUint32(4, payloadBytes.byteLength, true); packet.set(payloadBytes, 8); if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); } } // === DOWNLINK AUDIO PLAYBACK (EXTREME FIX v2) === // Uplink format: [sampleRate: Uint32LE 4 bytes] + [PCM Int16 data] private playRemoteAudio(audioPayload: Uint8Array) { try { if (!this.downlinkAudioCtx) { this.initDownlinkAudio(); (this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx.currentTime; } if (this.downlinkAudioCtx.state === 'suspended') { this.downlinkAudioCtx.resume(); } // Payload: [sampleRate(4 bytes LE)] + [PCM Int16 raw bytes] if (audioPayload.byteLength < 6) return; const dv = new DataView(audioPayload.buffer, audioPayload.byteOffset, audioPayload.byteLength); const sampleRate = dv.getUint32(0, true) || 16000; const int16Data = audioPayload.slice(4); if (int16Data.byteLength < 2) return; // Convert Int16 β†’ Float32 const int16 = new Int16Array(int16Data.buffer, int16Data.byteOffset, int16Data.byteLength / 2); const numSamples = int16.length; if (numSamples < 1) return; const float32 = new Float32Array(numSamples); for (let i = 0; i < numSamples; i++) { float32[i] = int16[i] / 32767; } // Create audio buffer at source sample rate and play at device rate const audioBuffer = this.downlinkAudioCtx.createBuffer(1, numSamples, sampleRate); audioBuffer.getChannelData(0).set(float32); const source = this.downlinkAudioCtx.createBufferSource(); source.buffer = audioBuffer; // ULTRA AUDIO: Gain boost for clarity (1.5x volume) const gainNode = this.downlinkAudioCtx.createGain(); gainNode.gain.value = 1.5; source.connect(gainNode); if (this.downlinkAudioDest) { gainNode.connect(this.downlinkAudioDest); } else { gainNode.connect(this.downlinkAudioCtx.destination); } // ULTRA Jitter Buffer: Smoother scheduling with 250ms tolerance const ctx = this.downlinkAudioCtx as any; const now = this.downlinkAudioCtx.currentTime; let playAt = ctx.nextPlayTime || now; const delay = playAt - now; if (delay > 0.25) { // Buffer too full β€” reset to near-realtime (50ms ahead) playAt = now + 0.03; ctx.nextPlayTime = playAt; } else if (delay < -0.05) { // We're behind β€” catch up smoothly playAt = now + 0.02; } if (playAt < now) { playAt = now + 0.02; } source.start(playAt); ctx.nextPlayTime = playAt + audioBuffer.duration; } catch (e) { // Silently ignore audio playback errors } } // ========================================== // PKEPX (PANCA KONSTITUSI) ZOOM-KILLER FEATURES // ========================================== /** * PKEPX Sovereign Controls: Host Broadcast */ public broadcastSovereignSignal(type: 'MUTE_ALL' | 'KICK_PEER' | 'LOCK_MATRIX', targetId?: number) { if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return; const payload = JSON.stringify({ pkepxType: 'SOVEREIGN', command: type, targetId: targetId }); this.sendQuantumData(payload); console.log(`[PKEPX SOVEREIGN] Sinyal ${type} dipancarkan ke matriks.`); } private handleSovereignSignal(sig: any, senderId: number) { console.log(`[PKEPX SOVEREIGN] Menerima titah dari ${senderId}:`, sig.command); if (sig.command === 'MUTE_ALL') { this.toggleMic(false); if (this.onSovereignSignal) this.onSovereignSignal('MUTED_BY_HOST'); } else if (sig.command === 'KICK_PEER' && sig.targetId === this.participantId) { console.warn('[PKEPX SOVEREIGN] Anda diusir dari matriks!'); this.shutdown(); if (this.onSovereignSignal) this.onSovereignSignal('KICKED_BY_HOST'); } else if (sig.command === 'LOCK_MATRIX') { if (this.onSovereignSignal) this.onSovereignSignal('MATRIX_LOCKED'); } } /** * PKEPX Quantum Resonance: Emit 3D Reaction */ public emitResonance(reactionType: string) { if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return; const payload = JSON.stringify({ pkepxType: 'RESONANCE', payload: reactionType }); this.sendQuantumData(payload); if (this.onQuantumResonance) this.onQuantumResonance(this.participantId, reactionType); } /** * BUG FIX: Send presence heartbeat so other participants detect us immediately. * This sends a FRAME_HEARTBEAT (type=5) with our display name. */ public sendPresenceHeartbeat() { const namePayload = this.displayName ? new TextEncoder().encode(JSON.stringify({ type: 'NAME_ANNOUNCE', name: this.displayName })) : new Uint8Array(0); const packet = new Uint8Array(8 + namePayload.length); packet[0] = 5; // FRAME_HEARTBEAT packet[1] = 0; const view = new DataView(packet.buffer); view.setUint16(2, this.participantId, true); view.setUint32(4, namePayload.length, true); if (namePayload.length > 0) packet.set(namePayload, 8); this.sendFrame(packet); console.log(`[QUANTUM HEARTBEAT] Presence sent: ${this.displayName || 'ID:' + this.participantId}`); } /** * BUG FIX: Periodic heartbeat loop (every 3s) so participants stay visible * even without camera/mic active. */ private startHeartbeatLoop() { if (this.heartbeatTimer) clearInterval(this.heartbeatTimer); this.heartbeatTimer = setInterval(() => { this.sendPresenceHeartbeat(); }, 3000); } /** * Announce display name to all participants in the room. * Sent via FRAME_CONTROL (type 3) as JSON payload. */ public announceDisplayName() { if (!this.displayName) return; const payload = JSON.stringify({ pkepxType: 'NAME_ANNOUNCE', name: this.displayName, participantId: this.participantId }); this.sendQuantumData(payload); } /** * Set/rename display name (like Zoom rename). * Immediately broadcasts to all participants. */ public setDisplayName(newName: string) { this.displayName = newName; this.announceDisplayName(); console.log(`[QUANTUM MATRIX] Display name set to: ${newName}`); } /** * PKEPX Fractal Matrix: Breakout Rooms Warp Engine */ public async warpToMatrix(newRoomId: string) { console.log(`[PKEPX FRACTAL WARP] Melompat ke Sub-Matrix: ${newRoomId}`); this.shutdown(); // Close current QUIC/WS properly // Re-ignite after 500ms to allow TCP flush setTimeout(() => { this.ignite(newRoomId, "/xcu-engine").then(() => { if (this.onSovereignSignal) this.onSovereignSignal('WARP_COMPLETE', newRoomId); }).catch(e => console.error("Warp Gagal:", e)); }, 500); } public disconnect() { this.isRunning = false; this.stopVideoSyncLoop(); if (this.downlinkAudioEl) { this.downlinkAudioEl.remove(); this.downlinkAudioEl = null; } if (this.ws) { try { this.ws.close(1000, 'LEAVING'); } catch(_) { this.ws.close(); } this.ws = null; } if (this.transport) { this.transport.close(); } this.videoDecoders.forEach((decoder) => { if (decoder.state !== "closed") decoder.close(); }); this.videoDecoders.clear(); this.canvasCtxMap.clear(); if (this.videoEncoder && this.videoEncoder.state !== "closed") { this.videoEncoder.close(); } if (this.mediaStream) { this.mediaStream.getTracks().forEach((t) => { t.stop(); }); } console.log("[QUANTUM MATRIX] Sistem Terputus."); } public shutdown() { this.isRunning = false; this.stopVideoSyncLoop(); if (this.downlinkAudioEl) { this.downlinkAudioEl.remove(); this.downlinkAudioEl = null; } if (this.natPingInterval) clearInterval(this.natPingInterval); this.stopVAD(); if (this.heartbeatTimer) { clearInterval(this.heartbeatTimer); this.heartbeatTimer = null; } if (this.ghostCleanupInterval) { clearInterval(this.ghostCleanupInterval); this.ghostCleanupInterval = null; } if (this.ws) { try { this.ws.close(1000, 'LEAVING'); } catch(_) { this.ws.close(); } this.ws = null; } if (this.transport) { this.transport.close(); } this.videoDecoders.forEach((decoder) => { if (decoder.state !== "closed") decoder.close(); }); this.videoDecoders.clear(); this.canvasCtxMap.clear(); if (this.videoEncoder && this.videoEncoder.state !== "closed") { this.videoEncoder.close(); } if (this.mediaStream) { this.mediaStream.getTracks().forEach((t) => { t.stop(); }); } console.log("[QUANTUM MATRIX] Sistem Terputus."); } private startVAD() { if (!this.mediaStream) return; const audioTracks = this.mediaStream.getAudioTracks(); if (audioTracks.length === 0) return; try { const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext; if (!AudioContextClass) return; this.vadAudioCtx = new AudioContextClass(); if (this.vadAudioCtx.state === "suspended") { this.vadAudioCtx.resume().catch(() => {}); } const source = this.vadAudioCtx.createMediaStreamSource(new MediaStream([audioTracks[0]])); this.vadAnalyser = this.vadAudioCtx.createAnalyser(); this.vadAnalyser.fftSize = 256; source.connect(this.vadAnalyser); const dataArray = new Uint8Array(this.vadAnalyser.frequencyBinCount); this.vadInterval = setInterval(() => { if (!this.vadAnalyser) return; this.vadAnalyser.getByteFrequencyData(dataArray); let sum = 0; for (let i = 0; i < dataArray.length; i++) { sum += dataArray[i]; } const average = sum / dataArray.length; let level = average / 128.0; if (level > 1.0) level = 1.0; if (this.onAudioLevel) { this.onAudioLevel(this.isMicMuted ? 0 : level); } }, 100); } catch (e) { console.warn("[VAD] Failed to initialize Voice Activity Detection", e); } } private stopVAD() { if (this.vadInterval) { clearInterval(this.vadInterval); this.vadInterval = null; } if (this.vadAudioCtx) { this.vadAudioCtx.close().catch(() => {}); this.vadAudioCtx = null; } this.vadAnalyser = null; } }