TSM.ID
2357 lines
85 KiB
TypeScript
2357 lines
85 KiB
TypeScript
// [TSM.ID].[11031972] — All Rights Reserved. Proprietary & Confidential.
|
||
/* eslint-disable */
|
||
// @ts-nocheck
|
||
import { XCUPulsarCodec } from './xcu-pulsar-codec';
|
||
import { XCUResonanceCodec } from './xcu-resonance-codec';
|
||
|
||
// Frame type constants for XCU Binary Protocol v3
|
||
const FRAME_VIDEO_DELTA = 0;
|
||
const FRAME_VIDEO_KEY = 1;
|
||
const FRAME_AUDIO = 2;
|
||
const FRAME_CONTROL = 3;
|
||
const FRAME_HEARTBEAT = 5;
|
||
const FRAME_LEDGER = 6;
|
||
const FRAME_PULSAR = 10;
|
||
|
||
class XCUWebGLFilter {
|
||
public canvas: HTMLCanvasElement;
|
||
private gl: WebGLRenderingContext;
|
||
private program: WebGLProgram;
|
||
private positionLocation: number;
|
||
private texCoordLocation: number;
|
||
private texture: WebGLTexture;
|
||
private resolutionLocation: WebGLUniformLocation;
|
||
private timeLocation: WebGLUniformLocation;
|
||
private modeLocation: WebGLUniformLocation;
|
||
|
||
constructor(width: number, height: number) {
|
||
this.canvas = document.createElement('canvas');
|
||
this.canvas.width = width;
|
||
this.canvas.height = height;
|
||
const gl = this.canvas.getContext('webgl')!;
|
||
this.gl = gl;
|
||
|
||
const vs = `
|
||
attribute vec2 a_position;
|
||
attribute vec2 a_texCoord;
|
||
varying vec2 v_texCoord;
|
||
void main() {
|
||
gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0);
|
||
v_texCoord = a_texCoord;
|
||
}
|
||
`;
|
||
|
||
const fs = `
|
||
precision mediump float;
|
||
uniform sampler2D u_image;
|
||
uniform vec2 u_resolution;
|
||
uniform float u_time;
|
||
uniform int u_mode;
|
||
varying vec2 v_texCoord;
|
||
|
||
float rand(vec2 co){ return fract(sin(dot(co.xy ,vec2(12.9898,78.233))) * 43758.5453); }
|
||
|
||
void main() {
|
||
vec2 uv = v_texCoord;
|
||
vec4 color = texture2D(u_image, uv);
|
||
vec2 px = 1.0 / u_resolution;
|
||
|
||
// Mode 1, 3, 5: Beauty Filter (Fast 5-tap Bilateral)
|
||
if (u_mode == 1 || u_mode == 3 || u_mode == 5) {
|
||
vec4 s1 = texture2D(u_image, uv + vec2(px.x, 0.0));
|
||
vec4 s2 = texture2D(u_image, uv + vec2(-px.x, 0.0));
|
||
vec4 s3 = texture2D(u_image, uv + vec2(0.0, px.y));
|
||
vec4 s4 = texture2D(u_image, uv + vec2(0.0, -px.y));
|
||
|
||
vec4 sum = color;
|
||
float w = 1.0;
|
||
|
||
float d1 = exp(-pow(length(s1.rgb - color.rgb), 2.0) * 50.0); sum += s1 * d1; w += d1;
|
||
float d2 = exp(-pow(length(s2.rgb - color.rgb), 2.0) * 50.0); sum += s2 * d2; w += d2;
|
||
float d3 = exp(-pow(length(s3.rgb - color.rgb), 2.0) * 50.0); sum += s3 * d3; w += d3;
|
||
float d4 = exp(-pow(length(s4.rgb - color.rgb), 2.0) * 50.0); sum += s4 * d4; w += d4;
|
||
|
||
color = sum / w;
|
||
color.rgb = mix(color.rgb, color.rgb * 1.05 + 0.02, 0.5);
|
||
}
|
||
|
||
// Naive Edge Segmentation
|
||
float s00 = length(texture2D(u_image, uv + vec2(-px.x, -px.y)).rgb);
|
||
float s22 = length(texture2D(u_image, uv + vec2(px.x, px.y)).rgb);
|
||
float edge = abs(s00 - s22);
|
||
float distCenter = distance(uv, vec2(0.5, 0.5));
|
||
bool isBg = (distCenter > 0.35 && edge < 0.15);
|
||
|
||
// Mode 2, 3: Hologram BG
|
||
if ((u_mode == 2 || u_mode == 3) && isBg) {
|
||
float m = fract(uv.y * 10.0 - u_time * 2.0 + rand(vec2(uv.x, 0.0)));
|
||
vec3 bg = vec3(0.0, m * 0.5, m * 0.2);
|
||
color.rgb = mix(color.rgb, bg, 0.85);
|
||
}
|
||
|
||
// Mode 4, 5: Bokeh Blur BG (Fast 5-tap box blur)
|
||
if ((u_mode == 4 || u_mode == 5) && isBg) {
|
||
vec4 b1 = texture2D(u_image, uv + vec2(px.x*4.0, 0.0));
|
||
vec4 b2 = texture2D(u_image, uv + vec2(-px.x*4.0, 0.0));
|
||
vec4 b3 = texture2D(u_image, uv + vec2(0.0, px.y*4.0));
|
||
vec4 b4 = texture2D(u_image, uv + vec2(0.0, -px.y*4.0));
|
||
vec4 blurSum = color + b1 + b2 + b3 + b4;
|
||
color.rgb = mix(color.rgb, (blurSum / 5.0).rgb, 0.95);
|
||
}
|
||
|
||
gl_FragColor = color;
|
||
}
|
||
`;
|
||
|
||
const createShader = (type: number, source: string) => {
|
||
const shader = gl.createShader(type)!;
|
||
gl.shaderSource(shader, source);
|
||
gl.compileShader(shader);
|
||
return shader;
|
||
};
|
||
|
||
const vShader = createShader(gl.VERTEX_SHADER, vs);
|
||
const fShader = createShader(gl.FRAGMENT_SHADER, fs);
|
||
this.program = gl.createProgram()!;
|
||
gl.attachShader(this.program, vShader);
|
||
gl.attachShader(this.program, fShader);
|
||
gl.linkProgram(this.program);
|
||
|
||
this.positionLocation = gl.getAttribLocation(this.program, 'a_position');
|
||
this.texCoordLocation = gl.getAttribLocation(this.program, 'a_texCoord');
|
||
this.resolutionLocation = gl.getUniformLocation(this.program, 'u_resolution')!;
|
||
this.timeLocation = gl.getUniformLocation(this.program, 'u_time')!;
|
||
this.modeLocation = gl.getUniformLocation(this.program, 'u_mode')!;
|
||
|
||
const positionBuffer = gl.createBuffer();
|
||
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
||
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
|
||
-1.0, -1.0, 1.0, -1.0, -1.0, 1.0,
|
||
-1.0, 1.0, 1.0, -1.0, 1.0, 1.0
|
||
]), gl.STATIC_DRAW);
|
||
|
||
const texCoordBuffer = gl.createBuffer();
|
||
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
|
||
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
|
||
0.0, 1.0, 1.0, 1.0, 0.0, 0.0,
|
||
0.0, 0.0, 1.0, 1.0, 1.0, 0.0
|
||
]), gl.STATIC_DRAW);
|
||
|
||
this.texture = gl.createTexture()!;
|
||
gl.bindTexture(gl.TEXTURE_2D, this.texture);
|
||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
||
|
||
gl.useProgram(this.program);
|
||
gl.enableVertexAttribArray(this.positionLocation);
|
||
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
||
gl.vertexAttribPointer(this.positionLocation, 2, gl.FLOAT, false, 0, 0);
|
||
|
||
gl.enableVertexAttribArray(this.texCoordLocation);
|
||
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
|
||
gl.vertexAttribPointer(this.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
|
||
}
|
||
|
||
public render(video: HTMLVideoElement, mode: number) {
|
||
const gl = this.gl;
|
||
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
|
||
gl.bindTexture(gl.TEXTURE_2D, this.texture);
|
||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
|
||
|
||
gl.useProgram(this.program);
|
||
gl.uniform2f(this.resolutionLocation, gl.canvas.width, gl.canvas.height);
|
||
gl.uniform1f(this.timeLocation, performance.now() / 1000.0);
|
||
gl.uniform1i(this.modeLocation, mode);
|
||
|
||
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
||
}
|
||
}
|
||
|
||
export class XCUQuantumMatrix {
|
||
private transport: WebTransport | null = null;
|
||
private stream: WebTransportBidirectionalStream | null = null;
|
||
private streamWriter: WritableStreamDefaultWriter | null = null;
|
||
private streamReader: ReadableStreamDefaultReader | null = null;
|
||
private videoDecoders: Map<number, VideoDecoder> = new Map();
|
||
private canvasCtxMap: Map<number, CanvasRenderingContext2D> = new Map();
|
||
private firstKeyFrameReceived: Map<number, boolean> = new Map();
|
||
private videoEncoder: VideoEncoder | null = null;
|
||
private isRunning: boolean = false;
|
||
private natPingInterval: any = null;
|
||
private mediaStream: MediaStream | null = null;
|
||
private ws: WebSocket | null = null;
|
||
private _frameCount: number = 0;
|
||
public onAudioLevel: ((level: number) => void) | null = null;
|
||
private vadAudioCtx: AudioContext | null = null;
|
||
private vadAnalyser: AnalyserNode | null = null;
|
||
private vadInterval: any = null;
|
||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||
public participantRole: "PANELIST" | "AUDIENCE" = "PANELIST";
|
||
public videoEngineMode: "auto" | "canvas" | "webcodecs" = "auto";
|
||
public audioEngineMode: "auto" | "pcm" | "xcu-neural" | "xcu-resonance" = "auto";
|
||
public activeVideoCodec: string = "STANDBY";
|
||
public activeAudioCodec: string = "STANDBY";
|
||
public targetFps: 15 | 30 | 60 = 30; // Default 30fps — user can set via Matrix UI
|
||
public fpsConfirmed60: boolean = false; // AutoPilot: true setelah konfirmasi device support 60fps
|
||
private fpsProbeCount: number = 0; // Counter untuk probe 60fps
|
||
public currentBandwidth: number = 0;
|
||
public displayName: string = ''; // Display name for this participant (email or custom)
|
||
private autoPilotInterval: any = null;
|
||
private isSwappingEncoder: boolean = false;
|
||
private audioEncoder: unknown = null;
|
||
private audioDecoders: Map<number, unknown> = new Map();
|
||
private trackProcessor: unknown = null;
|
||
private trackGenerator: unknown = null;
|
||
private activeCodecStr: string = "avc1.42E01F";
|
||
|
||
public useVirtualBg: number = 0; // 0=off, 2=Hologram, 4=Bokeh
|
||
public useBeautyFilter: boolean = false;
|
||
|
||
private pulsarEncoder: XCUPulsarCodec | null = null;
|
||
private pulsarDecoders: Map<number, XCUPulsarCodec> = new Map();
|
||
private pulsarCanvas: HTMLCanvasElement | null = null;
|
||
private pulsarCtx: CanvasRenderingContext2D | null = null;
|
||
private usePulsarCodec: boolean = false; // Primary codec flag
|
||
|
||
private participantLastSeen: Map<number, number> = new Map();
|
||
private ghostCleanupInterval: ReturnType<typeof setInterval> | null = null;
|
||
private isMicMuted: boolean = false;
|
||
private isEncodingCanvas: boolean = false;
|
||
private reconnectAttempt: number = 0; // Exponential backoff counter
|
||
private maxReconnectDelay: number = 30000; // Max 30s between reconnects
|
||
private adaptiveQuality: number = 0.92; // Smart JPEG quality (0.5-0.95)
|
||
private adaptiveResScale: number = 1.0; // Dynamic resolution scale (0.5-1.0)
|
||
private framesDropped: number = 0; // Frame drop counter for telemetry
|
||
private lastBandwidthEstimate: number = 10; // Mbps from real TX rate
|
||
private audioJitterMap: Map<number, number> = new Map();
|
||
private videoRenderQueue: Map<number, { frame: any, targetTime: number, isWebCodec: boolean }[]> = new Map();
|
||
private videoSyncLoopActive: boolean = false;
|
||
|
||
private startVideoSyncLoop() {
|
||
if (this.videoSyncLoopActive) return;
|
||
this.videoSyncLoopActive = true;
|
||
console.log("[QUANTUM MATRIX] A/V Sync Render Loop Started [TSM.ID].[11031972]");
|
||
|
||
const loop = () => {
|
||
if (!this.videoSyncLoopActive) return;
|
||
requestAnimationFrame(loop);
|
||
const now = performance.now();
|
||
|
||
this.videoRenderQueue.forEach((queue, senderId) => {
|
||
if (queue.length === 0) return;
|
||
|
||
queue.sort((a, b) => a.targetTime - b.targetTime);
|
||
|
||
const ctx = this.canvasCtxMap.get(senderId);
|
||
if (!ctx) {
|
||
queue.forEach(item => {
|
||
if (item.isWebCodec && item.frame.close) {
|
||
try { item.frame.close(); } catch(e) {}
|
||
}
|
||
});
|
||
queue.length = 0;
|
||
return;
|
||
}
|
||
|
||
while (queue.length > 0) {
|
||
const item = queue[0];
|
||
if (now >= item.targetTime) {
|
||
queue.shift();
|
||
|
||
if (now - item.targetTime > 150) {
|
||
if (item.isWebCodec && item.frame.close) {
|
||
try { item.frame.close(); } catch(e) {}
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (item.isWebCodec) {
|
||
const vf = item.frame as VideoFrame;
|
||
ctx.canvas.width = vf.displayWidth;
|
||
ctx.canvas.height = vf.displayHeight;
|
||
ctx.drawImage(vf, 0, 0);
|
||
vf.close();
|
||
} else {
|
||
const img = item.frame as HTMLImageElement;
|
||
ctx.canvas.width = img.width;
|
||
ctx.canvas.height = img.height;
|
||
ctx.drawImage(img, 0, 0);
|
||
}
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
});
|
||
};
|
||
requestAnimationFrame(loop);
|
||
}
|
||
|
||
private stopVideoSyncLoop() {
|
||
this.videoSyncLoopActive = false;
|
||
this.videoRenderQueue.forEach((queue) => {
|
||
queue.forEach(item => {
|
||
if (item.isWebCodec && item.frame.close) {
|
||
try { item.frame.close(); } catch(e) {}
|
||
}
|
||
});
|
||
queue.length = 0;
|
||
});
|
||
this.videoRenderQueue.clear();
|
||
}
|
||
|
||
public onRemoteAudio?: (participantId: number, stream: MediaStream) => void;
|
||
public onModuleUnlocked?: (moduleId: number) => void;
|
||
public onQuantumResonance?: (senderId: number, reactionType: string) => void;
|
||
public onSovereignSignal?: (type: string, payload?: any) => void;
|
||
|
||
public isDesktop: boolean = false;
|
||
public pulsarCodec: any = null;
|
||
public resonanceCodec: XCUResonanceCodec = new XCUResonanceCodec();
|
||
// REAL Traffic Counters — NO DUMMY (counts actual ws.send/ws.onmessage bytes)
|
||
public trafficStats = {
|
||
tx: { video: 0, audio: 0, control: 0, total: 0 },
|
||
rx: { video: 0, audio: 0, control: 0, total: 0 },
|
||
rates: {
|
||
txVideo: 0, txAudio: 0, txTotal: 0,
|
||
rxVideo: 0, rxAudio: 0, rxTotal: 0,
|
||
},
|
||
_lastSnapshot: {
|
||
ts: 0,
|
||
txVideo: 0, txAudio: 0, txTotal: 0,
|
||
rxVideo: 0, rxAudio: 0, rxTotal: 0,
|
||
},
|
||
startTime: 0,
|
||
wsState: 'CLOSED' as string,
|
||
};
|
||
private trafficInterval: ReturnType<typeof setInterval> | null = null;
|
||
|
||
public pulsarWasmMemory: WebAssembly.Memory | null = null;
|
||
|
||
public downlinkAudioCtx: AudioContext | null = null;
|
||
public downlinkAudioDest: MediaStreamAudioDestinationNode | null = null;
|
||
private downlinkAudioEl: HTMLAudioElement | null = null;
|
||
|
||
private initDownlinkAudio() {
|
||
if (!this.downlinkAudioCtx) {
|
||
const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext;
|
||
this.downlinkAudioCtx = new AudioContextClass();
|
||
(this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx!.currentTime;
|
||
|
||
this.downlinkAudioDest = this.downlinkAudioCtx!.createMediaStreamDestination();
|
||
this.downlinkAudioEl = document.createElement('audio');
|
||
this.downlinkAudioEl.autoplay = true;
|
||
this.downlinkAudioEl.srcObject = this.downlinkAudioDest.stream;
|
||
document.body.appendChild(this.downlinkAudioEl);
|
||
this.downlinkAudioEl.play().catch(e => console.warn("[AEC] Audio auto-play dicekal browser", e));
|
||
console.log("[AEC MATRIX] Web Audio to DOM Pipeline Active");
|
||
}
|
||
}
|
||
private e2eeKey: CryptoKey | null = null;
|
||
private e2eeKeyStr: string = "";
|
||
|
||
public async setE2EEKey(keyStr: string) {
|
||
if (!keyStr || keyStr === "NO_KEY" || keyStr === "none") {
|
||
this.e2eeKey = null;
|
||
this.e2eeKeyStr = "";
|
||
console.log("[QUANTUM MATRIX] E2EE Kriptografi DINONAKTIFKAN.");
|
||
return;
|
||
}
|
||
|
||
try {
|
||
// Derivasi Password ke CryptoKey AES-GCM (PBKDF2) - True Hardware Acceleration
|
||
const enc = new TextEncoder();
|
||
const keyMaterial = await crypto.subtle.importKey(
|
||
"raw",
|
||
enc.encode(keyStr),
|
||
"PBKDF2",
|
||
false,
|
||
["deriveBits", "deriveKey"]
|
||
);
|
||
|
||
// Salt tetap agar sinkron antar perangkat
|
||
const salt = enc.encode("XCOM_ULTRA_QUANTUM_SALT_2026");
|
||
|
||
this.e2eeKey = await crypto.subtle.deriveKey(
|
||
{
|
||
name: "PBKDF2",
|
||
salt: salt,
|
||
iterations: 100000,
|
||
hash: "SHA-256"
|
||
},
|
||
keyMaterial,
|
||
{ name: "AES-GCM", length: 256 },
|
||
false,
|
||
["encrypt", "decrypt"]
|
||
);
|
||
this.e2eeKeyStr = keyStr;
|
||
console.log("[QUANTUM MATRIX] 🛡️ E2EE Kriptografi AKTIF (AES-GCM 256-bit)");
|
||
} catch (_e) {
|
||
console.error("[QUANTUM MATRIX] Gagal merakit E2EE Key:", _e);
|
||
}
|
||
}
|
||
|
||
constructor(
|
||
private roomName: string,
|
||
public participantId: number,
|
||
private jwtToken: string // Entitlement Token
|
||
) {
|
||
this.isDesktop = !/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(
|
||
navigator.userAgent
|
||
);
|
||
console.log(`[QUANTUM MATRIX] Node ${this.participantId} Initialized.`);
|
||
|
||
// ANTI 3-SECOND AUDIO BUG: Eagerly create AudioContext so it can be resumed by global click events
|
||
try {
|
||
this.initDownlinkAudio();
|
||
(this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx.currentTime;
|
||
} catch (_e) {
|
||
console.warn("AudioContext failed to initialize early", _e);
|
||
}
|
||
}
|
||
|
||
// Callbacks
|
||
public onParticipantJoined: ((id: number) => void) | null = null;
|
||
public onParticipantLeft: ((id: number) => void) | null = null;
|
||
public onParticipantNameReceived: ((id: number, name: string) => void) | null = null;
|
||
public onQuantumDataReceived:
|
||
| ((senderId: number, payload: string) => void)
|
||
| null = null;
|
||
public onLocalStream: ((stream: MediaStream) => void) | null = null;
|
||
public onActiveSpeakerChanged: ((speakerId: number) => void) | null = null;
|
||
public onAudioLevel: ((level: number) => void) | null = null;
|
||
private vadLocalAudioCtx: AudioContext | null = null;
|
||
private uplinkAudioCtx: AudioContext | null = null;
|
||
private uplinkScriptNode: ScriptProcessorNode | null = null;
|
||
private uplinkAudioSource: MediaStreamAudioSourceNode | null = null;
|
||
|
||
public registerCanvas(participantId: number, canvasId: string) {
|
||
const canvas = document.getElementById(canvasId) as HTMLCanvasElement;
|
||
if (canvas) {
|
||
const ctx = canvas.getContext("2d");
|
||
if (ctx) this.canvasCtxMap.set(participantId, ctx);
|
||
console.log(
|
||
`[QUANTUM MATRIX] Canvas didaftarkan untuk Partisipan ${participantId}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
|
||
/** Try to find and register canvas by participant ID */
|
||
private tryAutoRegisterCanvas(participantId: number): boolean {
|
||
if (this.canvasCtxMap.has(participantId)) return true;
|
||
const canvasId = `quantum-matrix-${participantId}`;
|
||
const canvas = document.getElementById(canvasId) as HTMLCanvasElement;
|
||
if (canvas) {
|
||
const ctx = canvas.getContext("2d");
|
||
if (ctx) {
|
||
this.canvasCtxMap.set(participantId, ctx);
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
public async ignite(roomName: string, serverUrl: string = "/xcu-engine", quantumHash?: string) {
|
||
this.isRunning = true;
|
||
this.startVideoSyncLoop();
|
||
console.log("[QUANTUM MATRIX] Ignite v3.0 — WebTransport QUIC Primary + WebSocket Fallback");
|
||
|
||
// Attempt WebTransport QUIC first, then fall back to WebSocket
|
||
const host = window.location.hostname; // mesh.ultramodul.xyz
|
||
const wtUrl = `https://${host}:8443`;
|
||
|
||
let useWebTransport = false;
|
||
|
||
// === WebTransport QUIC (PRIMARY) ===
|
||
if (typeof WebTransport !== 'undefined') {
|
||
try {
|
||
console.log(`[QUANTUM MATRIX] Attempting WebTransport QUIC: ${wtUrl}`);
|
||
|
||
// Fetch server cert hash for serverCertificateHashes (needed for self-signed or pinning)
|
||
let certHashes: any[] | undefined;
|
||
try {
|
||
const certRes = await fetch('/api/v1/system/cert');
|
||
const certData = await certRes.json();
|
||
if (certData.hash) {
|
||
// Convert hex hash to Uint8Array
|
||
const hashBytes = new Uint8Array(certData.hash.match(/.{1,2}/g).map((b: string) => parseInt(b, 16)));
|
||
certHashes = [{ algorithm: 'sha-256', value: hashBytes.buffer }];
|
||
console.log('[QUANTUM MATRIX] Cert hash loaded for WebTransport pinning');
|
||
}
|
||
} catch(e) {
|
||
console.log('[QUANTUM MATRIX] No cert hash, using system trust');
|
||
}
|
||
|
||
const wtOptions: any = {};
|
||
// Only use serverCertificateHashes if available and needed
|
||
// For Let's Encrypt certs, browser trusts them natively
|
||
|
||
const wt = new WebTransport(wtUrl, wtOptions);
|
||
await Promise.race([
|
||
wt.ready,
|
||
new Promise((_, reject) => setTimeout(() => reject(new Error('WebTransport timeout')), 5000))
|
||
]);
|
||
|
||
console.log("[QUANTUM MATRIX] ✅ WebTransport QUIC CONNECTED! Zero Head-of-Line Blocking ACTIVE");
|
||
this.transport = wt;
|
||
this.trafficStats.wsState = 'QUIC';
|
||
useWebTransport = true;
|
||
this.reconnectAttempt = 0; // Reset backoff on success
|
||
|
||
// Setup datagram reader
|
||
const reader = wt.datagrams.readable.getReader();
|
||
const writer = wt.datagrams.writable.getWriter();
|
||
this.streamWriter = writer;
|
||
|
||
// Send registration datagram: [type=99, flags=0, pId_lo, pId_hi, room_len, ...room_bytes]
|
||
const roomBytes = new TextEncoder().encode(roomName);
|
||
const regPayload = new Uint8Array(5 + roomBytes.length);
|
||
regPayload[0] = 99; // CONTROL
|
||
regPayload[1] = 0;
|
||
regPayload[2] = this.participantId & 0xFF;
|
||
regPayload[3] = (this.participantId >> 8) & 0xFF;
|
||
regPayload[4] = roomBytes.length;
|
||
regPayload.set(roomBytes, 5);
|
||
await writer.write(regPayload);
|
||
this.trafficStats.tx.control += regPayload.length; this.trafficStats.tx.total += regPayload.length;
|
||
|
||
// Start NAT Keep-Alive Ping
|
||
this.natPingInterval = setInterval(() => {
|
||
if (this.streamWriter) {
|
||
const ping = new Uint8Array([99, 1]);
|
||
this.streamWriter.write(ping).catch(() => {});
|
||
}
|
||
}, 3000);
|
||
|
||
// Read datagrams in loop
|
||
(async () => {
|
||
try {
|
||
while (true) {
|
||
const { value, done } = await reader.read();
|
||
if (done) break;
|
||
if (!value || value.byteLength < 4) continue;
|
||
|
||
// REAL RX counter
|
||
const rxLen = value.byteLength;
|
||
this.trafficStats.rx.total += rxLen;
|
||
const frameType = value[0];
|
||
if (frameType === 1 || frameType === 10) this.trafficStats.rx.video += rxLen;
|
||
else if (frameType === 2) this.trafficStats.rx.audio += rxLen;
|
||
else this.trafficStats.rx.control += rxLen;
|
||
|
||
this.handleIncomingFrame(value);
|
||
}
|
||
} catch(e) {
|
||
console.warn('[QUANTUM MATRIX] WebTransport datagram read ended:', e);
|
||
}
|
||
})();
|
||
|
||
// Handle connection close — with exponential backoff
|
||
wt.closed.then(() => {
|
||
console.log('[QUANTUM MATRIX] WebTransport closed');
|
||
this.trafficStats.wsState = 'CLOSED';
|
||
if (this.isRunning) {
|
||
const delay = Math.min(1000 * Math.pow(1.5, this.reconnectAttempt), this.maxReconnectDelay);
|
||
this.reconnectAttempt++;
|
||
console.log(`[QUANTUM MATRIX] Reconnecting WebTransport in ${(delay/1000).toFixed(1)}s (attempt ${this.reconnectAttempt})...`);
|
||
setTimeout(() => this.ignite(roomName, serverUrl, quantumHash), delay);
|
||
}
|
||
}).catch(() => {
|
||
if (this.natPingInterval) clearInterval(this.natPingInterval);
|
||
this.trafficStats.wsState = 'CLOSED';
|
||
if (this.isRunning) {
|
||
const delay = Math.min(1000 * Math.pow(1.5, this.reconnectAttempt), this.maxReconnectDelay);
|
||
this.reconnectAttempt++;
|
||
console.log(`[QUANTUM MATRIX] Reconnecting WebTransport in ${(delay/1000).toFixed(1)}s (attempt ${this.reconnectAttempt})...`);
|
||
setTimeout(() => this.ignite(roomName, serverUrl, quantumHash), delay);
|
||
}
|
||
});
|
||
|
||
wt.closed.then(() => {
|
||
if (this.natPingInterval) clearInterval(this.natPingInterval);
|
||
}).catch(() => {});
|
||
|
||
this.startQuantumAutoPilot();
|
||
this.startTrafficMonitor();
|
||
this.startGhostCleanup();
|
||
// BUG FIX: Send heartbeat IMMEDIATELY on connect for instant visibility
|
||
this.sendPresenceHeartbeat();
|
||
this.startHeartbeatLoop();
|
||
|
||
} catch(e) {
|
||
console.warn(`[QUANTUM MATRIX] WebTransport failed: ${e}. Falling back to WebSocket...`);
|
||
useWebTransport = false;
|
||
}
|
||
}
|
||
|
||
// === WebSocket Fallback (SECONDARY) ===
|
||
if (!useWebTransport) {
|
||
try {
|
||
const wsHost = window.location.host;
|
||
const wsProto = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||
const wsUrl = `${wsProto}//${wsHost}/ws/${roomName}`;
|
||
|
||
console.log("[QUANTUM MATRIX] WebSocket Fallback URL:", wsUrl);
|
||
|
||
const ws = new WebSocket(wsUrl);
|
||
ws.binaryType = "arraybuffer";
|
||
this.ws = ws;
|
||
this.trafficStats.wsState = 'WS-CONNECTING';
|
||
|
||
let resolveReady: () => void;
|
||
let rejectReady: (e: unknown) => void;
|
||
const readyPromise = new Promise<void>((res, rej) => {
|
||
resolveReady = res;
|
||
rejectReady = rej;
|
||
});
|
||
|
||
ws.onopen = () => {
|
||
console.log("[QUANTUM MATRIX] WebSocket Fallback CONNECTED (TCP)");
|
||
this.trafficStats.wsState = 'WS-OPEN';
|
||
this.reconnectAttempt = 0; // Reset backoff on success
|
||
ws.send(JSON.stringify({
|
||
type: "qcg_handshake",
|
||
token: this.jwtToken,
|
||
displayName: this.displayName || ''
|
||
}));
|
||
resolveReady!();
|
||
};
|
||
ws.onerror = (e) => {
|
||
console.error("[QUANTUM MATRIX] WebSocket Error:", e);
|
||
this.trafficStats.wsState = 'WS-ERROR';
|
||
rejectReady!(e);
|
||
};
|
||
ws.onclose = () => {
|
||
console.log("[QUANTUM MATRIX] WebSocket Closed");
|
||
this.trafficStats.wsState = 'WS-CLOSED';
|
||
if (this.isRunning) {
|
||
const delay = Math.min(1000 * Math.pow(1.5, this.reconnectAttempt), this.maxReconnectDelay);
|
||
this.reconnectAttempt++;
|
||
console.log(`[QUANTUM MATRIX] Reconnecting WebSocket in ${(delay/1000).toFixed(1)}s (attempt ${this.reconnectAttempt})...`);
|
||
setTimeout(() => this.ignite(roomName, serverUrl, quantumHash), delay);
|
||
}
|
||
};
|
||
ws.onmessage = (event) => {
|
||
if (typeof event.data === "string") {
|
||
try {
|
||
const msg = JSON.parse(event.data);
|
||
if (msg.type === 'participant_list') {
|
||
console.log('[QUANTUM MATRIX] Participant list:', msg.participants);
|
||
}
|
||
} catch(e) {}
|
||
return;
|
||
}
|
||
const data = new Uint8Array(event.data);
|
||
if (data.byteLength < 4) return;
|
||
|
||
// REAL RX counter
|
||
const rxLen = data.byteLength;
|
||
this.trafficStats.rx.total += rxLen;
|
||
const frameType = data[0];
|
||
if (frameType === 1 || frameType === 10) this.trafficStats.rx.video += rxLen;
|
||
else if (frameType === 2) this.trafficStats.rx.audio += rxLen;
|
||
else this.trafficStats.rx.control += rxLen;
|
||
|
||
this.handleIncomingFrame(data);
|
||
};
|
||
|
||
await readyPromise;
|
||
this.startQuantumAutoPilot();
|
||
this.startTrafficMonitor();
|
||
this.startGhostCleanup();
|
||
// BUG FIX: Send heartbeat IMMEDIATELY on connect for instant visibility
|
||
this.sendPresenceHeartbeat();
|
||
this.startHeartbeatLoop();
|
||
|
||
} catch(e) {
|
||
console.error("[QUANTUM MATRIX] Both WebTransport and WebSocket failed:", e);
|
||
this.trafficStats.wsState = 'FAILED';
|
||
}
|
||
}
|
||
|
||
// 3. Start video capture pipeline
|
||
if (this.mediaStream) {
|
||
this.setupVideoCapturePipeline();
|
||
}
|
||
}
|
||
|
||
/** Unified incoming frame handler for both WebTransport and WebSocket */
|
||
private async handleIncomingFrame(data: Uint8Array) {
|
||
if (data.byteLength < 8) return;
|
||
const dataView = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||
const frameType = dataView.getUint8(0);
|
||
const quality = dataView.getUint8(1);
|
||
const senderId = dataView.getUint16(2, true);
|
||
const frameLength = dataView.getUint32(4, true);
|
||
|
||
if (senderId === this.participantId) return; // Skip own frames
|
||
this.participantLastSeen.set(senderId, Date.now());
|
||
|
||
if (data.byteLength < 8 + frameLength) return;
|
||
const payloadData = data.slice(8, 8 + frameLength);
|
||
|
||
if (frameType === 5) { // FRAME_HEARTBEAT
|
||
if (!this.canvasCtxMap.has(senderId)) {
|
||
if (this.onParticipantJoined) this.onParticipantJoined(senderId);
|
||
}
|
||
// Parse display name from heartbeat payload (if present)
|
||
if (payloadData.length > 0) {
|
||
try {
|
||
const nameStr = new TextDecoder().decode(payloadData);
|
||
if (nameStr && nameStr.length > 0 && nameStr.length < 256) {
|
||
const parsed = JSON.parse(nameStr);
|
||
if (parsed.type === 'NAME_ANNOUNCE' && parsed.name) {
|
||
if (this.onParticipantNameReceived) this.onParticipantNameReceived(senderId, parsed.name);
|
||
}
|
||
}
|
||
} catch(_) { /* Not JSON, ignore */ }
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (frameType === 10 || frameType === 1) { // FRAME_PULSAR
|
||
let jpegPayload = payloadData;
|
||
let isDecryptionFailed = false;
|
||
|
||
if (quality === 2) {
|
||
if (this.e2eeKey && payloadData.length > 12) {
|
||
const iv = payloadData.slice(0, 12);
|
||
const cipher = payloadData.slice(12);
|
||
try {
|
||
const plainBuf = await window.crypto.subtle.decrypt(
|
||
{ name: "AES-GCM", iv: iv },
|
||
this.e2eeKey,
|
||
cipher
|
||
);
|
||
jpegPayload = new Uint8Array(plainBuf);
|
||
} catch (_e) {
|
||
isDecryptionFailed = true;
|
||
}
|
||
} else {
|
||
isDecryptionFailed = true;
|
||
}
|
||
}
|
||
|
||
if (isDecryptionFailed) {
|
||
this.tryAutoRegisterCanvas(senderId);
|
||
const ctx = this.canvasCtxMap.get(senderId);
|
||
if (ctx) {
|
||
const w = ctx.canvas.width || 640;
|
||
const h = ctx.canvas.height || 360;
|
||
const idata = ctx.createImageData(w, h);
|
||
const d32 = new Uint32Array(idata.data.buffer);
|
||
for(let i=0; i<d32.length; i++) {
|
||
d32[i] = Math.random() < 0.5 ? 0xff000000 : 0xffffffff;
|
||
}
|
||
ctx.putImageData(idata, 0, 0);
|
||
ctx.fillStyle = 'red';
|
||
ctx.font = 'bold 24px sans-serif';
|
||
ctx.fillText("🔒 E2EE ENCRYPTED", 20, 40);
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (jpegPayload.length < 100) return;
|
||
|
||
if (!this.participantLastSeen.has(senderId) || !this.canvasCtxMap.has(senderId)) {
|
||
if (this.onParticipantJoined) this.onParticipantJoined(senderId);
|
||
}
|
||
|
||
// High-Performance Blob Renderer (Full Resolution)
|
||
this.tryAutoRegisterCanvas(senderId);
|
||
const jpegBlob = new Blob([jpegPayload], { type: "image/jpeg" });
|
||
const bmpUrl = URL.createObjectURL(jpegBlob);
|
||
const img = new Image();
|
||
img.onload = () => {
|
||
const jitter = this.audioJitterMap.get(senderId) || 50;
|
||
let queue = this.videoRenderQueue.get(senderId);
|
||
if (!queue) { queue = []; this.videoRenderQueue.set(senderId, queue); }
|
||
queue.push({
|
||
frame: img,
|
||
targetTime: performance.now() + jitter,
|
||
isWebCodec: false
|
||
});
|
||
URL.revokeObjectURL(bmpUrl);
|
||
};
|
||
img.onerror = () => URL.revokeObjectURL(bmpUrl);
|
||
img.src = bmpUrl;
|
||
} else if (frameType === 2) {
|
||
// AUDIO frame
|
||
let audioPayload = payloadData;
|
||
let isDecryptionFailed = false;
|
||
|
||
if (quality === 2) {
|
||
if (this.e2eeKey && payloadData.length > 12) {
|
||
const iv = payloadData.slice(0, 12);
|
||
const cipher = payloadData.slice(12);
|
||
try {
|
||
const plainBuf = await window.crypto.subtle.decrypt(
|
||
{ name: "AES-GCM", iv: iv },
|
||
this.e2eeKey,
|
||
cipher
|
||
);
|
||
audioPayload = new Uint8Array(plainBuf);
|
||
} catch (_e) {
|
||
isDecryptionFailed = true;
|
||
}
|
||
} else {
|
||
isDecryptionFailed = true;
|
||
}
|
||
}
|
||
|
||
if (isDecryptionFailed) return;
|
||
// Play audio if method exists
|
||
this.playRemoteAudio(senderId, audioPayload);
|
||
} else if (frameType === 6) {
|
||
// FRAME_LEDGER — JSON data (Chat, Resonance, Name Announce, etc.)
|
||
try {
|
||
const jsonStr = new TextDecoder().decode(payloadData);
|
||
const parsed = JSON.parse(jsonStr);
|
||
if (parsed.pkepxType === 'NAME_ANNOUNCE' && parsed.name) {
|
||
if (this.onParticipantNameReceived) this.onParticipantNameReceived(senderId, parsed.name);
|
||
} else if (parsed.pkepxType === 'RESONANCE' && parsed.payload) {
|
||
if (this.onQuantumResonance) this.onQuantumResonance(senderId, parsed.payload);
|
||
} else if (parsed.pkepxType === 'SOVEREIGN_SIGNAL') {
|
||
if (this.onSovereignSignal) this.onSovereignSignal(parsed.signalType, parsed.payload);
|
||
}
|
||
// Also forward to generic data handler
|
||
if (this.onQuantumDataReceived) this.onQuantumDataReceived(senderId, jsonStr);
|
||
} catch(_) { /* Not valid JSON */ }
|
||
}
|
||
}
|
||
|
||
/** Send data via WebTransport (QUIC datagram) or WebSocket (TCP) */
|
||
public async sendFrame(payload: Uint8Array) {
|
||
// REAL TX counter
|
||
this.trafficStats.tx.total += payload.byteLength;
|
||
const frameType = payload[0];
|
||
if (frameType === 1 || frameType === 10) this.trafficStats.tx.video += payload.byteLength;
|
||
else if (frameType === 2) this.trafficStats.tx.audio += payload.byteLength;
|
||
else this.trafficStats.tx.control += payload.byteLength;
|
||
|
||
// Send via WebTransport QUIC datagram (primary) or WebSocket (fallback)
|
||
if (this.streamWriter) {
|
||
try {
|
||
await this.streamWriter.write(payload);
|
||
return;
|
||
} catch(e) {
|
||
// WebTransport write failed, try WebSocket
|
||
}
|
||
}
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
|
||
this.ws.send(payload);
|
||
}
|
||
}
|
||
|
||
|
||
public async hotSwapVideoEngine(mode: "auto" | "canvas" | "webcodecs") {
|
||
if (this.videoEngineMode === mode) return;
|
||
console.log(`[QUANTUM HOT-SWAP] Mengalihkan Video Engine ke: ${mode}`);
|
||
this.videoEngineMode = mode;
|
||
if (this.mediaStream) {
|
||
await this.deactivateUplink();
|
||
await this.activateUplink('camera');
|
||
}
|
||
}
|
||
|
||
public async hotSwapAudioEngine(mode: "auto" | "pcm" | "xcu-neural" | "xcu-resonance") {
|
||
if (this.audioEngineMode === mode) return;
|
||
console.log(`[QUANTUM HOT-SWAP] Mengalihkan Audio Engine ke: ${mode}`);
|
||
this.audioEngineMode = mode;
|
||
if (this.mediaStream) {
|
||
await this.deactivateUplink();
|
||
await this.activateUplink('camera');
|
||
}
|
||
}
|
||
|
||
public unlockAudio() {
|
||
if (!this.downlinkAudioCtx) {
|
||
this.initDownlinkAudio();
|
||
(this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx.currentTime;
|
||
}
|
||
if (this.downlinkAudioCtx.state === "suspended") {
|
||
this.downlinkAudioCtx.resume().catch(() => {});
|
||
}
|
||
if (!this.uplinkAudioCtx) {
|
||
this.uplinkAudioCtx = new AudioContext();
|
||
}
|
||
if (this.uplinkAudioCtx.state === "suspended") {
|
||
this.uplinkAudioCtx.resume().catch(() => {});
|
||
}
|
||
console.log("[QUANTUM MATRIX] Audio Contexts Pre-Created and Unlocked via User Gesture");
|
||
}
|
||
|
||
public toggleMic(enabled: boolean) {
|
||
this.isMicMuted = !enabled;
|
||
if (this.mediaStream) {
|
||
this.mediaStream.getAudioTracks().forEach((track) => {
|
||
track.enabled = enabled;
|
||
});
|
||
}
|
||
console.log(`[QUANTUM UPLINK] Mikrofon ${enabled ? "AKTIF" : "MATI"} — pipeline terjaga untuk mencegah bug Mac Safari/Chrome.`);
|
||
}
|
||
|
||
public resumeAudioContext() {
|
||
if (this.downlinkAudioCtx && this.downlinkAudioCtx.state === "suspended") {
|
||
this.downlinkAudioCtx.resume().then(() => {
|
||
console.log("[QUANTUM MATRIX] AudioContext berhasil dibangunkan secara manual!");
|
||
}).catch(e => {
|
||
console.warn("[QUANTUM MATRIX] Gagal membangunkan AudioContext:", e);
|
||
});
|
||
}
|
||
if (this.uplinkAudioCtx && this.uplinkAudioCtx.state === "suspended") {
|
||
this.uplinkAudioCtx.resume().catch(() => {});
|
||
}
|
||
if (this.vadAudioCtx && this.vadAudioCtx.state === "suspended") {
|
||
this.vadAudioCtx.resume().catch(() => {});
|
||
}
|
||
}
|
||
|
||
public setEffects(virtualBgMode: number, beautyFilter: boolean) {
|
||
this.useVirtualBg = virtualBgMode;
|
||
this.useBeautyFilter = beautyFilter;
|
||
console.log(`[QUANTUM EFFECTS] Virtual BG Mode: ${virtualBgMode}, Beauty: ${beautyFilter}`);
|
||
}
|
||
|
||
|
||
private async detectBestCodec(): Promise<string> {
|
||
const codecs = [
|
||
"av01.0.04M.08", // AV1
|
||
"vp09.00.10.08", // VP9
|
||
"vp8", // VP8
|
||
"avc1.42E01F" // H.264
|
||
];
|
||
for (const c of codecs) {
|
||
try {
|
||
const support = await VideoEncoder.isConfigSupported({
|
||
codec: c,
|
||
width: 1920,
|
||
height: 1080,
|
||
bitrate: 8_000_000,
|
||
framerate: 30
|
||
});
|
||
if (support.supported) {
|
||
console.log(`[QUANTUM WEBCODECS] Hardware GPU Codec Terdeteksi: ${c}`);
|
||
return c;
|
||
}
|
||
} catch (_e) {}
|
||
}
|
||
return "avc1.42E01F"; // Fallback H.264
|
||
}
|
||
|
||
private createDecoderForParticipant(senderId: number, codecStr: string = "avc1.42E01F") {
|
||
// Jika decoder sudah ada dan codec-nya berbeda, tutup dulu
|
||
if (this.videoDecoders.has(senderId)) {
|
||
try { this.videoDecoders.get(senderId)!.close(); } catch(e){}
|
||
this.videoDecoders.delete(senderId);
|
||
}
|
||
|
||
const decoder = new VideoDecoder({
|
||
output: (frame: unknown) => {
|
||
this.tryAutoRegisterCanvas(senderId);
|
||
const jitter = this.audioJitterMap.get(senderId) || 50;
|
||
let queue = this.videoRenderQueue.get(senderId);
|
||
if (!queue) { queue = []; this.videoRenderQueue.set(senderId, queue); }
|
||
queue.push({
|
||
frame: frame,
|
||
targetTime: performance.now() + jitter,
|
||
isWebCodec: true
|
||
});
|
||
},
|
||
error: (e: unknown) => console.error(`Decoder Error untuk ${senderId}:`, e),
|
||
});
|
||
|
||
try {
|
||
decoder.configure({ codec: codecStr, codedWidth: 1280, codedHeight: 720 });
|
||
this.videoDecoders.set(senderId, decoder);
|
||
this.firstKeyFrameReceived.set(senderId, false);
|
||
console.log(`[QUANTUM WEBCODECS] Hardware Decoder (${codecStr}) siap untuk Partisipan ${senderId}`);
|
||
} catch (_e) {
|
||
console.error("[QUANTUM WEBCODECS] Gagal konfigurasi decoder:", e);
|
||
}
|
||
|
||
// Notifikasi React untuk merender tile
|
||
if (this.onParticipantJoined) {
|
||
this.onParticipantJoined(senderId);
|
||
}
|
||
}
|
||
|
||
public async activateUplink(source: "camera" | "screen" = "camera", facingMode: "user" | "environment" = "user") {
|
||
if (this.mediaStream) {
|
||
console.warn(
|
||
"[QUANTUM UPLINK] Kamera/Layar sudah aktif! Mengabaikan perintah ganda untuk mencegah kebocoran memori.",
|
||
);
|
||
return;
|
||
}
|
||
console.log(
|
||
`[QUANTUM UPLINK] Memulai Injeksi ${source === "screen" ? "Layar" : "Kamera"} ke WebCodecs...`,
|
||
);
|
||
try {
|
||
if (source === "screen") {
|
||
this.mediaStream = await navigator.mediaDevices.getDisplayMedia({
|
||
video: { width: { ideal: 1920 }, height: { ideal: 1080 }, frameRate: { ideal: this.targetFps } },
|
||
audio: false,
|
||
});
|
||
} else {
|
||
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
||
video: { width: { ideal: 1920 }, height: { ideal: 1080 }, frameRate: { ideal: this.targetFps }, facingMode: facingMode },
|
||
audio: {
|
||
echoCancellation: true,
|
||
noiseSuppression: true,
|
||
autoGainControl: true
|
||
},
|
||
});
|
||
}
|
||
|
||
if (this.onLocalStream) {
|
||
this.onLocalStream(this.mediaStream);
|
||
}
|
||
this.startVAD();
|
||
|
||
// AUDIENCE mode: skip encoding
|
||
if (this.participantRole === "AUDIENCE") {
|
||
console.log("[UPLINK] AUDIENCE mode - no video upload.");
|
||
await this.sendRoleSignal();
|
||
return;
|
||
}
|
||
|
||
const videoTrack = this.mediaStream.getVideoTracks()[0];
|
||
|
||
// === Canvas JPEG Pipeline: Works on ALL browsers ===
|
||
const captureVideo = document.createElement("video");
|
||
captureVideo.srcObject = new MediaStream([videoTrack]);
|
||
captureVideo.muted = true;
|
||
captureVideo.playsInline = true;
|
||
captureVideo.autoplay = true;
|
||
try { await captureVideo.play(); } catch { /* autoplay blocked */ }
|
||
|
||
// Adaptive canvas resolution based on bandwidth
|
||
const conn = (navigator as any).connection;
|
||
const dl = conn ? conn.downlink : 10;
|
||
let captureW = 1280, captureH = 720; // Default HD
|
||
if (dl > 5) { captureW = 1920; captureH = 1080; } // FHD for good bandwidth
|
||
else if (dl < 1.5) { captureW = 854; captureH = 480; } // SD for low bandwidth
|
||
const captureCvs = document.createElement("canvas");
|
||
captureCvs.width = captureW;
|
||
captureCvs.height = captureH;
|
||
const captureCtx = captureCvs.getContext("2d")!;
|
||
console.log(`[UPLINK] Canvas JPEG pipeline active (${captureW}x${captureH} @ ${this.targetFps}fps) [TSM.ID].[11031972]`);
|
||
|
||
let lastVideoFrame = 0;
|
||
const captureLoop = (now: number) => {
|
||
if (!this.isRunning) return;
|
||
requestAnimationFrame(captureLoop);
|
||
const fpsInterval = 1000 / this.targetFps;
|
||
if (now - lastVideoFrame < fpsInterval) return;
|
||
lastVideoFrame = now;
|
||
|
||
if (this.videoEngineMode === "webcodecs") return;
|
||
if (!this.streamWriter && (!this.ws || this.ws.readyState !== WebSocket.OPEN)) return;
|
||
|
||
// === ULTRA BACKPRESSURE CONTROL ===
|
||
// If WebSocket send buffer > 64KB, skip this frame (prevent 18KB backlog!)
|
||
if (this.ws && this.ws.bufferedAmount > 65536) {
|
||
this.framesDropped++;
|
||
// Smart downgrade: reduce quality when buffer is full
|
||
if (this.adaptiveQuality > 0.5) {
|
||
this.adaptiveQuality = Math.max(0.5, this.adaptiveQuality - 0.05);
|
||
console.log(`[AUTOPILOT] ⚡ Backpressure! Quality → ${(this.adaptiveQuality*100).toFixed(0)}% (dropped: ${this.framesDropped})`);
|
||
}
|
||
if (this.adaptiveResScale > 0.5) {
|
||
this.adaptiveResScale = Math.max(0.5, this.adaptiveResScale - 0.1);
|
||
}
|
||
return;
|
||
}
|
||
|
||
// === SMART RESOLUTION SCALING ===
|
||
const effectiveW = Math.round(captureW * this.adaptiveResScale);
|
||
const effectiveH = Math.round(captureH * this.adaptiveResScale);
|
||
if (captureCvs.width !== effectiveW || captureCvs.height !== effectiveH) {
|
||
captureCvs.width = effectiveW;
|
||
captureCvs.height = effectiveH;
|
||
}
|
||
|
||
try {
|
||
let mode = 0;
|
||
if (this.useBeautyFilter && this.useVirtualBg === 2) mode = 3;
|
||
else if (this.useBeautyFilter && this.useVirtualBg === 4) mode = 5;
|
||
else if (this.useVirtualBg > 0) mode = this.useVirtualBg;
|
||
else if (this.useBeautyFilter) mode = 1;
|
||
|
||
if (mode > 0) {
|
||
if (!(this as any).webglFilter) (this as any).webglFilter = new XCUWebGLFilter(effectiveW, effectiveH);
|
||
(this as any).webglFilter.render(captureVideo, mode);
|
||
captureCtx.drawImage((this as any).webglFilter.canvas, 0, 0, effectiveW, effectiveH);
|
||
} else {
|
||
if (captureVideo.readyState >= 2) {
|
||
captureCtx.drawImage(captureVideo, 0, 0, effectiveW, effectiveH);
|
||
}
|
||
}
|
||
|
||
if (this.isEncodingCanvas) return; // Prevent OOM stacking
|
||
this.isEncodingCanvas = true;
|
||
captureCvs.toBlob((blob) => {
|
||
this.isEncodingCanvas = false;
|
||
if (!blob) return;
|
||
// Check both WS and WT
|
||
if (!this.streamWriter && (!this.ws || this.ws.readyState !== WebSocket.OPEN)) return;
|
||
blob.arrayBuffer().then(async (buf) => {
|
||
let jpegData = new Uint8Array(buf);
|
||
let packetLen = 8 + jpegData.length;
|
||
let isEncrypted = 0;
|
||
const iv = new Uint8Array(12);
|
||
|
||
if (this.e2eeKey) {
|
||
window.crypto.getRandomValues(iv);
|
||
const cipherBuffer = await window.crypto.subtle.encrypt(
|
||
{ name: "AES-GCM", iv: iv },
|
||
this.e2eeKey,
|
||
jpegData
|
||
);
|
||
jpegData = new Uint8Array(cipherBuffer);
|
||
packetLen = 8 + 12 + jpegData.length;
|
||
isEncrypted = 2;
|
||
}
|
||
|
||
const packet = new Uint8Array(packetLen);
|
||
packet[0] = 10; // FRAME_PULSAR
|
||
packet[1] = isEncrypted ? 2 : 1;
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
|
||
if (isEncrypted) {
|
||
view.setUint32(4, 12 + jpegData.length, true);
|
||
packet.set(iv, 8);
|
||
packet.set(jpegData, 20);
|
||
} else {
|
||
view.setUint32(4, jpegData.length, true);
|
||
packet.set(jpegData, 8);
|
||
}
|
||
|
||
this.sendFrame(packet);
|
||
this._frameCount++;
|
||
});
|
||
}, "image/jpeg", this.adaptiveQuality);
|
||
} catch (err) { console.warn("[TX] capture error", err); }
|
||
};
|
||
requestAnimationFrame(captureLoop);
|
||
|
||
// === Audio PCM Pipeline — ULTRA TUNE-UP ===
|
||
const audioTracks = this.mediaStream.getAudioTracks();
|
||
if (audioTracks.length > 0) {
|
||
if (!this.uplinkAudioCtx) this.uplinkAudioCtx = new AudioContext();
|
||
this.uplinkAudioSource = this.uplinkAudioCtx.createMediaStreamSource(new MediaStream([audioTracks[0]]));
|
||
// Smaller buffer = lower latency (2048 samples = ~42ms @ 48kHz)
|
||
this.uplinkScriptNode = this.uplinkAudioCtx.createScriptProcessor(2048, 1, 1);
|
||
this.uplinkScriptNode.onaudioprocess = (ev: AudioProcessingEvent) => {
|
||
if (this.isMicMuted) {
|
||
const out = ev.outputBuffer.getChannelData(0);
|
||
for (let i = 0; i < out.length; i++) out[i] = 0;
|
||
return;
|
||
}
|
||
const float32 = ev.inputBuffer.getChannelData(0);
|
||
|
||
// SILENCE DETECTION: Skip sending if audio is nearly silent (saves bandwidth)
|
||
let rms = 0;
|
||
for (let i = 0; i < float32.length; i += 16) rms += float32[i] * float32[i];
|
||
rms = Math.sqrt(rms / (float32.length / 16));
|
||
if (rms < 0.005) {
|
||
const out = ev.outputBuffer.getChannelData(0);
|
||
for (let i = 0; i < out.length; i++) out[i] = 0;
|
||
return; // Skip silent frames
|
||
}
|
||
|
||
// DOWNSAMPLE 48kHz → 16kHz (3x bandwidth reduction)
|
||
const srcRate = this.uplinkAudioCtx!.sampleRate;
|
||
const targetRate = 16000;
|
||
const ratio = srcRate / targetRate;
|
||
const downLen = Math.floor(float32.length / ratio);
|
||
const int16 = new Int16Array(downLen);
|
||
for (let i = 0; i < downLen; i++) {
|
||
// Linear interpolation for cleaner downsampling
|
||
const srcIdx = i * ratio;
|
||
const idx0 = Math.floor(srcIdx);
|
||
const idx1 = Math.min(idx0 + 1, float32.length - 1);
|
||
const frac = srcIdx - idx0;
|
||
const sample = float32[idx0] * (1 - frac) + float32[idx1] * frac;
|
||
int16[i] = Math.max(-32768, Math.min(32767, Math.round(sample * 32767)));
|
||
}
|
||
|
||
const out = ev.outputBuffer.getChannelData(0);
|
||
for (let i = 0; i < out.length; i++) out[i] = 0;
|
||
|
||
const pcmBytes = new Uint8Array(int16.buffer);
|
||
|
||
if (this.e2eeKey) {
|
||
const iv = window.crypto.getRandomValues(new Uint8Array(12));
|
||
window.crypto.subtle.encrypt({ name: "AES-GCM", iv: iv }, this.e2eeKey, pcmBytes)
|
||
.then(cipherBuffer => {
|
||
const cipherBytes = new Uint8Array(cipherBuffer);
|
||
const packet = new Uint8Array(8 + 4 + 12 + cipherBytes.length);
|
||
packet[0] = 2;
|
||
packet[1] = 2;
|
||
const av = new DataView(packet.buffer);
|
||
av.setUint16(2, this.participantId, true);
|
||
av.setUint32(4, 4 + 12 + cipherBytes.length, true);
|
||
av.setUint32(8, targetRate, true);
|
||
packet.set(iv, 12);
|
||
packet.set(cipherBytes, 24);
|
||
this.sendFrame(packet);
|
||
}).catch(e => console.error("Audio Encrypt Error", e));
|
||
} else {
|
||
const packet = new Uint8Array(8 + 4 + pcmBytes.length);
|
||
packet[0] = 2;
|
||
packet[1] = 0;
|
||
const av = new DataView(packet.buffer);
|
||
av.setUint16(2, this.participantId, true);
|
||
av.setUint32(4, 4 + pcmBytes.length, true);
|
||
av.setUint32(8, targetRate, true);
|
||
packet.set(pcmBytes, 12);
|
||
this.sendFrame(packet);
|
||
}
|
||
};
|
||
this.uplinkAudioSource.connect(this.uplinkScriptNode);
|
||
this.uplinkScriptNode.connect(this.uplinkAudioCtx.destination);
|
||
(window as unknown as { [key: string]: unknown }).uplinkScriptNode = this.uplinkScriptNode;
|
||
console.log("[UPLINK] Audio pipeline active (16kHz downsampled, silence gated) [TSM.ID].[11031972]");
|
||
|
||
this.startVadLoop();
|
||
}
|
||
|
||
} catch (e: unknown) {
|
||
console.error("[UPLINK] Camera error:", e);
|
||
}
|
||
}
|
||
|
||
|
||
/** Calculate real bytes/sec rates from cumulative counters */
|
||
|
||
// Ghost Participant Cleanup — auto-remove stale participants after 8s of no data
|
||
private startGhostCleanup() {
|
||
if (this.ghostCleanupInterval) clearInterval(this.ghostCleanupInterval);
|
||
this.ghostCleanupInterval = setInterval(() => {
|
||
const now = Date.now();
|
||
const GHOST_TIMEOUT = 5000; // 5s — faster cleanup for accurate participant list
|
||
this.participantLastSeen.forEach((lastSeen, senderId) => {
|
||
if (now - lastSeen > GHOST_TIMEOUT) {
|
||
console.log(`[GHOST CLEANUP] Removing stale participant ${senderId} (${Math.round((now - lastSeen)/1000)}s stale)`);
|
||
this.participantLastSeen.delete(senderId);
|
||
this.canvasCtxMap.delete(senderId);
|
||
if (this.videoDecoders.has(senderId)) {
|
||
try { this.videoDecoders.get(senderId)!.close(); } catch(e) {}
|
||
this.videoDecoders.delete(senderId);
|
||
}
|
||
if (this.onParticipantLeft) {
|
||
this.onParticipantLeft(senderId);
|
||
}
|
||
}
|
||
});
|
||
}, 5000);
|
||
}
|
||
|
||
public startTrafficMonitor() {
|
||
this.trafficStats.startTime = performance.now();
|
||
this.trafficStats._lastSnapshot.ts = performance.now();
|
||
if (this.trafficInterval) clearInterval(this.trafficInterval);
|
||
this.trafficInterval = setInterval(() => {
|
||
const now = performance.now();
|
||
const elapsed = (now - this.trafficStats._lastSnapshot.ts) / 1000;
|
||
if (elapsed < 0.5) return;
|
||
const snap = this.trafficStats._lastSnapshot;
|
||
this.trafficStats.rates.txVideo = Math.round((this.trafficStats.tx.video - snap.txVideo) / elapsed);
|
||
this.trafficStats.rates.txAudio = Math.round((this.trafficStats.tx.audio - snap.txAudio) / elapsed);
|
||
this.trafficStats.rates.txTotal = Math.round((this.trafficStats.tx.total - snap.txTotal) / elapsed);
|
||
this.trafficStats.rates.rxVideo = Math.round((this.trafficStats.rx.video - snap.rxVideo) / elapsed);
|
||
this.trafficStats.rates.rxAudio = Math.round((this.trafficStats.rx.audio - snap.rxAudio) / elapsed);
|
||
this.trafficStats.rates.rxTotal = Math.round((this.trafficStats.rx.total - snap.rxTotal) / elapsed);
|
||
snap.txVideo = this.trafficStats.tx.video;
|
||
snap.txAudio = this.trafficStats.tx.audio;
|
||
snap.txTotal = this.trafficStats.tx.total;
|
||
snap.rxVideo = this.trafficStats.rx.video;
|
||
snap.rxAudio = this.trafficStats.rx.audio;
|
||
snap.rxTotal = this.trafficStats.rx.total;
|
||
snap.ts = now;
|
||
this.trafficStats.wsState = this.ws?.readyState === WebSocket.OPEN ? 'OPEN' : this.ws?.readyState === WebSocket.CONNECTING ? 'CONNECTING' : 'CLOSED';
|
||
}, 1000);
|
||
}
|
||
|
||
public stopTrafficMonitor() {
|
||
if (this.trafficInterval) { clearInterval(this.trafficInterval); this.trafficInterval = null; }
|
||
}
|
||
|
||
public startQuantumAutoPilot() {
|
||
if (this.autoPilotInterval) clearInterval(this.autoPilotInterval);
|
||
this.autoPilotInterval = setInterval(async () => {
|
||
const conn = (navigator as any).connection;
|
||
const downlink = conn ? conn.downlink : 10; // Mbps
|
||
this.currentBandwidth = downlink;
|
||
|
||
// === BANDWIDTH ESTIMATION (OPTION 1 & 2) ===
|
||
// Option 1: Standard Network API (Downlink) + Traffic Jitter
|
||
let bwScore = 3; // Default FHD
|
||
if (downlink < 0.5) bwScore = 0; // Base/Audio Only
|
||
else if (downlink < 1.5) bwScore = 1; // SD
|
||
else if (downlink < 4) bwScore = 2; // HD
|
||
|
||
// Option 2: Target Buffer Throttle (Dynamic RX Drop simulation)
|
||
// Jika RX video macet (Buffer tersendat), paksa turunkan skor!
|
||
if (this.trafficStats.rates.rxVideo < 5000 && this.participantRole === 'AUDIENCE') {
|
||
// Terindikasi video membeku walau downlink besar
|
||
bwScore = Math.max(0, bwScore - 1);
|
||
}
|
||
|
||
// Kirim Telemetry Datagram ke XCU Router (Ring-0 Simulator)
|
||
if (this.streamWriter) {
|
||
const bwTelemetry = new Uint8Array(4);
|
||
bwTelemetry[0] = 10; // Tipe: Bandwidth Telemetry
|
||
bwTelemetry[1] = bwScore; // Skor 0-3
|
||
bwTelemetry[2] = this.participantId & 0xFF;
|
||
bwTelemetry[3] = (this.participantId >> 8) & 0xFF;
|
||
this.streamWriter.write(bwTelemetry).catch(() => {});
|
||
}
|
||
|
||
if (this.isSwappingEncoder) return;
|
||
|
||
// --- AUTO VIDEO (XCU Pulsar = INSTANT BOOT, WebCodecs = PERFORMANCE UPGRADE) ---
|
||
// PKX Konstitusi 5 Pasal 3: XCU Pulsar STARTS FIRST, kemudian upgrade jika hardware tersedia
|
||
if (this.videoEngineMode === 'auto') {
|
||
if (!this.usePulsarCodec && !this.videoEncoder) {
|
||
// BOOT PERTAMA: XCU Pulsar langsung aktif tanpa delay
|
||
this.usePulsarCodec = true;
|
||
this.activeVideoCodec = 'XCU PULSAR (Delta)';
|
||
console.log(`[AUTO-PILOT] XCU Pulsar BOOT PERTAMA — instant start`);
|
||
} else if (this.usePulsarCodec && this.videoEncoder && downlink > 1) {
|
||
// UPGRADE: WebCodecs hardware tersedia + bandwidth cukup → upgrade untuk performa
|
||
this.isSwappingEncoder = true;
|
||
let targetCodec = "avc1.42E01F"; // H.264 baseline (paling kompatibel)
|
||
let targetBitrate = 800_000;
|
||
|
||
if (downlink > 10) {
|
||
targetCodec = "vp09.00.10.08"; // VP9
|
||
targetBitrate = 2_500_000;
|
||
} else if (downlink > 5) {
|
||
targetCodec = "vp8";
|
||
targetBitrate = 1_500_000;
|
||
} else if (downlink > 2) {
|
||
targetCodec = "avc1.42E01F"; // H.264
|
||
targetBitrate = 1_000_000;
|
||
}
|
||
|
||
console.log(`[AUTO-PILOT] UPGRADE dari Pulsar ke WebCodecs ${targetCodec} (${downlink} Mbps)`);
|
||
try {
|
||
try { (this.videoEncoder as any).close(); } catch(_e) {}
|
||
this.activeCodecStr = targetCodec;
|
||
this.activeVideoCodec = `WebCodecs ${targetCodec.split('.')[0].toUpperCase()}`;
|
||
this.videoEncoder = new (window as any).VideoEncoder({
|
||
output: (chunk: any, metadata: any) => this.handleVideoChunk(chunk, metadata),
|
||
error: (e: any) => {
|
||
console.error("[AUTO-PILOT] WebCodecs error, fallback ke Pulsar", e);
|
||
this.usePulsarCodec = true;
|
||
this.activeVideoCodec = 'XCU PULSAR (Delta)';
|
||
}
|
||
});
|
||
(this.videoEncoder as any).configure({
|
||
codec: this.activeCodecStr,
|
||
width: 1920, height: 1080, bitrate: targetBitrate, framerate: this.targetFps
|
||
});
|
||
this.usePulsarCodec = false; // Switch berhasil
|
||
} catch(e) {
|
||
// WebCodecs tidak support: tetap Pulsar (PKX fallback)
|
||
this.usePulsarCodec = true;
|
||
this.activeVideoCodec = 'XCU PULSAR (Delta)';
|
||
console.log('[AUTO-PILOT] WebCodecs tidak tersedia, Pulsar tetap aktif');
|
||
}
|
||
this.isSwappingEncoder = false;
|
||
} else if (!this.usePulsarCodec && this.videoEncoder && downlink < 0.5) {
|
||
// DOWNGRADE: Bandwidth sangat rendah → kembali ke Pulsar
|
||
this.usePulsarCodec = true;
|
||
this.activeVideoCodec = 'XCU PULSAR (Delta)';
|
||
console.log(`[AUTO-PILOT] Bandwidth rendah (${downlink} Mbps), kembali ke Pulsar`);
|
||
}
|
||
}
|
||
|
||
// --- AUTO AUDIO (XCU Resonance = INSTANT BOOT, Opus = PERFORMANCE UPGRADE) ---
|
||
// PKX: Resonance STARTS FIRST, upgrade ke Opus jika bandwidth stabil
|
||
if (this.audioEngineMode === 'auto') {
|
||
if (!this.audioEncoder && downlink > 2) {
|
||
// UPGRADE: Bandwidth cukup → Opus untuk kualitas audio lebih baik
|
||
this.isSwappingEncoder = true;
|
||
try {
|
||
this.audioEncoder = new (window as any).AudioEncoder({
|
||
output: (chunk: any) => this.handleAudioChunk(chunk),
|
||
error: (e: any) => {
|
||
console.error("[AUTO-PILOT] Opus gagal, Resonance tetap aktif", e);
|
||
this.audioEncoder = null;
|
||
this.activeAudioCodec = 'XCU RESONANCE (300bps)';
|
||
}
|
||
});
|
||
(this.audioEncoder as any).configure({
|
||
codec: 'opus', sampleRate: 48000, numberOfChannels: 1, bitrate: 32000
|
||
});
|
||
this.activeAudioCodec = 'XCU NEURAL (Opus 32k)';
|
||
console.log(`[AUTO-PILOT] UPGRADE Audio ke Opus (${downlink} Mbps)`);
|
||
} catch(e) {
|
||
this.audioEncoder = null;
|
||
this.activeAudioCodec = 'XCU RESONANCE (300bps)';
|
||
}
|
||
this.isSwappingEncoder = false;
|
||
} else if (this.audioEncoder && downlink < 0.5) {
|
||
// DOWNGRADE: Bandwidth rendah → kembali ke Resonance
|
||
try { (this.audioEncoder as any).close(); } catch(_e) {}
|
||
this.audioEncoder = null;
|
||
this.activeAudioCodec = 'XCU RESONANCE (300bps)';
|
||
console.log(`[AUTO-PILOT] Bandwidth rendah, kembali ke Resonance`);
|
||
} else if (!this.audioEncoder) {
|
||
this.activeAudioCodec = 'XCU RESONANCE (300bps)';
|
||
}
|
||
}
|
||
|
||
// --- AUTO FPS (30fps → 60fps probe — PKX Extreme Smooth) ---
|
||
if (this.targetFps === 30 && !this.fpsConfirmed60 && downlink > 3) {
|
||
this.fpsProbeCount++;
|
||
if (this.fpsProbeCount >= 4) { // 4 cycles × 3 sec = 12 detik stabil
|
||
this.fpsConfirmed60 = true;
|
||
this.targetFps = 60;
|
||
console.log(`[AUTO-PILOT] 60fps CONFIRMED! Bandwidth stabil ${downlink} Mbps selama 12 detik. Upgrade ke 60fps.`);
|
||
// Re-configure video track jika memungkinkan
|
||
try {
|
||
const videoTrack = this.mediaStream?.getVideoTracks()[0];
|
||
if (videoTrack) {
|
||
videoTrack.applyConstraints({ frameRate: { ideal: 60 } }).catch(() => {});
|
||
}
|
||
} catch(_e) {}
|
||
}
|
||
} else if (this.targetFps === 30 && downlink <= 3) {
|
||
this.fpsProbeCount = 0; // Reset probe jika bandwidth tidak stabil
|
||
}
|
||
|
||
// --- DOWNGRADE FPS jika bandwidth sangat rendah ---
|
||
if (this.targetFps === 60 && downlink < 1) {
|
||
this.targetFps = 30;
|
||
this.fpsConfirmed60 = false;
|
||
this.fpsProbeCount = 0;
|
||
console.log(`[AUTO-PILOT] Bandwidth drop (${downlink} Mbps). Downgrade ke 30fps.`);
|
||
}
|
||
}, 3000);
|
||
|
||
// === ULTRA AUTOPILOT: Smart Quality Recovery ===
|
||
// Gradually restore quality when bandwidth improves
|
||
setInterval(() => {
|
||
const wsBuffered = this.ws ? this.ws.bufferedAmount : 0;
|
||
const txRate = this.trafficStats.rates.txTotal; // bytes/sec
|
||
|
||
// Estimate real bandwidth from TX rate
|
||
this.lastBandwidthEstimate = txRate > 0 ? (txRate * 8 / 1_000_000) : this.currentBandwidth;
|
||
|
||
// Buffer is clear + bandwidth ok → gradually RESTORE quality
|
||
if (wsBuffered < 8192 && this.adaptiveQuality < 0.92) {
|
||
this.adaptiveQuality = Math.min(0.92, this.adaptiveQuality + 0.02);
|
||
console.log(`[AUTOPILOT] 📈 Quality recovering → ${(this.adaptiveQuality*100).toFixed(0)}%`);
|
||
}
|
||
if (wsBuffered < 4096 && this.adaptiveResScale < 1.0) {
|
||
this.adaptiveResScale = Math.min(1.0, this.adaptiveResScale + 0.05);
|
||
console.log(`[AUTOPILOT] 📈 Resolution recovering → ${(this.adaptiveResScale*100).toFixed(0)}%`);
|
||
}
|
||
|
||
// Severe congestion → aggressive downgrade
|
||
if (wsBuffered > 131072) { // >128KB stuck
|
||
this.adaptiveQuality = 0.5;
|
||
this.adaptiveResScale = 0.5;
|
||
if (this.targetFps > 15) {
|
||
this.targetFps = 15;
|
||
console.log(`[AUTOPILOT] 🚨 SEVERE CONGESTION! Quality→50%, Res→50%, FPS→15`);
|
||
}
|
||
}
|
||
}, 2000);
|
||
}
|
||
|
||
/**
|
||
* Set FPS manual dari Matrix UI
|
||
* @param fps - 15, 30, atau 60
|
||
* Setelah di-set manual, AutoPilot FPS probe di-nonaktifkan
|
||
*/
|
||
public setFps(fps: 15 | 30 | 60): void {
|
||
this.targetFps = fps;
|
||
this.fpsConfirmed60 = fps === 60; // Jika manual 60, tandai confirmed
|
||
this.fpsProbeCount = 999; // Disable auto probe
|
||
console.log(`[MATRIX] FPS manual di-set ke ${fps}fps oleh user`);
|
||
// Re-configure video track
|
||
try {
|
||
const videoTrack = this.mediaStream?.getVideoTracks()[0];
|
||
if (videoTrack) {
|
||
videoTrack.applyConstraints({ frameRate: { ideal: fps } }).catch(() => {});
|
||
}
|
||
} catch(_e) {}
|
||
}
|
||
|
||
private handleVideoChunk(chunk: any, metadata: any) {
|
||
if ((this.ws && this.ws.readyState === WebSocket.OPEN) || this.streamWriter) {
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN && this.ws.bufferedAmount > 65536) {
|
||
// Jaringan kewalahan (backlog TCP menumpuk). Buang frame delta!
|
||
if (chunk.type !== "key") return;
|
||
}
|
||
const chunkData = new Uint8Array(chunk.byteLength);
|
||
chunk.copyTo(chunkData);
|
||
const packetLen = 8 + chunkData.length;
|
||
const packet = new Uint8Array(packetLen);
|
||
packet[0] = chunk.type === "key" ? 3 : 4;
|
||
packet[1] = 0;
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
view.setUint32(4, packetLen - 8, true);
|
||
packet.set(chunkData, 8);
|
||
this.sendFrame(packet);
|
||
}
|
||
}
|
||
|
||
private handleAudioChunk(chunk: any) {
|
||
if ((this.ws && this.ws.readyState === WebSocket.OPEN) || this.streamWriter) {
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN && this.ws.bufferedAmount > 131072) {
|
||
// Jika antrean > 128KB, buang audio juga untuk reset TCP
|
||
return;
|
||
}
|
||
const chunkData = new Uint8Array(chunk.byteLength);
|
||
chunk.copyTo(chunkData);
|
||
const packetLen = 8 + 1 + chunkData.length;
|
||
const packet = new Uint8Array(packetLen);
|
||
packet[0] = 2; // FRAME_AUDIO
|
||
packet[1] = 1;
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
view.setUint32(4, packetLen - 8, true);
|
||
packet[8] = 1; // Opus Codec ID
|
||
packet.set(chunkData, 9);
|
||
this.sendFrame(packet);
|
||
}
|
||
}
|
||
|
||
public async deactivateUplink() {
|
||
console.log("[QUANTUM UPLINK] Mematikan Kamera (Downlink tetap aktif)...");
|
||
if (this.mediaStream) {
|
||
this.mediaStream.getTracks().forEach((t) => {
|
||
t.stop();
|
||
});
|
||
this.mediaStream = null;
|
||
}
|
||
if (this.videoEncoder && this.videoEncoder.state !== "closed") {
|
||
this.videoEncoder.close();
|
||
this.videoEncoder = null;
|
||
}
|
||
if (this.trackProcessor) {
|
||
try { /* Let GC handle trackProcessor */ } catch(e){}
|
||
this.trackProcessor = null;
|
||
}
|
||
}
|
||
|
||
private async encodeLoop(reader: ReadableStreamDefaultReader) {
|
||
while (this.isRunning) {
|
||
try {
|
||
const { done, value: frame } = await reader.read();
|
||
if (done || !frame) break;
|
||
|
||
if (this.videoEncoder && this.videoEncoder.state === "configured") {
|
||
if (this.videoEncoder.encodeQueueSize > 2) {
|
||
// Drop frame before encoding to prevent queue buildup (CPU bottleneck)
|
||
console.warn("[QUANTUM ENCODER] Skipping frame to prevent latency");
|
||
} else {
|
||
this.videoEncoder.encode(frame, { keyFrame: this._frameCount % 30 === 0 });
|
||
}
|
||
this._frameCount++;
|
||
}
|
||
if (frame) frame.close();
|
||
} catch {
|
||
// Stream terputus atau kamera dimatikan
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
private async startReceiver() {
|
||
if (!this.transport) return;
|
||
|
||
try {
|
||
console.log("[QUANTUM MATRIX] Menciptakan QUIC Bidirectional Stream...");
|
||
// Klien yang berinisiatif membuka Bi-Directional Stream pertama kali
|
||
this.stream = await this.transport.createBidirectionalStream();
|
||
this.streamWriter = this.stream.writable.getWriter();
|
||
this.streamReader = this.stream.readable.getReader();
|
||
|
||
let buffer = new Uint8Array(0);
|
||
|
||
while (this.isRunning) {
|
||
const { value, done } = await this.streamReader.read();
|
||
if (done) {
|
||
console.log("QUIC Stream Ditutup oleh Server.");
|
||
break;
|
||
}
|
||
|
||
if (value) {
|
||
// Gabungkan chunk yang baru datang ke dalam buffer
|
||
const newBuffer = new Uint8Array(buffer.length + value.length);
|
||
newBuffer.set(buffer);
|
||
newBuffer.set(value, buffer.length);
|
||
buffer = newBuffer;
|
||
|
||
// Ekstrak Frame berdasarkan 8-byte header
|
||
while (buffer.length >= 8) {
|
||
const frameType = buffer[0];
|
||
const quality = buffer[1];
|
||
const view = new DataView(
|
||
buffer.buffer,
|
||
buffer.byteOffset,
|
||
buffer.byteLength,
|
||
);
|
||
const senderId = view.getUint16(2, true);
|
||
const frameLength = view.getUint32(4, true);
|
||
|
||
if (buffer.length >= 8 + frameLength) {
|
||
const payloadData = buffer.slice(8, 8 + frameLength);
|
||
buffer = buffer.slice(8 + frameLength);
|
||
|
||
try {
|
||
if (senderId === this.participantId) continue;
|
||
this.participantLastSeen.set(senderId, Date.now());
|
||
|
||
if (frameType === FRAME_LEDGER) {
|
||
const textDecoder = new TextDecoder();
|
||
const payloadStr = textDecoder.decode(payloadData);
|
||
|
||
// Intersepsi PKEPX Signals
|
||
try {
|
||
const sig = JSON.parse(payloadStr);
|
||
if (sig.pkepxType === 'SOVEREIGN') {
|
||
this.handleSovereignSignal(sig, senderId);
|
||
continue;
|
||
} else if (sig.pkepxType === 'RESONANCE') {
|
||
if (this.onQuantumResonance) this.onQuantumResonance(senderId, sig.payload);
|
||
continue;
|
||
}
|
||
} catch(e) {}
|
||
|
||
if (this.onQuantumDataReceived) {
|
||
this.onQuantumDataReceived(senderId, payloadStr);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (frameType === FRAME_CONTROL && quality === 2) {
|
||
if (this.onActiveSpeakerChanged) {
|
||
this.onActiveSpeakerChanged(senderId);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (frameType === FRAME_HEARTBEAT) {
|
||
if (!this.pulsarDecoders.has(senderId) && !this.videoDecoders.has(senderId)) {
|
||
console.log("[PRESENCE] Remote participant detected via heartbeat:", senderId);
|
||
if (this.onParticipantJoined) {
|
||
this.onParticipantJoined(senderId);
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (frameType === FRAME_PULSAR) {
|
||
// JPEG image frame - decode with Image API (works everywhere)
|
||
let jpegPayload = payloadData;
|
||
let isDecryptionFailed = false;
|
||
|
||
if (quality === 2) {
|
||
if (this.e2eeKey && payloadData.length > 12) {
|
||
const iv = payloadData.slice(0, 12);
|
||
const cipher = payloadData.slice(12);
|
||
try {
|
||
const plainBuf = await window.crypto.subtle.decrypt(
|
||
{ name: "AES-GCM", iv: iv },
|
||
this.e2eeKey,
|
||
cipher
|
||
);
|
||
jpegPayload = new Uint8Array(plainBuf);
|
||
} catch (_e) {
|
||
isDecryptionFailed = true;
|
||
}
|
||
} else {
|
||
isDecryptionFailed = true; // No key to decrypt
|
||
}
|
||
}
|
||
|
||
if (isDecryptionFailed) {
|
||
// Render Quantum Noise (TV Semut)
|
||
this.tryAutoRegisterCanvas(senderId);
|
||
const ctx = this.canvasCtxMap.get(senderId);
|
||
if (ctx) {
|
||
const w = ctx.canvas.width || 640;
|
||
const h = ctx.canvas.height || 360;
|
||
const idata = ctx.createImageData(w, h);
|
||
const d32 = new Uint32Array(idata.data.buffer);
|
||
for(let i=0; i<d32.length; i++) {
|
||
d32[i] = Math.random() < 0.5 ? 0xff000000 : 0xffffffff;
|
||
}
|
||
ctx.putImageData(idata, 0, 0);
|
||
// Draw red lock
|
||
ctx.fillStyle = 'red';
|
||
ctx.font = 'bold 24px sans-serif';
|
||
ctx.fillText("🔒 E2EE ENCRYPTED", 20, 40);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (jpegPayload.length < 100) continue;
|
||
|
||
if (!this.participantLastSeen.has(senderId) || !this.canvasCtxMap.has(senderId)) {
|
||
if (this.onParticipantJoined) {
|
||
this.onParticipantJoined(senderId);
|
||
}
|
||
}
|
||
|
||
const jpegBlob = new Blob([jpegPayload], { type: "image/jpeg" });
|
||
const bmpUrl = URL.createObjectURL(jpegBlob);
|
||
const img = new Image();
|
||
img.onload = () => {
|
||
this.tryAutoRegisterCanvas(senderId);
|
||
const jitter = this.audioJitterMap.get(senderId) || 50;
|
||
let queue = this.videoRenderQueue.get(senderId);
|
||
if (!queue) { queue = []; this.videoRenderQueue.set(senderId, queue); }
|
||
queue.push({
|
||
frame: img,
|
||
targetTime: performance.now() + jitter,
|
||
isWebCodec: false
|
||
});
|
||
URL.revokeObjectURL(bmpUrl);
|
||
};
|
||
img.src = bmpUrl;
|
||
continue;
|
||
}
|
||
|
||
if (frameType === 2) { // FRAME_AUDIO
|
||
let audioPayload = payloadData;
|
||
let isDecryptionFailed = false;
|
||
|
||
if (quality === 2) {
|
||
if (this.e2eeKey && payloadData.length > 12) {
|
||
const iv = payloadData.slice(0, 12);
|
||
const cipher = payloadData.slice(12);
|
||
try {
|
||
const plainBuf = await window.crypto.subtle.decrypt(
|
||
{ name: "AES-GCM", iv: iv },
|
||
this.e2eeKey,
|
||
cipher
|
||
);
|
||
audioPayload = new Uint8Array(plainBuf);
|
||
} catch (_e) {
|
||
isDecryptionFailed = true;
|
||
}
|
||
} else {
|
||
isDecryptionFailed = true;
|
||
}
|
||
}
|
||
|
||
if (isDecryptionFailed) {
|
||
// Quantum Noise Audio (White Noise Intelijen)
|
||
if (audioPayload.length > 4) {
|
||
const fakePayload = new Uint8Array(audioPayload.length);
|
||
fakePayload.set(audioPayload.slice(0, 4), 0); // Keep header
|
||
for (let i = 4; i < fakePayload.length; i++) {
|
||
fakePayload[i] = Math.floor(Math.random() * 255);
|
||
}
|
||
audioPayload = fakePayload;
|
||
} else {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
if (!this.downlinkAudioCtx) {
|
||
this.initDownlinkAudio();
|
||
(this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx.currentTime;
|
||
}
|
||
// Jangan panggil resume di sini karena bukan berasal dari user gesture!
|
||
const ctx = this.downlinkAudioCtx as any;
|
||
|
||
if (audioPayload.length <= 4) continue;
|
||
|
||
// SAFE PARSING via DataView
|
||
const dv = new DataView(audioPayload.buffer, audioPayload.byteOffset, audioPayload.byteLength);
|
||
const senderSampleRate = dv.getUint32(0, true) || 48000;
|
||
|
||
const numSamples = Math.floor((audioPayload.length - 4) / 2);
|
||
const float32 = new Float32Array(numSamples);
|
||
|
||
let sum = 0;
|
||
for (let i = 0; i < numSamples; i++) {
|
||
const intSample = dv.getInt16(4 + i * 2, true);
|
||
float32[i] = intSample / 32768.0;
|
||
if (i % 10 === 0) sum += Math.abs(float32[i]);
|
||
}
|
||
|
||
// The buffer uses the exact sender sample rate, the hardware context will auto-resample
|
||
const audioBuffer = ctx.createBuffer(1, numSamples, senderSampleRate);
|
||
audioBuffer.getChannelData(0).set(float32);
|
||
const source = ctx.createBufferSource();
|
||
source.buffer = audioBuffer;
|
||
if (this.downlinkAudioDest) {
|
||
source.connect(this.downlinkAudioDest);
|
||
} else {
|
||
source.connect(ctx.destination);
|
||
}
|
||
|
||
const delay = ctx.nextPlayTime - ctx.currentTime;
|
||
this.audioJitterMap.set(senderId, Math.max(0, delay * 1000));
|
||
if (delay > 0.15) {
|
||
// EXTREME TUNE-UP: Jika delay > 150ms, BUANG paket ini agar tetap real-time & cegah GEMURUH!
|
||
ctx.nextPlayTime = ctx.currentTime + 0.05;
|
||
continue; // Drop packet
|
||
}
|
||
if (ctx.nextPlayTime < ctx.currentTime) {
|
||
ctx.nextPlayTime = ctx.currentTime + 0.05; // 50ms initial buffer
|
||
}
|
||
source.start(ctx.nextPlayTime);
|
||
ctx.nextPlayTime += audioBuffer.duration;
|
||
|
||
if (this.onAudioLevel) {
|
||
this.onAudioLevel(Math.min(100, (sum / (numSamples/10)) * 500));
|
||
}
|
||
if (this.onActiveSpeakerChanged && sum > 5) {
|
||
this.onActiveSpeakerChanged(senderId);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (frameType === FRAME_VIDEO_DELTA || frameType === FRAME_VIDEO_KEY) {
|
||
if (!this.videoDecoders.has(senderId)) {
|
||
this.createDecoderForParticipant(senderId);
|
||
}
|
||
if (frameType === FRAME_VIDEO_DELTA && !this.firstKeyFrameReceived.get(senderId)) {
|
||
continue;
|
||
}
|
||
if (frameType === FRAME_VIDEO_KEY) {
|
||
this.firstKeyFrameReceived.set(senderId, true);
|
||
}
|
||
this.tryAutoRegisterCanvas(senderId);
|
||
const decoder = this.videoDecoders.get(senderId);
|
||
if (decoder && decoder.state === "configured") {
|
||
if (decoder.decodeQueueSize > 5) {
|
||
if (frameType === FRAME_VIDEO_DELTA) {
|
||
// DROP delta frame if decoder is backed up to catch up to real-time!
|
||
this.firstKeyFrameReceived.set(senderId, false);
|
||
continue;
|
||
}
|
||
}
|
||
const chunk = new EncodedVideoChunk({
|
||
type: frameType === FRAME_VIDEO_KEY ? "key" : "delta",
|
||
timestamp: performance.now() * 1000,
|
||
data: payloadData,
|
||
});
|
||
decoder.decode(chunk);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (frameType === 99) { // QUANTUM CIPHER GATE: WASM BYTECODE INJECTION
|
||
// Asymmetric Bytecode Streaming dari XCU Core
|
||
const dv2 = new DataView(payloadData.buffer, payloadData.byteOffset, payloadData.byteLength);
|
||
const moduleId = dv2.getUint16(2, true);
|
||
const wasmBytes = payloadData.slice(4); // Payload murni WebAssembly
|
||
console.log(`[QUANTUM MATRIX] QCG: Menerima Injeksi Bytecode Modul ${moduleId} (${wasmBytes.byteLength} bytes). Merakit...`);
|
||
|
||
try {
|
||
const _module = await WebAssembly.instantiate(wasmBytes, {});
|
||
console.log(`[QUANTUM MATRIX] QCG: Modul ${moduleId} Berhasil Dirakit di Memori!`);
|
||
if (moduleId === 43) {
|
||
this.pulsarCodec = module.instance.exports;
|
||
this.pulsarWasmMemory = this.pulsarCodec.memory;
|
||
}
|
||
if (this.onModuleUnlocked) {
|
||
this.onModuleUnlocked(moduleId);
|
||
}
|
||
} catch (_e) {
|
||
console.error(`[QUANTUM MATRIX] QCG: Gagal merakit modul ${moduleId} (Hack Attempt?)`, e);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
} catch (frameErr) {
|
||
console.error("[QUANTUM DECODER] Frame parse error (ignored):", frameErr);
|
||
}
|
||
} else {
|
||
// Butuh lebih banyak byte untuk membentuk frame utuh
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch (e: unknown) {
|
||
console.error("[QUANTUM MATRIX] Kegagalan Aliran (Stream Failure):", e);
|
||
}
|
||
}
|
||
|
||
// Fase 37: Voice Activity Detection (VAD) Loop - QUANTUM WASM EDITION
|
||
private startVadLoop() {
|
||
try {
|
||
if (!this.uplinkAudioCtx || !this.uplinkAudioSource) return;
|
||
|
||
// Jika WASM Modul 43 belum turun, gunakan Analyser JS biasa sebagai fallback
|
||
const useWasm = this.pulsarCodec && this.pulsarCodec.pulsar_vad && this.pulsarWasmMemory;
|
||
|
||
const analyser = this.uplinkAudioCtx.createAnalyser();
|
||
analyser.fftSize = 1024; // Gunakan 1024 frame agar cukup untuk WASM
|
||
this.uplinkAudioSource.connect(analyser);
|
||
|
||
const jsDataArray = new Uint8Array(analyser.frequencyBinCount);
|
||
const f32DataArray = new Float32Array(analyser.fftSize);
|
||
let speakingFrames = 0;
|
||
|
||
setInterval(() => {
|
||
let energyScore = 0;
|
||
|
||
if (useWasm) {
|
||
// 1. Eksekusi Asimetris: Dapatkan Pointer Memori WASM
|
||
analyser.getFloatTimeDomainData(f32DataArray);
|
||
|
||
const bufferPtr = this.pulsarCodec.get_buffer_ptr();
|
||
const capacity = this.pulsarCodec.get_buffer_capacity();
|
||
|
||
// 2. Tembus Batas RAM: Tulis gelombang suara murni (PCM f32) ke memori WebAssembly
|
||
const wasmMemArray = new Float32Array(this.pulsarWasmMemory!.buffer, bufferPtr, capacity);
|
||
const copyLen = Math.min(f32DataArray.length, capacity);
|
||
wasmMemArray.set(f32DataArray.subarray(0, copyLen));
|
||
|
||
// 3. Kalkulasi Matematis oleh Sang Algojo Rust
|
||
energyScore = this.pulsarCodec.pulsar_vad(copyLen);
|
||
|
||
} else {
|
||
// Fallback Mode JS
|
||
analyser.getByteFrequencyData(jsDataArray);
|
||
const sum = jsDataArray.reduce((a, b) => a + b, 0);
|
||
energyScore = sum / jsDataArray.length;
|
||
}
|
||
|
||
// Tampilkan ke UI
|
||
if (this.onAudioLevel) {
|
||
// Wasm score: 0.0 to ~10.0+. JS score: 0 to 255.
|
||
const normalized = useWasm ? Math.min(100, energyScore * 10) : energyScore;
|
||
this.onAudioLevel(normalized);
|
||
}
|
||
|
||
// Ambang Deteksi
|
||
const threshold = useWasm ? 0.5 : 5;
|
||
if (energyScore > threshold) {
|
||
speakingFrames++;
|
||
if (speakingFrames === 3) {
|
||
if (this.streamWriter && this.isRunning) {
|
||
this.sendDirectorSignal(2);
|
||
}
|
||
}
|
||
} else {
|
||
speakingFrames = 0;
|
||
}
|
||
}, 100);
|
||
} catch (e: unknown) {
|
||
console.warn(
|
||
"[VAD] Audio Context tidak didukung atau dicekal browser.",
|
||
e,
|
||
);
|
||
}
|
||
}
|
||
|
||
private async sendDirectorSignal(actionQuality: number) {
|
||
if (!this.streamWriter) return;
|
||
const packet = new Uint8Array(8);
|
||
packet[0] = 3; // Control
|
||
packet[1] = actionQuality;
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
view.setUint32(4, 0, true);
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); }
|
||
}
|
||
|
||
// Fase 34: Zero-CPU Omniscient Recording
|
||
public async triggerQuantumRecording() {
|
||
if (!this.streamWriter) return;
|
||
const packet = new Uint8Array(8);
|
||
packet[0] = 3; // Control
|
||
packet[1] = 6; // Set Action: Record
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
view.setUint32(4, 0, true);
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); }
|
||
console.log(
|
||
"[VAULT MATRIX] Sinyal Perekaman Kuantum Langsung Ke SSD Server Dikirim.",
|
||
);
|
||
}
|
||
|
||
// Fase 35: SVC Downgrade (Anti-Lag Manual/Otomatis)
|
||
public async activateAntiLagDowngrade() {
|
||
if (this.streamWriter) {
|
||
console.log("[SVC MATRIX] Mengirim sinyal Downgrade ke Server Rust...");
|
||
const controlPacket = new Uint8Array(8);
|
||
controlPacket[0] = 3; // Type: Control
|
||
controlPacket[1] = 0; // Quality: Low (Drop paket 1080p)
|
||
const view = new DataView(controlPacket.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
view.setUint32(4, 0, true); // Payload Length: 0
|
||
|
||
await this.streamWriter.write(controlPacket);
|
||
}
|
||
}
|
||
|
||
// Fase 39: Webinar Asimetris (Audience)
|
||
public async joinAsAudience() {
|
||
this.participantRole = "AUDIENCE";
|
||
await this.sendRoleSignal();
|
||
console.log(
|
||
"[WEBINAR MATRIX] Beralih ke Mode AUDIENCE (Uplink Dimatikan).",
|
||
);
|
||
}
|
||
|
||
private async sendRoleSignal() {
|
||
if (!this.streamWriter) return;
|
||
const packet = new Uint8Array(8);
|
||
packet[0] = 3; // Control
|
||
packet[1] = 4; // Set Role
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
packet[4] = this.participantRole === "AUDIENCE" ? 1 : 0; // 1=Audience, 0=Panelist
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); }
|
||
}
|
||
|
||
// Fase 39: Podcast Mode (Matikan Video di Routing Level)
|
||
public async setPodcastMode(isPodcast: boolean) {
|
||
if (!this.streamWriter) return;
|
||
const packet = new Uint8Array(8);
|
||
packet[0] = 3; // Control
|
||
packet[1] = 5; // Set Mode
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
packet[4] = isPodcast ? 1 : 0; // 1=Podcast, 0=Webinar
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); }
|
||
console.log(
|
||
`[PODCAST MATRIX] Sinyal ${isPodcast ? "PODCAST" : "WEBINAR"} dikirim ke Core.`,
|
||
);
|
||
}
|
||
|
||
// Fase 40: The Quantum Ledger (Pengganti WebSocket Node.js)
|
||
// Menembakkan Chat, JSON, Emoji, Koordinat langsung via QUIC
|
||
public async sendQuantumData(payload: string) {
|
||
if (!this.streamWriter) return;
|
||
|
||
const textEncoder = new TextEncoder();
|
||
const payloadBytes = textEncoder.encode(payload);
|
||
|
||
const packet = new Uint8Array(8 + payloadBytes.byteLength);
|
||
packet[0] = 6; // Type: Ledger Data
|
||
packet[1] = 0; // Quality: N/A
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
view.setUint32(4, payloadBytes.byteLength, true);
|
||
|
||
packet.set(payloadBytes, 8);
|
||
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(packet); }
|
||
}
|
||
|
||
|
||
// === DOWNLINK AUDIO PLAYBACK (EXTREME FIX v2) ===
|
||
// Uplink format: [sampleRate: Uint32LE 4 bytes] + [PCM Int16 data]
|
||
private playRemoteAudio(audioPayload: Uint8Array) {
|
||
try {
|
||
if (!this.downlinkAudioCtx) {
|
||
this.initDownlinkAudio();
|
||
(this.downlinkAudioCtx as any).nextPlayTime = this.downlinkAudioCtx.currentTime;
|
||
}
|
||
if (this.downlinkAudioCtx.state === 'suspended') {
|
||
this.downlinkAudioCtx.resume();
|
||
}
|
||
|
||
// Payload: [sampleRate(4 bytes LE)] + [PCM Int16 raw bytes]
|
||
if (audioPayload.byteLength < 6) return;
|
||
const dv = new DataView(audioPayload.buffer, audioPayload.byteOffset, audioPayload.byteLength);
|
||
const sampleRate = dv.getUint32(0, true) || 16000;
|
||
const int16Data = audioPayload.slice(4);
|
||
|
||
if (int16Data.byteLength < 2) return;
|
||
|
||
// Convert Int16 → Float32
|
||
const int16 = new Int16Array(int16Data.buffer, int16Data.byteOffset, int16Data.byteLength / 2);
|
||
const numSamples = int16.length;
|
||
if (numSamples < 1) return;
|
||
|
||
const float32 = new Float32Array(numSamples);
|
||
for (let i = 0; i < numSamples; i++) {
|
||
float32[i] = int16[i] / 32767;
|
||
}
|
||
|
||
// Create audio buffer at source sample rate and play at device rate
|
||
const audioBuffer = this.downlinkAudioCtx.createBuffer(1, numSamples, sampleRate);
|
||
audioBuffer.getChannelData(0).set(float32);
|
||
|
||
const source = this.downlinkAudioCtx.createBufferSource();
|
||
source.buffer = audioBuffer;
|
||
|
||
// ULTRA AUDIO: Gain boost for clarity (1.5x volume)
|
||
const gainNode = this.downlinkAudioCtx.createGain();
|
||
gainNode.gain.value = 1.5;
|
||
source.connect(gainNode);
|
||
if (this.downlinkAudioDest) {
|
||
gainNode.connect(this.downlinkAudioDest);
|
||
} else {
|
||
gainNode.connect(this.downlinkAudioCtx.destination);
|
||
}
|
||
|
||
// ULTRA Jitter Buffer: Smoother scheduling with 250ms tolerance
|
||
const ctx = this.downlinkAudioCtx as any;
|
||
const now = this.downlinkAudioCtx.currentTime;
|
||
let playAt = ctx.nextPlayTime || now;
|
||
|
||
const delay = playAt - now;
|
||
if (delay > 0.25) {
|
||
// Buffer too full — reset to near-realtime (50ms ahead)
|
||
playAt = now + 0.03;
|
||
ctx.nextPlayTime = playAt;
|
||
} else if (delay < -0.05) {
|
||
// We're behind — catch up smoothly
|
||
playAt = now + 0.02;
|
||
}
|
||
if (playAt < now) {
|
||
playAt = now + 0.02;
|
||
}
|
||
|
||
source.start(playAt);
|
||
ctx.nextPlayTime = playAt + audioBuffer.duration;
|
||
} catch (e) {
|
||
// Silently ignore audio playback errors
|
||
}
|
||
}
|
||
|
||
// ==========================================
|
||
// PKEPX (PANCA KONSTITUSI) ZOOM-KILLER FEATURES
|
||
// ==========================================
|
||
|
||
/**
|
||
* PKEPX Sovereign Controls: Host Broadcast
|
||
*/
|
||
public broadcastSovereignSignal(type: 'MUTE_ALL' | 'KICK_PEER' | 'LOCK_MATRIX', targetId?: number) {
|
||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
||
const payload = JSON.stringify({
|
||
pkepxType: 'SOVEREIGN',
|
||
command: type,
|
||
targetId: targetId
|
||
});
|
||
this.sendQuantumData(payload);
|
||
console.log(`[PKEPX SOVEREIGN] Sinyal ${type} dipancarkan ke matriks.`);
|
||
}
|
||
|
||
private handleSovereignSignal(sig: any, senderId: number) {
|
||
console.log(`[PKEPX SOVEREIGN] Menerima titah dari ${senderId}:`, sig.command);
|
||
if (sig.command === 'MUTE_ALL') {
|
||
this.toggleMic(false);
|
||
if (this.onSovereignSignal) this.onSovereignSignal('MUTED_BY_HOST');
|
||
} else if (sig.command === 'KICK_PEER' && sig.targetId === this.participantId) {
|
||
console.warn('[PKEPX SOVEREIGN] Anda diusir dari matriks!');
|
||
this.shutdown();
|
||
if (this.onSovereignSignal) this.onSovereignSignal('KICKED_BY_HOST');
|
||
} else if (sig.command === 'LOCK_MATRIX') {
|
||
if (this.onSovereignSignal) this.onSovereignSignal('MATRIX_LOCKED');
|
||
}
|
||
}
|
||
|
||
/**
|
||
* PKEPX Quantum Resonance: Emit 3D Reaction
|
||
*/
|
||
public emitResonance(reactionType: string) {
|
||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
||
const payload = JSON.stringify({
|
||
pkepxType: 'RESONANCE',
|
||
payload: reactionType
|
||
});
|
||
this.sendQuantumData(payload);
|
||
if (this.onQuantumResonance) this.onQuantumResonance(this.participantId, reactionType);
|
||
}
|
||
|
||
/**
|
||
* BUG FIX: Send presence heartbeat so other participants detect us immediately.
|
||
* This sends a FRAME_HEARTBEAT (type=5) with our display name.
|
||
*/
|
||
public sendPresenceHeartbeat() {
|
||
const namePayload = this.displayName
|
||
? new TextEncoder().encode(JSON.stringify({ type: 'NAME_ANNOUNCE', name: this.displayName }))
|
||
: new Uint8Array(0);
|
||
|
||
const packet = new Uint8Array(8 + namePayload.length);
|
||
packet[0] = 5; // FRAME_HEARTBEAT
|
||
packet[1] = 0;
|
||
const view = new DataView(packet.buffer);
|
||
view.setUint16(2, this.participantId, true);
|
||
view.setUint32(4, namePayload.length, true);
|
||
if (namePayload.length > 0) packet.set(namePayload, 8);
|
||
|
||
this.sendFrame(packet);
|
||
console.log(`[QUANTUM HEARTBEAT] Presence sent: ${this.displayName || 'ID:' + this.participantId}`);
|
||
}
|
||
|
||
/**
|
||
* BUG FIX: Periodic heartbeat loop (every 3s) so participants stay visible
|
||
* even without camera/mic active.
|
||
*/
|
||
private startHeartbeatLoop() {
|
||
if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
|
||
this.heartbeatTimer = setInterval(() => {
|
||
this.sendPresenceHeartbeat();
|
||
}, 3000);
|
||
}
|
||
|
||
/**
|
||
* Announce display name to all participants in the room.
|
||
* Sent via FRAME_CONTROL (type 3) as JSON payload.
|
||
*/
|
||
public announceDisplayName() {
|
||
if (!this.displayName) return;
|
||
const payload = JSON.stringify({
|
||
pkepxType: 'NAME_ANNOUNCE',
|
||
name: this.displayName,
|
||
participantId: this.participantId
|
||
});
|
||
this.sendQuantumData(payload);
|
||
}
|
||
|
||
/**
|
||
* Set/rename display name (like Zoom rename).
|
||
* Immediately broadcasts to all participants.
|
||
*/
|
||
public setDisplayName(newName: string) {
|
||
this.displayName = newName;
|
||
this.announceDisplayName();
|
||
console.log(`[QUANTUM MATRIX] Display name set to: ${newName}`);
|
||
}
|
||
|
||
/**
|
||
* PKEPX Fractal Matrix: Breakout Rooms Warp Engine
|
||
*/
|
||
public async warpToMatrix(newRoomId: string) {
|
||
console.log(`[PKEPX FRACTAL WARP] Melompat ke Sub-Matrix: ${newRoomId}`);
|
||
this.shutdown(); // Close current QUIC/WS properly
|
||
// Re-ignite after 500ms to allow TCP flush
|
||
setTimeout(() => {
|
||
this.ignite(newRoomId, "/xcu-engine").then(() => {
|
||
if (this.onSovereignSignal) this.onSovereignSignal('WARP_COMPLETE', newRoomId);
|
||
}).catch(e => console.error("Warp Gagal:", e));
|
||
}, 500);
|
||
}
|
||
|
||
public disconnect() {
|
||
this.isRunning = false;
|
||
this.stopVideoSyncLoop();
|
||
if (this.downlinkAudioEl) {
|
||
this.downlinkAudioEl.remove();
|
||
this.downlinkAudioEl = null;
|
||
}
|
||
if (this.ws) {
|
||
try { this.ws.close(1000, 'LEAVING'); } catch(_) { this.ws.close(); }
|
||
this.ws = null;
|
||
}
|
||
if (this.transport) {
|
||
this.transport.close();
|
||
}
|
||
this.videoDecoders.forEach((decoder) => {
|
||
if (decoder.state !== "closed") decoder.close();
|
||
});
|
||
this.videoDecoders.clear();
|
||
this.canvasCtxMap.clear();
|
||
if (this.videoEncoder && this.videoEncoder.state !== "closed") {
|
||
this.videoEncoder.close();
|
||
}
|
||
if (this.mediaStream) {
|
||
this.mediaStream.getTracks().forEach((t) => {
|
||
t.stop();
|
||
});
|
||
}
|
||
console.log("[QUANTUM MATRIX] Sistem Terputus.");
|
||
}
|
||
|
||
public shutdown() {
|
||
this.isRunning = false;
|
||
this.stopVideoSyncLoop();
|
||
if (this.downlinkAudioEl) {
|
||
this.downlinkAudioEl.remove();
|
||
this.downlinkAudioEl = null;
|
||
}
|
||
if (this.natPingInterval) clearInterval(this.natPingInterval);
|
||
this.stopVAD();
|
||
if (this.heartbeatTimer) { clearInterval(this.heartbeatTimer); this.heartbeatTimer = null; }
|
||
if (this.ghostCleanupInterval) { clearInterval(this.ghostCleanupInterval); this.ghostCleanupInterval = null; }
|
||
if (this.ws) {
|
||
try { this.ws.close(1000, 'LEAVING'); } catch(_) { this.ws.close(); }
|
||
this.ws = null;
|
||
}
|
||
if (this.transport) {
|
||
this.transport.close();
|
||
}
|
||
this.videoDecoders.forEach((decoder) => {
|
||
if (decoder.state !== "closed") decoder.close();
|
||
});
|
||
this.videoDecoders.clear();
|
||
this.canvasCtxMap.clear();
|
||
if (this.videoEncoder && this.videoEncoder.state !== "closed") {
|
||
this.videoEncoder.close();
|
||
}
|
||
if (this.mediaStream) {
|
||
this.mediaStream.getTracks().forEach((t) => {
|
||
t.stop();
|
||
});
|
||
}
|
||
console.log("[QUANTUM MATRIX] Sistem Terputus.");
|
||
}
|
||
|
||
private startVAD() {
|
||
if (!this.mediaStream) return;
|
||
const audioTracks = this.mediaStream.getAudioTracks();
|
||
if (audioTracks.length === 0) return;
|
||
|
||
try {
|
||
const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext;
|
||
if (!AudioContextClass) return;
|
||
this.vadAudioCtx = new AudioContextClass();
|
||
if (this.vadAudioCtx.state === "suspended") {
|
||
this.vadAudioCtx.resume().catch(() => {});
|
||
}
|
||
const source = this.vadAudioCtx.createMediaStreamSource(new MediaStream([audioTracks[0]]));
|
||
this.vadAnalyser = this.vadAudioCtx.createAnalyser();
|
||
this.vadAnalyser.fftSize = 256;
|
||
source.connect(this.vadAnalyser);
|
||
|
||
const dataArray = new Uint8Array(this.vadAnalyser.frequencyBinCount);
|
||
|
||
this.vadInterval = setInterval(() => {
|
||
if (!this.vadAnalyser) return;
|
||
this.vadAnalyser.getByteFrequencyData(dataArray);
|
||
let sum = 0;
|
||
for (let i = 0; i < dataArray.length; i++) {
|
||
sum += dataArray[i];
|
||
}
|
||
const average = sum / dataArray.length;
|
||
let level = average / 128.0;
|
||
if (level > 1.0) level = 1.0;
|
||
|
||
if (this.onAudioLevel) {
|
||
this.onAudioLevel(this.isMicMuted ? 0 : level);
|
||
}
|
||
}, 100);
|
||
} catch (e) {
|
||
console.warn("[VAD] Failed to initialize Voice Activity Detection", e);
|
||
}
|
||
}
|
||
|
||
private stopVAD() {
|
||
if (this.vadInterval) {
|
||
clearInterval(this.vadInterval);
|
||
this.vadInterval = null;
|
||
}
|
||
if (this.vadAudioCtx) {
|
||
this.vadAudioCtx.close().catch(() => {});
|
||
this.vadAudioCtx = null;
|
||
}
|
||
this.vadAnalyser = null;
|
||
}
|
||
}
|