media-handler.js

/**
 * MediaHandler: Manages Audio/Video capture and playback
 */
class MediaHandler {
  constructor() {
    this.audioContext = null;
    this.mediaStream = null;
    this.audioWorkletNode = null;
    this.videoStream = null;
    this.videoInterval = null;
    this.nextStartTime = 0;
    this.scheduledSources = [];
    this.isRecording = false;
    this.videoCanvas = document.createElement('canvas');
    this.canvasCtx = this.videoCanvas.getContext('2d');
  }

  async initializeAudio() {
    if (!this.audioContext) {
      this.audioContext =
          new (window.AudioContext || window.webkitAudioContext)();
      await this.audioContext.audioWorklet.addModule(
          '/static/pcm-processor.js');
    }
    if (this.audioContext.state === 'suspended') {
      await this.audioContext.resume();
    }
  }

  async startAudio(onAudioData) {
    await this.initializeAudio();

    try {
      this.mediaStream = await navigator.mediaDevices.getUserMedia({
        audio: true,
      });
      const source =
          this.audioContext.createMediaStreamSource(this.mediaStream);
      this.audioWorkletNode =
          new AudioWorkletNode(this.audioContext, 'pcm-processor');

      this.audioWorkletNode.port.onmessage = (event) => {
        if (this.isRecording) {
          const downsampled = this.downsampleBuffer(
              event.data, this.audioContext.sampleRate, 16000);
          const pcm16 = this.convertFloat32ToInt16(downsampled);
          onAudioData(pcm16);
        }
      };

      source.connect(this.audioWorkletNode);
      // Mute local feedback
      const muteGain = this.audioContext.createGain();
      muteGain.gain.value = 0;
      this.audioWorkletNode.connect(muteGain);
      muteGain.connect(this.audioContext.destination);

      this.isRecording = true;
    } catch (e) {
      console.error('Error starting audio:', e);
      throw e;
    }
  }

  stopAudio() {
    this.isRecording = false;
    if (this.mediaStream) {
      this.mediaStream.getTracks().forEach((t) => t.stop());
      this.mediaStream = null;
    }
    if (this.audioWorkletNode) {
      this.audioWorkletNode.disconnect();
      this.audioWorkletNode = null;
    }
  }

  async startVideo(videoElement, onFrame) {
    try {
      this.videoStream = await navigator.mediaDevices.getUserMedia({
        video: true,
      });
      videoElement.srcObject = this.videoStream;

      this.videoInterval = setInterval(() => {
        this.captureFrame(videoElement, onFrame);
      }, 1000);  // 1 FPS
    } catch (e) {
      console.error('Error starting video:', e);
      throw e;
    }
  }

  /**
   * Start video and also upload frames to the robot backend at 5 FPS.
   * This feeds the fake backend so the robot camera pipeline works.
   */
  async startVideoWithRobotFeed(videoElement, onFrame) {
    try {
      this.videoStream = await navigator.mediaDevices.getUserMedia({
        video: true,
      });
      videoElement.srcObject = this.videoStream;

      // 1 FPS for Gemini direct image input
      this.videoInterval = setInterval(() => {
        this.captureFrame(videoElement, onFrame);
      }, 1000);

      // 5 FPS upload to fake backend via proxy
      const uploadCanvas = document.createElement('canvas');
      uploadCanvas.width = 384;
      uploadCanvas.height = 384;
      const uploadCtx = uploadCanvas.getContext('2d');

      this._robotFeedInterval = setInterval(() => {
        if (!this.videoStream) return;
        uploadCtx.drawImage(videoElement, 0, 0, 384, 384);
        uploadCanvas.toBlob(async (blob) => {
          if (!blob) return;
          try {
            await fetch('/api/upload_frame', {
              method: 'POST',
              headers: {'Content-Type': 'image/jpeg'},
              body: blob,
            });
          } catch (e) {
            // silent — backend may not be ready yet
          }
        }, 'image/jpeg', 0.8);
      }, 200);  // 5 FPS

    } catch (e) {
      console.error('Error starting video:', e);
      throw e;
    }
  }

  async startScreen(videoElement, onFrame, onEnded) {
    try {
      this.videoStream = await navigator.mediaDevices.getDisplayMedia({
        video: true,
      });
      videoElement.srcObject = this.videoStream;

      // Handle stream ending (e.g. user clicks "Stop sharing" in browser UI)
      this.videoStream.getVideoTracks()[0].onended = () => {
        this.stopVideo(videoElement);
        if (onEnded) onEnded();
      };

      this.videoInterval = setInterval(() => {
        this.captureFrame(videoElement, onFrame);
      }, 1000);  // 1 FPS
    } catch (e) {
      console.error('Error starting screen share:', e);
      throw e;
    }
  }

  stopVideo(videoElement) {
    if (this.videoStream) {
      this.videoStream.getTracks().forEach((t) => t.stop());
      this.videoStream = null;
    }
    if (this.videoInterval) {
      clearInterval(this.videoInterval);
      this.videoInterval = null;
    }
    if (this._robotFeedInterval) {
      clearInterval(this._robotFeedInterval);
      this._robotFeedInterval = null;
    }
    if (videoElement) {
      videoElement.srcObject = null;
    }
  }

  captureFrame(videoElement, onFrame) {
    if (!this.videoStream) return;
    this.videoCanvas.width = 640;
    this.videoCanvas.height = 480;
    this.canvasCtx.drawImage(videoElement, 0, 0, 640, 480);
    const base64 = this.videoCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
    onFrame(base64);
  }

  playAudio(arrayBuffer) {
    if (!this.audioContext) return;
    if (this.audioContext.state === 'suspended') {
      this.audioContext.resume();
    }

    let bufferToUse = arrayBuffer;
    if (arrayBuffer.byteLength % 2 !== 0) {
      console.warn(`Audio buffer length is odd (${
          arrayBuffer.byteLength}), slicing to even length.`);
      bufferToUse = arrayBuffer.slice(0, arrayBuffer.byteLength - 1);
    }
    const pcmData = new Int16Array(bufferToUse);
    const float32Data = new Float32Array(pcmData.length);
    for (let i = 0; i < pcmData.length; i++) {
      float32Data[i] = pcmData[i] / 32768.0;
    }

    const buffer = this.audioContext.createBuffer(1, float32Data.length, 24000);
    buffer.getChannelData(0).set(float32Data);

    const source = this.audioContext.createBufferSource();
    source.buffer = buffer;
    source.connect(this.audioContext.destination);

    const now = this.audioContext.currentTime;
    this.nextStartTime = Math.max(now, this.nextStartTime);
    source.start(this.nextStartTime);
    this.nextStartTime += buffer.duration;

    this.scheduledSources.push(source);
    source.onended = () => {
      const idx = this.scheduledSources.indexOf(source);
      if (idx > -1) this.scheduledSources.splice(idx, 1);
    };
  }

  stopAudioPlayback() {
    this.scheduledSources.forEach((s) => {
      try {
        s.stop();
      } catch (e) {
      }
    });
    this.scheduledSources = [];
    if (this.audioContext) {
      this.nextStartTime = this.audioContext.currentTime;
    }
  }

  // Utils
  downsampleBuffer(buffer, sampleRate, outSampleRate) {
    if (outSampleRate === sampleRate) return buffer;
    const ratio = sampleRate / outSampleRate;
    const newLength = Math.round(buffer.length / ratio);
    const result = new Float32Array(newLength);
    let offsetResult = 0;
    let offsetBuffer = 0;
    while (offsetResult < result.length) {
      const nextOffsetBuffer = Math.round((offsetResult + 1) * ratio);
      let accum = 0, count = 0;
      for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length;
           i++) {
        accum += buffer[i];
        count++;
      }
      result[offsetResult] = accum / count;
      offsetResult++;
      offsetBuffer = nextOffsetBuffer;
    }
    return result;
  }

  convertFloat32ToInt16(buffer) {
    let l = buffer.length;
    const buf = new Int16Array(l);
    while (l--) {
      buf[l] = Math.min(1, Math.max(-1, buffer[l])) * 0x7fff;
    }
    return buf.buffer;
  }
}