Add WebRTC configuration and allow two way audio

src/components/ha-web-rtc-player.ts

@@ -2,14 +2,21 @@ import {
   css,
   CSSResultGroup,
   html,
+  nothing,
   LitElement,
   PropertyValues,
   TemplateResult,
 } from "lit";
+import { mdiMicrophone, mdiMicrophoneOff } from "@mdi/js";
 import { customElement, property, state, query } from "lit/decorators";
 import { isComponentLoaded } from "../common/config/is_component_loaded";
 import { fireEvent } from "../common/dom/fire_event";
-import { handleWebRtcOffer, WebRtcAnswer } from "../data/camera";
+import {
+  handleWebRtcOffer,
+  WebRtcAnswer,
+  fetchWebRtcConfig,
+  closeWebRtcStream,
+} from "../data/camera";
 import { fetchWebRtcSettings } from "../data/rtsp_to_webrtc";
 import type { HomeAssistant } from "../types";
 import "./ha-alert";
@@ -48,21 +55,44 @@ class HaWebRtcPlayer extends LitElement {
 
   private _remoteStream?: MediaStream;
 
+  private _localReturnAudioTrack?: MediaStreamTrack;
+
+  public toggleMic() {
+    this._localReturnAudioTrack!.enabled =
+      !this._localReturnAudioTrack!.enabled;
+    this.requestUpdate();
+  }
+
   protected override render(): TemplateResult {
     if (this._error) {
       return html`<ha-alert alert-type="error">${this._error}</ha-alert>`;
     }
-    return html`
-      <video
-        id="remote-stream"
-        ?autoplay=${this.autoPlay}
-        .muted=${this.muted}
-        ?playsinline=${this.playsInline}
-        ?controls=${this.controls}
-        .poster=${this.posterUrl}
-        @loadeddata=${this._loadedData}
-      ></video>
-    `;
+    const video_html = html` <video
+      id="remote-stream"
+      ?autoplay=${this.autoPlay}
+      .muted=${this.muted}
+      ?playsinline=${this.playsInline}
+      ?controls=${this.controls}
+      .poster=${this.posterUrl}
+      @loadeddata=${this._loadedData}
+    ></video>`;
+    const mic_html = !this._localReturnAudioTrack
+      ? nothing
+      : html`
+          <div class="controls">
+            <mwc-button @click=${this.toggleMic} halign id="toggle_mic">
+              <ha-svg-icon
+                .path=${this._localReturnAudioTrack!.enabled
+                  ? mdiMicrophone
+                  : mdiMicrophoneOff}
+              ></ha-svg-icon>
+              ${this._localReturnAudioTrack!.enabled
+                ? "Turn off mic"
+                : "Turn on mic"}
+            </mwc-button>
+          </div>
+        `;
+    return html`${video_html}${mic_html}`;
   }
 
   public override connectedCallback() {
@@ -90,14 +120,41 @@ class HaWebRtcPlayer extends LitElement {
   private async _startWebRtc(): Promise<void> {
     this._error = undefined;
 
-    const configuration = await this._fetchPeerConfiguration();
-    const peerConnection = new RTCPeerConnection(configuration);
+    let peer_configuration;
+    const web_rtc_config = await fetchWebRtcConfig(this.hass!, this.entityid);
+    if (web_rtc_config && web_rtc_config.rtc_configuration) {
+      peer_configuration = web_rtc_config.rtc_configuration;
+    } else {
+      peer_configuration = await this._fetchPeerConfiguration();
+    }
+    const peerConnection = new RTCPeerConnection(peer_configuration);
     // Some cameras (such as nest) require a data channel to establish a stream
     // however, not used by any integrations.
     peerConnection.createDataChannel("dataSendChannel");
-    peerConnection.addTransceiver("audio", { direction: "recvonly" });
-    peerConnection.addTransceiver("video", { direction: "recvonly" });
 
+    // If the stream supports two way audio and the client is configured to allow it
+    // N.B. connections must be secure for microphone access.
+    if (
+      web_rtc_config &&
+      web_rtc_config.audio_direction === "sendrecv" &&
+      navigator.mediaDevices
+    ) {
+      const tracks = await this.getMediaTracks("user", {
+        video: false,
+        audio: true,
+      });
+      if (tracks && tracks.length > 0) {
+        this._localReturnAudioTrack = tracks[0];
+        // Start with mic off
+        this._localReturnAudioTrack.enabled = false;
+      }
+      peerConnection.addTransceiver(this._localReturnAudioTrack!, {
+        direction: "sendrecv",
+      });
+    } else {
+      peerConnection.addTransceiver("audio", { direction: "recvonly" });
+    }
+    peerConnection.addTransceiver("video", { direction: "recvonly" });
     const offerOptions: RTCOfferOptions = {
       offerToReceiveAudio: true,
       offerToReceiveVideo: true,
@@ -155,6 +212,20 @@ class HaWebRtcPlayer extends LitElement {
     this._peerConnection = peerConnection;
   }
 
+  private async getMediaTracks(media, constraints) {
+    try {
+      const stream =
+        media === "user"
+          ? await navigator.mediaDevices.getUserMedia(constraints)
+          : await navigator.mediaDevices.getDisplayMedia(constraints);
+      return stream.getTracks();
+    } catch (e) {
+      // eslint-disable-next-line no-console
+      console.warn(e);
+      return [];
+    }
+  }
+
   private async _fetchPeerConfiguration(): Promise<RTCConfiguration> {
     if (!isComponentLoaded(this.hass!, "rtsp_to_webrtc")) {
       return {};
@@ -172,13 +243,16 @@ class HaWebRtcPlayer extends LitElement {
     };
   }
 
-  private _cleanUp() {
+  private async _cleanUp() {
     if (this._remoteStream) {
       this._remoteStream.getTracks().forEach((track) => {
         track.stop();
       });
       this._remoteStream = undefined;
     }
+    if (this._localReturnAudioTrack) {
+      this._localReturnAudioTrack.stop();
+    }
     if (this._videoEl) {
       this._videoEl.removeAttribute("src");
       this._videoEl.load();
@@ -186,6 +260,7 @@ class HaWebRtcPlayer extends LitElement {
     if (this._peerConnection) {
       this._peerConnection.close();
       this._peerConnection = undefined;
+      await closeWebRtcStream(this.hass, this.entityid);
     }
   }
 

src/data/camera.ts

@@ -43,6 +43,12 @@ export interface WebRtcAnswer {
   answer: string;
 }
 
+export interface WebRtcConfig {
+  rtc_configuration: RTCConfiguration;
+  video_direction: string;
+  audio_direction: string;
+}
+
 export const cameraUrlWithWidthHeight = (
   base_url: string,
   width: number,
@@ -105,6 +111,18 @@ export const handleWebRtcOffer = (
     offer: offer,
   });
 
+export const fetchWebRtcConfig = (hass: HomeAssistant, entityId: string) =>
+  hass.callWS<WebRtcConfig>({
+    type: "camera/web_rtc_config",
+    entity_id: entityId,
+  });
+
+export const closeWebRtcStream = (hass: HomeAssistant, entityId: string) =>
+  hass.callWS({
+    type: "camera/web_rtc_close",
+    entity_id: entityId,
+  });
+
 export const fetchCameraPrefs = (hass: HomeAssistant, entityId: string) =>
   hass.callWS<CameraPreferences>({
     type: "camera/get_prefs",