Group Calls (WIP)

src/client.ts

@@ -23,7 +23,7 @@ import { EventEmitter } from "events";
 import { ISyncStateData, SyncApi } from "./sync";
 import { EventStatus, IContent, IDecryptOptions, IEvent, MatrixEvent } from "./models/event";
 import { StubStore } from "./store/stub";
-import { createNewMatrixCall, MatrixCall } from "./webrtc/call";
+import { createNewMatrixCall, MatrixCall, ConstraintsType, getUserMediaContraints, CallType } from "./webrtc/call";
 import { Filter, IFilterDefinition } from "./filter";
 import { CallEventHandler } from './webrtc/callEventHandler';
 import * as utils from './utils';
@@ -144,6 +144,7 @@ import { IHierarchyRoom, ISpaceSummaryEvent, ISpaceSummaryRoom } from "./@types/
 import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules";
 import { IThreepid } from "./@types/threepids";
 import { CryptoStore } from "./crypto/store/base";
+import { GroupCall } from "./webrtc/groupCall";
 
 export type Store = IStore;
 export type SessionStore = WebStorageSessionStore;
@@ -694,6 +695,8 @@ export class MatrixClient extends EventEmitter {
     public iceCandidatePoolSize = 0; // XXX: Intended private, used in code.
     public idBaseUrl: string;
     public baseUrl: string;
+    private localAVStreamType: ConstraintsType;
+    private localAVStream: MediaStream;
 
     // Note: these are all `protected` to let downstream consumers make mistakes if they want to.
     // We don't technically support this usage, but have reasons to do this.
@@ -1258,15 +1261,68 @@ export class MatrixClient extends EventEmitter {
         this.supportsCallTransfer = support;
     }
 
+    public async getLocalVideoStream() {
+        if (this.localAVStreamType === ConstraintsType.Video) {
+            return this.localAVStream.clone();
+        }
+
+        const constraints = getUserMediaContraints(ConstraintsType.Video);
+        logger.log("Getting user media with constraints", constraints);
+        const mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
+        this.localAVStreamType = ConstraintsType.Video;
+        this.localAVStream = mediaStream;
+        return mediaStream;
+    }
+
+    public async getLocalAudioStream() {
+        if (this.localAVStreamType === ConstraintsType.Audio) {
+            return this.localAVStream.clone();
+        }
+
+        const constraints = getUserMediaContraints(ConstraintsType.Audio);
+        logger.log("Getting user media with constraints", constraints);
+        const mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
+        this.localAVStreamType = ConstraintsType.Audio;
+        this.localAVStream = mediaStream;
+        return mediaStream;
+    }
+
+    public stopLocalMediaStream() {
+        if (this.localAVStream) {
+            for (const track of this.localAVStream.getTracks()) {
+                track.stop();
+            }
+
+            this.localAVStreamType = null;
+            this.localAVStream = null;
+        }
+    }
+
     /**
      * Creates a new call.
      * The place*Call methods on the returned call can be used to actually place a call
      *
      * @param {string} roomId The room the call is to be placed in.
+     * @param {string} invitee The user to call in the given room.
      * @return {MatrixCall} the call or null if the browser doesn't support calling.
      */
-    public createCall(roomId: string): MatrixCall {
-        return createNewMatrixCall(this, roomId);
+    public createCall(roomId: string, invitee?: string): MatrixCall {
+        return createNewMatrixCall(this, roomId, { invitee });
+    }
+
+    /**
+     * Creates a new group call.
+     *
+     * @param {string} roomId The room the call is to be placed in.
+     * @return {GroupCall} the call or null if the browser doesn't support calling.
+     */
+    public createGroupCall(
+        roomId: string,
+        type: CallType,
+        dataChannelsEnabled?: boolean,
+        dataChannelOptions?: RTCDataChannelInit,
+    ): GroupCall {
+        return new GroupCall(this, roomId, type, dataChannelsEnabled, dataChannelOptions);
     }
 
     /**
@@ -6107,11 +6163,11 @@ export class MatrixClient extends EventEmitter {
     public register(
         username: string,
         password: string,
-        sessionId: string,
+        sessionId: string | null,
         auth: any,
-        bindThreepids: any,
-        guestAccessToken: string,
-        inhibitLogin: boolean,
+        bindThreepids?: any,
+        guestAccessToken?: string,
+        inhibitLogin?: boolean,
         callback?: Callback,
     ): Promise<any> { // TODO: Types (many)
         // backwards compat

src/matrix.ts

@@ -51,8 +51,14 @@ export {
     createNewMatrixCall,
     setAudioInput as setMatrixCallAudioInput,
     setVideoInput as setMatrixCallVideoInput,
+    CallType,
 } from "./webrtc/call";
 
+// TODO: This export is temporary and is only used for the local call feed for conference calls
+// Ideally conference calls will become a first-class concept and we will have a local call feed with
+// a lifecycle that matches the conference call, not individual calls to members.
+export { CallFeed } from "./webrtc/callFeed";
+
 // expose the underlying request object so different environments can use
 // different request libs (e.g. request or browser-request)
 let requestInstance;

src/webrtc/call.ts

@@ -67,6 +67,7 @@ import { ISendEventResponse } from "../@types/requests";
 
 interface CallOpts {
     roomId?: string;
+    invitee?: string;
     client?: any; // Fix when client is TSified
     forceTURN?: boolean;
     turnServers?: Array<TurnServer>;
@@ -128,6 +129,7 @@ export enum CallEvent {
 
     AssertedIdentityChanged = 'asserted_identity_changed',
 
+    DataChannel = 'datachannel',
     LengthChanged = 'length_changed'
 }
 
@@ -212,7 +214,7 @@ export enum CallErrorCode {
     Transfered = 'transferred',
 }
 
-enum ConstraintsType {
+export enum ConstraintsType {
     Audio = "audio",
     Video = "video",
 }
@@ -273,6 +275,7 @@ export class MatrixCall extends EventEmitter {
     public roomId: string;
     public type: CallType = null;
     public callId: string;
+    public invitee?: string;
     public state = CallState.Fledgling;
     public hangupParty: CallParty;
     public hangupReason: string;
@@ -302,6 +305,7 @@ export class MatrixCall extends EventEmitter {
     private opponentPartyId: string;
     private opponentCaps: CallCapabilities;
     private inviteTimeout: number;
+    private iceDisconnectedTimeout: number;
 
     // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold
     // This flag represents whether we want the other party to be on hold
@@ -331,6 +335,7 @@ export class MatrixCall extends EventEmitter {
     constructor(opts: CallOpts) {
         super();
         this.roomId = opts.roomId;
+        this.invitee = opts.invitee;
         this.client = opts.client;
         this.forceTURN = opts.forceTURN;
         this.ourPartyId = this.client.deviceId;
@@ -354,9 +359,8 @@ export class MatrixCall extends EventEmitter {
     public async placeVoiceCall(): Promise<void> {
         logger.debug("placeVoiceCall");
         this.checkForErrorListener();
-        const constraints = getUserMediaContraints(ConstraintsType.Audio);
         this.type = CallType.Voice;
-        await this.placeCallWithConstraints(constraints);
+        await this.placeCallWithConstraints(ConstraintsType.Audio);
     }
 
     /**
@@ -366,9 +370,15 @@ export class MatrixCall extends EventEmitter {
     public async placeVideoCall(): Promise<void> {
         logger.debug("placeVideoCall");
         this.checkForErrorListener();
-        const constraints = getUserMediaContraints(ConstraintsType.Video);
         this.type = CallType.Video;
-        await this.placeCallWithConstraints(constraints);
+        await this.placeCallWithConstraints(ConstraintsType.Video);
+    }
+
+    public createDataChannel(label: string, options: RTCDataChannelInit) {
+        logger.debug("createDataChannel");
+        const dataChannel = this.peerConn.createDataChannel(label, options);
+        this.emit(CallEvent.DataChannel, dataChannel);
+        return dataChannel;
     }
 
     public getOpponentMember(): RoomMember {
@@ -558,6 +568,10 @@ export class MatrixCall extends EventEmitter {
     }
 
     private deleteAllFeeds(): void {
+        for (const feed of this.feeds) {
+            feed.dispose();
+        }
+
         this.feeds = [];
         this.emit(CallEvent.FeedsChanged, this.feeds);
     }
@@ -571,6 +585,7 @@ export class MatrixCall extends EventEmitter {
             return;
         }
 
+        feed.dispose();
         this.feeds.splice(this.feeds.indexOf(feed), 1);
         this.emit(CallEvent.FeedsChanged, this.feeds);
     }
@@ -697,7 +712,14 @@ export class MatrixCall extends EventEmitter {
             this.waitForLocalAVStream = true;
 
             try {
-                const mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
+                let mediaStream: MediaStream;
+
+                if (this.type === CallType.Voice) {
+                    mediaStream = await this.client.getLocalAudioStream();
+                } else {
+                    mediaStream = await this.client.getLocalVideoStream();
+                }
+
                 this.waitForLocalAVStream = false;
                 this.gotUserMediaForAnswer(mediaStream);
             } catch (e) {
@@ -1384,6 +1406,10 @@ export class MatrixCall extends EventEmitter {
             lifetime: CALL_TIMEOUT_MS,
         } as MCallInviteNegotiate;
 
+        if (eventType === EventType.CallInvite && this.invitee) {
+            content.invitee = this.invitee;
+        }
+
         // clunky because TypeScript can't follow the types through if we use an expression as the key
         if (this.state === CallState.CreateOffer) {
             content.offer = this.peerConn.localDescription;
@@ -1483,13 +1509,18 @@ export class MatrixCall extends EventEmitter {
         // ideally we'd consider the call to be connected when we get media but
         // chrome doesn't implement any of the 'onstarted' events yet
         if (this.peerConn.iceConnectionState == 'connected') {
+            clearTimeout(this.iceDisconnectedTimeout);
             this.setState(CallState.Connected);
             this.callLengthInterval = setInterval(() => {
                 this.callLength++;
                 this.emit(CallEvent.LengthChanged, this.callLength);
             }, 1000);
         } else if (this.peerConn.iceConnectionState == 'failed') {
             this.hangup(CallErrorCode.IceFailed, false);
+        } else if (this.peerConn.iceConnectionState == 'disconnected') {
+            this.iceDisconnectedTimeout = setTimeout(() => {
+                this.hangup(CallErrorCode.IceFailed, false);
+            }, 30 * 1000);
         }
     };
 
@@ -1511,6 +1542,10 @@ export class MatrixCall extends EventEmitter {
         stream.addEventListener("removetrack", () => this.deleteFeedByStream(stream));
     };
 
+    private onDataChannel = (ev: RTCDataChannelEvent): void => {
+        this.emit(CallEvent.DataChannel, ev.channel);
+    };
+
     /**
      * This method removes all video/rtx codecs from screensharing video
      * transceivers. This is necessary since they can cause problems. Without
@@ -1826,8 +1861,7 @@ export class MatrixCall extends EventEmitter {
         }
     }
 
-    private async placeCallWithConstraints(constraints: MediaStreamConstraints): Promise<void> {
-        logger.log("Getting user media with constraints", constraints);
+    private async placeCallWithConstraints(constraintsType: ConstraintsType): Promise<void> {
         // XXX Find a better way to do this
         this.client.callEventHandler.calls.set(this.callId, this);
         this.setState(CallState.WaitLocalMedia);
@@ -1845,7 +1879,14 @@ export class MatrixCall extends EventEmitter {
         this.peerConn = this.createPeerConnection();
 
         try {
-            const mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
+            let mediaStream: MediaStream;
+
+            if (constraintsType === ConstraintsType.Audio) {
+                mediaStream = await this.client.getLocalAudioStream();
+            } else {
+                mediaStream = await this.client.getLocalVideoStream();
+            }
+
             this.gotUserMediaForInvite(mediaStream);
         } catch (e) {
             this.getUserMediaFailed(e);
@@ -1867,6 +1908,7 @@ export class MatrixCall extends EventEmitter {
         pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange);
         pc.addEventListener('track', this.onTrack);
         pc.addEventListener('negotiationneeded', this.onNegotiationNeeded);
+        pc.addEventListener('datachannel', this.onDataChannel);
 
         return pc;
     }
@@ -1962,7 +2004,7 @@ function setTracksEnabled(tracks: Array<MediaStreamTrack>, enabled: boolean): vo
     }
 }
 
-function getUserMediaContraints(type: ConstraintsType): MediaStreamConstraints {
+export function getUserMediaContraints(type: ConstraintsType): MediaStreamConstraints {
     const isWebkit = !!navigator.webkitGetUserMedia;
 
     switch (type) {
@@ -2085,6 +2127,7 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO
     const opts = {
         client: client,
         roomId: roomId,
+        invitee: options && options.invitee,
         turnServers: client.getTurnServers(),
         // call level options
         forceTURN: client.forceTURN || optionsForceTURN,

src/webrtc/callEventHandler.ts

@@ -151,6 +151,10 @@ export class CallEventHandler {
                 );
             }
 
+            if (content.invitee && content.invitee !== this.client.getUserId()) {
+                return; // This invite was meant for another user in the room
+            }
+
             const timeUntilTurnCresExpire = this.client.getTurnServersExpiry() - Date.now();
             logger.info("Current turn creds expire in " + timeUntilTurnCresExpire + " ms");
             call = createNewMatrixCall(
@@ -190,6 +194,7 @@ export class CallEventHandler {
                 if (
                     call.roomId === thisCall.roomId &&
                     thisCall.direction === CallDirection.Outbound &&
+                    call.invitee === thisCall.invitee &&
                     isCalling
                 ) {
                     existingCall = thisCall;

src/webrtc/callEventTypes.ts

@@ -53,6 +53,7 @@ export interface MCallInviteNegotiate extends MCallBase {
     description: RTCSessionDescription;
     lifetime: number;
     capabilities?: CallCapabilities;
+    invitee?: string;
     [SDPStreamMetadataKey]: SDPStreamMetadata;
 }
 

src/webrtc/callFeed.ts

@@ -35,7 +35,8 @@ export class CallFeed extends EventEmitter {
     private analyser: AnalyserNode;
     private frequencyBinCount: Float32Array;
     private speakingThreshold = SPEAKING_THRESHOLD;
-    private speaking = false;
+    public speaking = false;
+    private volumeLooperTimeout: number;
 
     constructor(
         public stream: MediaStream,
@@ -166,7 +167,7 @@ export class CallFeed extends EventEmitter {
     private volumeLooper(): void {
         if (!this.analyser) return;
 
-        setTimeout(() => {
+        this.volumeLooperTimeout = setTimeout(() => {
             if (!this.measuringVolumeActivity) return;
 
             this.analyser.getFloatFrequencyData(this.frequencyBinCount);
@@ -188,4 +189,8 @@ export class CallFeed extends EventEmitter {
             this.volumeLooper();
         }, POLLING_INTERVAL);
     }
+
+    public dispose(): void {
+        clearTimeout(this.volumeLooperTimeout);
+    }
 }