feat(RTC): add support for creating non-standard tracks (#2409)

* feat(RTC): add support for creating non-standard tracks

* fix(RTC): add additional checks for creating tracks via mediastream

* fix(RTC): simplify options to create track

* fix(RTC): fix tests

* fix(RTC): update sourceId documentation

Author: DanielMcAssey

JitsiMeetJS.spec.ts

@@ -0,0 +1,39 @@
+import JitsiMeetJS from './JitsiMeetJS';
+import { VideoType } from './service/RTC/VideoType';
+import { MediaType } from './service/RTC/MediaType';
+import { JitsiTrackErrors } from './JitsiTrackErrors';
+
+describe('JitsiMeetJS', () => {
+    describe('createLocalTracksFromMediaStreams', () => {
+        it('creates a local track from a media stream', () => {
+            const canvas = document.createElement('canvas');
+
+            const canvasStream = canvas.captureStream(5);
+            const trackInfo = {
+                stream: canvasStream,
+                sourceType: 'canvas',
+                mediaType: MediaType.VIDEO,
+                videoType: VideoType.DESKTOP
+            };
+            const newTracks = JitsiMeetJS.createLocalTracksFromMediaStreams([ trackInfo ]);
+
+            expect(newTracks).toBeDefined();
+            expect(newTracks.length).toBe(1);
+        });
+
+        it('throws an error if track is not the correct media type', () => {
+            const canvas = document.createElement('canvas');
+
+            const canvasStream = canvas.captureStream(5);
+            const trackInfo = {
+                stream: canvasStream,
+                sourceType: 'canvas',
+                mediaType: MediaType.AUDIO,
+                videoType: VideoType.DESKTOP
+            };
+
+            expect(() => JitsiMeetJS.createLocalTracksFromMediaStreams([ trackInfo ]))
+                .toThrowError(JitsiTrackErrors.TRACK_NO_STREAM_TRACKS_FOUND);
+        });
+    });
+});

JitsiMeetJS.ts

@@ -36,6 +36,7 @@ import * as ConnectionQualityEvents
 import * as E2ePingEvents from './service/e2eping/E2ePingEvents';
 import { createGetUserMediaEvent } from './service/statistics/AnalyticsEvents';
 import *  as RTCStatsEvents from './modules/RTCStats/RTCStatsEvents';
+import { VideoType } from './service/RTC/VideoType';
 
 const logger = Logger.getLogger(__filename);
 
@@ -90,6 +91,13 @@ interface IJitsiMeetJSOptions {
     }
 }
 
+interface ICreateLocalTrackFromMediaStreamOptions {
+    stream: MediaStream,
+    sourceType: string,
+    mediaType: MediaType,
+    videoType?: VideoType
+}
+
 /**
  * The public API of the Jitsi Meet library (a.k.a. {@code JitsiMeetJS}).
  */
@@ -421,6 +429,31 @@ export default {
             });
     },
 
+    /**
+     * Manually create JitsiLocalTrack's from the provided track info, by exposing the RTC method
+     *
+     * @param {Array<ICreateLocalTrackFromMediaStreamOptions>} tracksInfo - array of track information
+     * @returns {Array<JitsiLocalTrack>} - created local tracks
+     */
+    createLocalTracksFromMediaStreams(tracksInfo) {
+        return RTC.createLocalTracks(tracksInfo.map((trackInfo) => {
+            const tracks = trackInfo.stream.getTracks()
+                .filter(track => track.kind === trackInfo.mediaType);
+
+            if (!tracks || tracks.length === 0) {
+                throw new JitsiTrackError(JitsiTrackErrors.TRACK_NO_STREAM_TRACKS_FOUND, null, null);
+            }
+
+            if (tracks.length > 1) {
+                throw new JitsiTrackError(JitsiTrackErrors.TRACK_TOO_MANY_TRACKS_IN_STREAM, null, null);
+            }
+
+            trackInfo.track = tracks[0];
+
+            return trackInfo;
+        }));
+    },
+
     /**
      * Create a TrackVADEmitter service that connects an audio track to an VAD (voice activity detection) processor in
      * order to obtain VAD scores for individual PCM audio samples.

JitsiTrackErrors.spec.ts

@@ -16,6 +16,8 @@ describe( "/JitsiTrackErrors members", () => {
         TRACK_IS_DISPOSED,
         TRACK_NO_STREAM_FOUND,
         UNSUPPORTED_RESOLUTION,
+        TRACK_TOO_MANY_TRACKS_IN_STREAM,
+        TRACK_NO_STREAM_TRACKS_FOUND,
         JitsiTrackErrors,
         ...others
     } = exported;
@@ -33,6 +35,8 @@ describe( "/JitsiTrackErrors members", () => {
         expect( TRACK_IS_DISPOSED ).toBe( 'track.track_is_disposed' );
         expect( TRACK_NO_STREAM_FOUND ).toBe( 'track.no_stream_found' );
         expect( UNSUPPORTED_RESOLUTION ).toBe( 'gum.unsupported_resolution' );
+        expect( TRACK_TOO_MANY_TRACKS_IN_STREAM ).toBe( 'track.too_many_tracks_in_stream' );
+        expect( TRACK_NO_STREAM_TRACKS_FOUND ).toBe( 'track.no_stream_tracks_found' );
 
         expect( JitsiTrackErrors ).toBeDefined();
 
@@ -48,10 +52,12 @@ describe( "/JitsiTrackErrors members", () => {
         expect( JitsiTrackErrors.TRACK_IS_DISPOSED ).toBe( 'track.track_is_disposed' );
         expect( JitsiTrackErrors.TRACK_NO_STREAM_FOUND ).toBe( 'track.no_stream_found' );
         expect( JitsiTrackErrors.UNSUPPORTED_RESOLUTION ).toBe( 'gum.unsupported_resolution' );
+        expect( JitsiTrackErrors.TRACK_TOO_MANY_TRACKS_IN_STREAM ).toBe( 'track.too_many_tracks_in_stream' );
+        expect( JitsiTrackErrors.TRACK_NO_STREAM_TRACKS_FOUND ).toBe( 'track.no_stream_tracks_found' );
     } );
 
     it( "unknown members", () => {
         const keys = Object.keys( others );
         expect( keys ).withContext( `Extra members: ${ keys.join( ", " ) }` ).toEqual( [] );
     } );
-} );
+} );

JitsiTrackErrors.ts

@@ -68,7 +68,17 @@ export enum JitsiTrackErrors {
      * An error which indicates that requested video resolution is not supported
      * by a webcam.
      */
-    UNSUPPORTED_RESOLUTION = 'gum.unsupported_resolution'
+    UNSUPPORTED_RESOLUTION = 'gum.unsupported_resolution',
+
+    /**
+     * An error which indicates that there are too many tracks in the provided media stream
+     */
+    TRACK_TOO_MANY_TRACKS_IN_STREAM = 'track.too_many_tracks_in_stream',
+
+    /**
+     * An error which indicates that no tracks were found in the media stream
+     */
+    TRACK_NO_STREAM_TRACKS_FOUND = 'track.no_stream_tracks_found',
 }
 
 // exported for backward compatibility
@@ -84,3 +94,5 @@ export const TIMEOUT = JitsiTrackErrors.TIMEOUT;
 export const TRACK_IS_DISPOSED = JitsiTrackErrors.TRACK_IS_DISPOSED;
 export const TRACK_NO_STREAM_FOUND = JitsiTrackErrors.TRACK_NO_STREAM_FOUND;
 export const UNSUPPORTED_RESOLUTION = JitsiTrackErrors.UNSUPPORTED_RESOLUTION;
+export const TRACK_TOO_MANY_TRACKS_IN_STREAM = JitsiTrackErrors.TRACK_TOO_MANY_TRACKS_IN_STREAM;
+export const TRACK_NO_STREAM_TRACKS_FOUND = JitsiTrackErrors.TRACK_NO_STREAM_TRACKS_FOUND;

modules/RTC/JitsiLocalTrack.js

@@ -47,8 +47,8 @@ export default class JitsiLocalTrack extends JitsiTrack {
      * @param {number} trackInfo.resolution - The the video resolution if it's a video track
      * @param {string} trackInfo.deviceId - The ID of the local device for this track.
      * @param {string} trackInfo.facingMode - Thehe camera facing mode used in getUserMedia call (for mobile only).
-     * @param {sourceId} trackInfo.sourceId - The id of the desktop sharing source. NOTE: defined for desktop sharing
-     * tracks only.
+     * @param {sourceId} trackInfo.sourceId - The id of the desktop sharing source, which is the Chrome media source ID,
+     * returned by Desktop Picker on Electron. NOTE: defined for desktop sharing tracks only.
      */
     constructor({
         deviceId,

types/hand-crafted/JitsiMeetJS.d.ts

@@ -122,6 +122,8 @@ export type JitsiMeetJSType = {
 
   getActiveAudioDevice: () => Promise<Object>; // TODO: can we improve on object?
 
+  createLocalTracksFromMediaStreams: ( tracksInfo: unknown[] ) => JitsiLocalTrack[]; // TODO:
+
   // isDeviceListAvailable: () => boolean; // obsosete
 
   // isDeviceChangeAvailable: ( deviceType: string ) => boolean; // obsosete