This is an automated email from the ASF dual-hosted git repository. solomax pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/openmeetings.git
commit 92a8a51a6943c5fc673189e7284ca7ede8111f35 Author: Maxim Solodovnik <[email protected]> AuthorDate: Sun Nov 27 11:56:54 2022 +0700 [OPENMEETINGS-2253] RTC related JS code is simplified; deprecated kurento-utils-js is dropped --- openmeetings-web/src/main/front/room/src/video.js | 214 +++----- .../src/main/front/settings/package.json | 5 +- .../src/main/front/settings/src/WebRtcPeer.js | 592 +++++++++++++++++++++ .../src/main/front/settings/src/index.js | 9 +- .../src/main/front/settings/src/mic-level.js | 6 +- .../src/main/front/settings/src/settings.js | 140 ++--- .../src/main/front/settings/src/video-util.js | 23 +- 7 files changed, 745 insertions(+), 244 deletions(-) diff --git a/openmeetings-web/src/main/front/room/src/video.js b/openmeetings-web/src/main/front/room/src/video.js index 676c9d551..3477c4bae 100644 --- a/openmeetings-web/src/main/front/room/src/video.js +++ b/openmeetings-web/src/main/front/room/src/video.js @@ -105,9 +105,7 @@ module.exports = class Video { data.aDest = data.aCtx.createMediaStreamDestination(); data.analyser.connect(data.aDest); _stream = data.aDest.stream; - stream.getVideoTracks().forEach(function(track) { - _stream.addTrack(track); - }); + stream.getVideoTracks().forEach(track => _stream.addTrack(track)); } } state.data = data; @@ -131,86 +129,69 @@ module.exports = class Video { }); }); } - function __attachListener(state) { - if (!state.disposed && state.data.rtcPeer) { - const pc = state.data.rtcPeer.peerConnection; - pc.onconnectionstatechange = function(event) { - console.warn(`!!RTCPeerConnection state changed: ${pc.connectionState}, user: ${sd.user.displayName}, uid: ${sd.uid}`); - switch(pc.connectionState) { - case "connected": - if (sd.self) { - // The connection has become fully connected - OmUtil.alert('info', `Connection to Media server has been established`, 3000);//notify user - } - break; - case "disconnected": - case "failed": - //connection has been dropped - OmUtil.alert('warning', `Media server connection for user ${sd.user.displayName} is ${pc.connectionState}, will try to re-connect`, 3000);//notify user - _refresh(); - break; - case "closed": - // The connection has been closed - break; + function __connectionStateChangeListener(state) { + const pc = state.data.rtcPeer.pc; + console.warn(`!!RTCPeerConnection state changed: ${pc.connectionState}, user: ${sd.user.displayName}, uid: ${sd.uid}`); + switch(pc.connectionState) { + case "connected": + if (sd.self) { + // The connection has become fully connected + OmUtil.alert('info', `Connection to Media server has been established`, 3000);//notify user } - } + break; + case "disconnected": + case "failed": + //connection has been dropped + OmUtil.alert('warning', `Media server connection for user ${sd.user.displayName} is ${pc.connectionState}, will try to re-connect`, 3000);//notify user + _refresh(); + break; + case "closed": + // The connection has been closed + break; } } function __createSendPeer(msg, state, cnts) { state.options = { - videoStream: state.stream + mediaStream: state.stream , mediaConstraints: cnts - , onicecandidate: self.onIceCandidate + , onIceCandidate: self.onIceCandidate + , onConnectionStateChange: () => __connectionStateChangeListener(state) }; - if (!isSharing) { - state.options.localVideo = __getVideo(state); - } + const vid = __getVideo(state); + vid.srcObject = state.stream; + const data = state.data; - data.rtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerSendonly( - VideoUtil.addIceServers(state.options, msg) - , function (error) { - if (state.disposed || true === data.rtcPeer.cleaned) { - return; + data.rtcPeer = new WebRtcPeerSendonly(VideoUtil.addIceServers(state.options, msg)); + if (data.analyser) { + level = new MicLevel(); + level.meter(data.analyser, lm, _micActivity, OmUtil.error); + } + data.rtcPeer.createOffer() + .then(sdpOffer => { + data.rtcPeer.processLocalOffer(sdpOffer); + OmUtil.log('Invoking Sender SDP offer callback function'); + const bmsg = { + id : 'broadcastStarted' + , uid: sd.uid + , sdpOffer: sdpOffer.sdp + }, vtracks = state.stream.getVideoTracks(); + if (vtracks && vtracks.length > 0) { + const vts = vtracks[0].getSettings(); + vidSize.width = vts.width; + vidSize.height = vts.height; + bmsg.width = vts.width; + bmsg.height = vts.height; + bmsg.fps = vts.frameRate; } - if (error) { - return OmUtil.error(error); + VideoMgrUtil.sendMessage(bmsg); + if (isSharing) { + Sharer.setShareState(Sharer.SHARE_STARTED); } - if (data.analyser) { - level = new MicLevel(); - level.meter(data.analyser, lm, _micActivity, OmUtil.error); + if (isRecording) { + Sharer.setRecState(Sharer.SHARE_STARTED); } - data.rtcPeer.generateOffer(function(genErr, offerSdp) { - if (state.disposed || true === data.rtcPeer.cleaned) { - return; - } - if (genErr) { - return OmUtil.error('Sender sdp offer error ' + genErr); - } - OmUtil.log('Invoking Sender SDP offer callback function'); - const bmsg = { - id : 'broadcastStarted' - , uid: sd.uid - , sdpOffer: offerSdp - }, vtracks = state.stream.getVideoTracks(); - if (vtracks && vtracks.length > 0) { - const vts = vtracks[0].getSettings(); - vidSize.width = vts.width; - vidSize.height = vts.height; - bmsg.width = vts.width; - bmsg.height = vts.height; - bmsg.fps = vts.frameRate; - } - VideoMgrUtil.sendMessage(bmsg); - if (isSharing) { - Sharer.setShareState(Sharer.SHARE_STARTED); - } - if (isRecording) { - Sharer.setRecState(Sharer.SHARE_STARTED); - } - }); - }); - data.rtcPeer.cleaned = false; - __attachListener(state); + }) + .catch(error => OmUtil.error(error)); } function _createSendPeer(msg, state) { if (isSharing || isRecording) { @@ -222,36 +203,23 @@ module.exports = class Video { function _createResvPeer(msg, state) { __createVideo(state); const options = VideoUtil.addIceServers({ - remoteVideo : __getVideo(state) - , onicecandidate : self.onIceCandidate + mediaConstraints: {audio: true, video: true} + , onIceCandidate : self.onIceCandidate + , onConnectionStateChange: () => __connectionStateChangeListener(state) }, msg); const data = state.data; - data.rtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerRecvonly( - options - , function(error) { - if (state.disposed || true === data.rtcPeer.cleaned) { - return; - } - if (error) { - return OmUtil.error(error); - } - data.rtcPeer.generateOffer(function(genErr, offerSdp) { - if (state.disposed || true === data.rtcPeer.cleaned) { - return; - } - if (genErr) { - return OmUtil.error('Receiver sdp offer error ' + genErr); - } - OmUtil.log('Invoking Receiver SDP offer callback function'); - VideoMgrUtil.sendMessage({ - id : 'addListener' - , sender: sd.uid - , sdpOffer: offerSdp - }); + data.rtcPeer = new WebRtcPeerRecvonly(options); + data.rtcPeer.createOffer() + .then(sdpOffer => { + data.rtcPeer.processLocalOffer(sdpOffer); + OmUtil.log('Invoking Receiver SDP offer callback function'); + VideoMgrUtil.sendMessage({ + id : 'addListener' + , sender: sd.uid + , sdpOffer: sdpOffer.sdp }); - }); - data.rtcPeer.cleaned = false; - __attachListener(state); + }) + .catch(genErr => OmUtil.error('Receiver sdp offer error ' + genErr)); } function _handleMicStatus(state) { if (!footer || !footer.is(':visible')) { @@ -513,7 +481,6 @@ module.exports = class Video { delete state.options.videoStream; delete state.options.mediaConstraints; delete state.options.onicecandidate; - delete state.options.localVideo; state.options = null; } _cleanData(state.data); @@ -557,7 +524,7 @@ module.exports = class Video { const data = state.data , videoEl = state.video[0]; if (data.rtcPeer && (!videoEl.srcObject || !videoEl.srcObject.active)) { - videoEl.srcObject = sd.self ? data.rtcPeer.getLocalStream() : data.rtcPeer.getRemoteStream(); + videoEl.srcObject = data.rtcPeer.stream; } } }); @@ -567,39 +534,30 @@ module.exports = class Video { if (!state || state.disposed || !state.data.rtcPeer || state.data.rtcPeer.cleaned) { return; } - state.data.rtcPeer.processAnswer(answer, function (error) { - if (true === this.cleaned) { - return; - } - const video = __getVideo(state); - if (this.peerConnection.signalingState === 'stable' && video && video.paused) { - video.play().catch(function (err) { - if ('NotAllowedError' === err.name) { - VideoUtil.askPermission(function () { - video.play(); - }); - } - }); - return; - } - if (error) { - OmUtil.error(error, true); - } - }); + state.data.rtcPeer.processRemoteAnswer(answer) + .then(() => { + const video = __getVideo(state); + const rStream = state.data.rtcPeer.pc.getRemoteStreams()[0]; + if (rStream) { + video.srcObject = rStream; + } + if (state.data.rtcPeer.pc.signalingState === 'stable' && video && video.paused) { + video.play().catch(err => { + if ('NotAllowedError' === err.name) { + VideoUtil.askPermission(() => video.play()); + } + }); + } + }) + .catch(error => OmUtil.error(error, true)); } function _processIceCandidate(candidate) { const state = states.length > 0 ? states[0] : null; if (!state || state.disposed || !state.data.rtcPeer || state.data.rtcPeer.cleaned) { return; } - state.data.rtcPeer.addIceCandidate(candidate, function (error) { - if (true === this.cleaned) { - return; - } - if (error) { - OmUtil.error('Error adding candidate: ' + error, true); - } - }); + state.data.rtcPeer.addIceCandidate(candidate) + .catch(error => OmUtil.error('Error adding candidate: ' + error, true)); } function _init(_msg) { sd = _msg.stream; diff --git a/openmeetings-web/src/main/front/settings/package.json b/openmeetings-web/src/main/front/settings/package.json index c45a30778..8437ecc37 100644 --- a/openmeetings-web/src/main/front/settings/package.json +++ b/openmeetings-web/src/main/front/settings/package.json @@ -16,7 +16,8 @@ "tinyify": "^3.1.0" }, "dependencies": { - "adapterjs": "^0.15.5", - "kurento-utils": "^6.16.0" + "freeice": "2.2.2", + "uuid": "^9.0.0", + "webrtc-adapter": "^8.2.0" } } diff --git a/openmeetings-web/src/main/front/settings/src/WebRtcPeer.js b/openmeetings-web/src/main/front/settings/src/WebRtcPeer.js new file mode 100644 index 000000000..d40d4b015 --- /dev/null +++ b/openmeetings-web/src/main/front/settings/src/WebRtcPeer.js @@ -0,0 +1,592 @@ +/* + * (C) Copyright 2017-2022 OpenVidu (https://openvidu.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// taken from here: +// https://github.com/OpenVidu/openvidu/blob/master/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts +// and monkey-patched + +const freeice = require('freeice'); + +const ExceptionEventName = { + /** + * The [ICE connection state](https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/iceConnectionState) + * of an [RTCPeerConnection](https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection) reached `failed` status. + * + * This is a terminal error that won't have any kind of possible recovery. If the client is still connected to OpenVidu Server, + * then an automatic reconnection process of the media stream is immediately performed. If the ICE connection has broken due to + * a total network drop, then no automatic reconnection process will be possible. + * + * {@link ExceptionEvent} objects with this {@link ExceptionEvent.name} will have as {@link ExceptionEvent.origin} property a {@link Stream} object. + */ + ICE_CONNECTION_FAILED: 'ICE_CONNECTION_FAILED', + + /** + * The [ICE connection state](https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/iceConnectionState) + * of an [RTCPeerConnection](https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection) reached `disconnected` status. + * + * This is not a terminal error, and it is possible for the ICE connection to be reconnected. If the client is still connected to + * OpenVidu Server and after certain timeout the ICE connection has not reached a success or terminal status, then an automatic + * reconnection process of the media stream is performed. If the ICE connection has broken due to a total network drop, then no + * automatic reconnection process will be possible. + * + * You can customize the timeout for the reconnection attempt with property {@link OpenViduAdvancedConfiguration.iceConnectionDisconnectedExceptionTimeout}, + * which by default is 4000 milliseconds. + * + * {@link ExceptionEvent} objects with this {@link ExceptionEvent.name} will have as {@link ExceptionEvent.origin} property a {@link Stream} object. + */ + ICE_CONNECTION_DISCONNECTED: 'ICE_CONNECTION_DISCONNECTED', +}; + +class WebRtcPeer { + constructor(configuration) { + this.remoteCandidatesQueue = []; + this.localCandidatesQueue = []; + this.iceCandidateList = []; + this.candidategatheringdone = false; + + // Same as WebRtcPeerConfiguration but without optional fields. + this.configuration = { + ...configuration, + iceServers: !!configuration.iceServers && configuration.iceServers.length > 0 ? configuration.iceServers : freeice(), + mediaStream: configuration.mediaStream !== undefined ? configuration.mediaStream : null, + mode: !!configuration.mode ? configuration.mode : 'sendrecv', + id: !!configuration.id ? configuration.id : this.generateUniqueId() + }; + // prettier-ignore + OmUtil.log(`[WebRtcPeer] configuration:\n${JSON.stringify(this.configuration, null, 2)}`); + + this.pc = new RTCPeerConnection({ iceServers: this.configuration.iceServers }); + + this._iceCandidateListener = (event) => { + if (event.candidate !== null) { + // `RTCPeerConnectionIceEvent.candidate` is supposed to be an RTCIceCandidate: + // https://w3c.github.io/webrtc-pc/#dom-rtcpeerconnectioniceevent-candidate + // + // But in practice, it is actually an RTCIceCandidateInit that can be used to + // obtain a proper candidate, using the RTCIceCandidate constructor: + // https://w3c.github.io/webrtc-pc/#dom-rtcicecandidate-constructor + const candidateInit = event.candidate; + const iceCandidate = new RTCIceCandidate(candidateInit); + + this.configuration.onIceCandidate(iceCandidate); + if (iceCandidate.candidate !== '') { + this.localCandidatesQueue.push(iceCandidate); + } + } + }; + this.pc.addEventListener('icecandidate', this._iceCandidateListener); + + this._signalingStateChangeListener = () => { + if (this.pc.signalingState === 'stable') { + // SDP Offer/Answer finished. Add stored remote candidates. + while (this.iceCandidateList.length > 0) { + let candidate = this.iceCandidateList.shift(); + this.pc.addIceCandidate(candidate); + } + } + }; + this.pc.addEventListener('signalingstatechange', this._signalingStateChangeListener); + if (this.configuration.onConnectionStateChange) { + this.pc.addEventListener('connectionstatechange', this.configuration.onConnectionStateChange); + } + } + + getId() { + return this.configuration.id; + } + + /** + * This method frees the resources used by WebRtcPeer + */ + dispose() { + OmUtil.log('Disposing WebRtcPeer'); + if (this.pc) { + if (this.pc.signalingState === 'closed') { + return; + } + this.pc.removeEventListener('icecandidate', this._iceCandidateListener); + this._iceCandidateListener = undefined; + this.pc.removeEventListener('signalingstatechange', this._signalingStateChangeListener); + this._signalingStateChangeListener = undefined; + if (this._iceConnectionStateChangeListener) { + this.pc.removeEventListener('iceconnectionstatechange', this._iceConnectionStateChangeListener); + } + if (this.configuration.onConnectionStateChange) { + this.pc.removeEventListener('connectionstatechange', this.configuration.onConnectionStateChange); + } + this.configuration = {}; + this.pc.close(); + this.remoteCandidatesQueue = []; + this.localCandidatesQueue = []; + } + } + + /** + * Creates an SDP offer from the local RTCPeerConnection to send to the other peer. + * Only if the negotiation was initiated by this peer. + */ + async createOffer() { + // TODO: Delete this conditional when all supported browsers are + // modern enough to implement the Transceiver methods. + if (!('addTransceiver' in this.pc)) { + OmUtil.error( + '[createOffer] Method RTCPeerConnection.addTransceiver() is NOT available; using LEGACY offerToReceive{Audio,Video}' + ); + return this.createOfferLegacy(); + } else { + OmUtil.log('[createOffer] Method RTCPeerConnection.addTransceiver() is available; using it'); + } + + // Spec doc: https://w3c.github.io/webrtc-pc/#dom-rtcpeerconnection-addtransceiver + + if (this.configuration.mode !== 'recvonly') { + // To send media, assume that all desired media tracks have been + // already added by higher level code to our MediaStream. + + if (!this.configuration.mediaStream) { + throw new Error( + `[WebRtcPeer.createOffer] Direction is '${this.configuration.mode}', but no stream was configured to be sent` + ); + } + + for (const track of this.configuration.mediaStream.getTracks()) { + const tcInit = { + direction: this.configuration.mode, + streams: [this.configuration.mediaStream] + }; + + if (track.kind === 'video' && this.configuration.simulcast) { + // Check if the requested size is enough to ask for 3 layers. + const trackSettings = track.getSettings(); + const trackConsts = track.getConstraints(); + + const trackWidth = typeof(trackSettings.width) === 'object' ? trackConsts.width.ideal : trackConsts.width || 0; + const trackHeight = typeof(trackSettings.height) === 'object' ? trackConsts.height.ideal : trackConsts.height || 0; + OmUtil.info(`[createOffer] Video track dimensions: ${trackWidth}x${trackHeight}`); + + const trackPixels = trackWidth * trackHeight; + let maxLayers = 0; + if (trackPixels >= 960 * 540) { + maxLayers = 3; + } else if (trackPixels >= 480 * 270) { + maxLayers = 2; + } else { + maxLayers = 1; + } + + tcInit.sendEncodings = []; + for (let l = 0; l < maxLayers; l++) { + const layerDiv = 2 ** (maxLayers - l - 1); + + const encoding = { + rid: 'rdiv' + layerDiv.toString(), + + // @ts-ignore -- Property missing from DOM types. + scalabilityMode: 'L1T1' + }; + + if (['detail', 'text'].includes(track.contentHint)) { + // Prioritize best resolution, for maximum picture detail. + encoding.scaleResolutionDownBy = 1.0; + + // @ts-ignore -- Property missing from DOM types. + encoding.maxFramerate = Math.floor(30 / layerDiv); + } else { + encoding.scaleResolutionDownBy = layerDiv; + } + + tcInit.sendEncodings.push(encoding); + } + } + + const tc = this.pc.addTransceiver(track, tcInit); + + if (track.kind === 'video') { + let sendParams = tc.sender.getParameters(); + let needSetParams = false; + + if (sendParams.degradationPreference && !sendParams.degradationPreference.length) { + // degradationPreference for video: "balanced", "maintain-framerate", "maintain-resolution". + // https://www.w3.org/TR/2018/CR-webrtc-20180927/#dom-rtcdegradationpreference + if (['detail', 'text'].includes(track.contentHint)) { + sendParams.degradationPreference = 'maintain-resolution'; + } else { + sendParams.degradationPreference = 'balanced'; + } + + OmUtil.info(`[createOffer] Video sender Degradation Preference set: ${sendParams.degradationPreference}`); + + // FIXME: Firefox implements degradationPreference on each individual encoding! + // (set it on every element of the sendParams.encodings array) + + needSetParams = true; + } + + // FIXME: Check that the simulcast encodings were applied. + // Firefox doesn't implement `RTCRtpTransceiverInit.sendEncodings` + // so the only way to enable simulcast is with `RTCRtpSender.setParameters()`. + // + // This next block can be deleted when Firefox fixes bug #1396918: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1396918 + // + // NOTE: This is done in a way that is compatible with all browsers, to save on + // browser-conditional code. The idea comes from WebRTC Adapter.js: + // * https://github.com/webrtcHacks/adapter/issues/998 + // * https://github.com/webrtcHacks/adapter/blob/v7.7.0/src/js/firefox/firefox_shim.js#L231-L255 + if (this.configuration.simulcast) { + if (sendParams.encodings.length !== tcInit.sendEncodings.length) { + sendParams.encodings = tcInit.sendEncodings; + + needSetParams = true; + } + } + + if (needSetParams) { + OmUtil.log(`[createOffer] Setting new RTCRtpSendParameters to video sender`); + try { + await tc.sender.setParameters(sendParams); + } catch (error) { + let message = `[WebRtcPeer.createOffer] Cannot set RTCRtpSendParameters to video sender`; + if (error instanceof Error) { + message += `: ${error.message}`; + } + throw new Error(message); + } + } + } + } + } else { + // To just receive media, create new recvonly transceivers. + for (const kind of ['audio', 'video']) { + // Check if the media kind should be used. + if (!this.configuration.mediaConstraints[kind]) { + continue; + } + + this.configuration.mediaStream = new MediaStream(); + this.pc.addTransceiver(kind, { + direction: this.configuration.mode, + streams: [this.configuration.mediaStream] + }); + } + } + + let sdpOffer; + try { + sdpOffer = await this.pc.createOffer(); + } catch (error) { + let message = `[WebRtcPeer.createOffer] Browser failed creating an SDP Offer`; + if (error instanceof Error) { + message += `: ${error.message}`; + } + throw new Error(message); + } + + return sdpOffer; + } + + /** + * Creates an SDP answer from the local RTCPeerConnection to send to the other peer + * Only if the negotiation was initiated by the other peer + */ + createAnswer() { + return new Promise((resolve, reject) => { + // TODO: Delete this conditional when all supported browsers are + // modern enough to implement the Transceiver methods. + if ('getTransceivers' in this.pc) { + OmUtil.log('[createAnswer] Method RTCPeerConnection.getTransceivers() is available; using it'); + + // Ensure that the PeerConnection already contains one Transceiver + // for each kind of media. + // The Transceivers should have been already created internally by + // the PC itself, when `pc.setRemoteDescription(sdpOffer)` was called. + + for (const kind of ['audio', 'video']) { + // Check if the media kind should be used. + if (!this.configuration.mediaConstraints[kind]) { + continue; + } + + let tc = this.pc.getTransceivers().find((tc) => tc.receiver.track.kind === kind); + + if (tc) { + // Enforce our desired direction. + tc.direction = this.configuration.mode; + } else { + return reject(new Error(`${kind} requested, but no transceiver was created from remote description`)); + } + } + + this.pc + .createAnswer() + .then((sdpAnswer) => resolve(sdpAnswer)) + .catch((error) => reject(error)); + } else { + // TODO: Delete else branch when all supported browsers are + // modern enough to implement the Transceiver methods + + let offerAudio, + offerVideo = true; + if (!!this.configuration.mediaConstraints) { + offerAudio = + typeof this.configuration.mediaConstraints.audio === 'boolean' ? this.configuration.mediaConstraints.audio : true; + offerVideo = + typeof this.configuration.mediaConstraints.video === 'boolean' ? this.configuration.mediaConstraints.video : true; + const constraints = { + offerToReceiveAudio: offerAudio, + offerToReceiveVideo: offerVideo + }; + (this.pc).createAnswer(constraints) + .then((sdpAnswer) => resolve(sdpAnswer)) + .catch((error) => reject(error)); + } + } + + // else, there is nothing to do; the legacy createAnswer() options do + // not offer any control over which tracks are included in the answer. + }); + } + + /** + * This peer initiated negotiation. Step 1/4 of SDP offer-answer protocol + */ + processLocalOffer(offer) { + return new Promise((resolve, reject) => { + this.pc + .setLocalDescription(offer) + .then(() => { + const localDescription = this.pc.localDescription; + if (!!localDescription) { + OmUtil.log('Local description set', localDescription.sdp); + return resolve(); + } else { + return reject('Local description is not defined'); + } + }) + .catch((error) => reject(error)); + }); + } + + /** + * Other peer initiated negotiation. Step 2/4 of SDP offer-answer protocol + */ + processRemoteOffer(sdpOffer) { + return new Promise((resolve, reject) => { + const offer = { + type: 'offer', + sdp: sdpOffer + }; + OmUtil.log('SDP offer received, setting remote description', offer); + + if (this.pc.signalingState === 'closed') { + return reject('RTCPeerConnection is closed when trying to set remote description'); + } + this.setRemoteDescription(offer) + .then(() => resolve()) + .catch((error) => reject(error)); + }); + } + + /** + * Other peer initiated negotiation. Step 3/4 of SDP offer-answer protocol + */ + processLocalAnswer(answer) { + return new Promise((resolve, reject) => { + OmUtil.log('SDP answer created, setting local description'); + if (this.pc.signalingState === 'closed') { + return reject('RTCPeerConnection is closed when trying to set local description'); + } + this.pc + .setLocalDescription(answer) + .then(() => resolve()) + .catch((error) => reject(error)); + }); + } + + /** + * This peer initiated negotiation. Step 4/4 of SDP offer-answer protocol + */ + processRemoteAnswer(sdpAnswer) { + return new Promise((resolve, reject) => { + const answer = { + type: 'answer', + sdp: sdpAnswer + }; + OmUtil.log('SDP answer received, setting remote description'); + + if (this.pc.signalingState === 'closed') { + return reject('RTCPeerConnection is closed when trying to set remote description'); + } + this.setRemoteDescription(answer) + .then(() => { + resolve(); + }) + .catch((error) => reject(error)); + }); + } + + /** + * @hidden + */ + async setRemoteDescription(sdp) { + return this.pc.setRemoteDescription(sdp); + } + + /** + * Callback function invoked when an ICE candidate is received + */ + addIceCandidate(iceCandidate) { + return new Promise((resolve, reject) => { + OmUtil.log('Remote ICE candidate received', iceCandidate); + this.remoteCandidatesQueue.push(iceCandidate); + switch (this.pc.signalingState) { + case 'closed': + reject(new Error('PeerConnection object is closed')); + break; + case 'stable': + if (!!this.pc.remoteDescription) { + this.pc + .addIceCandidate(iceCandidate) + .then(() => resolve()) + .catch((error) => reject(error)); + } else { + this.iceCandidateList.push(iceCandidate); + resolve(); + } + break; + default: + this.iceCandidateList.push(iceCandidate); + resolve(); + } + }); + } + + addIceConnectionStateChangeListener(otherId) { + if (!this._iceConnectionStateChangeListener) { + this._iceConnectionStateChangeListener = () => { + const iceConnectionState = this.pc.iceConnectionState; + switch (iceConnectionState) { + case 'disconnected': + // Possible network disconnection + const msg1 = + 'IceConnectionState of RTCPeerConnection ' + + this.configuration.id + + ' (' + + otherId + + ') change to "disconnected". Possible network disconnection'; + logger.warn(msg1); + this.configuration.onIceConnectionStateException(ExceptionEventName.ICE_CONNECTION_DISCONNECTED, msg1); + break; + case 'failed': + const msg2 = 'IceConnectionState of RTCPeerConnection ' + this.configuration.id + ' (' + otherId + ') to "failed"'; + logger.error(msg2); + this.configuration.onIceConnectionStateException(ExceptionEventName.ICE_CONNECTION_FAILED, msg2); + break; + case 'closed': + OmUtil.log( + 'IceConnectionState of RTCPeerConnection ' + this.configuration.id + ' (' + otherId + ') change to "closed"' + ); + break; + case 'new': + OmUtil.log('IceConnectionState of RTCPeerConnection ' + this.configuration.id + ' (' + otherId + ') change to "new"'); + break; + case 'checking': + logger.log( + 'IceConnectionState of RTCPeerConnection ' + this.configuration.id + ' (' + otherId + ') change to "checking"' + ); + break; + case 'connected': + logger.log( + 'IceConnectionState of RTCPeerConnection ' + this.configuration.id + ' (' + otherId + ') change to "connected"' + ); + break; + case 'completed': + logger.log( + 'IceConnectionState of RTCPeerConnection ' + this.configuration.id + ' (' + otherId + ') change to "completed"' + ); + break; + } + }; + } + this.pc.addEventListener('iceconnectionstatechange', this._iceConnectionStateChangeListener); + } + + /** + * @hidden + */ + generateUniqueId() { + return uuidv4(); + } + + get stream() { + return this.pc.getLocalStreams()[0] || this.pc.getRemoteStreams()[0]; + } + + // LEGACY code + deprecatedPeerConnectionTrackApi() { + for (const track of this.configuration.mediaStream.getTracks()) { + this.pc.addTrack(track, this.configuration.mediaStream); + } + } + + // DEPRECATED LEGACY METHOD: Old WebRTC versions don't implement + // Transceivers, and instead depend on the deprecated + // "offerToReceiveAudio" and "offerToReceiveVideo". + createOfferLegacy() { + if (!!this.configuration.mediaStream) { + this.deprecatedPeerConnectionTrackApi(); + } + + const hasAudio = this.configuration.mediaConstraints.audio; + const hasVideo = this.configuration.mediaConstraints.video; + + const options = { + offerToReceiveAudio: this.configuration.mode !== 'sendonly' && hasAudio, + offerToReceiveVideo: this.configuration.mode !== 'sendonly' && hasVideo + }; + + OmUtil.log('[createOfferLegacy] RTCPeerConnection.createOffer() options:', JSON.stringify(options)); + + return this.pc.createOffer(options); + } +} + +class WebRtcPeerRecvonly extends WebRtcPeer { + constructor(configuration) { + configuration.mode = 'recvonly'; + super(configuration); + } +}; + +class WebRtcPeerSendonly extends WebRtcPeer { + constructor(configuration) { + configuration.mode = 'sendonly'; + super(configuration); + } +}; + +class WebRtcPeerSendrecv extends WebRtcPeer { + constructor(configuration) { + configuration.mode = 'sendrecv'; + super(configuration); + } +}; + +module.exports = { + WebRtcPeerRecvonly: WebRtcPeerRecvonly, + WebRtcPeerSendonly: WebRtcPeerSendonly +}; diff --git a/openmeetings-web/src/main/front/settings/src/index.js b/openmeetings-web/src/main/front/settings/src/index.js index 982236461..7edcb067f 100644 --- a/openmeetings-web/src/main/front/settings/src/index.js +++ b/openmeetings-web/src/main/front/settings/src/index.js @@ -1,5 +1,8 @@ /* Licensed under the Apache License, Version 2.0 (the "License") http://www.apache.org/licenses/LICENSE-2.0 */ const VideoUtil = require('./video-util'); +require('webrtc-adapter'); +const {v4: uuidv4} = require('uuid'); +const {WebRtcPeerRecvonly, WebRtcPeerSendonly} = require('./WebRtcPeer'); if (window.hasOwnProperty('isSecureContext') === false) { window.isSecureContext = window.location.protocol == 'https:' || ["localhost", "127.0.0.1"].indexOf(window.location.hostname) !== -1; @@ -10,9 +13,9 @@ Object.assign(window, { , VIDWIN_SEL: VideoUtil.VIDWIN_SEL , VID_SEL: VideoUtil.VID_SEL , MicLevel: require('./mic-level') + , WebRtcPeerRecvonly: WebRtcPeerRecvonly + , WebRtcPeerSendonly: WebRtcPeerSendonly , VideoSettings: require('./settings') - // AdapterJS is not added for now - , kurentoUtils: require('kurento-utils') - , uuidv4: require('uuid/v4') + , uuidv4: uuidv4 }); diff --git a/openmeetings-web/src/main/front/settings/src/mic-level.js b/openmeetings-web/src/main/front/settings/src/mic-level.js index 32668f69b..9fb3edc00 100644 --- a/openmeetings-web/src/main/front/settings/src/mic-level.js +++ b/openmeetings-web/src/main/front/settings/src/mic-level.js @@ -5,11 +5,7 @@ module.exports = class MicLevel { constructor() { let ctx, mic, analyser, vol = .0, vals = new RingBuffer(100); - this.meterPeer = (rtcPeer, cnvs, _micActivity, _error, connectAudio) => { - if (!rtcPeer || ('function' !== typeof(rtcPeer.getLocalStream) && 'function' !== typeof(rtcPeer.getRemoteStream))) { - return; - } - const stream = rtcPeer.getLocalStream() || rtcPeer.getRemoteStream(); + this.meterStream = (stream, cnvs, _micActivity, _error, connectAudio) => { if (!stream || stream.getAudioTracks().length < 1) { return; } diff --git a/openmeetings-web/src/main/front/settings/src/settings.js b/openmeetings-web/src/main/front/settings/src/settings.js index 633eaf61e..6c16d4cc3 100644 --- a/openmeetings-web/src/main/front/settings/src/settings.js +++ b/openmeetings-web/src/main/front/settings/src/settings.js @@ -1,7 +1,6 @@ /* Licensed under the Apache License, Version 2.0 (the "License") http://www.apache.org/licenses/LICENSE-2.0 */ const MicLevel = require('./mic-level'); const VideoUtil = require('./video-util'); -const kurentoUtils = require('kurento-utils'); const DEV_AUDIO = 'audioinput' , DEV_VIDEO = 'videoinput' @@ -149,7 +148,7 @@ function _setCntsDimensions(cnts) { //each bool OR https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackConstraints // min/ideal/max/exact/mandatory can also be used function _constraints(sd, callback) { - _getDevConstraints(function(devCnts){ + _getDevConstraints(function(devCnts) { const cnts = {}; if (devCnts.video && false === o.audioOnly && VideoUtil.hasCam(sd) && s.video.cam > -1) { cnts.video = { @@ -202,39 +201,32 @@ function _readValues(msg, func) { _constraints(null, function(cnts) { if (cnts.video !== false || cnts.audio !== false) { const options = VideoUtil.addIceServers({ - localVideo: vid[0] - , mediaConstraints: cnts + mediaConstraints: cnts + , onIceCandidate: _onIceCandidate }, msg); - rtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerSendonly( - options - , function(error) { - if (error) { - if (true === rtcPeer.cleaned) { - return; - } - return OmUtil.error(error); - } + navigator.mediaDevices.getUserMedia(cnts) + .then(stream => { + vid[0].srcObject = stream; + options.mediaStream = stream; + + rtcPeer = new WebRtcPeerSendonly(options); if (cnts.audio) { lm.show(); level = new MicLevel(); - level.meterPeer(rtcPeer, lm, function(){}, OmUtil.error, false); + level.meterStream(stream, lm, function(){}, OmUtil.error, false); } else { lm.hide(); } - rtcPeer.generateOffer(function(error, _offerSdp) { - if (error) { - if (true === rtcPeer.cleaned) { - return; - } - return OmUtil.error('Error generating the offer'); - } - if (typeof(func) === 'function') { - func(_offerSdp, cnts); - } else { - _allowRec(true); - } - }); - }); + return rtcPeer.createOffer(); + }) + .then(sdpOffer => { + rtcPeer.processLocalOffer(sdpOffer); + if (typeof(func) === 'function') { + func(sdpOffer.sdp, cnts); + } else { + _allowRec(true); + } + }).catch(_ => OmUtil.error('Error generating the offer')); } if (!msg) { _updateRec(); @@ -384,75 +376,49 @@ function _onKMessage(m) { , video: cnts.video !== false , audio: cnts.audio !== false }, MsgBase); - rtcPeer.on('icecandidate', _onIceCandidate); }); break; - case 'canPlay': - { - const options = VideoUtil.addIceServers({ - remoteVideo: vid[0] - , mediaConstraints: {audio: true, video: true} - , onicecandidate: _onIceCandidate - }, m); - _clear(); - rtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerRecvonly( - options - , function(error) { - if (error) { - if (true === rtcPeer.cleaned) { - return; - } - return OmUtil.error(error); - } - rtcPeer.generateOffer(function(error, offerSdp) { - if (error) { - if (true === rtcPeer.cleaned) { - return; - } - return OmUtil.error('Error generating the offer'); - } - OmUtil.sendMessage({ - id : 'play' - , sdpOffer: offerSdp - }, MsgBase); - }); - }); - } + case 'canPlay': { + const options = VideoUtil.addIceServers({ + mediaConstraints: {audio: true, video: true} + , onIceCandidate: _onIceCandidate + }, m); + _clear(); + rtcPeer = new WebRtcPeerRecvonly(options); + rtcPeer.createOffer() + .then(sdpOffer => { + rtcPeer.processLocalOffer(sdpOffer); + OmUtil.sendMessage({ + id : 'play' + , sdpOffer: sdpOffer.sdp + }, MsgBase); + }) + .catch(_ => OmUtil.error('Error generating the offer')); + } break; case 'playResponse': OmUtil.log('Play SDP answer received from server. Processing ...'); - rtcPeer.processAnswer(m.sdpAnswer, function(error) { - if (error) { - if (true === rtcPeer.cleaned) { - return; - } - return OmUtil.error(error); - } - lm.show(); - level = new MicLevel(); - level.meterPeer(rtcPeer, lm, function(){}, OmUtil.error, true); - }); + + rtcPeer.processRemoteAnswer(m.sdpAnswer) + .then(() => { + const stream = rtcPeer.stream; + if (stream) { + vid[0].srcObject = stream; + lm.show(); + level = new MicLevel(); + level.meterStream(stream, lm, function(){}, OmUtil.error, true); + }; + }) + .catch(error => OmUtil.error(error)); break; case 'startResponse': OmUtil.log('SDP answer received from server. Processing ...'); - rtcPeer.processAnswer(m.sdpAnswer, function(error) { - if (error) { - if (true === rtcPeer.cleaned) { - return; - } - return OmUtil.error(error); - } - }); + rtcPeer.processRemoteAnswer(m.sdpAnswer) + .catch(error => OmUtil.error(error)); break; case 'iceCandidate': - rtcPeer.addIceCandidate(m.candidate, function(error) { - if (error) { - if (true === rtcPeer.cleaned) { - return; - } - return OmUtil.error('Error adding candidate: ' + error); - } - }); + rtcPeer.addIceCandidate(m.candidate) + .catch(error => OmUtil.error('Error adding candidate: ' + error)); break; case 'recording': timer.show().find('.time').text(m.time); diff --git a/openmeetings-web/src/main/front/settings/src/video-util.js b/openmeetings-web/src/main/front/settings/src/video-util.js index 2d9c28c3f..d13f1e5c9 100644 --- a/openmeetings-web/src/main/front/settings/src/video-util.js +++ b/openmeetings-web/src/main/front/settings/src/video-util.js @@ -184,11 +184,10 @@ function _cleanStream(stream) { stream.getTracks().forEach(track => track.stop()); } } -function _cleanPeer(peer) { - if (!!peer) { - peer.cleaned = true; +function _cleanPeer(rtcPeer) { + if (!!rtcPeer) { try { - const pc = peer.peerConnection; + const pc = rtcPeer.pc; if (!!pc) { pc.getSenders().forEach(sender => { try { @@ -208,22 +207,8 @@ function _cleanPeer(peer) { OmUtil.log('Failed to clean receiver' + e); } }); - pc.onconnectionstatechange = null; - pc.ontrack = null; - pc.onremovetrack = null; - pc.onremovestream = null; - pc.onicecandidate = null; - pc.oniceconnectionstatechange = null; - pc.onsignalingstatechange = null; - pc.onicegatheringstatechange = null; - pc.onnegotiationneeded = null; } - peer.dispose(); - peer.removeAllListeners('icecandidate'); - delete peer.generateOffer; - delete peer.processAnswer; - delete peer.processOffer; - delete peer.addIceCandidate; + rtcPeer.dispose(); } catch(e) { //no-op }
