From 7848c5770b210818806c7f6a55e6017083f2dc91 Mon Sep 17 00:00:00 2001 From: Vittorio Palmisano Date: Sun, 7 Nov 2021 14:42:55 +0100 Subject: [PATCH 1/3] feat(server): Experimental WHIP support --- server/lib/Room.js | 10 ++ server/lib/interactiveServer.js | 2 +- server/package.json | 4 +- server/server.js | 162 ++++++++++++++++++++++++++++++-- 4 files changed, 167 insertions(+), 11 deletions(-) diff --git a/server/lib/Room.js b/server/lib/Room.js index 1d5bade6a..dc04bb3bc 100644 --- a/server/lib/Room.js +++ b/server/lib/Room.js @@ -258,6 +258,16 @@ class Room extends EventEmitter return this._mediasoupRouter.rtpCapabilities; } + /** + * Get a Broadcaster. + * + * @type {String} broadcasterId - Broadcaster id. + */ + getBroadcaster({ broadcasterId }) + { + return this._broadcasters.get(broadcasterId); + } + /** * Create a Broadcaster. This is for HTTP API requests (see server.js). * diff --git a/server/lib/interactiveServer.js b/server/lib/interactiveServer.js index 0bda590d7..9303cf8b7 100644 --- a/server/lib/interactiveServer.js +++ b/server/lib/interactiveServer.js @@ -542,7 +542,7 @@ class Interactive useColors : true, useGlobal : true, ignoreUndefined : false, - preview : false, + preview : false }); this._isTerminalOpen = true; diff --git a/server/package.json b/server/package.json index 5ea33f366..cdae8079a 100644 --- a/server/package.json +++ b/server/package.json @@ -20,8 +20,10 @@ "express": "^4.17.1", "heapdump": "^0.3.15", "mediasoup": "github:versatica/mediasoup#v3", + "mediasoup-client": "github:versatica/mediasoup-client#v3", "pidusage": "^2.0.21", - "protoo-server": "^4.0.5" + "protoo-server": "^4.0.5", + "sdp-transform": "^2.14.1" }, "devDependencies": { "eslint": "^6.8.0", diff --git a/server/server.js b/server/server.js index ef28f4197..3c7caf2d0 100755 --- a/server/server.js +++ b/server/server.js @@ -16,12 +16,17 @@ const url = require('url'); const protoo = require('protoo-server'); const mediasoup = require('mediasoup'); const express = require('express'); -const bodyParser = require('body-parser'); const { AwaitQueue } = require('awaitqueue'); const Logger = require('./lib/Logger'); const Room = require('./lib/Room'); const interactiveServer = require('./lib/interactiveServer'); const interactiveClient = require('./lib/interactiveClient'); +const sdpTransform = require('sdp-transform'); +const sdpCommonUtils = require('mediasoup-client/lib/handlers/sdp/commonUtils'); +const ortc = require('mediasoup-client/lib/ortc'); +const { RemoteSdp } = require('mediasoup-client/lib/handlers/sdp/RemoteSdp'); +const sdpUnifiedPlanUtils = require('mediasoup-client/lib/handlers/sdp/unifiedPlanUtils'); +const utils = require('mediasoup-client/lib/utils'); const logger = new Logger(); @@ -134,22 +139,25 @@ async function createExpressApp() expressApp = express(); - expressApp.use(bodyParser.json()); + expressApp.use(express.json()); + expressApp.use(express.text({ + type : [ + 'application/sdp', + 'text/plain' + ] + })); /** * For every API request, verify that the roomId in the path matches and * existing room. */ expressApp.param( - 'roomId', (req, res, next, roomId) => + 'roomId', async (req, res, next, roomId) => { // The room must exist for all API requests. if (!rooms.has(roomId)) { - const error = new Error(`room with id "${roomId}" not found`); - - error.status = 404; - throw error; + await getOrCreateRoom({ roomId }); } req.room = rooms.get(roomId); @@ -233,7 +241,7 @@ async function createExpressApp() broadcasterId, type, rtcpMux, - comedia, + comedia, sctpCapabilities }); @@ -363,7 +371,7 @@ async function createExpressApp() next(error); } }); - + /** * POST API to create a mediasoup DataProducer associated to a Broadcaster. * The exact Transport in which the DataProducer must be created is signaled in @@ -395,6 +403,142 @@ async function createExpressApp() } }); + /** + * WHIP post handler. + */ + expressApp.post( + '/whip/:roomId/:broadcasterId', async (req, res, next) => + { + logger.info('whip POST', req.params, req.headers, req.body); + const { broadcasterId } = req.params; + + try + { + const localSdpObject = sdpTransform.parse(req.body); + + const rtpCapabilities = sdpCommonUtils.extractRtpCapabilities( + { sdpObject: localSdpObject }); + const dtlsParameters = sdpCommonUtils.extractDtlsParameters( + { sdpObject: localSdpObject }); + + const routerRtpCapabilities = req.room.getRouterRtpCapabilities(); + const extendedRtpCapabilities = ortc.getExtendedRtpCapabilities( + rtpCapabilities, routerRtpCapabilities); + + const sendingRtpParametersByKind = + { + audio : ortc.getSendingRtpParameters('audio', extendedRtpCapabilities), + video : ortc.getSendingRtpParameters('video', extendedRtpCapabilities) + }; + const sendingRemoteRtpParametersByKind = + { + audio : ortc.getSendingRemoteRtpParameters('audio', extendedRtpCapabilities), + video : ortc.getSendingRemoteRtpParameters('video', extendedRtpCapabilities) + }; + + // Create a broadcaster, if it not exists. + if (!req.room.getBroadcaster({ broadcasterId })) + { + await req.room.createBroadcaster({ + id : broadcasterId, + displayName : 'test', + device : { name: 'device' }, + rtpCapabilities + }); + } + + // Create a WebRTC transport. + const transport = await req.room.createBroadcasterTransport({ + broadcasterId, + type : 'webrtc' + }); + + // Connect the WebRTC transport. + await req.room.connectBroadcasterTransport({ + broadcasterId, + transportId : transport.id, + dtlsParameters + }); + + const remoteSdp = new RemoteSdp({ + iceParameters : transport.iceParameters, + iceCandidates : transport.iceCandidates, + dtlsParameters : transport.dtlsParameters, + sctpParameters : transport.sctpParameters + }); + + // Publish audio and video. + for (const { type, mid } of localSdpObject.media) + { + const mediaSectionIdx = remoteSdp.getNextMediaSectionIdx(); + const offerMediaObject = localSdpObject.media[mediaSectionIdx.idx]; + + const sendingRtpParameters = + utils.clone(sendingRtpParametersByKind[type], {}); + + const sendingRemoteRtpParameters = + utils.clone(sendingRemoteRtpParametersByKind[type], {}); + + // Set MID. + sendingRtpParameters.mid = mid; + + // Set RTCP CNAME. + sendingRtpParameters.rtcp.cname = + sdpCommonUtils.getCname({ offerMediaObject }); + + // Set RTP encodings by parsing the SDP offer. + sendingRtpParameters.encodings = + sdpUnifiedPlanUtils.getRtpEncodings({ offerMediaObject }); + + remoteSdp.send({ + offerMediaObject, + reuseMid : mediaSectionIdx.reuseMid, + offerRtpParameters : sendingRtpParameters, + answerRtpParameters : sendingRemoteRtpParameters, + codecOptions : {}, + extmapAllowMixed : true + }); + + await req.room.createBroadcasterProducer({ + broadcasterId, + transportId : transport.id, + kind : type, + rtpParameters : sendingRtpParameters + }); + } + + const answer = remoteSdp.getSdp(); + + res.contentType('application/sdp').status(201) + .send(answer); + } + catch (error) + { + next(error); + } + }); + + /** + * WHIP delete handler. + */ + expressApp.delete( + '/whip/:roomId/:broadcasterId', async (req, res, next) => + { + logger.info('whip DELETE', req.params, req.headers); + const { broadcasterId } = req.params; + + try + { + req.room.deleteBroadcaster({ broadcasterId }); + res.contentType('text/plain').status(200) + .send(); + } + catch (error) + { + next(error); + } + }); + /** * Error handler. */ From 7593ce2dbc8f039542f63f28d2ff9976c1b7124d Mon Sep 17 00:00:00 2001 From: Vittorio Palmisano Date: Sun, 7 Nov 2021 14:53:58 +0100 Subject: [PATCH 2/3] changed display names --- server/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server.js b/server/server.js index 3c7caf2d0..19640837c 100755 --- a/server/server.js +++ b/server/server.js @@ -441,8 +441,8 @@ async function createExpressApp() { await req.room.createBroadcaster({ id : broadcasterId, - displayName : 'test', - device : { name: 'device' }, + displayName : 'WHIP broadcaster', + device : { name: 'WHIP device' }, rtpCapabilities }); } From e11b6242a7f4c87f34d30fef884ad52f328679b0 Mon Sep 17 00:00:00 2001 From: Vittorio Palmisano Date: Sun, 14 Nov 2021 17:32:52 +0100 Subject: [PATCH 3/3] WIP adding support for whip-js --- server/lib/Room.js | 34 +++++++++++++++++++++++++ server/package.json | 1 + server/server.js | 62 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/server/lib/Room.js b/server/lib/Room.js index dc04bb3bc..8cc660f55 100644 --- a/server/lib/Room.js +++ b/server/lib/Room.js @@ -520,6 +520,40 @@ class Room extends EventEmitter await transport.connect({ dtlsParameters }); } + /** + * Restart ICE for a Broadcaster mediasoup WebRtcTransport. + * + * @async + * + * @type {String} broadcasterId + * @type {String} transportId + */ + async restartBroadcasterTransportICE( + { + broadcasterId, + transportId + } + ) + { + const broadcaster = this._broadcasters.get(broadcasterId); + + if (!broadcaster) + throw new Error(`broadcaster with id "${broadcasterId}" does not exist`); + + const transport = broadcaster.data.transports.get(transportId); + + if (!transport) + throw new Error(`transport with id "${transportId}" does not exist`); + + if (transport.constructor.name !== 'WebRtcTransport') + { + throw new Error( + `transport with id "${transportId}" is not a WebRtcTransport`); + } + + return await transport.restartIce(); + } + /** * Create a mediasoup Producer associated to a Broadcaster. * diff --git a/server/package.json b/server/package.json index cdae8079a..6bc3baa5a 100644 --- a/server/package.json +++ b/server/package.json @@ -16,6 +16,7 @@ "awaitqueue": "^2.3.3", "body-parser": "^1.19.0", "colors": "^1.4.0", + "cors": "^2.8.5", "debug": "^4.3.1", "express": "^4.17.1", "heapdump": "^0.3.15", diff --git a/server/server.js b/server/server.js index 19640837c..3927ee43b 100755 --- a/server/server.js +++ b/server/server.js @@ -27,6 +27,7 @@ const ortc = require('mediasoup-client/lib/ortc'); const { RemoteSdp } = require('mediasoup-client/lib/handlers/sdp/RemoteSdp'); const sdpUnifiedPlanUtils = require('mediasoup-client/lib/handlers/sdp/unifiedPlanUtils'); const utils = require('mediasoup-client/lib/utils'); +const cors = require('cors'); const logger = new Logger(); @@ -143,9 +144,15 @@ async function createExpressApp() expressApp.use(express.text({ type : [ 'application/sdp', + 'application/trickle-ice-sdpfrag', 'text/plain' ] })); + expressApp.use( + cors({ + origin : true + }) + ); /** * For every API request, verify that the roomId in the path matches and @@ -437,7 +444,9 @@ async function createExpressApp() }; // Create a broadcaster, if it not exists. - if (!req.room.getBroadcaster({ broadcasterId })) + let broadcaster = req.room.getBroadcaster({ broadcasterId }); + + if (!broadcaster) { await req.room.createBroadcaster({ id : broadcasterId, @@ -445,6 +454,7 @@ async function createExpressApp() device : { name: 'WHIP device' }, rtpCapabilities }); + broadcaster = req.room.getBroadcaster({ broadcasterId }); } // Create a WebRTC transport. @@ -467,6 +477,8 @@ async function createExpressApp() sctpParameters : transport.sctpParameters }); + broadcaster.data.transports.get(transport.id).appData.remoteSdp = remoteSdp; + // Publish audio and video. for (const { type, mid } of localSdpObject.media) { @@ -480,7 +492,7 @@ async function createExpressApp() utils.clone(sendingRemoteRtpParametersByKind[type], {}); // Set MID. - sendingRtpParameters.mid = mid; + sendingRtpParameters.mid = String(mid); // Set RTCP CNAME. sendingRtpParameters.rtcp.cname = @@ -506,10 +518,54 @@ async function createExpressApp() rtpParameters : sendingRtpParameters }); } + const answer = remoteSdp.getSdp(); + + res.contentType('application/sdp') + .status(201) + .send(answer); + } + catch (error) + { + next(error); + } + }); + + /** + * WHIP patch handler. + */ + expressApp.patch( + '/whip/:roomId/:broadcasterId', async (req, res, next) => + { + logger.info('whip PATCH', req.params, req.headers, req.body); + const { broadcasterId } = req.params; + + try + { + const broadcaster = req.room.getBroadcaster({ broadcasterId }); + + if (!broadcaster) + throw Error(`broadcaster with id "${broadcasterId}" does not exist`); + + if (!broadcaster.data.transports.size) + throw Error(`broadcaster with id "${broadcasterId}" has no transports`); + + const transport = [ ...broadcaster.data.transports.values() ][0]; + const { remoteSdp } = transport.appData; + + if (!remoteSdp) + throw Error(`broadcaster with id "${broadcasterId}" has no remote SDP set`); + + const iceParameters = await req.room.restartBroadcasterTransportICE({ + broadcasterId, + transportId : transport.id + }); + + remoteSdp.updateIceParameters(iceParameters); const answer = remoteSdp.getSdp(); - res.contentType('application/sdp').status(201) + res.contentType('application/sdp') + .status(200) .send(answer); } catch (error)