NonFunctional Backup

.vscode/settings.json

@@ -17,6 +17,7 @@
         "dbpath",
         "favicon",
         "ijmap",
+        "kurento",
         "mkdir",
         "mongod",
         "msapplication",

server/package.json

@@ -18,17 +18,20 @@
   "author": "Simon Giesel",
   "license": "ISC",
   "dependencies": {
-    "@nestjs/common": "^6.11.11",
+    "@nestjs/common": "^7.0.8",
-    "@nestjs/core": "^6.11.11",
+    "@nestjs/core": "^7.0.8",
-    "@nestjs/platform-fastify": "^6.11.11",
+    "@nestjs/platform-fastify": "^7.0.8",
-    "@nestjs/swagger": "^4.5.2",
+    "@nestjs/platform-socket.io": "^7.0.8",
+    "@nestjs/swagger": "^4.5.3",
+    "@nestjs/websockets": "^7.0.8",
     "class-transformer": "^0.2.3",
-    "class-validator": "^0.11.1",
+    "class-validator": "^0.12.1",
     "dotenv": "^8.2.0",
     "fastify": "^2.13.1",
     "fastify-swagger": "^2.5.1",
     "handlebars": "^4.7.6",
-    "mongoose": "^5.9.9",
+    "kurento-client": "^6.13.0",
+    "mongoose": "^5.9.10",
     "point-of-view": "^3.8.0",
     "reflect-metadata": "^0.1.13"
   },
@@ -36,9 +39,10 @@
     "@babel/core": "^7.9.0",
     "@babel/preset-env": "^7.9.5",
     "@types/mongoose": "^5.7.12",
-    "@types/node": "^13.13.0",
+    "@types/node": "^13.13.2",
-    "@typescript-eslint/eslint-plugin": "^2.28.0",
+    "@types/socket.io": "^2.1.4",
-    "@typescript-eslint/parser": "^2.28.0",
+    "@typescript-eslint/eslint-plugin": "^2.29.0",
+    "@typescript-eslint/parser": "^2.29.0",
     "babel-loader": "^8.1.0",
     "concurrently": "^5.1.0",
     "dateformat": "^3.0.3",
@@ -51,10 +55,12 @@
     "gulp-sass": "^4.0.2",
     "gulp-typescript": "^6.0.0-alpha.1",
     "gulp-uglifycss": "^1.1.0",
+    "kurento-utils": "^6.13.1",
     "material-design-icons": "^3.0.1",
     "nodemon": "^2.0.3",
     "normalize-scss": "^7.0.1",
-    "ts-node": "^8.8.2",
+    "socket.io-client": "^2.3.0",
+    "ts-node": "^8.9.0",
     "typeface-roboto": "0.0.75",
     "typescript": "^3.8.3",
     "webpack-stream": "^5.2.1"

server/src/app/app.module.ts

@@ -1,8 +1,9 @@
 import { AppController } from './controllers/app.controller';
+import { EventsGateway } from './events/events.gateway';
 import { Module } from '@nestjs/common';
 
 @Module({
-    providers: [],
+    providers: [EventsGateway],
     controllers: [AppController],
     exports: [],
 })

server/src/app/events/events.gateway.ts

@@ -0,0 +1,178 @@
+import * as kurento from 'kurento-client';
+import { ConnectedSocket, MessageBody, SubscribeMessage, WebSocketGateway, WebSocketServer } from '@nestjs/websockets';
+import { Server, Socket } from 'socket.io';
+import { Logger } from '@nestjs/common';
+
+@WebSocketGateway()
+export class EventsGateway {
+    @WebSocketServer()
+    server: Server;
+
+    private kurentoClient = null;
+    private sessions = {};
+    private candidatesQueue = {};
+
+    @SubscribeMessage('start')
+    handleStart(
+        @MessageBody() data: string,
+        @ConnectedSocket() client: Socket,
+    ): void {
+        const message = JSON.parse(data);
+        this.start('helloWorld', client, message.sdpOffer, (error, sdpAnswer)=> {
+            Logger.log('Starting Response:::');
+            Logger.log(error, 'ERROR');
+            if(error) {
+                return client.send(JSON.stringify({
+                    id: 'error',
+                    message: error,
+                }));
+            }
+            client.emit('startResponse', JSON.stringify({
+                sdpAnswer: sdpAnswer,
+            }));
+        });
+    }
+
+    @SubscribeMessage('stop')
+    handleStop(
+    ): void {
+        this.stop('helloWorld');
+    }
+
+    @SubscribeMessage('onIceCandidate')
+    handleOnIceCandidate(@MessageBody() data: string): void{
+        const message = JSON.parse(data);
+        this.onIceCandidate('helloWorld', message.candidate);
+    }
+
+    private getKurentoClient(callback) {
+        if(this.kurentoClient !== null) {
+            return callback(null, this.kurentoClient);
+        }
+
+        kurento('ws://cliffbreak.de:8888/kurento', (error, _kurentoClient) => {
+            if(error) {
+                Logger.error('Could not find media server at address ' + 'ws://cliffbreak.de:8888/kurento', 'EventsGateway');
+                return callback('Could not find media server at address' + 'ws://cliffbreak.de:8888/kurento'
+                        + '. Exiting with error ' + error);
+            }
+
+            this.kurentoClient = _kurentoClient;
+            callback(null, this.kurentoClient);
+        });
+    }
+
+    private start(sessionId, ws, sdpOffer, callback) {
+        if(!sessionId) {
+            return callback('Cannot use undefined sessionId');
+        }
+
+        this.getKurentoClient((error, kurentoClient) => {
+            if(error) {
+                return callback(error);
+            }
+
+            kurentoClient.create('MediaPipeline', (error, pipeline) => {
+                if(error) {
+                    return callback(error);
+                }
+                Logger.log(error, 'Error');
+                Logger.log(pipeline, 'Pipeline');
+
+                this.createMediaElements(pipeline, ws, (error, webRtcEndpoint) => {
+                    if(error) {
+                        pipeline.release();
+                        return callback(error);
+                    }
+
+                    if(this.candidatesQueue[sessionId]) {
+                        while(this.candidatesQueue[sessionId].length) {
+                            const candidate = this.candidatesQueue[sessionId].shift();
+                            webRtcEndpoint.addIceCandidate(candidate);
+                        }
+                    }
+
+                    this.connectMediaElements(webRtcEndpoint, (error) =>{
+                        if(error) {
+                            pipeline.release();
+                            return callback(error);
+                        }
+
+                        webRtcEndpoint.on('OnIceCandidate', (event) =>{
+                            const candidate = kurento.getComplexType('IceCandidate')(event.candidate);
+                            ws.send(JSON.stringify({
+                                id: 'iceCandidate',
+                                candidate: candidate,
+                            }));
+                        });
+
+                        webRtcEndpoint.processOffer(sdpOffer, (error, sdpAnswer) => {
+                            if(error) {
+                                pipeline.release();
+                                return callback(error);
+                            }
+
+                            this.sessions[sessionId] = {
+                                'pipeline': pipeline,
+                                'webRtcEndpoint': webRtcEndpoint,
+                            };
+                            return callback(null, sdpAnswer);
+                        });
+
+                        webRtcEndpoint.gatherCandidates((error) => {
+                            if(error) {
+                                return callback(error);
+                            }
+                        });
+                    });
+                });
+            });
+        });
+    }
+
+    private createMediaElements(pipeline, ws, callback) {
+        pipeline.create('WebRtcEndpoint', (error, webRtcEndpoint) => {
+            if(error) {
+                return callback(error);
+            }
+
+            return callback(null, webRtcEndpoint);
+        });
+    }
+
+    private connectMediaElements(webRtcEndpoint, callback) {
+        webRtcEndpoint.connect(webRtcEndpoint, (error) => {
+            if(error) {
+                return callback(error);
+            }
+            return callback(null);
+        });
+    }
+
+    private stop(sessionId) {
+        if(this.sessions[sessionId]) {
+            const pipeline = this.sessions[sessionId].pipeline;
+            Logger.log('Releasing pipeline', 'EventsGateway');
+            pipeline.release();
+
+            delete this.sessions[sessionId];
+            delete this.candidatesQueue[sessionId];
+        }
+    }
+
+    private onIceCandidate(sessionId, _candidate) {
+        const candidate = kurento.getComplexType('IceCandidate')(_candidate);
+
+        if(this.sessions[sessionId]) {
+            Logger.log('Sending candidate', 'EventsGateway:onICECandidate');
+            const webRtcEndpoint = this.sessions[sessionId].webRtcEndpoint;
+            webRtcEndpoint.addIceCandidate(candidate);
+        } else {
+            Logger.log('Queueing candidate', 'EventsGateway:onICECandidate');
+            if(!this.candidatesQueue[sessionId]) {
+                this.candidatesQueue[sessionId] = [];
+            }
+            this.candidatesQueue[sessionId].push(candidate);
+        }
+    }
+}

webapp/src/js/app.js

@@ -1,5 +1,12 @@
+import io from 'socket.io-client';
+import kurentoUtils from 'kurento-utils';
+import { EventEmitter } from 'events';
+
 (async () => {
 
+    // RTCPeerConnection
+    let pc;
+
     // Send permission dialog to client
     const openMediaDevices = async (constraints) => {
         return await navigator.mediaDevices.getUserMedia(constraints);
@@ -54,11 +61,222 @@
             }
         }
 
-        document.querySelector('button#startVideo').addEventListener('click', _ => {
+        // document.querySelector('button#startVideo').addEventListener('click', () => playVideoFromCamera());
-            playVideoFromCamera();
-        });
 
     } catch (error) {
         console.error('Error accessing media devices.', error);
     }
+
+
+    // console.log('Make Call');
+    const socket = io();
+    socket.on('connect', function () {
+        socket.on('startResponse', startResponse);
+    });
+
+    //     socket.emit('events', { test: 'test' });
+    //     socket.emit('start', {});
+    // });
+
+    // socket.on('exception', function (data) { // Not working?
+    //     console.log('event', data);
+    // });
+
+    // socket.on('disconnect', function () {
+    //     console.log('Disconnected');
+    //     socket.close(); // Good idea?
+    // });
+
+    var webRtcPeer;
+
+    document.querySelector('button#startCall').addEventListener('click', () => start());
+    document.querySelector('button#stopCall').addEventListener('click', () => stop());
+
+    async function start() {
+        console.log('Starting video call ...')
+
+        console.log('Creating WebRtcPeer and generating local sdp offer ...');
+
+        pc = new RTCPeerConnection({
+            iceServers: [
+                {
+                    urls: 'stun:stun.cliffbreak.de',
+                    credential: '8dd2a7643a840ff4bb10e82069b6feceaf6d344a19a590ffc062fa10ea34687b',
+                },
+            ]
+        });
+
+        const dataChannel = pc.createDataChannel('test'); //Unneeded?
+        let videoStream;
+
+        try {
+            const videoSelect = document.querySelector('select#availableCameras');
+            const stream = await navigator.mediaDevices.getUserMedia({
+                video: {
+                    deviceId: videoSelect.options[videoSelect.selectedIndex].value,
+                    // width: 1920,
+                    // height: 1080,
+                    frameRate: 30,
+                }
+            });
+            const videoElement = document.querySelector('video#localVideo');
+            console.log(stream);
+            videoElement.srcObject = stream;
+            videoStream = stream;
+        } catch (error) {
+            console.error('Error opening video camera.', error);
+        }
+
+        let candidategatheringdone = false;
+        const candidatesQueueOut = [];
+        pc.onicecandidate = (event) => {
+            const candidate = event.candidate;
+            if (EventEmitter.listenerCount('icecandidate') || EventEmitter
+                .listenerCount('candidategatheringdone')) {
+                candidategatheringdone = false;
+            } else if (!candidategatheringdone) {
+                candidatesQueueOut.push(candidate);
+                if (!candidate) {
+                    candidategatheringdone = true;
+                    console.log(candidatesQueueOut);
+                    while (candidatesQueueOut.length) {
+                        const candidate = candidatesQueueOut.shift();
+                        if (!!candidate)
+                            onIceCandidate(candidate);
+                    }
+                    for (const track of videoStream.getVideoTracks()) {
+                        pc.addTrack(track, videoStream);
+                    }
+
+                    // pc.addStream(videoStream);
+                }
+            }
+        };
+
+        pc.createOffer().then((offer) => {
+            console.log('Created SDP offer');
+            console.log(offer);
+            const newOffer = new RTCSessionDescription({
+                type: offer.type,
+                sdp: removeFIDFromOffer(offer.sdp) + getSimulcastInfo(videoStream),
+            });
+            return pc.setLocalDescription(newOffer);
+        }).then(() => {
+            const localDescription = pc.localDescription;
+            console.log('Local description set: ' + localDescription.sdp);
+            onOffer(null, localDescription.sdp);
+        }).catch(onOffer);
+
+
+        function removeFIDFromOffer(sdp) {
+            var n = sdp.indexOf("a=ssrc-group:FID");
+
+            if (n > 0) {
+                return sdp.slice(0, n);
+            } else {
+                return sdp;
+            }
+        }
+
+        function getSimulcastInfo(videoStream) {
+            var videoTracks = videoStream.getVideoTracks();
+            if (!videoTracks.length) {
+                logger.warn('No video tracks available in the video stream')
+                return ''
+            }
+            var lines = [
+                'a=x-google-flag:conference',
+                'a=ssrc-group:SIM 1 2 3',
+                'a=ssrc:1 cname:localVideo',
+                'a=ssrc:1 msid:' + videoStream.id + ' ' + videoTracks[0].id,
+                'a=ssrc:1 mslabel:' + videoStream.id,
+                'a=ssrc:1 label:' + videoTracks[0].id,
+                'a=ssrc:2 cname:localVideo',
+                'a=ssrc:2 msid:' + videoStream.id + ' ' + videoTracks[0].id,
+                'a=ssrc:2 mslabel:' + videoStream.id,
+                'a=ssrc:2 label:' + videoTracks[0].id,
+                'a=ssrc:3 cname:localVideo',
+                'a=ssrc:3 msid:' + videoStream.id + ' ' + videoTracks[0].id,
+                'a=ssrc:3 mslabel:' + videoStream.id,
+                'a=ssrc:3 label:' + videoTracks[0].id
+            ];
+
+            lines.push('');
+
+            return lines.join('\n');
+        }
+
+        // var options = {
+        //     localVideo: document.querySelector('video#localVideo'),
+        //     remoteVideo: document.querySelector('video#remoteVideo'),
+        //     onicecandidate: onIceCandidate
+        // }
+
+        // webRtcPeer = kurentoUtils.WebRtcPeer.WebRtcPeerSendrecv(options, function (error) {
+        //     if (error) return onError(error);
+        //     this.generateOffer(onOffer);
+        // });
+
+    }
+
+    function onIceCandidate(candidate) {
+        console.log('Local candidate' + JSON.stringify(candidate));
+        socket.emit('onIceCandidate', JSON.stringify({ candidate }));
+    }
+
+    function onOffer(error, offerSdp) {
+        if (error) return onError(error);
+
+        console.info('Invoking SDP offer callback function ' + location.host);
+        socket.emit('start', JSON.stringify({ sdpOffer: offerSdp }));
+    }
+
+    function onError(error) {
+        console.error(error);
+    }
+
+    function startResponse(message) {
+        console.log('SDP answer received from server. Processing ...');
+        const sdpAnswer = JSON.parse(message).sdpAnswer;
+        // webRtcPeer.processAnswer(sdpAnswer);
+        let answer = new RTCSessionDescription({
+            type: 'answer',
+            sdp: sdpAnswer,
+        });
+
+
+        pc.setRemoteDescription(answer, () => {
+            const remoteVideo = document.querySelector('video#remoteVideo');
+            // remoteVideo.pause();
+
+
+            // pc.getRemoteStreams = function () {
+            //     var stream = new MediaStream();
+            //     pc.getReceivers().forEach(function (sender) {
+            //         stream.addTrack(sender.track);
+            //     });
+            //     return [stream];
+            // };
+
+            // console.table(pc.getRemoteStreams());
+            // const stream = pc.getRemoteStreams()[0];
+            // console.log(stream);
+            pc.ontrack = function (event) {
+                console.log('ontrack');
+                console.log(event);
+                remoteVideo.srcObject = event.streams[0];
+            }
+            // remoteVideo.load();
+        });
+    }
+
+    function stop() {
+        console.log('Stopping video call ...');
+        socket.emit('stop');
+        // TODO:::
+        if (webRtcPeer) {
+            webRtcPeer.dispose();
+            webRtcPeer = null;
+        }
+    }
 })();

webapp/src/views/layouts/main.hbs

@@ -8,6 +8,7 @@
     </content>
     {{>footer}}
     <script src="/assets/js/app.js" async></script>
+    <script src="/assets/js/vendors~app.js" async></script>
 </body>
 
 </html>

webapp/src/views/room.hbs

@@ -1,7 +1,7 @@
-This is a room
-
 <p>RoomId: {{roomId}}</p>
 <select name="availableCameras" id="availableCameras"></select>
 <select name="availableMicrophones" id="availableMicrophones"></select>
-<button name="startVideo" id="startVideo">Start Video</button>
+<button name="startCall" id="startCall">Start Call</button>
-<video id="localVideo" autoplay playsinline />
+<button name="stopCall" id="stopCall">Stop Call</button>
+<video id="localVideo" autoplay playsinline></video>
+<video id="remoteVideo" autoplay playsinline></video>