Cliffbreak/CliffbreakMeet
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

NonFunctional Backup

Browse source
This commit is contained in:
Simon Giesel 2020-04-23 14:24:36 +02:00
parent 067996357a
commit 556d6e3ab8
8 changed files with 1738 additions and 861 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.vscode/settings.json vendored
View file

@ -17,6 +17,7 @@
"dbpath",
"favicon",
"ijmap",
"kurento",
"mkdir",
"mongod",
"msapplication",

2156
server/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

26
server/package.json
View file

@ -18,17 +18,20 @@
"author": "Simon Giesel",
"license": "ISC",
"dependencies": {
"@nestjs/common": "^6.11.11",
"@nestjs/core": "^6.11.11",
"@nestjs/platform-fastify": "^6.11.11",
"@nestjs/swagger": "^4.5.2",
"@nestjs/common": "^7.0.8",
"@nestjs/core": "^7.0.8",
"@nestjs/platform-fastify": "^7.0.8",
"@nestjs/platform-socket.io": "^7.0.8",
"@nestjs/swagger": "^4.5.3",
"@nestjs/websockets": "^7.0.8",
"class-transformer": "^0.2.3",
"class-validator": "^0.11.1",
"class-validator": "^0.12.1",
"dotenv": "^8.2.0",
"fastify": "^2.13.1",
"fastify-swagger": "^2.5.1",
"handlebars": "^4.7.6",
"mongoose": "^5.9.9",
"kurento-client": "^6.13.0",
"mongoose": "^5.9.10",
"point-of-view": "^3.8.0",
"reflect-metadata": "^0.1.13"
},
@ -36,9 +39,10 @@
"@babel/core": "^7.9.0",
"@babel/preset-env": "^7.9.5",
"@types/mongoose": "^5.7.12",
"@types/node": "^13.13.0",
"@typescript-eslint/eslint-plugin": "^2.28.0",
"@typescript-eslint/parser": "^2.28.0",
"@types/node": "^13.13.2",
"@types/socket.io": "^2.1.4",
"@typescript-eslint/eslint-plugin": "^2.29.0",
"@typescript-eslint/parser": "^2.29.0",
"babel-loader": "^8.1.0",
"concurrently": "^5.1.0",
"dateformat": "^3.0.3",
@ -51,10 +55,12 @@
"gulp-sass": "^4.0.2",
"gulp-typescript": "^6.0.0-alpha.1",
"gulp-uglifycss": "^1.1.0",
"kurento-utils": "^6.13.1",
"material-design-icons": "^3.0.1",
"nodemon": "^2.0.3",
"normalize-scss": "^7.0.1",
"ts-node": "^8.8.2",
"socket.io-client": "^2.3.0",
"ts-node": "^8.9.0",
"typeface-roboto": "0.0.75",
"typescript": "^3.8.3",
"webpack-stream": "^5.2.1"

3
server/src/app/app.module.ts
View file

@ -1,8 +1,9 @@
import { AppController } from './controllers/app.controller';
import { EventsGateway } from './events/events.gateway';
import { Module } from '@nestjs/common';
@Module({
providers: [],
providers: [EventsGateway],
controllers: [AppController],
exports: [],
})

178
server/src/app/events/events.gateway.ts Normal file
View file

@ -0,0 +1,178 @@
import * as kurento from 'kurento-client';
import { ConnectedSocket, MessageBody, SubscribeMessage, WebSocketGateway, WebSocketServer } from '@nestjs/websockets';
import { Server, Socket } from 'socket.io';
import { Logger } from '@nestjs/common';
@WebSocketGateway()
export class EventsGateway {
@WebSocketServer()
server: Server;
private kurentoClient = null;
private sessions = {};
private candidatesQueue = {};
@SubscribeMessage('start')
handleStart(
@MessageBody() data: string,
@ConnectedSocket() client: Socket,
): void {
const message = JSON.parse(data);
this.start('helloWorld', client, message.sdpOffer, (error, sdpAnswer)=> {
Logger.log('Starting Response:::');
Logger.log(error, 'ERROR');
if(error) {
return client.send(JSON.stringify({
id: 'error',
message: error,
}));
}
client.emit('startResponse', JSON.stringify({
sdpAnswer: sdpAnswer,
}));
});
}
@SubscribeMessage('stop')
handleStop(
): void {
this.stop('helloWorld');
}
@SubscribeMessage('onIceCandidate')
handleOnIceCandidate(@MessageBody() data: string): void{
const message = JSON.parse(data);
this.onIceCandidate('helloWorld', message.candidate);
}
private getKurentoClient(callback) {
if(this.kurentoClient !== null) {
return callback(null, this.kurentoClient);
}
kurento('ws://cliffbreak.de:8888/kurento', (error, _kurentoClient) => {
if(error) {
Logger.error('Could not find media server at address ' + 'ws://cliffbreak.de:8888/kurento', 'EventsGateway');
return callback('Could not find media server at address' + 'ws://cliffbreak.de:8888/kurento'
+ '. Exiting with error ' + error);
}
this.kurentoClient = _kurentoClient;
callback(null, this.kurentoClient);
});
}
private start(sessionId, ws, sdpOffer, callback) {
if(!sessionId) {
return callback('Cannot use undefined sessionId');
}
this.getKurentoClient((error, kurentoClient) => {
if(error) {
return callback(error);
}
kurentoClient.create('MediaPipeline', (error, pipeline) => {
if(error) {
return callback(error);
}
Logger.log(error, 'Error');
Logger.log(pipeline, 'Pipeline');
this.createMediaElements(pipeline, ws, (error, webRtcEndpoint) => {
if(error) {
pipeline.release();
return callback(error);
}
if(this.candidatesQueue[sessionId]) {
while(this.candidatesQueue[sessionId].length) {
const candidate = this.candidatesQueue[sessionId].shift();
webRtcEndpoint.addIceCandidate(candidate);
}
}
this.connectMediaElements(webRtcEndpoint, (error) =>{
if(error) {
pipeline.release();
return callback(error);
}
webRtcEndpoint.on('OnIceCandidate', (event) =>{
const candidate = kurento.getComplexType('IceCandidate')(event.candidate);
ws.send(JSON.stringify({
id: 'iceCandidate',
candidate: candidate,
}));
});
webRtcEndpoint.processOffer(sdpOffer, (error, sdpAnswer) => {
if(error) {
pipeline.release();
return callback(error);
}
this.sessions[sessionId] = {
'pipeline': pipeline,
'webRtcEndpoint': webRtcEndpoint,
};
return callback(null, sdpAnswer);
});
webRtcEndpoint.gatherCandidates((error) => {
if(error) {
return callback(error);
}
});
});
});
});
});
}
private createMediaElements(pipeline, ws, callback) {
pipeline.create('WebRtcEndpoint', (error, webRtcEndpoint) => {
if(error) {
return callback(error);
}
return callback(null, webRtcEndpoint);
});
}
private connectMediaElements(webRtcEndpoint, callback) {
webRtcEndpoint.connect(webRtcEndpoint, (error) => {
if(error) {
return callback(error);
}
return callback(null);
});
}
private stop(sessionId) {
if(this.sessions[sessionId]) {
const pipeline = this.sessions[sessionId].pipeline;
Logger.log('Releasing pipeline', 'EventsGateway');
pipeline.release();
delete this.sessions[sessionId];
delete this.candidatesQueue[sessionId];
}
}
private onIceCandidate(sessionId, _candidate) {
const candidate = kurento.getComplexType('IceCandidate')(_candidate);
if(this.sessions[sessionId]) {
Logger.log('Sending candidate', 'EventsGateway:onICECandidate');
const webRtcEndpoint = this.sessions[sessionId].webRtcEndpoint;
webRtcEndpoint.addIceCandidate(candidate);
} else {
Logger.log('Queueing candidate', 'EventsGateway:onICECandidate');
if(!this.candidatesQueue[sessionId]) {
this.candidatesQueue[sessionId] = [];
}
this.candidatesQueue[sessionId].push(candidate);
}
}
}

224
webapp/src/js/app.js
View file

@ -1,5 +1,12 @@
import io from 'socket.io-client';
import kurentoUtils from 'kurento-utils';
import { EventEmitter } from 'events';
(async () => {
// RTCPeerConnection
let pc;
// Send permission dialog to client
const openMediaDevices = async (constraints) => {
return await navigator.mediaDevices.getUserMedia(constraints);
@ -54,11 +61,222 @@
}
}
document.querySelector('button#startVideo').addEventListener('click', _ => {
playVideoFromCamera();
});
// document.querySelector('button#startVideo').addEventListener('click', () => playVideoFromCamera());
} catch (error) {
console.error('Error accessing media devices.', error);
}
// console.log('Make Call');
const socket = io();
socket.on('connect', function () {
socket.on('startResponse', startResponse);
});
// socket.emit('events', { test: 'test' });
// socket.emit('start', {});
// });
// socket.on('exception', function (data) { // Not working?
// console.log('event', data);
// });
// socket.on('disconnect', function () {
// console.log('Disconnected');
// socket.close(); // Good idea?
// });
var webRtcPeer;
document.querySelector('button#startCall').addEventListener('click', () => start());
document.querySelector('button#stopCall').addEventListener('click', () => stop());
async function start() {
console.log('Starting video call ...')
console.log('Creating WebRtcPeer and generating local sdp offer ...');
pc = new RTCPeerConnection({
iceServers: [
{
urls: 'stun:stun.cliffbreak.de',
credential: '8dd2a7643a840ff4bb10e82069b6feceaf6d344a19a590ffc062fa10ea34687b',
},
]
});
const dataChannel = pc.createDataChannel('test'); //Unneeded?
let videoStream;
try {
const videoSelect = document.querySelector('select#availableCameras');
const stream = await navigator.mediaDevices.getUserMedia({
video: {
deviceId: videoSelect.options[videoSelect.selectedIndex].value,
// width: 1920,
// height: 1080,
frameRate: 30,
}
});
const videoElement = document.querySelector('video#localVideo');
console.log(stream);
videoElement.srcObject = stream;
videoStream = stream;
} catch (error) {
console.error('Error opening video camera.', error);
}
let candidategatheringdone = false;
const candidatesQueueOut = [];
pc.onicecandidate = (event) => {
const candidate = event.candidate;
if (EventEmitter.listenerCount('icecandidate') || EventEmitter
.listenerCount('candidategatheringdone')) {
candidategatheringdone = false;
} else if (!candidategatheringdone) {
candidatesQueueOut.push(candidate);
if (!candidate) {
candidategatheringdone = true;
console.log(candidatesQueueOut);
while (candidatesQueueOut.length) {
const candidate = candidatesQueueOut.shift();
if (!!candidate)
onIceCandidate(candidate);
}
for (const track of videoStream.getVideoTracks()) {
pc.addTrack(track, videoStream);
}
// pc.addStream(videoStream);
}
}
};
pc.createOffer().then((offer) => {
console.log('Created SDP offer');
console.log(offer);
const newOffer = new RTCSessionDescription({
type: offer.type,
sdp: removeFIDFromOffer(offer.sdp) + getSimulcastInfo(videoStream),
});
return pc.setLocalDescription(newOffer);
}).then(() => {
const localDescription = pc.localDescription;
console.log('Local description set: ' + localDescription.sdp);
onOffer(null, localDescription.sdp);
}).catch(onOffer);
function removeFIDFromOffer(sdp) {
var n = sdp.indexOf("a=ssrc-group:FID");
if (n > 0) {
return sdp.slice(0, n);
} else {
return sdp;
}
}
function getSimulcastInfo(videoStream) {
var videoTracks = videoStream.getVideoTracks();
if (!videoTracks.length) {
logger.warn('No video tracks available in the video stream')
return ''
}
var lines = [
'a=x-google-flag:conference',
'a=ssrc-group:SIM 1 2 3',
'a=ssrc:1 cname:localVideo',
'a=ssrc:1 msid:' + videoStream.id + ' ' + videoTracks[0].id,
'a=ssrc:1 mslabel:' + videoStream.id,
'a=ssrc:1 label:' + videoTracks[0].id,
'a=ssrc:2 cname:localVideo',
'a=ssrc:2 msid:' + videoStream.id + ' ' + videoTracks[0].id,
'a=ssrc:2 mslabel:' + videoStream.id,
'a=ssrc:2 label:' + videoTracks[0].id,
'a=ssrc:3 cname:localVideo',
'a=ssrc:3 msid:' + videoStream.id + ' ' + videoTracks[0].id,
'a=ssrc:3 mslabel:' + videoStream.id,
'a=ssrc:3 label:' + videoTracks[0].id
];
lines.push('');
return lines.join('\n');
}
// var options = {
// localVideo: document.querySelector('video#localVideo'),
// remoteVideo: document.querySelector('video#remoteVideo'),
// onicecandidate: onIceCandidate
// }
// webRtcPeer = kurentoUtils.WebRtcPeer.WebRtcPeerSendrecv(options, function (error) {
// if (error) return onError(error);
// this.generateOffer(onOffer);
// });
}
function onIceCandidate(candidate) {
console.log('Local candidate' + JSON.stringify(candidate));
socket.emit('onIceCandidate', JSON.stringify({ candidate }));
}
function onOffer(error, offerSdp) {
if (error) return onError(error);
console.info('Invoking SDP offer callback function ' + location.host);
socket.emit('start', JSON.stringify({ sdpOffer: offerSdp }));
}
function onError(error) {
console.error(error);
}
function startResponse(message) {
console.log('SDP answer received from server. Processing ...');
const sdpAnswer = JSON.parse(message).sdpAnswer;
// webRtcPeer.processAnswer(sdpAnswer);
let answer = new RTCSessionDescription({
type: 'answer',
sdp: sdpAnswer,
});
pc.setRemoteDescription(answer, () => {
const remoteVideo = document.querySelector('video#remoteVideo');
// remoteVideo.pause();
// pc.getRemoteStreams = function () {
// var stream = new MediaStream();
// pc.getReceivers().forEach(function (sender) {
// stream.addTrack(sender.track);
// });
// return [stream];
// };
// console.table(pc.getRemoteStreams());
// const stream = pc.getRemoteStreams()[0];
// console.log(stream);
pc.ontrack = function (event) {
console.log('ontrack');
console.log(event);
remoteVideo.srcObject = event.streams[0];
}
// remoteVideo.load();
});
}
function stop() {
console.log('Stopping video call ...');
socket.emit('stop');
// TODO:::
if (webRtcPeer) {
webRtcPeer.dispose();
webRtcPeer = null;
}
}
})();

1
webapp/src/views/layouts/main.hbs
View file

@ -8,6 +8,7 @@
</content>
{{>footer}}
<script src="/assets/js/app.js" async></script>
<script src="/assets/js/vendors~app.js" async></script>
</body>
</html>

8
webapp/src/views/room.hbs
View file

@ -1,7 +1,7 @@
This is a room
<p>RoomId: {{roomId}}</p>
<select name="availableCameras" id="availableCameras"></select>
<select name="availableMicrophones" id="availableMicrophones"></select>
<button name="startVideo" id="startVideo">Start Video</button>
<video id="localVideo" autoplay playsinline />
<button name="startCall" id="startCall">Start Call</button>
<button name="stopCall" id="stopCall">Stop Call</button>
<video id="localVideo" autoplay playsinline></video>
<video id="remoteVideo" autoplay playsinline></video>