bug fixes and added more audio codecs

thirdparty/CodecsHandler.js

@@ -28,6 +28,10 @@ Copyright (c) 2012-2020 [Muaz Khan](https://github.com/muaz-khan)
 var CodecsHandler = (function() {
     function preferCodec(sdp, codecName) {
 		
+		if (codecName){
+			codecName = codecName.toLowerCase();
+		}
+		
         var info = splitLines(sdp);
         if (!info.videoCodecNumbers) {
             return sdp;
@@ -41,18 +45,45 @@ var CodecsHandler = (function() {
             return sdp;
         } else if (codecName === 'av1' && info.av1LineNumber === info.videoCodecNumbers[0]) {
             return sdp;
-		} else if (codecName === 'red' && info.redLineNumber === info.videoCodecNumbers[0]) {
-            return sdp;
-		} else if (codecName === 'fec' && info.fecLineNumber === info.videoCodecNumbers[0]) {
-            return sdp;
-        }
+		}
+		//} else if (codecName === 'red' && info.redLineNumber === info.videoCodecNumbers[0]) {
+        //    return sdp;
+		//} else if (codecName === 'fec' && info.fecLineNumber === info.videoCodecNumbers[0]) {
+        //    return sdp;
+        //}
 
         sdp = preferCodecHelper(sdp, codecName, info);
 
         return sdp;
     }
+	
+	
+	function preferAudioCodec(sdp, codecName) {
+		if (codecName){
+			codecName = codecName.toLowerCase();
+		}
+        var info = splitAudioLines(sdp);
+        if (!info.audioCodecNumbers) {
+            return sdp;
+		} else if (codecName === 'opus' && info.opusLineNumber === info.audioCodecNumbers[0]) {
+            return sdp;
+		} else if (codecName === 'isac' && info.isacLineNumber === info.audioCodecNumbers[0]) {
+            return sdp;
+		} else if (codecName === 'g722' && info.g722LineNumber === info.audioCodecNumbers[0]) {
+            return sdp;
+		} else if (codecName === 'pcmu' && info.pcmuLineNumber === info.audioCodecNumbers[0]) {
+            return sdp;
+		} else if (codecName === 'pcma' && info.pcmaLineNumber === info.audioCodecNumbers[0]) {
+            return sdp;
+		} else if (codecName === 'red' && info.redLineNumber === info.audioCodecNumbers[0]) {
+            return sdp;
+        }
 
-    function preferCodecHelper(sdp, codec, info, ignore) {
+        sdp = preferAudioCodecHelper(sdp, codecName, info);
+        return sdp;
+    }
+
+    function preferCodecHelper(sdp, codec, info) {
         var preferCodecNumber = '';
 
         if (codec === 'vp8') {
@@ -82,28 +113,26 @@ var CodecsHandler = (function() {
                 return sdp;
             }
             preferCodecNumber = info.av1LineNumber;
+		} else {
+			return sdp;
+		}
+		//} else if (codec === 'red') {
+      //      if (!info.redLineNumber) {
+      //          return sdp;
+      //      }
+       //     preferCodecNumber = info.redLineNumber;
 			
-		} else if (codec === 'red') {
-            if (!info.redLineNumber) {
-                return sdp;
-            }
-            preferCodecNumber = info.redLineNumber;
-			
-		} else if (codec === 'fec') {
-            if (!info.fecLineNumber) {
-                return sdp;
-            }
-            preferCodecNumber = info.fecLineNumber;
-        }
+		//} else if (codec === 'fec') {
+      //      if (!info.fecLineNumber) {
+       //         return sdp;
+       //     }
+       //     preferCodecNumber = info.fecLineNumber;
+      //  }
 
         var newLine = info.videoCodecNumbersOriginal.split('SAVPF')[0] + 'SAVPF ';
 
         var newOrder = [preferCodecNumber];
-
-        if (ignore) {
-            newOrder = [];
-        }
-
+		
         info.videoCodecNumbers.forEach(function(codecNumber) {
             if (codecNumber === preferCodecNumber) return;
             newOrder.push(codecNumber);
@@ -114,6 +143,55 @@ var CodecsHandler = (function() {
         sdp = sdp.replace(info.videoCodecNumbersOriginal, newLine);
         return sdp;
     }
+	
+	function preferAudioCodecHelper(sdp, codec, info) {
+        var preferCodecNumber = '';
+
+        if (codec === 'opus') {
+            if (!info.opusLineNumber) {
+                return sdp;
+            }
+            preferCodecNumber = info.opusLineNumber;
+        } else if (codec === 'isac') {
+            if (!info.isacLineNumber) {
+                return sdp;
+            }
+            preferCodecNumber = info.isacLineNumber;
+		} else if (codec === 'g722') {
+            if (!info.g722LineNumber) {
+                return sdp;
+            }
+            preferCodecNumber = info.g722LineNumber;	
+        } else if (codec === 'pcmu') {
+            if (!info.pcmuLineNumber) {
+                return sdp;
+            }
+            preferCodecNumber = info.pcmuLineNumber;
+		 } else if (codec === 'pcma') {
+            if (!info.pcmaLineNumber) {
+                return sdp;
+            }
+            preferCodecNumber = info.pcmaLineNumber;	
+		} else if (codec === 'red') {
+            if (!info.redLineNumber) {
+                return sdp;
+            }
+            preferCodecNumber = info.redLineNumber;
+		}
+        var newLine = info.audioCodecNumbersOriginal.split('SAVPF')[0] + 'SAVPF ';
+
+        var newOrder = [preferCodecNumber];
+
+        info.audioCodecNumbers.forEach(function(codecNumber) {
+            if (codecNumber === preferCodecNumber) return;
+            newOrder.push(codecNumber);
+        });
+
+        newLine += newOrder.join(' ');
+
+        sdp = sdp.replace(info.audioCodecNumbersOriginal, newLine);
+        return sdp;
+    }
 
     function splitLines(sdp) {
         var info = {};
@@ -127,36 +205,82 @@ var CodecsHandler = (function() {
                     info.videoCodecNumbersOriginal = line;
                 });
             }
+			
+			var LINE = line.toUpperCase();
 
-            if (line.indexOf('VP8/90000') !== -1 && !info.vp8LineNumber) {
+            if (LINE.indexOf('VP8/90000') !== -1 && !info.vp8LineNumber) {
                 info.vp8LineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
             }
 
-            if (line.indexOf('VP9/90000') !== -1 && !info.vp9LineNumber) {
+            if (LINE.indexOf('VP9/90000') !== -1 && !info.vp9LineNumber) {
                 info.vp9LineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
             }
 
-            if (line.indexOf('H264/90000') !== -1 && !info.h264LineNumber) {
+            if (LINE.indexOf('H264/90000') !== -1 && !info.h264LineNumber) {
                 info.h264LineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
             }
 			
-			if (line.indexOf('H265/90000') !== -1 && !info.h265LineNumber) {
+			if (LINE.indexOf('H265/90000') !== -1 && !info.h265LineNumber) {
                 info.h265LineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
             }
 			
-            if (line.indexOf('AV1X/90000') !== -1 && !info.av1LineNumber) {
+            if (LINE.indexOf('AV1X/90000') !== -1 && !info.av1LineNumber) {
                 info.av1LineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
-            } else if (line.indexOf('AV1/90000') !== -1 && !info.av1LineNumber) {
+            } else if (LINE.indexOf('AV1/90000') !== -1 && !info.av1LineNumber) {
                 info.av1LineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
             }
 			
-			if (line.indexOf('red/90000') !== -1 && !info.redLineNumber) {
+			//if (LINE.indexOf('RED/90000') !== -1 && !info.redLineNumber) {
+          //      info.redLineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
+         //   }
+			
+		//	if (LINE.indexOf('FEC/90000') !== -1 && !info.fecLineNumber) {
+         //       info.fecLineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
+          //  }
+        });
+
+        return info;
+    }
+	
+	function splitAudioLines(sdp) {
+        var info = {};
+        sdp.split('\n').forEach(function(line) {
+            if (line.indexOf('m=audio') === 0) {
+                info.audioCodecNumbers = [];
+                line.split('SAVPF')[1].split(' ').forEach(function(codecNumber) {
+                    codecNumber = codecNumber.trim();
+                    if (!codecNumber || !codecNumber.length) return;
+                    info.audioCodecNumbers.push(codecNumber);
+                    info.audioCodecNumbersOriginal = line;
+                });
+            }
+			
+			var LINE = line.toLowerCase();
+
+            if (LINE.indexOf('opus/48000') !== -1 && !info.opusLineNumber) {
+                info.opusLineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
+            }
+
+            if (LINE.indexOf('isac/32000') !== -1 && !info.isacLineNumber) {
+                info.isacLineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
+            }
+
+            if (LINE.indexOf('g722/8000') !== -1 && !info.g722LineNumber) {
+                info.g722LineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
+            }
+			
+			if (LINE.indexOf('pcmu/8000') !== -1 && !info.pcmuLineNumber) {
+                info.pcmuLineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
+            }
+			
+			if (LINE.indexOf('pcma/8000') !== -1 && !info.pcmaLineNumber) {
+                info.pcmaLineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
+            }
+			
+			if (LINE.indexOf('red/48000') !== -1 && !info.redLineNumber) {
                 info.redLineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
             }
 			
-			if (line.indexOf('ulpfec/90000') !== -1 && !info.fecLineNumber) {
-                info.fecLineNumber = line.replace('a=rtpmap:', '').split(' ')[0];
-            }
         });
 
         return info;
@@ -494,25 +618,25 @@ var CodecsHandler = (function() {
 				appendOpusNext += ';cbr=' + params.cbr; // default is 0 (vbr)
 			}
         }
+		
+		if (typeof params.dtx != 'undefined') {
+			if (params.dtx){
+				if (sdpLines[opusFmtpLineIndex].split("usedtx=").length==1){
+					appendOpusNext += ';usedtx=1';
+				}
+			}
+        }
 
         if (typeof params.useinbandfec != 'undefined') {  // useful for handling packet loss
 			if (sdpLines[opusFmtpLineIndex].split("useinbandfec=").length==1){
 				appendOpusNext += ';useinbandfec=' + params.useinbandfec;  // Defaults to 0
+			} else {
+				sdpLines[opusFmtpLineIndex] = sdpLines[opusFmtpLineIndex].replace("useinbandfec="+(params.useinbandfec ? 0 : 1), "useinbandfec="+params.useinbandfec);
 			}
         }
 
-        if (typeof params.usedtx != 'undefined') {  // Default is 0
-			if (sdpLines[opusFmtpLineIndex].split("usedtx=").length==1){
-				appendOpusNext += ';usedtx=' + params.usedtx; // if decoder prefers the use of DTX.
-			}
-        }
-
-        
-
         sdpLines[opusFmtpLineIndex] = sdpLines[opusFmtpLineIndex].concat(appendOpusNext);
-
         sdp = sdpLines.join('\r\n');
-		
         return sdp;
     }
 	
@@ -548,6 +672,73 @@ var CodecsHandler = (function() {
 		}
         return 32768;
     }
+	
+	function modifyDescLyra(modifiedSDP) { // WIP 
+		if (!modifiedSDP.includes("m=audio")){ // don't bother modifying if no audio line found
+			return modifiedSDP;
+		}
+		///// Snippet based on Apache 2.0 licenced code. Source: https://github.com/Flash-Meeting/lyra-webrtc //////////
+		modifiedSDP = modifiedSDP.replace("SAVPF 111", "SAVPF 109 111").replace("a=rtpmap:111", "a=rtpmap:109 L16/16000/1\r\na=fmtp:109 ptime=20\r\na=rtpmap:111");
+		modifiedSDP = modifiedSDP.replace("a=rtpmap:106 CN/32000\r\n", "").replace("a=rtpmap:105 CN/16000\r\n", "").replace("a=rtpmap:13 CN/8000\r\n", "").replace(" 106 105 13", "");
+		///////////////////////////////
+		return modifiedSDP;
+	}
+
+	function modifyDescPCM(modifiedSDP, rate=32000, stereo=false, ptimeOverride=false) {
+		if (!modifiedSDP.includes("m=audio")){ // don't bother modifying if no audio line found
+			return modifiedSDP;
+		} 
+		var ptime = 10; 
+		if (ptimeOverride){
+			ptime = parseInt(ptimeOverride); // 10 seems to work with 48000, so might as well make it default
+		}
+		ptime = parseInt(ptime/10)*10;
+		if (ptime<10){
+			ptime = 10;
+		}
+		rate = parseInt(rate) || 32000;
+		
+		
+		if (!stereo && (rate>=48000)){
+			rate = 48000; // 44100 doesn't want to work for me, so we'll skip it.
+			ptime = 10; // 48000 only works with ptime=10
+		} else if (!stereo && rate>=44100){
+			rate = 44100; // 44100 doesn't want to work for me, so we'll skip it.
+			ptime = 10;
+		} else if (rate>=32000){
+			rate = 32000;
+			if (stereo){
+				ptime=10; // can be ptime = 20 if not stereo
+			} else if (ptime>20){
+				ptime=20;
+			}
+		} else if (rate>=16000){
+			rate = 16000;
+			if (stereo){
+				if (ptime>20){
+					ptime=20; // can be ptime = 20 if not stereo
+				}
+			} else if (ptime>40){
+				ptime=40;
+			}
+		} else {
+			rate = 8000;
+			if (stereo){
+				if (ptime>40){
+					ptime=40; // can be ptime = 20 if not stereo
+				}
+			}
+		}
+		
+		if (stereo){
+			modifiedSDP = modifiedSDP.replace("SAVPF 111", "SAVPF 109 111").replace("a=rtpmap:111", "a=rtpmap:109 L16/"+rate+"/2\r\na=fmtp:109 ptime="+ptime+"\r\na=rtpmap:111");
+		} else {
+			modifiedSDP = modifiedSDP.replace("SAVPF 111", "SAVPF 109 111").replace("a=rtpmap:111", "a=rtpmap:109 L16/"+rate+"/1\r\na=fmtp:109 ptime="+ptime+"\r\na=rtpmap:111");
+		}
+		
+		modifiedSDP = modifiedSDP.replace("a=rtpmap:106 CN/32000\r\n", "").replace("a=rtpmap:105 CN/16000\r\n", "").replace("a=rtpmap:13 CN/8000\r\n", "").replace(" 106 105 13", "");
+		return modifiedSDP;
+	}
 
 	
     return {
@@ -556,6 +747,10 @@ var CodecsHandler = (function() {
 		disablePLI: disablePLI,
 		
 		disableREMB: disableREMB,
+		
+		modifyDescPCM: modifyDescPCM,
+		
+		modifyDescLyra: modifyDescLyra,
         
 		getVideoBitrates: function(sdp) {
             return getVideoBitrates(sdp);
@@ -572,7 +767,9 @@ var CodecsHandler = (function() {
 			return getOpusBitrate(sdp);
 		},
 
-        preferCodec: preferCodec
+        preferCodec: preferCodec,
+		
+		preferAudioCodec: preferAudioCodec
     };
 })();
 

thirdparty/lyra/README.md

@@ -0,0 +1,25 @@
+## WebRTC snippets for Lyra sourced from meeting.dev
+
+Apache 2.0 Licenced
+
+Sourced from: https://meeting.dev/lab/lyra-webrtc/loopback.html
+
+
+## NOTE FROM STEVE
+ At the moment Lyra isn't actually being used, but I'm trying to get it working with VDO.Ninja. It may be removed at some future point - steve
+
+## TFLITE MODELS SOURCED FROM GOOGLE ON GITHUB
+
+google/lyra is licensed under the
+Apache License 2.0
+https://github.com/google/lyra
+https://github.com/google/lyra/tree/f079e8c4dd1c61c87de1852178976ee3bdf15561/model_coeffs
+
+
+## Further Acknowledgments 
+
+Thanks to [the team that developed Lyra](https://ai.googleblog.com/2021/02/lyra-new-very-low-bitrate-codec-for.html) and to [mayitayew for making it work with WASM](https://github.com/mayitayew/soundstream-wasm).
+
+We are using a modified Lyra WASM. It is available [here](https://github.com/Flash-Meeting/lyra-wasm)
+
+

thirdparty/lyra/model_coeffs/README.md

@@ -0,0 +1,6 @@
+TFLITE MODELS SOURCED FROM GOOGLE ON GITHUB
+
+google/lyra is licensed under the
+Apache License 2.0
+https://github.com/google/lyra
+https://github.com/google/lyra/tree/f079e8c4dd1c61c87de1852178976ee3bdf15561/model_coeffs