Imported RTP changes 2 from userconcept.

Author: mo-g

Libraries/ecma-rtp/codec.mjs

@@ -15,3 +15,79 @@
  * along with ecma-rtp.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+
+import alawmulaw from 'alawmulaw';
+import waveResampler from 'wave-resampler';
+
+/**
+ * Superclass to represent an audio Codec.
+ */
+class Codec {
+    constructor () {
+        this.samples = []
+    }
+
+    transcode () {
+
+    }
+
+    /**
+     * We need to do some testing to see how resampling options affect STT because
+     * 
+     * Cubic without Low Pass Filter:
+     * generate: 0.283ms
+     * transcode: 0.144ms
+     * resample: 0.442ms
+     * 
+     * Cubic with LPF:
+     * generate: 0.375ms
+     * transcode: 0.123ms
+     * resample: 6.605ms
+     * 
+     * That's on 20ms audio on an i5 9500. Interestingly it scales x2 per x10 duration,
+     * so it's not terrible - but if we can shave time off, so much the better
+     */
+    static resampleOld (samples) {
+        var floatSamples=Float32Array.from(Float32Array.from(samples).map(x=>x/0x8000));
+        var newSamples = waveResampler.resample(floatSamples, 16000, 8000); // RETURNS A FLOAT64 NOT AN INT16 READ THE DOCS , {method: "cubic", LPF: false}
+        return Int16Array.from(newSamples.map(x => (x>0 ? x*0x7FFF : x*0x8000)));
+    }
+
+    static resample (samples) {
+        var decoded = alawmulaw.mulaw.decode(samples);
+        var resampled = waveResampler.resample(decoded, 16000, 8000, {method: "cubic", LPF: false});
+        return Int16Array.from(resampled);
+    }
+
+    set samples (samples) {
+        this.samples.push(this.transcode(samples));
+    }
+
+    get samples () {
+        return this.samples.pop;
+    }
+}
+
+/**
+ * Represents mulaw (as used by Combadges)
+ */
+class μLaw extends Codec {
+    constructor ({sourceRate = 16000,
+                  sourceDepth = 16,
+                  targetRate = 8000,
+                  targetDepth = 8} = {}) {
+        super();
+    }
+
+
+
+    transcode (samples, callback) {
+        if (samples instanceof Uint8Array) {
+            return alawmulaw.mulaw.decode(samples);
+        } else if (samples instanceof Int16Array) {
+            return alawmulaw.mulaw.encode(samples);
+        }
+    }
+}
+
+export { μLaw, Codec }

Libraries/ecma-rtp/package.json

@@ -1,5 +1,5 @@
 {
-    "name": "ecma-rtp",
+    "name": "@combadge/ecma-rtp",
     "version": "0.0.5",
     "type": "module",
     "description": "An RTP Library for Spin Doctor, written in ECMAScript",

Libraries/ecma-rtp/rtp.mjs

@@ -23,6 +23,8 @@
 
 
 import alawmulaw from 'alawmulaw'; // For now, move to codec.mjs later.
+import dgram from 'dgram';
+import { Codec } from './codec.mjs';
 
 const Payload = {
     Mulaw8K: 0,
@@ -275,14 +277,73 @@ class RTPHeader {
 }
 
 /**
- * Temporary class - replace with RTPPacket instances when you work out how.
+ * Should provide a source and sink for RTP Packets. For now, hardcode to the correct values for Combadge audio.
+ * 
+ * Down the line, we should generalise this to allow the library to be reused elsewhere -
+ * (since I wouldn't be writing it if there was another approachable JS RTP library)
  */
-class RTPStream {
-    constructor() {
+class RTPServer {
+    constructor(listenAddress, portNumber, consumer = undefined) {
         this.sampleCount = new Number(160); // At 8 bits/sample, this can be used for both incrementing the timestamp AND counting bytes.
         this.header = new RTPHeader(this.sampleCount);
+        this._consumer = undefined;
+
+        if (portNumber % 2 != 0) {
+            throw `RTP Ports must be even. Odd-numbered ports are reserved for RTCP. Invalid port ${portNumber} passed.`;
+        }
+
+        if (consumer) {
+            this._consumer = consumer;
+        }
+
+        this.udpServer = dgram.createSocket('udp4');
+        this.udpServer.bind(portNumber, listenAddress)
+        this.udpServer.on('listening', () => {
+            const address = this.udpServer.address();
+            console.log(`RTP Server spawned at ${address.address}:${address.port}`);
+        })
+        this.udpServer.on('message', (message, clientInfo) => {  
+            var packet = RTPPacket.from(message);
+            this.receivePacket(packet);
+        })
+
+        /*
+        this.transcoding = true;
+        var floatSamples=Float32Array.from(Float32Array.from(this.recSamples).map(x=>x/0x8000));
+        var newSamples = waveResampler.resample(floatSamples, 8000, 16000); // RETURNS A FLOAT64 NOT AN INT16 READ THE DOCS
+        var samples1616=Int16Array.from(newSamples.map(x => (x>0 ? x*0x7FFF : x*0x8000)));
+        var wav16buffer = new Buffer.from(samples1616.buffer);
+        console.log("Result:", model.stt(wav16buffer));
+        this.transcoding = false;
+        */
+    }
+
+    /**
+     * Function to pass in a function to receive audio from the server.
+     */
+    set consumer (consumer) {
+        this._consumer = consumer;
+    }
+
+    get consumer () {
+        return this._consumer;
+    }
+
+    /** Do something with the packet, then forward it to the Consumer if set, or cache it if not. */
+    receivePacket (packet) {
+        var samples = Codec.resampleOld(packet.payload);
+        this.consumer.receiveSamples(samples);
     }
 
+    /**
+     * Send audio to a remote device. We'll be explicit about target, rather than just using send() from the responder
+     * because otherwise I can already see an exploit where a snooper sends 0-length packets to ensure they're always the most recent address.
+     * socket.send(msg[, offset, length][, port][, address][, callback])
+     */
+    sendPacket (packet, address, port) {
+        packetData = packet.toBuffer()
+        this.udpServer.send(packetData, 0, packetData.length, port, address)
+    }
 
     /**
      * Return a completed media packet without transcoding.
@@ -308,25 +369,4 @@ class RTPStream {
     }
 }
 
-class MediaQueue {
-    constructor() {
-        this.inQueue = new Array();
-        this.outQueue = new Array();
-    }
-
-    /**
-     * Add a packet to the queue for RTP decoding
-     */
-    addReceivedPacket (packet, callback) {
-        return null;
-    }
-
-    /**
-     * Add a media stream for RTP encoding
-     */
-    addTransmitMedia (samples, callback) {
-        return null;
-    }
-}
-
-export { Payload, RTPHeader, RTPStream, RTPPacket, MediaQueue };
+export { Payload, RTPHeader, RTPPacket, RTPServer };

Libraries/robin-agent/non_verbal.mjs

@@ -0,0 +1,92 @@
+/**
+ * Robin-agent - an interactive voice agent for Spin Doctor
+ * Copyright (C) 2021-2022 The Combadge Project by mo-g
+ *
+ * Robin-agent is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * Robin-agent is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Robin-agent.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * tone({
+  freq: Tones["E♭6"],
+  lengthInSecs: 0.2,
+  volume: tone.MAX_16/2,
+  rate: 16000,
+  shape: 'sine'
+})
+ */
+
+
+import { Tones, FixedPitch, Silence} from '@combadge/ecma-tonegenerator'
+
+
+/**
+ * Superclass for all nonverbal sound generators. By default, we should generate audio in the same codec and profile as Robin uses internally.
+ * 
+ * 
+ */
+class Sound {
+    constructor (notes) {
+        this.tones = notes.map(function (note) {
+            var tone = new FixedPitch({frequency:Tones[note],
+                            bitDepth: 16,
+                            sampleRate: 16000});
+            var silence = new Silence({bitDepth: 16,
+                                       sampleRate: 16000});
+            
+            return [tone.approximate({duration:500}), silence.accurate({duration:500})];
+        });
+
+    }
+}
+
+/**
+ * Click, B5, Click, B5, Click, D6
+ */
+class ListeningChirp extends Sound {
+    constructor () {
+        notes = ["B5", "B5", "D6"];
+        super(notes);
+    }
+
+    get output () {
+        output = [];
+        this.tones.forEach(function (tone) {
+            tone.forEach(function (sample) {
+                output.push(sample);
+            })
+        });
+        return new Int16Array(output);
+    }
+}
+
+/**
+ * 
+ */
+class OfflineChirp extends Sound {
+    constructor () {
+        notes = ["B5", "B5"];
+        super(notes);
+    }
+}
+
+/**
+ * B5, D6, B5
+ */
+class BotswainWhistle extends Sound {
+    constructor () {
+        notes = ["B5", "D6", "B5"];
+        super(notes);
+    }
+}
+
+export { ListeningChirp, OfflineChirp, BotswainWhistle };

Libraries/robin-agent/package.json

@@ -1,5 +1,5 @@
 {
-    "name": "robin-agent",
+    "name": "@combadge/robin-agent",
     "version": "0.0.1",
     "type": "module",
     "description": "An interactive voice agent for Spin Doctor.",

Libraries/robin-agent/robin.mjs

@@ -25,8 +25,6 @@ const scorerFile = "./huge-vocabulary.scorer";
 
 const vadTimeOut = 300;
 const vadAggressiveness = vad.Mode.VERY_AGGRESSIVE;
-
-
 const vadInstance = new vad(vadAggressiveness);
 
 
@@ -39,32 +37,27 @@ console.error('Loaded model.');
 /**
  * Agent provides an instance of an automated responder to take voice commands and send commands back to badges for action
  * 
- * Currently stubbed.
+ * The API for this and ecma-rtp should be made more-or-less independent, so you can feed this any source of numbers and it should work - from file, ecma-rtp, or something else.
+ * We should also (for now) hard code a specific codec and profile, and transcode in the RTP library.
  */
 class Agent {
-    constructor (callback = undefined) {
+    constructor (callback = undefined, audioResponder = undefined) {
         this.samples = [];
         this.recSamples = [];
         this.transcoding = false;
         this._callback = callback;
+        this._audioResponder = audioResponder;
     }
 
     recogniseBatch () {
-        console.log("Transcoding, please wait.")
-        this.transcoding = true;
-        var floatSamples=Float32Array.from(Float32Array.from(this.recSamples).map(x=>x/0x8000));
-        var newSamples = waveResampler.resample(floatSamples, 8000, 16000); // RETURNS A FLOAT64 NOT AN INT16 READ THE DOCS
-        var samples1616=Int16Array.from(newSamples.map(x => (x>0 ? x*0x7FFF : x*0x8000)));
-        var wav16buffer = new Buffer.from(samples1616.buffer);
-        console.log("Result:", model.stt(wav16buffer));
+        console.log(this.recSamples);
+        console.log("Result:", model.stt(new Buffer.from(this.recSamples)));
         this.transcoding = false;
     }
 
     receiveSamples (samples) {
-        if (this.transcoding == false) {
-            this.samples.push(...samples);
-        }
-        if (this.samples.length > 50000) {
+        this.samples.push(...samples);
+        if (this.samples.length > 60000) {
             this.recSamples = this.samples;
             this.samples = [];
             this.recogniseBatch();
@@ -80,6 +73,18 @@ class Agent {
     get callback () {
         return this._callback;
     }
+
+    /**
+     * Expects a function(Buffer) to pass back audio to, which will be transcoded
+     * and encapsulated as an RTP packet before sending to the client.
+     */
+    set audioResponder (audioResponder = undefined) {
+        this._audioResponder = audioResponder;
+    }
+
+    get audioResponder () {
+        return this._audioResponder;
+    }
 }
 
 export { Agent };