Skip to content

Commit

Permalink
fix: Audio recording
Browse files Browse the repository at this point in the history
  • Loading branch information
@aiko-chan-ai
aiko-chan-ai committed Sep 18, 2024
1 parent bc12422 commit 70f7acb
Show file tree
Hide file tree
Showing 2 changed files with 72 additions and 65 deletions.
13 changes: 8 additions & 5 deletions src/client/voice/dispatcher/BaseDispatcher.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ const kill = require('tree-kill');
const secretbox = require('../util/Secretbox');

const CHANNELS = 2;
const MAX_NONCE_SIZE = 2 ** 32 - 1;

const MAX_UINT_16 = 2 ** 16 - 1;
const MAX_UINT_32 = 2 ** 32 - 1;

const extensions = [{ id: 5, len: 2, val: 0 }];

Expand Down Expand Up @@ -115,7 +117,7 @@ class BaseDispatcher extends Writable {
getNewSequence() {
const currentSeq = this.sequence;
this.sequence++;
if (this.sequence > 2 ** 16 - 1) this.sequence = 0;
if (this.sequence > MAX_UINT_16) this.sequence = 0;
return currentSeq;
}

Expand Down Expand Up @@ -239,8 +241,9 @@ class BaseDispatcher extends Writable {
if ((!this.pausedSince || this._silence) && this._writeCallback) this._writeCallback();
}, next).unref();
this.timestamp += this.TIMESTAMP_INC;
if (this.timestamp >= 2 ** 32) this.timestamp = 0;
if (this.timestamp > MAX_UINT_32) this.timestamp = 0;
this.count++;
if (this.count > MAX_UINT_16) this.count = 0;
}

_final(callback) {
Expand All @@ -261,7 +264,7 @@ class BaseDispatcher extends Writable {
/**
* Creates a one-byte extension header
* https://www.rfc-editor.org/rfc/rfc5285#section-4.2
* @returns {Buffer} extension header
* @returns {Buffer} <Buffer be de 00 01>
*/
createHeaderExtension() {
/**
Expand Down Expand Up @@ -317,7 +320,7 @@ class BaseDispatcher extends Writable {
const { secret_key, mode } = this.player.voiceConnection.authentication;
// Both supported encryption methods want the nonce to be an incremental integer
this._nonce++;
if (this._nonce > MAX_NONCE_SIZE) this._nonce = 0;
if (this._nonce > MAX_UINT_32) this._nonce = 0;
if (!this._nonceBuffer) {
this.resetNonceBuffer();
}
Expand Down
124 changes: 64 additions & 60 deletions src/client/voice/receiver/PacketHandler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ class PacketHandler extends EventEmitter {
constructor(receiver) {
super();
this.receiver = receiver;
this.nonce = null;
this.streams = new Map();
this.videoStreams = new Map(); // Placeholder
this.speakingTimeouts = new Map();
}

resetNonce() {
this.nonce =
this.receiver.connection.authentication.mode === 'aead_aes256_gcm_rtpsize' ? Buffer.alloc(12) : Buffer.alloc(24);
getNonceBuffer() {
return this.receiver.connection.authentication.mode === 'aead_aes256_gcm_rtpsize'
? Buffer.alloc(12)
: Buffer.alloc(24);
}

get connection() {
Expand All @@ -55,23 +55,13 @@ class PacketHandler extends EventEmitter {
return stream;
}

makeTestVideoStream(user) {
if (this.videoStreams.has(user)) return this.videoStreams.get(user);
const stream = new Readable();
stream.on('end', () => this.videoStreams.delete(user));
this.videoStreams.set(user, stream);
return stream;
}

parseBuffer(buffer) {
const { secret_key, mode } = this.receiver.connection.authentication;
// Open packet
if (!secret_key) return new Error('secret_key cannot be null or undefined');
if (!this.nonce) {
this.resetNonce();
}
const nonce = this.getNonceBuffer();
// Copy the last 4 bytes of unpadded nonce to the padding of (12 - 4) or (24 - 4) bytes
buffer.copy(this.nonce, 0, buffer.length - UNPADDED_NONCE_LENGTH);
buffer.copy(nonce, 0, buffer.length - UNPADDED_NONCE_LENGTH);

let headerSize = 12;
const first = buffer.readUint8();
Expand All @@ -90,7 +80,7 @@ class PacketHandler extends EventEmitter {
let packet;
switch (mode) {
case 'aead_aes256_gcm_rtpsize': {
const decipheriv = crypto.createDecipheriv('aes-256-gcm', secret_key, this.nonce);
const decipheriv = crypto.createDecipheriv('aes-256-gcm', secret_key, nonce);
decipheriv.setAAD(header);
decipheriv.setAuthTag(authTag);

Expand All @@ -102,7 +92,7 @@ class PacketHandler extends EventEmitter {
packet = secretbox.methods.crypto_aead_xchacha20poly1305_ietf_decrypt(
Buffer.concat([encrypted, authTag]),
header,
this.nonce,
nonce,
secret_key,
);

Expand All @@ -114,41 +104,29 @@ class PacketHandler extends EventEmitter {
}
}

// Strip RTP Header Extensions (one-byte only)
if (packet[0] === 0xbe && packet[1] === 0xde) {
const headerExtensionLength = packet.readUInt16BE(2);
packet = packet.subarray(4 + 4 * headerExtensionLength);
} else if (buffer.slice(12, 14).compare(HEADER_EXTENSION_BYTE) === 0) {
// Strip decrypted RTP Header Extension if present
// Strip decrypted RTP Header Extension if present
if (buffer.slice(12, 14).compare(HEADER_EXTENSION_BYTE) === 0) {
const headerExtensionLength = buffer.slice(14).readUInt16BE();
packet = packet.subarray(4 * headerExtensionLength);
}

// Ex VP8
// <Buffer 90 80 80 00 30 b7 01 9d 01 2a 80 07 38 04 0b c7 08 85 85 88 99 84 88 3f 82 00 06 16 04 f7 06 81 64 9f 6b db 9b 27 38 7b 27 38 7b 27 38 7b 27 38 7b 27 ... 1154 more bytes>
// 90 80: payloadDescriptorBuf (90 80 if first frame | 80 80 else)
// 80 00: pictureIdBuf
// n bytes: chunk raw (Ivf splitter)

return packet;
}

push(buffer) {
audioReceiver(buffer) {
const ssrc = buffer.readUInt32BE(8);
const userStat = this.connection.ssrcMap.get(ssrc) || this.connection.ssrcMap.get(ssrc - 1); // Maybe video_ssrc ?
const userStat = this.connection.ssrcMap.get(ssrc);

if (!userStat) return;

let opusPacket;
const streamInfo = this.streams.get(userStat.userId);
const videoStreamInfo = this.videoStreams.get(userStat.userId);

// If the user is in video, we need to check if the packet is just silence
if (userStat.hasVideo) {
opusPacket = this.parseBuffer(buffer);
if (opusPacket instanceof Error) {
// Only emit an error if we were actively receiving packets from this user
if (streamInfo || videoStreamInfo) {
if (streamInfo) {
this.emit('error', opusPacket);
}
return;
Expand All @@ -160,28 +138,25 @@ class PacketHandler extends EventEmitter {
}

let speakingTimeout = this.speakingTimeouts.get(ssrc);
// Only for voice... idk
if (this.connection.ssrcMap.has(ssrc)) {
if (typeof speakingTimeout === 'undefined') {
// Ensure at least the speaking bit is set.
// As the object is by reference, it's only needed once per client re-connect.
if (userStat.speaking === 0) {
userStat.speaking = Speaking.FLAGS.SPEAKING;
}
this.connection.onSpeaking({ user_id: userStat.userId, ssrc: ssrc, speaking: userStat.speaking });
speakingTimeout = setTimeout(() => {
try {
this.connection.onSpeaking({ user_id: userStat.userId, ssrc: ssrc, speaking: 0 });
clearTimeout(speakingTimeout);
this.speakingTimeouts.delete(ssrc);
} catch {
// Connection already closed, ignore
}
}, DISCORD_SPEAKING_DELAY).unref();
this.speakingTimeouts.set(ssrc, speakingTimeout);
} else {
speakingTimeout.refresh();
if (typeof speakingTimeout === 'undefined') {
// Ensure at least the speaking bit is set.
// As the object is by reference, it's only needed once per client re-connect.
if (userStat.speaking === 0) {
userStat.speaking = Speaking.FLAGS.SPEAKING;
}
this.connection.onSpeaking({ user_id: userStat.userId, ssrc: ssrc, speaking: userStat.speaking });
speakingTimeout = setTimeout(() => {
try {
this.connection.onSpeaking({ user_id: userStat.userId, ssrc: ssrc, speaking: 0 });
clearTimeout(speakingTimeout);
this.speakingTimeouts.delete(ssrc);
} catch {
// Connection already closed, ignore
}
}, DISCORD_SPEAKING_DELAY).unref();
this.speakingTimeouts.set(ssrc, speakingTimeout);
} else {
speakingTimeout.refresh();
}

if (streamInfo) {
Expand All @@ -194,9 +169,33 @@ class PacketHandler extends EventEmitter {
}
}
stream.push(opusPacket);
if (opusPacket === null) {
// ! null marks EOF for stream
stream.destroy();
}
}

videoReceiver(buffer) {
const ssrc = buffer.readUInt32BE(8);
const userStat = this.connection.ssrcMap.get(ssrc - 1); // Video_ssrc

if (!userStat) return;
this.parseBuffer(buffer);

let opusPacket;

const videoStreamInfo = this.videoStreams.get(userStat.userId);

// If the user is in video, we need to check if the packet is just silence
if (userStat.hasVideo) {
opusPacket = this.parseBuffer(buffer);
if (opusPacket instanceof Error) {
// Only emit an error if we were actively receiving packets from this user
if (videoStreamInfo) {
this.emit('error', opusPacket);
}
return;
}
if (SILENCE_FRAME.equals(opusPacket)) {
// If this is a silence frame, pretend we never received it
return;
}
}

Expand All @@ -209,9 +208,14 @@ class PacketHandler extends EventEmitter {
return;
}
}
stream.push(opusPacket); // VP8 ? idk
stream.push(opusPacket);
}
}

push(buffer) {
this.audioReceiver(buffer);
this.videoReceiver(buffer);
}
}

module.exports = PacketHandler;

0 comments on commit 70f7acb

Please sign in to comment.