From c33e37a5c63ee51096f7047f6ab182e4ac4277f2 Mon Sep 17 00:00:00 2001 From: Uri Shaked Date: Thu, 12 Sep 2024 21:15:07 +0300 Subject: [PATCH 01/16] feat: audio support (WiP) --- index.html | 1 + src/AudioPlayer.ts | 64 ++++++++++++++++++++++++++++++++++ src/examples/stripes/project.v | 11 ++++-- src/index.ts | 19 ++++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 src/AudioPlayer.ts diff --git a/index.html b/index.html index 8c2a931..125c9a8 100644 --- a/index.html +++ b/index.html @@ -53,6 +53,7 @@ + diff --git a/src/AudioPlayer.ts b/src/AudioPlayer.ts new file mode 100644 index 0000000..c020bd6 --- /dev/null +++ b/src/AudioPlayer.ts @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2024, Tiny Tapeout LTD +// Author: Uri Shaked + +const CHUNKS_PER_SECOND = 10; + +export class AudioPlayer { + private counter = 0; + readonly audioCtx = new AudioContext(); + + private readonly gainNode = this.audioCtx.createGain(); + private chunkBuffer = new AudioBuffer({ + length: this.audioCtx.sampleRate / CHUNKS_PER_SECOND, + numberOfChannels: 1, + sampleRate: this.audioCtx.sampleRate, + }); + + private chunk = this.chunkBuffer.getChannelData(0); + private node: AudioBufferSourceNode | null = null; + private prevValue = 0; + private playedSamples = 0; + private lastSample = 0; + + constructor(private readonly clockFrequency: number) { + this.gainNode.connect(this.audioCtx.destination); + } + + feed(value: number) { + this.counter++; + if (this.prevValue === value) { + return; + } + + const currentTime = this.counter / this.clockFrequency; + const { sampleRate } = this.audioCtx; + let currentSample = Math.floor(currentTime * sampleRate) - this.playedSamples; + if (currentSample - this.lastSample > sampleRate / 20) { + this.lastSample = currentSample; + currentSample = 0; + } else { + this.lastSample = currentSample; + } + if (currentSample > this.chunk.length) { + this.playedSamples += this.chunk.length; + this.node = new AudioBufferSourceNode(this.audioCtx, { buffer: this.chunkBuffer }); + this.node.connect(this.gainNode); + this.node.start(); + currentSample %= this.chunk.length; + this.chunkBuffer = new AudioBuffer({ + length: sampleRate / CHUNKS_PER_SECOND, + numberOfChannels: 1, + sampleRate, + }); + this.chunk = this.chunkBuffer.getChannelData(0); + this.chunk.fill(this.prevValue, 0, currentSample); + } + this.chunk.fill(value, currentSample); + this.prevValue = value; + } + + resume() { + this.audioCtx.resume(); + } +} diff --git a/src/examples/stripes/project.v b/src/examples/stripes/project.v index f63027e..53b5801 100644 --- a/src/examples/stripes/project.v +++ b/src/examples/stripes/project.v @@ -25,17 +25,24 @@ module tt_um_vga_example( wire video_active; wire [9:0] pix_x; wire [9:0] pix_y; + wire sound; // TinyVGA PMOD assign uo_out = {hsync, B[0], G[0], R[0], vsync, B[1], G[1], R[1]}; + assign uio_out = {sound, 7'b0}; // Unused outputs assigned to 0. - assign uio_out = 0; - assign uio_oe = 0; + assign uio_oe = 8'hff; // Suppress unused signals warning wire _unused_ok = &{ena, ui_in, uio_in}; + reg [15:0] counter2; + assign sound = counter2[14]; + always @(posedge clk) begin + counter2 <= counter2 + 1; + end + reg [9:0] counter; hvsync_generator hvsync_gen( diff --git a/src/index.ts b/src/index.ts index d4f4d56..8b69425 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import { examples } from './examples'; import { exportProject } from './exportProject'; import { HDLModuleWASM } from './sim/hdlwasm'; import { compileVerilator } from './verilator/compile'; +import { AudioPlayer } from './AudioPlayer'; let currentProject = structuredClone(examples[0]); @@ -54,6 +55,18 @@ function getVGASignals() { }; } +function getAudioSignal() { + const uio_out = (jmod.state.uio_out ?? 0) as number; + const uio_oe = (jmod.state.uio_oe ?? 0) as number; + return (uio_out & uio_oe) >> 7; // uio[7] +} + +const audioPlayer = new AudioPlayer(25_000_000); +function updateAudio() { + const audio = getAudioSignal(); + audioPlayer.feed(audio); +} + let stopped = false; const fpsCounter = new FPSCounter(); @@ -102,6 +115,7 @@ function waitFor(condition: () => boolean, timeout = 10000) { let counter = 0; while (!condition() && counter < timeout) { jmod.tick2(1); + updateAudio(); counter++; } } @@ -125,6 +139,7 @@ function animationFrame(now: number) { for (let x = 0; x < 736; x++) { const offset = (y * 736 + x) * 4; jmod.tick2(1); + updateAudio(); const { hsync, vsync, r, g, b } = getVGASignals(); if (hsync) { break; @@ -175,6 +190,10 @@ document.querySelector('#download-button')?.addEventListener('click', () => { }); function toggleButton(index: number) { + if (index === 8) { + audioPlayer.resume(); + return; + } const bit = 1 << index; jmod.state.ui_in = jmod.state.ui_in ^ bit; if (jmod.state.ui_in & bit) { From ce83cdfb369bd9c5a28a27256dcdaacb3e7165c2 Mon Sep 17 00:00:00 2001 From: rej Date: Fri, 20 Sep 2024 20:37:05 +0200 Subject: [PATCH 02/16] audio: implemented WebWorker to resample output audio according to difference between browser FPS and expected (60) frame rate of VGA output from Verilator. New audio pipeline: 1) first generated audio samples are low-pass filtered (20 kHz) to approximately simulate Audio PMOD behavior and reduce occasional high-pitch noise especially when browser FPS significantly differs from 60 Hz 2) accumulate approximately --- public/resampler.js | 124 ++++++++++++++++++++++++++++++++++++++++++++ src/AudioPlayer.ts | 84 +++++++++++++----------------- src/index.ts | 46 +++++++++++++--- 3 files changed, 200 insertions(+), 54 deletions(-) create mode 100644 public/resampler.js diff --git a/public/resampler.js b/public/resampler.js new file mode 100644 index 0000000..acc8852 --- /dev/null +++ b/public/resampler.js @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2024, Tiny Tapeout LTD +// Author: Renaldas Zioma, Uri Shaked + +class AudioResamplerProcessor extends AudioWorkletProcessor { + constructor() { + super(); + + // Define buffer properties + this.ringBufferSize = 4096 * 4; // can store approximately 5 frames of audio data + this.ringBuffer = new Float32Array(this.ringBufferSize); // ring-buffer helps to amortise uneven framerate and + // keeps re-sampler from starving or overflowing + this.writeIndex = 0; // Index where new samples are written + this.readIndex = 0; // Index where samples are read + this.previousSample = 0.0; + this.ringBuffer.fill(this.previousSample); + + // Listen to messages from the main thread + this.port.onmessage = this.handleMessage.bind(this); + + this.expectedFPS = 60.0; + this.currentFPS = this.expectedFPS; + console.log("Audio WebWorker started"); + } + + handleMessage(event) { + const data = event.data; + + // Handle incoming audio samples and write to the ring buffer + if (data.type === 'samples') { + this.currentFPS = data.fps; + const samples = data.samples; + for (let i = 0; i < samples.length; i++) { + if ((this.writeIndex + 1) % this.ringBufferSize == this.readIndex) + { + console.log("Buffer is full. Dropping", samples.length - i, "incomming samples!"); + break; // Skip incomming samples when ring-buffer is full + } + if (this.writeIndex == this.readIndex) + this.ringBuffer[(this.writeIndex - 1) % this.ringBufferSize] = samples[i]; + this.ringBuffer[this.writeIndex] = samples[i]; + this.writeIndex = (this.writeIndex + 1) % this.ringBufferSize; // Wrap around + } + } + else if (data.type === 'reset') { + this.ringBuffer.fill(this.previousSample); + this.readIndex = 0; + this.writeIndex = 0; + } + } + + // Linear interpolation for resampling + interpolate(buffer, index1, index2, frac) { + return (1 - frac) * buffer[index1] + frac * buffer[index2]; + } + + // Process function that resamples the data from the ring buffer to match output size + process(inputs, outputs) { + const output = outputs[0]; // Mono output (1 channel) + const outputData = output[0]; // Get the output data array + + const playbackRate = this.currentFPS / this.expectedFPS; + const borderSamples = 2; + const samplesRequired = Math.round(outputData.length * playbackRate) + borderSamples; + + // example when samplesRequired = 8 + 2 border samples + // (border samples are marked as 'b' below) + // + // 3 subsequent invocations of process(): + // + // ringBuffer: b01234567b01234567b01234567b. + // process#0 ^........^ | <- sampling window + // process#1 ^........^ | <- sampling window + // process#2 ^........^| <- sampling window + // WRITE pointer--` + + // process#0 ^--READ pointer + // process#1 ^--READ pointer + // process#2 ^--READ pointer + // after process#2 ^--READ pointer + + const samplesAvailable = (this.writeIndex - this.readIndex + this.ringBufferSize) % this.ringBufferSize; + if (samplesAvailable < borderSamples) + { + for (let i = 0; i < outputData.length; i++) + outputData[i] = this.previousSample; + console.log("Buffer is empty. Using previous sample value " + this.previousSample.toFixed(3)); + return true; + } + + const samplesConsumed = Math.min(samplesRequired, samplesAvailable) - borderSamples; + + // Calculate resampling ratio + const ratio = samplesConsumed / outputData.length; + + // Fill the output buffer by resampling from the ring buffer + for (let i = 0; i < outputData.length; i++) { + const floatPos = 0.5 + ratio * (i + 0.5); // use sample centroids, thus +0.5 + const intPos = Math.floor(floatPos); + const nextIntPos = intPos + 1; + const frac = floatPos - intPos; // fractional part for interpolation + + if ((this.readIndex + nextIntPos) % this.ringBufferSize == this.writeIndex) + console.log(i, frac, intPos, nextIntPos, playbackRate, 'req: ', samplesRequired, 'avail: ', samplesAvailable, ratio, samplesConsumed); + + // Resample with linear interpolation + outputData[i] = this.interpolate(this.ringBuffer, + (this.readIndex + intPos) % this.ringBufferSize, + (this.readIndex + nextIntPos) % this.ringBufferSize, frac); + } + + this.previousSample = outputData[outputData.length - 1]; + + // Update readIndex to match how many samples were consumed + this.readIndex = (this.readIndex + samplesConsumed) % this.ringBufferSize; + + // Return true to keep the processor alive + return true; + } +} + +// Register the processor +registerProcessor('resampler', AudioResamplerProcessor); + diff --git a/src/AudioPlayer.ts b/src/AudioPlayer.ts index c020bd6..07130c8 100644 --- a/src/AudioPlayer.ts +++ b/src/AudioPlayer.ts @@ -1,64 +1,52 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2024, Tiny Tapeout LTD -// Author: Uri Shaked - -const CHUNKS_PER_SECOND = 10; +// Author: Renaldas Zioma, Uri Shaked export class AudioPlayer { - private counter = 0; - readonly audioCtx = new AudioContext(); + private audioCtx : AudioContext; + private resamplerNode : AudioWorkletNode; - private readonly gainNode = this.audioCtx.createGain(); - private chunkBuffer = new AudioBuffer({ - length: this.audioCtx.sampleRate / CHUNKS_PER_SECOND, - numberOfChannels: 1, - sampleRate: this.audioCtx.sampleRate, - }); + constructor(private readonly sampleRate: number, private readonly fps: number, private readonly bufferSize: number = 200) { + this.audioCtx = new AudioContext({sampleRate:sampleRate, latencyHint:'interactive'}); + this.audioCtx.audioWorklet.addModule(new URL('/resampler.js', import.meta.url)).then(() => { - private chunk = this.chunkBuffer.getChannelData(0); - private node: AudioBufferSourceNode | null = null; - private prevValue = 0; - private playedSamples = 0; - private lastSample = 0; + this.resamplerNode = new AudioWorkletNode(this.audioCtx, 'resampler'); + this.resamplerNode.connect(this.audioCtx.destination); - constructor(private readonly clockFrequency: number) { - this.gainNode.connect(this.audioCtx.destination); + this.audioCtx.resume().then(() => { + console.log('Audio playback started'); + }); + }); } - feed(value: number) { - this.counter++; - if (this.prevValue === value) { - return; + private writeIndex = 0; + readonly buffer = new Float32Array(this.bufferSize); // larger buffer reduces the communication overhead with the worker thread + // however, if buffer is too large it could lead to worker thread starving + feed(value: number, current_fps: number) { + if (this.writeIndex >= this.bufferSize) { + if (this.resamplerNode != null) + { + this.resamplerNode.port.postMessage({ + type: 'samples', + samples: this.buffer, + fps: current_fps, + }); + } + this.writeIndex = 0; } - - const currentTime = this.counter / this.clockFrequency; - const { sampleRate } = this.audioCtx; - let currentSample = Math.floor(currentTime * sampleRate) - this.playedSamples; - if (currentSample - this.lastSample > sampleRate / 20) { - this.lastSample = currentSample; - currentSample = 0; - } else { - this.lastSample = currentSample; - } - if (currentSample > this.chunk.length) { - this.playedSamples += this.chunk.length; - this.node = new AudioBufferSourceNode(this.audioCtx, { buffer: this.chunkBuffer }); - this.node.connect(this.gainNode); - this.node.start(); - currentSample %= this.chunk.length; - this.chunkBuffer = new AudioBuffer({ - length: sampleRate / CHUNKS_PER_SECOND, - numberOfChannels: 1, - sampleRate, - }); - this.chunk = this.chunkBuffer.getChannelData(0); - this.chunk.fill(this.prevValue, 0, currentSample); - } - this.chunk.fill(value, currentSample); - this.prevValue = value; + + this.buffer[this.writeIndex] = value; + this.writeIndex++; } resume() { this.audioCtx.resume(); + if (this.resamplerNode != null) + { + this.resamplerNode.port.postMessage({ + type: 'reset' + }); + } } } + diff --git a/src/index.ts b/src/index.ts index 1629030..fdaf8a5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -34,6 +34,8 @@ let jmod = new HDLModuleWASM(res.output.modules['TOP'], res.output.modules['@CON await jmod.init(); const uo_out_offset_in_jmod_databuf = jmod.globals.lookup("uo_out").offset; +const uio_out_offset_in_jmod_databuf = jmod.globals.lookup("uio_out").offset; +const uio_oe_offset_in_jmod_databuf = jmod.globals.lookup("uio_oe").offset; function reset() { const ui_in = jmod.state.ui_in; @@ -62,15 +64,35 @@ function getVGASignals() { } function getAudioSignal() { - const uio_out = (jmod.state.uio_out ?? 0) as number; - const uio_oe = (jmod.state.uio_oe ?? 0) as number; - return (uio_out & uio_oe) >> 7; // uio[7] + // see getVGASignals() implementation above for explanation about use of jmod.data8 + const uio_out = jmod.data8[uio_out_offset_in_jmod_databuf]; + const uio_oe = jmod.data8[uio_oe_offset_in_jmod_databuf]; + return (uio_out & uio_oe) >> 7; } -const audioPlayer = new AudioPlayer(25_000_000); +const expectedFPS = 60; +const sampleRate = 48_000*4;// @TODO: 192000 high sampleRate might not be supported on all platforms + // downsample to 48000 inside the AudioResamplerProcessor + // Empirically higher sampling rate helps with occasional high pitch noise. +const audioPlayer = new AudioPlayer(sampleRate, expectedFPS); + +const vgaClockRate = 25_175_000; +const ticksPerSample = vgaClockRate / sampleRate; + +let audioTickCounter = 0; +let audioSample = 0; +let lowPassFilter = 0; +let alphaLowPass20kHzAdjustedToFPS = 1.0; function updateAudio() { - const audio = getAudioSignal(); - audioPlayer.feed(audio); + const alpha = alphaLowPass20kHzAdjustedToFPS; + // @TODO: optimize the following line, floating operations here are currently slow! + lowPassFilter = alpha*lowPassFilter + (1.0-alpha)*getAudioSignal(); + audioSample += lowPassFilter; + if (++audioTickCounter < ticksPerSample) + return; + audioPlayer.feed(audioSample / ticksPerSample, fpsCounter.getFPS()); + audioTickCounter = 0; + audioSample = 0; } let stopped = false; @@ -131,6 +153,18 @@ function animationFrame(now: number) { fpsCounter.update(now); + // Need to simulate low pass filter of Audio PMOD + // with a likely cutoff around 20 kHz + // + // Time constant tau = 1 / (2 * π * cutoff_freq) = 1/(2*π* 20kHz) ~ 1 / 125664 + // = 1/(2*π*100kHz) ~ 1 / 628318 + // Sampling period Ts = (1 / sampling_freq) = 1/25MHz ~ 1 / 25175000 + // Alpha = tau / (tau + Ts) = 1 / (1 + tau / Ts) + const alphaLowPass10kHz = 0.998 // = 1 / (1 + 62832/25175000) ~ 1 / 1.002 + const alphaLowPass20kHz = 0.995 // = 1 / (1 + 125664/25175000) ~ 1 / 1.005 + const alphaLowPass100kHz = 0.9756 // = 1 / (1 + 628318/25175000) ~ 1 / 1.025 + alphaLowPass20kHzAdjustedToFPS = 1.0 / (1.0 + 0.005 * (fpsCounter.getFPS()/expectedFPS)); + if (fpsDisplay) { fpsDisplay.textContent = `${fpsCounter.getFPS().toFixed(0)}`; } From 81c743307350083692dc4778f427f6155af7164d Mon Sep 17 00:00:00 2001 From: rej Date: Fri, 20 Sep 2024 20:38:09 +0200 Subject: [PATCH 03/16] audio: music example from "Drop" demo for testing --- src/examples/index.ts | 3 +- src/examples/music/LICENSE.txt | 202 ++++++++++++++++++++++++++++ src/examples/music/index.ts | 12 ++ src/examples/music/project.v | 232 +++++++++++++++++++++++++++++++++ 4 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 src/examples/music/LICENSE.txt create mode 100644 src/examples/music/index.ts create mode 100644 src/examples/music/project.v diff --git a/src/examples/index.ts b/src/examples/index.ts index 282738d..feaffca 100644 --- a/src/examples/index.ts +++ b/src/examples/index.ts @@ -5,5 +5,6 @@ import { logo } from './logo'; import { conway } from './conway'; import { checkers } from './checkers'; import { drop } from './drop'; +import { music } from './music'; -export const examples: Project[] = [stripes, balls, logo, conway, checkers, drop]; +export const examples: Project[] = [music, stripes, balls, logo, conway, checkers, drop]; diff --git a/src/examples/music/LICENSE.txt b/src/examples/music/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/src/examples/music/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/examples/music/index.ts b/src/examples/music/index.ts new file mode 100644 index 0000000..f3cf304 --- /dev/null +++ b/src/examples/music/index.ts @@ -0,0 +1,12 @@ +import hvsync_generator_v from '../common/hvsync_generator.v?raw'; +import project_v from './project.v?raw'; + +export const music = { + name: 'Music', + author: 'Renaldas Zioma', + topModule: 'tt_um_vga_example', + sources: { + 'project.v': project_v, + 'hvsync_generator.v': hvsync_generator_v, + }, +}; diff --git a/src/examples/music/project.v b/src/examples/music/project.v new file mode 100644 index 0000000..e379354 --- /dev/null +++ b/src/examples/music/project.v @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2024 Renaldas Zioma, Erik Hemming + * SPDX-License-Identifier: Apache-2.0 + */ + +`default_nettype none + +`define C1 481; // 32.70375 Hz +`define Cs1 454; // 34.6475 Hz +`define D1 429; // 36.7075 Hz +`define Ds1 405; // 38.89125 Hz +`define E1 382; // 41.20375 Hz +`define F1 360; // 43.65375 Hz +`define Fs1 340; // 46.24875 Hz +`define G1 321; // 49.0 Hz +`define Gs1 303; // 51.9125 Hz +`define A1 286; // 55.0 Hz +`define As1 270; // 58.27 Hz +`define B1 255; // 61.735 Hz +`define C2 241; // 65.4075 Hz +`define Cs2 227; // 69.295 Hz +`define D2 214; // 73.415 Hz +`define Ds2 202; // 77.7825 Hz +`define E2 191; // 82.4075 Hz +`define F2 180; // 87.3075 Hz +`define Fs2 170; // 92.4975 Hz +`define G2 161; // 98.0 Hz +`define Gs2 152; // 103.825 Hz +`define A2 143; // 110.0 Hz +`define As2 135; // 116.54 Hz +`define B2 127; // 123.47 Hz +`define C3 120; // 130.815 Hz +`define Cs3 114; // 138.59 Hz +`define D3 107; // 146.83 Hz +`define Ds3 101; // 155.565 Hz +`define E3 95; // 164.815 Hz +`define F3 90; // 174.615 Hz +`define Fs3 85; // 184.995 Hz +`define G3 80; // 196.0 Hz +`define Gs3 76; // 207.65 Hz +`define A3 72; // 220.0 Hz +`define As3 68; // 233.08 Hz +`define B3 64; // 246.94 Hz +`define C4 60; // 261.63 Hz +`define Cs4 57; // 277.18 Hz +`define D4 54; // 293.66 Hz +`define Ds4 51; // 311.13 Hz +`define E4 48; // 329.63 Hz +`define F4 45; // 349.23 Hz +`define Fs4 43; // 369.99 Hz +`define G4 40; // 392.0 Hz +`define Gs4 38; // 415.3 Hz +`define A4 36; // 440.0 Hz +`define As4 34; // 466.16 Hz +`define B4 32; // 493.88 Hz +`define C5 30; // 523.26 Hz +`define Cs5 28; // 554.36 Hz +`define D5 27; // 587.32 Hz +`define Ds5 25; // 622.26 Hz +`define E5 24; // 659.26 Hz +`define F5 23; // 698.46 Hz +`define Fs5 21; // 739.98 Hz +`define G5 20; // 784.0 Hz +`define Gs5 19; // 830.6 Hz +`define A5 18; // 880.0 Hz +`define As5 17; // 932.32 Hz +`define B5 16; // 987.76 Hz + +module tt_um_vga_example( + input wire [7:0] ui_in, // Dedicated inputs + output wire [7:0] uo_out, // Dedicated outputs + input wire [7:0] uio_in, // IOs: Input path + output wire [7:0] uio_out, // IOs: Output path + output wire [7:0] uio_oe, // IOs: Enable path (active high: 0=input, 1=output) + input wire ena, // always 1 when the design is powered, so you can ignore it + input wire clk, // clock + input wire rst_n // reset_n - low to reset +); + + // VGA signals + wire hsync; + wire vsync; + wire [1:0] R; + wire [1:0] G; + wire [1:0] B; + wire video_active; + wire [9:0] x; + wire [9:0] y; + wire sound; + + // TinyVGA PMOD + assign {R,G,B} = {6{video_active * sound}}; + assign uo_out = {hsync, B[0], G[0], R[0], vsync, B[1], G[1], R[1]}; + assign uio_out = {sound, 7'b0}; + + // Unused outputs assigned to 0. + assign uio_oe = 8'hff; + + // Suppress unused signals warning + wire _unused_ok = &{ena, ui_in, uio_in}; + + hvsync_generator hvsync_gen( + .clk(clk), + .reset(~rst_n), + .hsync(hsync), + .vsync(vsync), + .display_on(video_active), + .hpos(x), + .vpos(y) + ); + + // reg [19:0] lfsr; + // wire feedback = lfsr[19] ^ lfsr[15] ^ lfsr[11] ^ lfsr[5] ^ lfsr[4] ^ lfsr[3] + 1; + reg [12:0] lfsr; + wire feedback = lfsr[12] ^ lfsr[8] ^ lfsr[2] ^ lfsr[0] + 1; + always @(posedge clk) begin + lfsr <= {lfsr[11:0], feedback}; + end + + // music + wire [2:0] part = frame_counter[9-:3]; + wire [12:0] timer = {frame_counter, frame_counter_frac}; + reg noise, noise_src = ^lfsr; + reg [2:0] noise_counter; + + wire square60hz = y < 262; // 60Hz square wave + wire square120hz = y[7]; // 120Hz square wave + wire square240hz = y[6]; // 240Hz square wave + wire square480hz = y[5]; // 480Hz square wave + wire [4:0] envelopeA = 5'd31 - timer[4:0]; // exp(t*-10) decays to 0 approximately in 32 frames [255 215 181 153 129 109 92 77 65 55 46 39 33 28 23 20 16 14 12 10 8 7 6 5 4 3 3 2 2] + wire [4:0] envelopeB = 5'd31 - timer[3:0]*2;// exp(t*-20) decays to 0 approximately in 16 frames [255 181 129 92 65 46 33 23 16 12 8 6 4 3] + wire envelopeP8 = (|timer[3:2])*5'd31;// pulse for 8 frames + wire beats_1_3 = timer[5:4] == 2'b10; + + // melody notes (in hsync): 151 26 40 60 _ 90 143 23 35 + // (x1.5 wrap-around progression) + reg [8:0] note2_freq; + reg [8:0] note2_counter; + reg note2; + + reg [7:0] note_freq; + reg [7:0] note_counter; + reg note; + wire [3:0] note_in = timer[7-:4]; // 8 notes, 32 frames per note each. 256 frames total, ~4 seconds + always @(note_in) + case(note_in) + 4'd0 : note_freq = `E2 + 4'd1 : note_freq = `E3 + 4'd2 : note_freq = `D3 + 4'd3 : note_freq = `E3 + 4'd4 : note_freq = `A2 + 4'd5 : note_freq = `B2 + 4'd6 : note_freq = `D3 + 4'd7 : note_freq = `E3 + 4'd8 : note_freq = `E2 + 4'd9 : note_freq = `E3 + 4'd10: note_freq = `D3 + 4'd11: note_freq = `E3 + 4'd12: note_freq = `G2 + 4'd13: note_freq = `E3 + 4'd14: note_freq = `Fs2 + 4'd15: note_freq = `E3 + endcase + + wire [2:0] note2_in = timer[8-:3]; // 8 notes, 32 frames per note each. 256 frames total, ~4 seconds + always @(note2_in) + case(note2_in) + 3'd0 : note2_freq = `B1 + 3'd1 : note2_freq = `A2 + 3'd2 : note2_freq = `E1 + 3'd3 : note2_freq = `A2 + 3'd4 : note2_freq = `B1 + 3'd5 : note2_freq = `A2 + 3'd6 : note2_freq = `D1 + 3'd7 : note2_freq = `Cs1 + endcase + + // wire kick = square60hz & (~|x[7:5] & x[4:0] < envelopeA); // 60Hz square wave with half second envelope + wire kick = square60hz & (x < envelopeA*4); // 60Hz square wave with half second envelope + wire snare = noise & (x >= 128 && x < 128+envelopeB*4); // noise with half second envelope + wire lead = note & (x >= 256 && x < 256+envelopeB*8); // ROM square wave with quarter second envelope + wire base = note2 & (x >= 512 && x < ((beats_1_3)?(512+8*4):(512+32*4))); + assign sound = { kick | (snare & beats_1_3 & part != 0) | (base) | (lead & part > 2) }; + + reg [11:0] frame_counter; + reg frame_counter_frac; + always @(posedge clk) begin + if (~rst_n) begin + frame_counter <= 0; + frame_counter_frac <= 0; + noise_counter <= 0; + note_counter <= 0; + note2_counter <= 0; + noise <= 0; + note <= 0; + note2 <= 0; + + end else begin + + if (x == 0 && y == 0) begin + {frame_counter, frame_counter_frac} <= {frame_counter,frame_counter_frac} + 1; + end + + // noise + if (x == 0) begin + if (noise_counter > 1) begin + noise_counter <= 0; + noise <= noise ^ noise_src; + end else + noise_counter <= noise_counter + 1'b1; + end + + // square wave + if (x == 0) begin + if (note_counter > note_freq) begin + note_counter <= 0; + note <= ~note; + end else begin + note_counter <= note_counter + 1'b1; + end + + if (note2_counter > note2_freq) begin + note2_counter <= 0; + note2 <= ~note2; + end else begin + note2_counter <= note2_counter + 1'b1; + end + end + end + end + +endmodule From c9bde3f68eaec409b0b28b025c245b3dcacbd9d1 Mon Sep 17 00:00:00 2001 From: rej Date: Fri, 20 Sep 2024 22:33:06 +0200 Subject: [PATCH 04/16] perf: ~20% speedup, moved 20 kHz low-pass filter after PWM sample averaging. From 22 ms (44FPS) down to 18 ms (55 FPS) in "music" example. --- src/index.ts | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/index.ts b/src/index.ts index fdaf8a5..70fd8f8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -79,18 +79,29 @@ const audioPlayer = new AudioPlayer(sampleRate, expectedFPS); const vgaClockRate = 25_175_000; const ticksPerSample = vgaClockRate / sampleRate; +const lowPassFrequency = 20_000; +const lowPassFilterSize = Math.ceil(sampleRate/lowPassFrequency); + let audioTickCounter = 0; let audioSample = 0; -let lowPassFilter = 0; -let alphaLowPass20kHzAdjustedToFPS = 1.0; + +let sampleQueueForLowPassFiter = new Float32Array(lowPassFilterSize); +let sampleQueueIndex = 0; + function updateAudio() { - const alpha = alphaLowPass20kHzAdjustedToFPS; - // @TODO: optimize the following line, floating operations here are currently slow! - lowPassFilter = alpha*lowPassFilter + (1.0-alpha)*getAudioSignal(); - audioSample += lowPassFilter; + audioSample += getAudioSignal(); if (++audioTickCounter < ticksPerSample) return; - audioPlayer.feed(audioSample / ticksPerSample, fpsCounter.getFPS()); + + const newSample = audioSample / ticksPerSample; + + sampleQueueForLowPassFiter[sampleQueueIndex++] = newSample; + sampleQueueIndex %= lowPassFilterSize; + let filteredSample = sampleQueueForLowPassFiter[0]; + for (let i = 1; i < lowPassFilterSize; i++) + filteredSample += sampleQueueForLowPassFiter[i]; + + audioPlayer.feed(filteredSample / lowPassFilterSize, fpsCounter.getFPS()); audioTickCounter = 0; audioSample = 0; } @@ -153,18 +164,6 @@ function animationFrame(now: number) { fpsCounter.update(now); - // Need to simulate low pass filter of Audio PMOD - // with a likely cutoff around 20 kHz - // - // Time constant tau = 1 / (2 * π * cutoff_freq) = 1/(2*π* 20kHz) ~ 1 / 125664 - // = 1/(2*π*100kHz) ~ 1 / 628318 - // Sampling period Ts = (1 / sampling_freq) = 1/25MHz ~ 1 / 25175000 - // Alpha = tau / (tau + Ts) = 1 / (1 + tau / Ts) - const alphaLowPass10kHz = 0.998 // = 1 / (1 + 62832/25175000) ~ 1 / 1.002 - const alphaLowPass20kHz = 0.995 // = 1 / (1 + 125664/25175000) ~ 1 / 1.005 - const alphaLowPass100kHz = 0.9756 // = 1 / (1 + 628318/25175000) ~ 1 / 1.025 - alphaLowPass20kHzAdjustedToFPS = 1.0 / (1.0 + 0.005 * (fpsCounter.getFPS()/expectedFPS)); - if (fpsDisplay) { fpsDisplay.textContent = `${fpsCounter.getFPS().toFixed(0)}`; } From 11e25e3ad0e2c6abdf7fda646e70df988eab5744 Mon Sep 17 00:00:00 2001 From: rej Date: Fri, 20 Sep 2024 22:39:43 +0200 Subject: [PATCH 05/16] audio(examples): option to choose speed for 60 or 30 fps in "music" example --- src/examples/music/project.v | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/examples/music/project.v b/src/examples/music/project.v index e379354..8b6046d 100644 --- a/src/examples/music/project.v +++ b/src/examples/music/project.v @@ -5,6 +5,9 @@ `default_nettype none +`define MUSIC_SPEED 1'b1; // for 60 FPS +// `define MUSIC_SPEED 2'd2; // for 30 FPS + `define C1 481; // 32.70375 Hz `define Cs1 454; // 34.6475 Hz `define D1 429; // 36.7075 Hz @@ -198,7 +201,7 @@ module tt_um_vga_example( end else begin if (x == 0 && y == 0) begin - {frame_counter, frame_counter_frac} <= {frame_counter,frame_counter_frac} + 1; + {frame_counter, frame_counter_frac} <= {frame_counter,frame_counter_frac} + `MUSIC_SPEED; end // noise From 6d04dfd4a36adc60117d724bfd0cf617c9513799 Mon Sep 17 00:00:00 2001 From: rej Date: Fri, 20 Sep 2024 23:25:53 +0200 Subject: [PATCH 06/16] audio: display audio latency next to FPS --- index.html | 5 ++++- public/resampler.js | 3 +++ src/AudioPlayer.ts | 7 +++++++ src/index.css | 8 ++++++++ src/index.ts | 5 +++++ 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/index.html b/index.html index 125c9a8..4f30ff5 100644 --- a/index.html +++ b/index.html @@ -42,7 +42,10 @@
- FPS: 00 +
+ Audio latency: 00 ms + FPS: 00 +
ui_in: diff --git a/public/resampler.js b/public/resampler.js index acc8852..b30da5d 100644 --- a/public/resampler.js +++ b/public/resampler.js @@ -41,6 +41,9 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { this.ringBuffer[this.writeIndex] = samples[i]; this.writeIndex = (this.writeIndex + 1) % this.ringBufferSize; // Wrap around } + + const samplesAvailable = (this.writeIndex - this.readIndex + this.ringBufferSize) % this.ringBufferSize; + this.port.postMessage(samplesAvailable); } else if (data.type === 'reset') { this.ringBuffer.fill(this.previousSample); diff --git a/src/AudioPlayer.ts b/src/AudioPlayer.ts index 07130c8..cb0d45b 100644 --- a/src/AudioPlayer.ts +++ b/src/AudioPlayer.ts @@ -13,12 +13,19 @@ export class AudioPlayer { this.resamplerNode = new AudioWorkletNode(this.audioCtx, 'resampler'); this.resamplerNode.connect(this.audioCtx.destination); + this.resamplerNode.port.onmessage = this.handleMessage.bind(this); + this.audioCtx.resume().then(() => { console.log('Audio playback started'); }); }); } + readonly latencyInMilliseconds = 0.0; + handleMessage(event) { + this.latencyInMilliseconds = event.data / this.sampleRate * 1000.0; + } + private writeIndex = 0; readonly buffer = new Float32Array(this.bufferSize); // larger buffer reduces the communication overhead with the worker thread // however, if buffer is too large it could lead to worker thread starving diff --git a/src/index.css b/src/index.css index a18763e..c884ed5 100644 --- a/src/index.css +++ b/src/index.css @@ -39,6 +39,14 @@ main { max-width: 50%; } +#audio-latency-display { + position: absolute; + top: 8px; + right: 80px; + font-variant-numeric: tabular-nums; + color: grey; +} + #fps-display { position: absolute; top: 8px; diff --git a/src/index.ts b/src/index.ts index 70fd8f8..a7fcf3f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -149,6 +149,7 @@ const canvas = document.querySelector('#vga-canvas'); const ctx = canvas?.getContext('2d'); const imageData = ctx?.createImageData(736, 520); const fpsDisplay = document.querySelector('#fps-count'); +const audioLatencyDisplay = document.querySelector('#audio-latency-ms'); function waitFor(condition: () => boolean, timeout = 10000) { let counter = 0; @@ -168,6 +169,10 @@ function animationFrame(now: number) { fpsDisplay.textContent = `${fpsCounter.getFPS().toFixed(0)}`; } + if (audioLatencyDisplay) { + audioLatencyDisplay.textContent = `${audioPlayer.latencyInMilliseconds.toFixed(0)}` + } + if (stopped || !imageData || !ctx) { return; } From 7a01ba552afb6e7575afbf37000c0cd50d092875 Mon Sep 17 00:00:00 2001 From: rej Date: Sat, 21 Sep 2024 09:33:33 +0200 Subject: [PATCH 07/16] audio: include latency of the AudioContext --- src/AudioPlayer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AudioPlayer.ts b/src/AudioPlayer.ts index cb0d45b..da52e8c 100644 --- a/src/AudioPlayer.ts +++ b/src/AudioPlayer.ts @@ -24,6 +24,7 @@ export class AudioPlayer { readonly latencyInMilliseconds = 0.0; handleMessage(event) { this.latencyInMilliseconds = event.data / this.sampleRate * 1000.0; + this.latencyInMilliseconds += this.audioCtx.outputLatency * 1000.0; } private writeIndex = 0; From e3fc6be33073d98063a3740a4975f61ebdc477f6 Mon Sep 17 00:00:00 2001 From: rej Date: Sat, 21 Sep 2024 13:23:43 +0200 Subject: [PATCH 08/16] audio: made "Audio" button toggle-able. Added ring-buffer pre-warm upon audio context resume to avoid starving the buffer. --- src/AudioPlayer.ts | 30 ++++++++++++++++++++++++++++-- src/index.ts | 21 +++++++++++++++++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/AudioPlayer.ts b/src/AudioPlayer.ts index da52e8c..82e0b95 100644 --- a/src/AudioPlayer.ts +++ b/src/AudioPlayer.ts @@ -6,7 +6,7 @@ export class AudioPlayer { private audioCtx : AudioContext; private resamplerNode : AudioWorkletNode; - constructor(private readonly sampleRate: number, private readonly fps: number, private readonly bufferSize: number = 200) { + constructor(private readonly sampleRate: number, private readonly fps: number, stateListener = null, private readonly bufferSize: number = 200) { this.audioCtx = new AudioContext({sampleRate:sampleRate, latencyHint:'interactive'}); this.audioCtx.audioWorklet.addModule(new URL('/resampler.js', import.meta.url)).then(() => { @@ -19,6 +19,8 @@ export class AudioPlayer { console.log('Audio playback started'); }); }); + + this.audioCtx.onstatechange = stateListener; } readonly latencyInMilliseconds = 0.0; @@ -39,6 +41,9 @@ export class AudioPlayer { samples: this.buffer, fps: current_fps, }); + if (this.resumeScheduled == 1) + this.audioCtx.resume(); + this.resumeScheduled--; } this.writeIndex = 0; } @@ -47,8 +52,10 @@ export class AudioPlayer { this.writeIndex++; } + private resumeScheduled = 0; resume() { - this.audioCtx.resume(); + this.resumeScheduled = 50; // pre-feed buffers before resuming playback + // to avoid starving playback if (this.resamplerNode != null) { this.resamplerNode.port.postMessage({ @@ -56,5 +63,24 @@ export class AudioPlayer { }); } } + + suspend() { + this.resumeScheduled = 0; + this.audioCtx.suspend(); + if (this.resamplerNode != null) + { + this.resamplerNode.port.postMessage({ + type: 'reset' + }); + } + } + + isRunning() { + return (this.audioCtx.state === "running"); + } + needsFeeding() { + return this.isRunning() | this.resumeScheduled > 0; + } + } diff --git a/src/index.ts b/src/index.ts index a7fcf3f..32b0b0c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,8 @@ import { AudioPlayer } from './AudioPlayer'; let currentProject = structuredClone(examples[0]); const inputButtons = Array.from(document.querySelectorAll('#input-values button')); +const audioButtonIndex = inputButtons.findIndex(e => e.innerHTML === "Audio"); + const codeEditorDiv = document.getElementById('code-editor'); const editor = monaco.editor.create(codeEditorDiv!, { @@ -74,7 +76,13 @@ const expectedFPS = 60; const sampleRate = 48_000*4;// @TODO: 192000 high sampleRate might not be supported on all platforms // downsample to 48000 inside the AudioResamplerProcessor // Empirically higher sampling rate helps with occasional high pitch noise. -const audioPlayer = new AudioPlayer(sampleRate, expectedFPS); +const audioPlayer = new AudioPlayer(sampleRate, expectedFPS, () => { + if (audioPlayer.isRunning()) + inputButtons[audioButtonIndex].classList.add('active'); + else + inputButtons[audioButtonIndex].classList.remove('active'); +}); +let enableAudioUpdate = audioPlayer.needsFeeding(); const vgaClockRate = 25_175_000; const ticksPerSample = vgaClockRate / sampleRate; @@ -89,6 +97,9 @@ let sampleQueueForLowPassFiter = new Float32Array(lowPassFilterSize); let sampleQueueIndex = 0; function updateAudio() { + if (!enableAudioUpdate) + return; + audioSample += getAudioSignal(); if (++audioTickCounter < ticksPerSample) return; @@ -138,6 +149,8 @@ editor.onDidChangeModelContent(async () => { jmod.dispose(); } inputButtons.map((b) => b.classList.remove('active')); + if (audioPlayer.isRunning()) + inputButtons[audioButtonIndex].classList.add('active'); jmod = new HDLModuleWASM(res.output.modules['TOP'], res.output.modules['@CONST-POOL@']); await jmod.init(); reset(); @@ -177,6 +190,7 @@ function animationFrame(now: number) { return; } + enableAudioUpdate = audioPlayer.needsFeeding(); const data = new Uint8Array(imageData.data.buffer); frameLoop: for (let y = 0; y < 520; y++) { waitFor(() => !getVGASignals().hsync); @@ -235,7 +249,10 @@ document.querySelector('#download-button')?.addEventListener('click', () => { function toggleButton(index: number) { if (index === 8) { - audioPlayer.resume(); + if (audioPlayer.isRunning()) + audioPlayer.suspend(); + else + audioPlayer.resume(); return; } const bit = 1 << index; From 7aa147a479366155c181ab9c5daf4aa2e727b444 Mon Sep 17 00:00:00 2001 From: rej Date: Sat, 21 Sep 2024 20:41:20 +0200 Subject: [PATCH 09/16] audio: made resume mechanism of prefeeding ring-buffer more consistent by including occupancy percentage of the buffer in the callback event data --- public/resampler.js | 3 ++- src/AudioPlayer.ts | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/public/resampler.js b/public/resampler.js index b30da5d..28adcb9 100644 --- a/public/resampler.js +++ b/public/resampler.js @@ -33,6 +33,7 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { for (let i = 0; i < samples.length; i++) { if ((this.writeIndex + 1) % this.ringBufferSize == this.readIndex) { + this.port.postMessage([this.ringBufferSize, 1.0]); console.log("Buffer is full. Dropping", samples.length - i, "incomming samples!"); break; // Skip incomming samples when ring-buffer is full } @@ -43,7 +44,7 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { } const samplesAvailable = (this.writeIndex - this.readIndex + this.ringBufferSize) % this.ringBufferSize; - this.port.postMessage(samplesAvailable); + this.port.postMessage([samplesAvailable, samplesAvailable / this.ringBufferSize]); } else if (data.type === 'reset') { this.ringBuffer.fill(this.previousSample); diff --git a/src/AudioPlayer.ts b/src/AudioPlayer.ts index 82e0b95..5f0356f 100644 --- a/src/AudioPlayer.ts +++ b/src/AudioPlayer.ts @@ -25,8 +25,16 @@ export class AudioPlayer { readonly latencyInMilliseconds = 0.0; handleMessage(event) { - this.latencyInMilliseconds = event.data / this.sampleRate * 1000.0; + const samplesInBuffer = event.data[0]; + this.latencyInMilliseconds = samplesInBuffer / this.sampleRate * 1000.0; this.latencyInMilliseconds += this.audioCtx.outputLatency * 1000.0; + + const bufferOccupancy = event.data[1]; + if (this.resumeScheduled && bufferOccupancy > 0.25) // resume playback once resampler's + { // buffer is at least 25% full + this.audioCtx.resume(); + this.resumeScheduled = false; + } } private writeIndex = 0; @@ -41,21 +49,18 @@ export class AudioPlayer { samples: this.buffer, fps: current_fps, }); - if (this.resumeScheduled == 1) - this.audioCtx.resume(); - this.resumeScheduled--; } this.writeIndex = 0; } - + this.buffer[this.writeIndex] = value; this.writeIndex++; } - private resumeScheduled = 0; + private resumeScheduled = false; resume() { - this.resumeScheduled = 50; // pre-feed buffers before resuming playback - // to avoid starving playback + // Pre-feed buffers before resuming playback to avoid starving playback + this.resumeScheduled = true; if (this.resamplerNode != null) { this.resamplerNode.port.postMessage({ @@ -65,7 +70,7 @@ export class AudioPlayer { } suspend() { - this.resumeScheduled = 0; + this.resumeScheduled = false; this.audioCtx.suspend(); if (this.resamplerNode != null) { @@ -79,7 +84,7 @@ export class AudioPlayer { return (this.audioCtx.state === "running"); } needsFeeding() { - return this.isRunning() | this.resumeScheduled > 0; + return this.isRunning() || this.resumeScheduled; } } From 0620f3f977a244275c16dcbece8bfb00921dd5f9 Mon Sep 17 00:00:00 2001 From: rej Date: Mon, 23 Sep 2024 15:01:43 +0200 Subject: [PATCH 10/16] perf(audio): ~2-3% speedup by moving fps calculation from getFPS() into update() function. getFPS() now is called several times per frame. --- src/FPSCounter.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/FPSCounter.ts b/src/FPSCounter.ts index 6db6c48..90f2d82 100644 --- a/src/FPSCounter.ts +++ b/src/FPSCounter.ts @@ -3,6 +3,7 @@ export class FPSCounter { private index = 0; private lastTime = -1; private pauseTime = -1; + readonly fps = 0; constructor() {} @@ -18,6 +19,12 @@ export class FPSCounter { this.samples[this.index++ % this.samples.length] = time - this.lastTime; } this.lastTime = time; + + if (this.index > 0) { + const slice = this.samples.slice(0, this.index); + const avgDelta = slice.reduce((a, b) => a + b, 0) / slice.length; + this.fps = 1000 / avgDelta; + } } pause(time: number) { @@ -34,12 +41,6 @@ export class FPSCounter { } getFPS() { - if (this.index === 0) { - // Not enough data yet - return 0; - } - const slice = this.samples.slice(0, this.index); - const avgDelta = slice.reduce((a, b) => a + b, 0) / slice.length; - return 1000 / avgDelta; + return this.fps; } } From 95c1dd8e1e5058137178af195d606ac01bbaa9c3 Mon Sep 17 00:00:00 2001 From: rej Date: Mon, 23 Sep 2024 20:34:02 +0200 Subject: [PATCH 11/16] audio: optional downsampler in case if browser's Audio context does not support 192 kHz sampling rate. --- public/resampler.js | 37 ++++++++++++++++++++++++++++++------- src/AudioPlayer.ts | 17 +++++++++++++++-- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/public/resampler.js b/public/resampler.js index 28adcb9..0ef5dfe 100644 --- a/public/resampler.js +++ b/public/resampler.js @@ -7,13 +7,16 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { super(); // Define buffer properties - this.ringBufferSize = 4096 * 4; // can store approximately 5 frames of audio data + this.downsampleFactor = 1; + this.ringBufferSize = 16_384 * this.downsampleFactor; // stores approximately 5 frames of audio data at 192 kHz this.ringBuffer = new Float32Array(this.ringBufferSize); // ring-buffer helps to amortise uneven framerate and // keeps re-sampler from starving or overflowing this.writeIndex = 0; // Index where new samples are written this.readIndex = 0; // Index where samples are read this.previousSample = 0.0; this.ringBuffer.fill(this.previousSample); + + this.downsampleBuffer = new Float32Array(128 * this.downsampleFactor); // Listen to messages from the main thread this.port.onmessage = this.handleMessage.bind(this); @@ -29,6 +32,7 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { // Handle incoming audio samples and write to the ring buffer if (data.type === 'samples') { this.currentFPS = data.fps; + this.downsampleFactor = data.downsampleFactor; const samples = data.samples; for (let i = 0; i < samples.length; i++) { if ((this.writeIndex + 1) % this.ringBufferSize == this.readIndex) @@ -65,7 +69,7 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { const playbackRate = this.currentFPS / this.expectedFPS; const borderSamples = 2; - const samplesRequired = Math.round(outputData.length * playbackRate) + borderSamples; + const samplesRequired = Math.round(outputData.length * playbackRate * this.downsampleFactor) + borderSamples; // example when samplesRequired = 8 + 2 border samples // (border samples are marked as 'b' below) @@ -94,11 +98,14 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { const samplesConsumed = Math.min(samplesRequired, samplesAvailable) - borderSamples; + if (this.downsampleBuffer.length != outputData.length * this.downsampleFactor); + this.downsampleBuffer = new Float32Array(outputData.length * this.downsampleFactor); + // Calculate resampling ratio - const ratio = samplesConsumed / outputData.length; + const ratio = samplesConsumed / this.downsampleBuffer.length; // Fill the output buffer by resampling from the ring buffer - for (let i = 0; i < outputData.length; i++) { + for (let i = 0; i < this.downsampleBuffer.length; i++) { const floatPos = 0.5 + ratio * (i + 0.5); // use sample centroids, thus +0.5 const intPos = Math.floor(floatPos); const nextIntPos = intPos + 1; @@ -108,18 +115,34 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { console.log(i, frac, intPos, nextIntPos, playbackRate, 'req: ', samplesRequired, 'avail: ', samplesAvailable, ratio, samplesConsumed); // Resample with linear interpolation - outputData[i] = this.interpolate(this.ringBuffer, + this.downsampleBuffer[i] = this.interpolate(this.ringBuffer, (this.readIndex + intPos) % this.ringBufferSize, (this.readIndex + nextIntPos) % this.ringBufferSize, frac); } + // Optional (if audio context does not support 192 kHz) downsample to output buffer + const N = this.downsampleFactor; + if (N > 1) { + for (let i = 0; i < outputData.length; i++) { + let acc = this.downsampleBuffer[i*N]; + for (let j = 1; j < N; j++) + acc += this.downsampleBuffer[i*N + j]; + outputData[i] = acc / N; + } + } else { + for (let i = 0; i < outputData.length; i++) + outputData[i] = this.downsampleBuffer[i]; + + } + + // Store last sample as a future fallback value in case + // if data would not be ready for the next process() call this.previousSample = outputData[outputData.length - 1]; // Update readIndex to match how many samples were consumed this.readIndex = (this.readIndex + samplesConsumed) % this.ringBufferSize; - // Return true to keep the processor alive - return true; + return true; // return true to keep the processor alive } } diff --git a/src/AudioPlayer.ts b/src/AudioPlayer.ts index 5f0356f..325226d 100644 --- a/src/AudioPlayer.ts +++ b/src/AudioPlayer.ts @@ -6,8 +6,20 @@ export class AudioPlayer { private audioCtx : AudioContext; private resamplerNode : AudioWorkletNode; - constructor(private readonly sampleRate: number, private readonly fps: number, stateListener = null, private readonly bufferSize: number = 200) { + private downsampleIntFactor = 1; + private downsampleFracFactor = 1; + + constructor(private readonly sampleRate: number, + private readonly fps: number, + stateListener = null, + private readonly bufferSize: number = 200) { this.audioCtx = new AudioContext({sampleRate:sampleRate, latencyHint:'interactive'}); + // Optional downsampling is used in case when audio context does not support 192 kHz + // for example when context playback rate is 44.1 kHz: + this.downsampleFracFactor = sampleRate / this.audioCtx.sampleRate;// 4.35 = 192_000 / 44_100 + this.downsampleIntFactor = Math.floor(this.downsampleFracFactor); // 4 + this.downsampleFracFactor /= this.downsampleIntFactor; // 1.088 ~~ 48_000 / 44_100 + this.audioCtx.audioWorklet.addModule(new URL('/resampler.js', import.meta.url)).then(() => { this.resamplerNode = new AudioWorkletNode(this.audioCtx, 'resampler'); @@ -47,7 +59,8 @@ export class AudioPlayer { this.resamplerNode.port.postMessage({ type: 'samples', samples: this.buffer, - fps: current_fps, + fps: current_fps * this.downsampleFracFactor, + downsampleFactor: this.downsampleIntFactor, }); } this.writeIndex = 0; From 99b7d04d36d3f16aac2a13f50a407acb06e1f00f Mon Sep 17 00:00:00 2001 From: rej Date: Mon, 23 Sep 2024 20:39:43 +0200 Subject: [PATCH 12/16] audio: comments, small code cleanup --- public/resampler.js | 3 --- src/index.ts | 8 +++----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/public/resampler.js b/public/resampler.js index 0ef5dfe..6902354 100644 --- a/public/resampler.js +++ b/public/resampler.js @@ -111,9 +111,6 @@ class AudioResamplerProcessor extends AudioWorkletProcessor { const nextIntPos = intPos + 1; const frac = floatPos - intPos; // fractional part for interpolation - if ((this.readIndex + nextIntPos) % this.ringBufferSize == this.writeIndex) - console.log(i, frac, intPos, nextIntPos, playbackRate, 'req: ', samplesRequired, 'avail: ', samplesAvailable, ratio, samplesConsumed); - // Resample with linear interpolation this.downsampleBuffer[i] = this.interpolate(this.ringBuffer, (this.readIndex + intPos) % this.ringBufferSize, diff --git a/src/index.ts b/src/index.ts index 32b0b0c..60afc9d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -73,9 +73,7 @@ function getAudioSignal() { } const expectedFPS = 60; -const sampleRate = 48_000*4;// @TODO: 192000 high sampleRate might not be supported on all platforms - // downsample to 48000 inside the AudioResamplerProcessor - // Empirically higher sampling rate helps with occasional high pitch noise. +const sampleRate = 192_000; // 192 kHz -- higher number gives a better quality const audioPlayer = new AudioPlayer(sampleRate, expectedFPS, () => { if (audioPlayer.isRunning()) inputButtons[audioButtonIndex].classList.add('active'); @@ -84,10 +82,10 @@ const audioPlayer = new AudioPlayer(sampleRate, expectedFPS, () => { }); let enableAudioUpdate = audioPlayer.needsFeeding(); -const vgaClockRate = 25_175_000; +const vgaClockRate = 25_175_000; // 25.175 MHz -- VGA pixel clock const ticksPerSample = vgaClockRate / sampleRate; -const lowPassFrequency = 20_000; +const lowPassFrequency = 20_000; // 20 kHz -- Audio PMOD low pass filter const lowPassFilterSize = Math.ceil(sampleRate/lowPassFrequency); let audioTickCounter = 0; From e158216f37fc9c0f06454db743f140a3f8d2a3ef Mon Sep 17 00:00:00 2001 From: rej Date: Mon, 23 Sep 2024 20:55:25 +0200 Subject: [PATCH 13/16] audio: handle (Safari) browsers that do not support AudioContext.outputLatency --- src/AudioPlayer.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/AudioPlayer.ts b/src/AudioPlayer.ts index 325226d..63eee09 100644 --- a/src/AudioPlayer.ts +++ b/src/AudioPlayer.ts @@ -37,9 +37,13 @@ export class AudioPlayer { readonly latencyInMilliseconds = 0.0; handleMessage(event) { + const getEffectiveLatency = (audioContext) => { + return audioContext.outputLatency || audioContext.baseLatency || 0; + } + const samplesInBuffer = event.data[0]; this.latencyInMilliseconds = samplesInBuffer / this.sampleRate * 1000.0; - this.latencyInMilliseconds += this.audioCtx.outputLatency * 1000.0; + this.latencyInMilliseconds += getEffectiveLatency(this.audioCtx) * 1000.0; const bufferOccupancy = event.data[1]; if (this.resumeScheduled && bufferOccupancy > 0.25) // resume playback once resampler's From 27f6ddeeeaa5caf380d0377b8f9fc6fec48b238a Mon Sep 17 00:00:00 2001 From: rej Date: Mon, 23 Sep 2024 21:55:40 +0200 Subject: [PATCH 14/16] audio: audioButtonIndex instead of hardcoded 8 --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 60afc9d..7b16196 100644 --- a/src/index.ts +++ b/src/index.ts @@ -246,7 +246,7 @@ document.querySelector('#download-button')?.addEventListener('click', () => { }); function toggleButton(index: number) { - if (index === 8) { + if (index === audioButtonIndex) { if (audioPlayer.isRunning()) audioPlayer.suspend(); else From e49dea514453c1d053ba24d3d244c15bfc0c514e Mon Sep 17 00:00:00 2001 From: rej Date: Mon, 23 Sep 2024 22:28:58 +0200 Subject: [PATCH 15/16] audio(examples): "music" example cleanup, removed unused variables, added some comments --- src/examples/music/project.v | 56 ++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/examples/music/project.v b/src/examples/music/project.v index 8b6046d..986e775 100644 --- a/src/examples/music/project.v +++ b/src/examples/music/project.v @@ -1,4 +1,7 @@ /* + * Music from "Drop" demo. + * Full version: https://github.com/rejunity/tt08-vga-drop + * * Copyright (c) 2024 Renaldas Zioma, Erik Hemming * SPDX-License-Identifier: Apache-2.0 */ @@ -111,40 +114,39 @@ module tt_um_vga_example( .hpos(x), .vpos(y) ); - - // reg [19:0] lfsr; - // wire feedback = lfsr[19] ^ lfsr[15] ^ lfsr[11] ^ lfsr[5] ^ lfsr[4] ^ lfsr[3] + 1; - reg [12:0] lfsr; - wire feedback = lfsr[12] ^ lfsr[8] ^ lfsr[2] ^ lfsr[0] + 1; - always @(posedge clk) begin - lfsr <= {lfsr[11:0], feedback}; - end - // music - wire [2:0] part = frame_counter[9-:3]; - wire [12:0] timer = {frame_counter, frame_counter_frac}; + wire [2:0] part = frame_counter[10-:3]; + wire [12:0] timer = frame_counter; reg noise, noise_src = ^lfsr; reg [2:0] noise_counter; - wire square60hz = y < 262; // 60Hz square wave - wire square120hz = y[7]; // 120Hz square wave - wire square240hz = y[6]; // 240Hz square wave - wire square480hz = y[5]; // 480Hz square wave + // envelopes wire [4:0] envelopeA = 5'd31 - timer[4:0]; // exp(t*-10) decays to 0 approximately in 32 frames [255 215 181 153 129 109 92 77 65 55 46 39 33 28 23 20 16 14 12 10 8 7 6 5 4 3 3 2 2] wire [4:0] envelopeB = 5'd31 - timer[3:0]*2;// exp(t*-20) decays to 0 approximately in 16 frames [255 181 129 92 65 46 33 23 16 12 8 6 4 3] - wire envelopeP8 = (|timer[3:2])*5'd31;// pulse for 8 frames wire beats_1_3 = timer[5:4] == 2'b10; - // melody notes (in hsync): 151 26 40 60 _ 90 143 23 35 - // (x1.5 wrap-around progression) - reg [8:0] note2_freq; - reg [8:0] note2_counter; - reg note2; + // kick wave + wire square60hz = y < 262; // standing 60Hz square wave + // snare noise + reg [12:0] lfsr; + wire feedback = lfsr[12] ^ lfsr[8] ^ lfsr[2] ^ lfsr[0] + 1; + always @(posedge clk) begin + lfsr <= {lfsr[11:0], feedback}; + end + + // lead wave counter reg [7:0] note_freq; reg [7:0] note_counter; reg note; - wire [3:0] note_in = timer[7-:4]; // 8 notes, 32 frames per note each. 256 frames total, ~4 seconds + + // bass wave counter + reg [8:0] note2_freq; + reg [8:0] note2_counter; + reg note2; + + // lead notes + wire [3:0] note_in = timer[7-:4]; // 16 notes, 16 frames per note each. 256 frames total, ~4 seconds always @(note_in) case(note_in) 4'd0 : note_freq = `E2 @@ -165,6 +167,7 @@ module tt_um_vga_example( 4'd15: note_freq = `E3 endcase + // bass notes wire [2:0] note2_in = timer[8-:3]; // 8 notes, 32 frames per note each. 256 frames total, ~4 seconds always @(note2_in) case(note2_in) @@ -178,19 +181,16 @@ module tt_um_vga_example( 3'd7 : note2_freq = `Cs1 endcase - // wire kick = square60hz & (~|x[7:5] & x[4:0] < envelopeA); // 60Hz square wave with half second envelope - wire kick = square60hz & (x < envelopeA*4); // 60Hz square wave with half second envelope - wire snare = noise & (x >= 128 && x < 128+envelopeB*4); // noise with half second envelope + wire kick = square60hz & (x < envelopeA*4); // 60Hz square wave with half second envelope + wire snare = noise & (x >= 128 && x < 128+envelopeB*4); // noise with half a second envelope wire lead = note & (x >= 256 && x < 256+envelopeB*8); // ROM square wave with quarter second envelope wire base = note2 & (x >= 512 && x < ((beats_1_3)?(512+8*4):(512+32*4))); assign sound = { kick | (snare & beats_1_3 & part != 0) | (base) | (lead & part > 2) }; reg [11:0] frame_counter; - reg frame_counter_frac; always @(posedge clk) begin if (~rst_n) begin frame_counter <= 0; - frame_counter_frac <= 0; noise_counter <= 0; note_counter <= 0; note2_counter <= 0; @@ -201,7 +201,7 @@ module tt_um_vga_example( end else begin if (x == 0 && y == 0) begin - {frame_counter, frame_counter_frac} <= {frame_counter,frame_counter_frac} + `MUSIC_SPEED; + frame_counter <= frame_counter + `MUSIC_SPEED; end // noise From 833653dda0cafb67e49965f2612373531ca62ee6 Mon Sep 17 00:00:00 2001 From: rej Date: Mon, 23 Sep 2024 22:31:25 +0200 Subject: [PATCH 16/16] audio(examples): removed test beep sound from "Stripes" example --- src/examples/stripes/project.v | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/examples/stripes/project.v b/src/examples/stripes/project.v index 53b5801..c96ec81 100644 --- a/src/examples/stripes/project.v +++ b/src/examples/stripes/project.v @@ -29,20 +29,14 @@ module tt_um_vga_example( // TinyVGA PMOD assign uo_out = {hsync, B[0], G[0], R[0], vsync, B[1], G[1], R[1]}; - assign uio_out = {sound, 7'b0}; // Unused outputs assigned to 0. - assign uio_oe = 8'hff; + assign uio_out = 0; + assign uio_oe = 0; // Suppress unused signals warning wire _unused_ok = &{ena, ui_in, uio_in}; - reg [15:0] counter2; - assign sound = counter2[14]; - always @(posedge clk) begin - counter2 <= counter2 + 1; - end - reg [9:0] counter; hvsync_generator hvsync_gen(