feat(sds): add message channel with buffers and send/receive logic

This commit creates the class for an SDS message channel, including
buffers for outgoing and incoming messages. Adds logic for sending
messages, receiving messages, delivering messages, and reviewing
acknowledgement status of messages. Also adds byte serialization
for bloom filters.

Author: adklempner

.cspell.json

@@ -64,6 +64,7 @@
     "kdfparams",
     "keccak",
     "keypair",
+    "lamport",
     "lastpub",
     "libauth",
     "libp",

package-lock.json

@@ -15,3 +15,5 @@ export * as proto_store from './generated/store_v3.js'
 export * as proto_peer_exchange from "./generated/peer_exchange.js";
 
 export * as proto_metadata from './generated/metadata.js'
+
+export * as proto_sds_message from './generated/sds_message.js'

packages/proto/src/index.ts

@@ -8,7 +8,7 @@ const config = {
     'loader=ts-node/esm'
   ],
   exit: true,
-  retries: 4
+  retries: 2
 };
 
 if (process.env.CI) {

packages/sds/.mocharc.cjs

@@ -59,6 +59,10 @@
         "node": ">=20"
     },
     "dependencies": {
+        "@noble/hashes": "^1.7.1",
+        "@waku/message-hash": "^0.1.17",
+        "@waku/proto": "^0.0.8",
+        "@waku/utils": "^0.0.21",
         "chai": "^5.1.2"
     },
     "devDependencies": {

packages/sds/package.json

@@ -1,7 +1,6 @@
 import { expect } from "chai";
 
-import { BloomFilter } from "./bloom.js";
-import { hashN } from "./nim_hashn/nim_hashn.mjs";
+import { BloomFilter, DefaultBloomFilter } from "./bloom.js";
 
 const n = 10000;
 const sampleChars =
@@ -20,13 +19,10 @@ describe("BloomFilter", () => {
   let testElements: string[];
 
   beforeEach(() => {
-    bloomFilter = new BloomFilter(
-      {
-        capacity: n,
-        errorRate: 0.001
-      },
-      hashN
-    );
+    bloomFilter = new DefaultBloomFilter({
+      capacity: n,
+      errorRate: 0.001
+    });
 
     testElements = new Array<string>(n);
 
@@ -55,15 +51,12 @@ describe("BloomFilter", () => {
     expect(bloomFilter.kHashes).to.equal(10);
     expect(bloomFilter.totalBits / n).to.equal(15);
 
-    const bloomFilter2 = new BloomFilter(
-      {
-        capacity: 10000,
-        errorRate: 0.001,
-        kHashes: 4,
-        forceNBitsPerElem: 20
-      },
-      hashN
-    );
+    const bloomFilter2 = new DefaultBloomFilter({
+      capacity: 10000,
+      errorRate: 0.001,
+      kHashes: 4,
+      forceNBitsPerElem: 20
+    });
     expect(bloomFilter2.kHashes).to.equal(4);
     expect(bloomFilter2.totalBits).to.equal(200000);
   });
@@ -107,20 +100,28 @@ describe("BloomFilter", () => {
       expect(bloomFilter.lookup(item)).to.equal(true);
     }
   });
+
+  it("should serialize and deserialize correctly", () => {
+    const serialized = bloomFilter.toBytes();
+    const deserialized = DefaultBloomFilter.fromBytes(
+      serialized,
+      bloomFilter.options
+    );
+    for (const item of testElements) {
+      expect(deserialized.lookup(item)).to.equal(true);
+    }
+  });
 });
 
 describe("BloomFilter with special patterns", () => {
   let bloomFilter: BloomFilter;
   const inserted: string[] = [];
 
   beforeEach(() => {
-    bloomFilter = new BloomFilter(
-      {
-        capacity: n,
-        errorRate: 0.001
-      },
-      hashN
-    );
+    bloomFilter = new DefaultBloomFilter({
+      capacity: n,
+      errorRate: 0.001
+    });
   });
 
   it("should handle special patterns correctly", () => {

packages/sds/src/bloom.spec.ts

@@ -1,3 +1,4 @@
+import { hashN } from "./nim_hashn/nim_hashn.mjs";
 import { getMOverNBitsForK } from "./probabilities.js";
 
 export interface BloomFilterOptions {
@@ -30,11 +31,15 @@ export class BloomFilter {
   public kHashes: number;
   public errorRate: number;
 
+  public options: BloomFilterOptions;
+
   private hashN: (item: string, n: number, maxValue: number) => number;
   public constructor(
     options: BloomFilterOptions,
     hashN: (item: string, n: number, maxValue: number) => number
   ) {
+    this.options = options;
+
     let nBitsPerElem: number;
     let k = options.kHashes ?? 0;
     const forceNBitsPerElem = options.forceNBitsPerElem ?? 0;
@@ -103,4 +108,39 @@ export class BloomFilter {
     }
     return true;
   }
+
+  public toBytes(): Uint8Array {
+    const buffer = new ArrayBuffer(this.data.length * 8);
+    const view = new DataView(buffer);
+    for (let i = 0; i < this.data.length; i++) {
+      view.setBigInt64(i * 8, this.data[i]);
+    }
+    return new Uint8Array(buffer);
+  }
+
+  public static fromBytes(
+    bytes: Uint8Array,
+    options: BloomFilterOptions,
+    hashN: (item: string, n: number, maxValue: number) => number
+  ): BloomFilter {
+    const bloomFilter = new BloomFilter(options, hashN);
+    const view = new DataView(bytes.buffer);
+    for (let i = 0; i < bloomFilter.data.length; i++) {
+      bloomFilter.data[i] = view.getBigUint64(i * 8, false);
+    }
+    return bloomFilter;
+  }
+}
+
+export class DefaultBloomFilter extends BloomFilter {
+  public constructor(options: BloomFilterOptions) {
+    super(options, hashN);
+  }
+
+  public static fromBytes(
+    bytes: Uint8Array,
+    options: BloomFilterOptions
+  ): DefaultBloomFilter {
+    return BloomFilter.fromBytes(bytes, options, hashN);
+  }
 }