Billboard deploy (#496)

* initial react setup and dm3-billboard-api fix

* setup new project using vite

- add messages list with auto scroll logic
- add new messages simulation button
- TODO: cleanup old package, boilerplate etc ..

* improve scroll behaviour and clean upcode

* add scrollbars, normalize css class names

* remove old billboard-app

* add message create area

- add timeout button
- add tolerance to scrolled to bottom detection

* add date formatting

* use svg colors from vars

* add viewers count, empty view and branding

- optimize some css
- add icons and logos
- fix some margins

* smaller UI fixes

* fix scrolling the whole page first

* style scroll-to-bottom button

* add viewers count, configure env vars

* move wrongly located component

* add random mack message, fix jumping avatar

* add experimental provider, fix some styles

- add experimental metamask provider
- fix create message styles for text area
- hide gear icon button, if no action is defined

* add websocket listener

- expose all props to toplevel component
- add event websocket listener for message
- check for existing messages
- fix small UI bugs
- remove support for .env vars

* migrate to socket.io

- add viewers count event
- fix problem with adding keys
- add warning to unique function

* make avatar configurable

* prepare npm release

- separate css from js
- create types
- configure vite in lib mode
- temporarily add registry for our scope "testscope/"

* add use auth hook

* add yarn dir to gitignore

* add dedicated profile endpoint

* adjust profilep

* add interface docs and update README

* add send msg

* ready for publish

* fix dm3 styles

* disable button when no message was entered

- add small timeout of 6 seconds
- add separate disabled state for timeout and disabled prop

* build: npm publish

* add create UserProfile in case it does not exit yet

* build: billboard client deploy

* build: heroku

* fix: build

* build: ds

* fix: build

* feat: new logger

---------

Co-authored-by: Karl Adler <[email protected]>
Co-authored-by: AlexNi245 <[email protected]>

Author: 3 people

.gitignore

@@ -57,4 +57,6 @@ packages/lib/**/dist
 !.yarn/plugins
 !.yarn/releases
 !.yarn/sdks
-!.yarn/versions
+!.yarn/versions
+
+packages/**/.yarn

.yarnrc.yml

@@ -5,3 +5,15 @@ plugins:
     spec: "@yarnpkg/plugin-workspace-tools"
 
 yarnPath: .yarn/releases/yarn-3.5.1.cjs
+
+unsafeHttpWhitelist:
+  - "localhost"
+
+# using https://verdaccio.org/ for the scope: `@testscope`
+# Needs to be removed before release
+# Update Scope in package.json
+npmScopes:
+  testscope:
+      npmPublishRegistry: http://localhost:4873
+      npmRegistryServer: http://localhost:4873
+      # npmAuthIdent: user:password

package.json

@@ -12,12 +12,13 @@
             "packages/lib/offchainResolver",
             "packages/lib/offchainResolver-api",
             "packages/offchain-resolver-backend",
-            "packages/lib/lib-billboard-api",
+            "packages/lib/billboard-api",
             "packages/backend",
             "packages/billboard-client",
             "packages/react",
             "packages/web",
             "packages/offchain-resolver",
+            "packages/billboard-widget",
             "packages/integration-tests"
         ],
         "nohoist": [
@@ -43,8 +44,8 @@
     "scripts": {
         "docker:up": "cd packages/backend && docker-compose up -d",
         "build": " yarn workspaces foreach -pt run build",
-        "start": "cd packages/backend && yarn start",
-        "resolver": "cd packages/offchain-resolver && yarn start",
+        "start": "yarn workspace dm3-backend start",
+        "resolver": "yarn workspace dm3-offchain-resolver-backend start",
         "lint": "eslint '*/**/*.{js,ts,tsx}' --fix",
         "format": "prettier --write '*/**/*.{js,ts,tsx,json}'",
         "format:check": "prettier --check '*/**/*.{js,ts,tsx,json}'",

packages/backend/package.json

@@ -30,7 +30,7 @@
         "docker:up": "docker-compose up -d",
         "start": "ts-node --transpile-only  ./src/index.ts",
         "test": "npm run docker:up && jest --coverage --runInBand --transformIgnorePatterns 'node_modules/(?!(dm3-lib-\\w*)/)'",
-        "build": "tsc && cp ./config.yml ./dist/config.yml | true",
+        "build": "yarn tsc && cp ./config.yml ./dist/config.yml | true",
         "createDeliveryServiceProfile": "node --no-warnings ./cli.js"
     },
     "devDependencies": {

packages/backend/src/cleanup/cleanUpPendingMessages.ts

@@ -7,7 +7,7 @@ const DEFAULT_CLEANUP_INTERVAL = 86400000;
 async function onCleanUpPendingMessages(db: IDatabase, ttl: number) {
     const now = new Date().getTime();
     const expiryDate = now - ttl;
-    log('[Clean up] Delete expired messages');
+    log('[Clean up] Delete expired messages', 'info');
     await db.deleteExpiredMessages(expiryDate);
 }
 
@@ -21,7 +21,7 @@ export function startCleanUpPendingMessagesJob(
     if (ttl <= 0) {
         return;
     }
-    log('[Clean up] Start Clean up job');
+    log('[Clean up] Start Clean up job', 'info');
     return setInterval(() => {
         onCleanUpPendingMessages(db, ttl);
     }, cleaningInterval);

packages/backend/src/config/getDeliveryServiceProperties.ts

@@ -19,7 +19,7 @@ export function getDeliveryServiceProperties(
     defaultDeliveryServiceProperties: DeliveryServiceProperties = DEFAULT_DELIVERY_SERVICE_PROPERTIES,
 ): DeliveryServiceProperties {
     if (!existsSync(path)) {
-        log('Config file not found. Default Config is used');
+        log('Config file not found. Default Config is used', 'info');
         return defaultDeliveryServiceProperties;
     }
     const yamlString = readFileSync(path, { encoding: 'utf-8' });

packages/backend/src/index.ts

@@ -83,5 +83,5 @@ app.use(express.static(path.join(__dirname, '../../web/build')));
 const port = process.env.PORT || '8080';
 
 server.listen(port, () => {
-    log('[Server] listening at port ' + port + ' and dir ' + __dirname);
+    log('[Server] listening at port ' + port + ' and dir ' + __dirname, 'info');
 });

packages/backend/src/messaging.ts

@@ -9,6 +9,7 @@ import {
     checkToken,
     incomingMessage,
 } from 'dm3-lib-delivery/dist.backend';
+import { log } from 'dm3-lib-shared';
 const pendingMessageSchema = {
     type: 'object',
     properties: {
@@ -23,7 +24,6 @@ const pendingMessageSchema = {
 export function onConnection(app: express.Application & WithLocals) {
     return (socket: Socket) => {
         socket.on('disconnect', () => {
-            app;
             app.locals.logger.info({
                 method: 'WS DISCONNECT',
                 socketId: socket.id,

packages/backend/src/rpc/methods/handleSubmitMessage.ts

@@ -4,10 +4,7 @@ import { validateSchema } from 'dm3-lib-shared/dist.backend';
 import 'dotenv/config';
 import express from 'express';
 import { WithLocals } from '../../types';
-
-interface SubmitMessage {
-    token: string;
-}
+import { log } from 'dm3-lib-shared';
 
 export async function handleSubmitMessage(
     req: express.Request & { app: WithLocals },
@@ -19,8 +16,15 @@ export async function handleSubmitMessage(
 
     const envelop = JSON.parse(stringifiedEnvelop);
 
-    if (!envelop || !token) {
-        return res.send(400);
+    if (!token) {
+        log('Auth token missing', 'error');
+        return res.status(400).send('Auth token missing');
+    }
+
+    if (!envelop) {
+        log('Envelop missing', 'error');
+
+        return res.status(400).send('Envelop missing');
     }
 
     const isSchemaValid = validateSchema(schema.MessageSubmission, {
@@ -58,7 +62,8 @@ export async function handleSubmitMessage(
             method: 'RPC SUBMIT MESSAGE',
             error,
         });
+        log('[handleSubmitMessage]' + error, 'error');
 
-        return res.send(400);
+        return res.status(400).send();
     }
 }

packages/billboard-client/package.json

@@ -8,6 +8,7 @@
         "axios": "^0.27.2",
         "body-parser": "^1.20.1",
         "connect-redis": "^6.1.3",
+        "cors": "^2.8.5",
         "dm3-lib-delivery": "^0.1.1",
         "dm3-lib-messaging": "^0.1.1",
         "dm3-lib-profile": "^0.1.1",
@@ -21,12 +22,13 @@
         "prettier": "^2.6.2",
         "redis": "^4.1.0",
         "socket.io": "^4.5.1",
+        "ts-node": "^10.9.1",
         "winston": "^3.8.2",
         "yaml": "^2.1.3"
     },
     "scripts": {
         "docker:up": "docker-compose up -d",
-        "start": "ts-node --transpile-only  ./src/index.ts",
+        "start": "yarn ts-node --transpile-only  ./src/index.ts",
         "test": "npm run docker:up && jest --coverage --runInBand --forceExit",
         "lint": "eslint '*/**/*.{js,ts,tsx}' --fix",
         "format": "prettier --write '*/**/*.{js,ts,tsx,json}'",

packages/billboard-client/src/api/external/getExternalApi.ts

@@ -1,12 +1,14 @@
 import express from 'express';
 import { v4 } from 'uuid';
+import cors from 'cors';
 import { getViewerCountHandler } from './handler/getViewerCount';
 import { getMessagesHandler } from './handler/getMessages';
 import { IDatabase } from '../../persitance/getDatabase';
 import { IViewerService } from '../../service/viewerService/IViewerService';
 
 export function getExternalApi(db: IDatabase, viewerService: IViewerService) {
     const app = express();
+    app.use(cors());
 
     app.post('/rpc', async (req: express.Request, res: express.Response) => {
         const { method, params } = req.body;

packages/billboard-client/src/api/internal/rest/getChallenge.ts

@@ -12,7 +12,7 @@ export async function getChallenge(
         const { data } = await axios.get(url);
         return data.challenge;
     } catch (e) {
-        log("can't get challenge from ds " + dsUrl);
+        log("can't get challenge from ds " + dsUrl, 'error');
         return null;
     }
 }

packages/billboard-client/src/api/internal/rest/getIncomingMessages.ts

@@ -11,15 +11,26 @@ import { log } from 'dm3-lib-shared';
 export async function getIncomingMessages(
     dsUrl: string,
     ensName: string,
+    token: string,
 ): Promise<EncryptionEnvelop[] | null> {
     try {
         const response: AxiosResponse<EncryptionEnvelop[]> = await axios.get(
-            `${dsUrl}/messages/incoming/${ensName}`,
+            `${dsUrl}/delivery/messages/incoming/${ensName}`,
+            {
+                headers: {
+                    Authorization: `Bearer ${token}`,
+                },
+            },
         );
 
         return response.data;
     } catch (e) {
-        log(`Failed to retrieve incoming messages for ds ${dsUrl}`);
+        log(
+            `Failed to retrieve incoming messages for ds ${dsUrl}` +
+                JSON.stringify(e),
+
+            'error',
+        );
         return null;
     }
 }

packages/billboard-client/src/api/internal/rest/getNewToken.ts

@@ -12,7 +12,7 @@ export async function getNewToken(
         const { data } = await axios.post(url, { signature });
         return data.token;
     } catch (e) {
-        log("can't get new token from ds");
+        log("can't get new token from ds: " + JSON.stringify(e), 'error');
         return null;
     }
 }

packages/billboard-client/src/api/internal/rest/submitUserProfile.ts

@@ -0,0 +1,22 @@
+import axios from 'axios';
+import { SignedUserProfile, normalizeEnsName } from 'dm3-lib-profile';
+import { log } from 'dm3-lib-shared';
+
+export async function submitUserProfile(
+    dsUrl: string,
+    ensName: string,
+    signedUserProfile: SignedUserProfile,
+): Promise<string | null> {
+    const url = `${dsUrl}/profile/${normalizeEnsName(ensName)}`;
+
+    try {
+        const { data } = await axios.post(url, signedUserProfile);
+        return data;
+    } catch (e) {
+        log(
+            "can't submit userProfile to ds " + dsUrl + ' ' + JSON.stringify(e),
+            'error',
+        );
+        return null;
+    }
+}

packages/billboard-client/src/api/internal/ws/getDeliveryServiceWSConnections.ts

@@ -1,29 +1,43 @@
 import { EncryptionEnvelop } from 'dm3-lib-messaging';
+import { DeliveryServiceProfile } from 'dm3-lib-profile';
 import { log } from 'dm3-lib-shared';
 
 import { io, Socket } from 'socket.io-client';
 
 export const getDeliveryServiceWSClient = (
-    deliveryServices: string[],
+    deliveryServices: (DeliveryServiceProfile & {
+        token: string;
+    })[],
     onMessage: (encelop: EncryptionEnvelop) => any,
+    ensName: string,
 ): Promise<Socket[]> => {
     return Promise.all(
-        deliveryServices.map(async (url: string) => {
+        deliveryServices.map(async (ds) => {
             const client = await new Promise<Socket>((res, rej) => {
-                const c = io(url, {});
+                const c = io(ds.url, {
+                    auth: {
+                        account: { ensName },
+                        token: ds.token,
+                    },
+                });
                 c.on('connect', () => {
-                    log(`Connected to Delivery Service ${url}`);
+                    log(`Connected to Delivery Service ${ds.url}`, 'info');
                     res(c);
                 });
                 c.on('connect_error', (err: any) => {
-                    log(`Connection error to Delivery Service ${url}`);
+                    log(
+                        `Connection error to Delivery Service ${ds.url} ` +
+                            JSON.stringify(err),
+                        'error',
+                    );
 
                     rej(err);
                 });
             });
+            log(`Register listener`, 'info');
             //register listners
             client.on('disconnect', () => {
-                log(`disconnected from deliveryService ${url}`);
+                log(`disconnected from deliveryService ${ds.url}`, 'info');
             });
             client.on('message', onMessage);
 

packages/billboard-client/src/getBillboardClientApp.ts

@@ -29,6 +29,7 @@ export const getBillboardClientApp = async (
 
     //Readng the ENV config file
     const config = ConfigService().readConfigFromEnv();
+    log('[getBillboardClientApp] config ' + JSON.stringify(config), 'debug');
 
     //Each Ens name provided in the config file is an billboard instance
     //Right now we're using one profile and hence one private key for all billboards
@@ -37,6 +38,10 @@ export const getBillboardClientApp = async (
         ensName,
         privateKey: config.privateKey,
     }));
+    log(
+        '[getBillboardClientApp] billboards ' + JSON.stringify(billboards),
+        'debug',
+    );
 
     //Register services
     const viewerService = await ViewerService(httpServer);
@@ -53,7 +58,7 @@ export const getBillboardClientApp = async (
     app.use(getExternalApi(db, viewerService));
 
     httpServer.listen(port, () => {
-        log('billboard client listening at port ' + port);
+        log('billboard client listening at port ' + port, 'info');
     });
 
     //In oder to finish the test everything has to be cleaned up

packages/billboard-client/src/index.ts

@@ -15,7 +15,11 @@ const main = async () => {
     }
     const provider = new ethers.providers.JsonRpcProvider(rpcUrl);
 
-    const database = await getDatabase(winston.createLogger());
+    const database = await getDatabase(
+        winston.createLogger({
+            transports: [new winston.transports.Console()],
+        }),
+    );
 
     await getBillboardClientApp(provider, database, Number(port));
 };

packages/billboard-client/src/persitance/getDatabase.ts

@@ -9,7 +9,7 @@ export enum RedisPrefix {
 }
 
 export async function getRedisClient(logger: winston.Logger) {
-    const url = process.env.REDIS_URL || 'redis://127.0.0.1:6368';
+    const url = process.env.REDIS_URL ?? 'redis://127.0.0.1:6368';
     const socketConf = {
         socket: {
             tls: true,

packages/billboard-client/src/persitance/getMessages/getMessages.test.ts

@@ -7,8 +7,17 @@ describe('getMessages', () => {
     let redisClient: Redis;
     let db: IDatabase;
     beforeEach(async () => {
-        redisClient = await getRedisClient(winston.createLogger());
-        db = await getDatabase(winston.createLogger(), redisClient);
+        redisClient = await getRedisClient(
+            winston.createLogger({
+                transports: [new winston.transports.Console()],
+            }),
+        );
+        db = await getDatabase(
+            winston.createLogger({
+                transports: [new winston.transports.Console()],
+            }),
+            redisClient,
+        );
         await redisClient.flushDb();
     });
 

packages/billboard-client/src/service/dsManager/DsManager.test.ts

@@ -363,7 +363,9 @@ describe('DsManager', () => {
             });
 
             axiosMock
-                .onGet('http://localhost:4060/messages/incoming/billboard1.eth')
+                .onGet(
+                    'http://localhost:4060/delivery/messages/incoming/billboard1.eth',
+                )
                 .reply(200, [
                     await mockChat1.createMessage('hello'),
                     await mockChat1.createMessage('world'),