Merge pull request #9425 from hicommonwealth/tim/merge-discord-bot

.github/workflows/CI.yml

@@ -346,7 +346,6 @@ jobs:
           chmod -R 755 ./libs/model/coverage/lcov.info
           chmod -R 755 ./libs/sitemaps/coverage/lcov.info
           chmod -R 755 ./packages/commonwealth/coverage/lcov.info
-          chmod -R 755 ./packages/discord-bot/coverage/lcov.info
           chmod -R 755 ./packages/scripts/coverage/lcov.info
           chmod -R 755 ./packages/snapshot-listener/coverage/lcov.info
 
@@ -355,7 +354,7 @@ jobs:
         with:
           flag-name: unit-test-coverage
           parallel: true
-          files: libs/adapters/coverage/lcov.info libs/core/coverage/lcov.info libs/model/coverage/lcov.info libs/sitemaps/coverage/lcov.info packages/commonwealth/coverage/lcov.info packages/discord-bot/coverage/lcov.info packages/scripts/coverage/lcov.info packages/snapshot-listener/coverage/lcov.info
+          files: libs/adapters/coverage/lcov.info libs/core/coverage/lcov.info libs/model/coverage/lcov.info libs/sitemaps/coverage/lcov.info packages/commonwealth/coverage/lcov.info packages/scripts/coverage/lcov.info packages/snapshot-listener/coverage/lcov.info
 
       - name: Run integration tests
         run: pnpm -F commonwealth test-integration --allowOnly=false

Dockerfile.commonwealth_base

@@ -12,10 +12,6 @@ WORKDIR /usr/src/app
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
 RUN pnpm run build
 
-RUN pnpm -F discord-bot run build
-RUN pnpm deploy -F discord-bot --prod /prod/discord-bot
-RUN mv /usr/src/app/packages/discord-bot/build /prod/discord-bot/build
-
 RUN pnpm -F snapshot-listener run build
 RUN pnpm deploy -F snapshot-listener --prod /prod/snapshot-listener
 RUN mv /usr/src/app/packages/snapshot-listener/build /prod/snapshot-listener/build
@@ -41,14 +37,4 @@ COPY ./scripts/datadog-entrypoint.sh /prod/snapshot-listener
 RUN chmod +x /prod/snapshot-listener/datadog-entrypoint.sh
 COPY ./scripts/get-max-old-space-size.sh /prod/snapshot-listener/scripts
 RUN chmod +x /prod/snapshot-listener/scripts/get-max-old-space-size.sh
-ENV PORT=$PORT
-
-FROM base AS discord-bot
-ENV NODE_ENV=production
-COPY --from=build /prod/discord-bot /prod/discord-bot
-WORKDIR /prod/discord-bot
-COPY ./scripts/datadog-entrypoint.sh /prod/discord-bot
-RUN chmod +x /prod/discord-bot/datadog-entrypoint.sh
-COPY ./scripts/get-max-old-space-size.sh /prod/discord-bot/scripts
-RUN chmod +x /prod/discord-bot/scripts/get-max-old-space-size.sh
 ENV PORT=$PORT

common_knowledge/Discobot.md

@@ -19,32 +19,14 @@
 
 “Discobot” refers to the set of entities and interactions that power the Commonwealth <> Discord integration. In particular, this is 3 things:
 
-1. Discord Listener (`/packages/discord-bot/discord-listener/discordListener.ts`), an app that handles incoming events from the Discord API and pushes these events to a RabbitMQ queue.
-2. Discord Consumer (`/packages/discord-bot/discord-consumer/discordConsumer.ts`), an app that handles events from the RabbitMQ queue and hits the CW API endpoint to create Threads and Comments.
+1. Discord Listener (`/packages/commonwealth/server/workers/discordBot/discordListener.ts`), an app that handles incoming events from the Discord API and pushes these events to a RabbitMQ queue.
+2. DiscordBotPolicy, a policy that handles events from the RabbitMQ queue to create Threads and Comments.
 3. RabbitMQ Instance: a queue has been set up called `discord-message`
 4. Commonwealth Manage Community Page (`/packages/commonwealth/…/community_metadata_rows.tsx`), where admins are able to add a bot connection and connect Forum Channels (in a connected Discord Server) to Topics (in the CW forum).
 
 The basic flow here:
 ![image (4)](./assets/Discobot-2.png)
 
-## Deployments
-
-### Production
-
-1. Discord Listener: deployed as a worker dyno on the [discobot-listener](https://dashboard.heroku.com/apps/discobot-listener/resources) Heroku app.
-2. Discord Consumer: deployed as a worker dyno on the [discobot-listener](https://dashboard.heroku.com/apps/discobot-listener/resources) Heroku app.
-
-### Staging
-
-1. Discord Listener: deployed as a worker dyno on the [discobot-listener-staging](https://dashboard.heroku.com/apps/discobot-listener-staging/resources) Heroku app.
-2. Discord Consumer: deployed as a worker dyno on the [discobot-listener-staging](https://dashboard.heroku.com/apps/discobot-listener-staging/resources) Heroku app.
-
-The staging Discobot app is used by the following environments for testing purposes:
-
-- `commonwealth-beta` Heroku app (i.e. QA)
-- `commonwealth-frick` Heroku app
-- `commonwealth-frack` Heroku app
-
 ## Discord Apps (Bots)
 
 All the Discord Bots that are used to build the Commonwealth Discobot functionality are accessible
@@ -65,27 +47,22 @@ All redirect URLs that the bot should support need to be inserted/
 
 ### Environment Variables (Local)
 
-#### In `packages/discord-bot/.env` create the following environment variables
+#### In `.env` create the following environment variables
 
 - `DISCORD_TOKEN`:This is the token of the staging Discord bot.
   - This variable cannot be found on the Discord developer portal (once created it is hidden). To get this
   variable view the config vars of the [`discobot-listener-staging` Heroku app](https://dashboard.heroku.com/apps/discobot-listener-staging/settings)
   or contact one of the following: Jake, Timothee, Ian
-- `CW_BOT_KEY`: This can be any random string, but it must match `CW_BOT_KEY` in `packages/commonwealth/.env`
-
-#### In `packages/commonwealth/.env` create the following environment variables
-
 - `DISCORD_CLIENT_ID`: this is the client ID of the Discord app.
   - For local test we use the staging Discord app/bot. The client ID can therefore be found on the [developer dashboard](https://discord.com/developers/applications/1027997517964644453/oauth2/general)
-  or by contacting Jake or Timothee.
-- `DISCORD_TOKEN`: this is the same as the `DISCORD_TOKEN` in `/discord-bot/.env`
-- `CW_BOT_KEY`: this is the same as the `CW_BOT_KEY` in `/discord-bot/.env`
+    or by contacting Jake or Timothee.
 
 ### Startup
 
 1. Start a local RabbitMQ instance by executing `pnpm start-rmq` in the root directory (requires Docker).
-2. In a separate terminal execute `pnpm start` in `packages/discord-bot/` to start the Discord Listener
-3. In a separate terminal execute `pnpm start-consumer` in `packages/discord-bot/` to start the Discord Consumer
+2. In a separate terminal execute `pnpm start-discord-listener` in `packages/commonwealth/` to start the Discord Listener
+3. In a separate terminal execute `pnpm start-consumer` in `packages/commonwealth/` to start the Commonwealth Consumer
+4. In a separate terminal execute `pnpm start-message-relayer`
 
 ## Staging and Production Setup
 
@@ -102,14 +79,9 @@ connect to. For the staging environments this will be the `CLOUDAMQP_URL` enviro
 [`commonwealth-frick` Heroku app](https://dashboard.heroku.com/apps/commonwealth-frick/settings). For production this is
 the `CLOUDAMQP_URL` environment variable in the [`commonwealthapp` Heroku app](https://dashboard.heroku.com/apps/commonwealth-beta/settings).
 - `CLOUDAMQP_APIKEY`: Same principle as `CLOUDAMQP_URL`.
-- `CW_BOT_KEY`: A strong password that matches `CW_BOT_KEY` in the [`commonwealthapp` Heroku app](https://dashboard.heroku.com/apps/commonwealth-beta/settings).
 - `DATABASE_URL`: Same principles as `CLOUDAMQP_URL` and `CLOUDAMQP_APIKEY` (copy variable from the relevant app).
 - `DISCORD_TOKEN`: This is the token of the staging or production Discord bot.
   - This variable cannot be found on the Discord developer portal (once created it is hidden).
-- `DL_BUILD`: Should be set to 'true'.
-  - This ensures that Heroku only builds the Discobot package and related code when deploying.
-- `PROCFILE`: Should be set to `packages/discord-bot/Procfile`
-  - This tells Heroku which Procfile in the repository to use when deploying the app since we use multi-procfile setup.
 - `SERVER_URL`: This should be set to the URL of the Heroku app the Discobot is associated to. This will be
 `https://commonwealth.im` for the production.
 
@@ -120,7 +92,6 @@ the `CLOUDAMQP_URL` environment variable in the [`commonwealthapp` Heroku app](h
       or the [production bot](https://discord.com/developers/applications/1133050809412763719/oauth2/general).
   The client ID can also be retrieved by contacting Jake or Timothee.
 - `DISCORD_TOKEN`: this is the same as the `DISCORD_TOKEN` in the associated `Discobot app` above.
-- `CW_BOT_KEY`: this is the same as the `CW_BOT_KEY` in the associated `Discobot app` above.
 
 ## Testing
 

common_knowledge/Environment-Variables.md

@@ -18,7 +18,6 @@ If you add a new environment variable, you must add documentation here. Please d
 - [COSMOS_GOV_V1](#cosmos_gov_v1)
 - [COSMOS_PROXY_REFERER](#cosmos_proxy_referer)
 - [COSMOS_REGISTRY_API](#cosmos_registry_api)
-- [CW_BOT_KEY](#cw_bot_key)
 - [DATABASE_CLEAN_HOUR](#database_clean_hour)
 - [DATABASE_LOG_TRACE](#database_log_trace)
 - [DATABASE_URI](#database_uri)
@@ -122,10 +121,6 @@ Community-maintained data source for Cosmos ecosystem blockchains. Pulls from a
 
 Owner: Mark Hagelberg.
 
-## CW_BOT_KEY
-
-Required for Common bots, e.g. Discobot. In development, can be set to any random identifier string, but must match the value of `CW_BOT_KEY` set in Discobot's .env file.
-
 ## DATABASE_CLEAN_HOUR
 
 When the cleaner runs is determined by the DATABASE_CLEAN_HOUR env var. The env var is a simple number between 0 and 24 indicating (in 24hr format) at what time the cleaner should execute the cleaning functions. If env var is not set, the database cleaner will not run.

libs/adapters/src/rabbitmq/configuration/rascalConfig.ts

@@ -91,18 +91,14 @@ export function getAllRascalConfigs(
   };
 
   const allExchanges: Record<keyof OmittedRascalExchanges, ExchangeConfig> = {
-    [RascalExchanges.Discobot]: {
-      type: 'fanout',
-      ...exchangeConfig,
-    },
     [RascalExchanges.MessageRelayer]: {
       type: 'topic',
       ...exchangeConfig,
     },
   };
 
   const allQueues: Record<keyof OmittedRascalQueue, QueueConfig> = {
-    [RascalQueues.DiscordListener]: {
+    [RascalQueues.DiscordBotPolicy]: {
       ...queueConfig,
       options: {
         arguments: queueOptions,
@@ -147,11 +143,19 @@ export function getAllRascalConfigs(
       destinationType: 'queue',
       bindingKey: RascalRoutingKeys.ChainEvent,
     },
-    [RascalBindings.DiscordListener]: {
-      source: RascalExchanges.Discobot,
-      destination: RascalQueues.DiscordListener,
+    [RascalBindings.DiscordBotPolicy]: {
+      source: RascalExchanges.MessageRelayer,
+      destination: RascalQueues.DiscordBotPolicy,
       destinationType: 'queue',
-      bindingKey: RascalRoutingKeys.DiscordListener,
+      bindingKeys: [
+        RascalRoutingKeys.DiscordThreadCreated,
+        RascalRoutingKeys.DiscordThreadTitleUpdated,
+        RascalRoutingKeys.DiscordThreadBodyUpdated,
+        RascalRoutingKeys.DiscordThreadDeleted,
+        RascalRoutingKeys.DiscordThreadCommentCreated,
+        RascalRoutingKeys.DiscordThreadCommentUpdated,
+        RascalRoutingKeys.DiscordThreadCommentDeleted,
+      ],
     },
     [RascalBindings.NotificationsProvider]: {
       source: RascalExchanges.MessageRelayer,
@@ -196,22 +200,13 @@ export function getAllRascalConfigs(
   };
 
   const allPublications: Record<RascalPublications, PublicationConfig> = {
-    [RascalPublications.DiscordListener]: {
-      exchange: RascalExchanges.Discobot,
-      routingKey: RascalRoutingKeys.DiscordListener,
-      ...publicationConfig,
-    },
     [RascalPublications.MessageRelayer]: {
       exchange: RascalExchanges.MessageRelayer,
       ...publicationConfig,
     },
   };
 
   const allSubscriptions: Record<RascalSubscriptions, SubscriptionConfig> = {
-    [RascalSubscriptions.DiscordListener]: {
-      queue: RascalQueues.DiscordListener,
-      ...subscriptionConfig,
-    },
     [RascalSubscriptions.ChainEvent]: {
       queue: RascalQueues.ChainEvent,
       ...subscriptionConfig,
@@ -228,6 +223,10 @@ export function getAllRascalConfigs(
       queue: RascalQueues.ContestProjection,
       ...subscriptionConfig,
     },
+    [RascalSubscriptions.DiscordBotPolicy]: {
+      queue: RascalQueues.DiscordBotPolicy,
+      ...subscriptionConfig,
+    },
     [RascalSubscriptions.NotificationsSettings]: {
       queue: RascalQueues.NotificationsSettings,
       ...subscriptionConfig,

libs/adapters/src/rabbitmq/rabbitMQConfig.ts

@@ -9,11 +9,8 @@ import {
   RascalSubscriptions,
 } from './types';
 
-// TODO: Move configs to specific services
-
 export enum RascalConfigServices {
   CommonwealthService = 'commonwealth',
-  DiscobotService = 'discobot',
 }
 
 /**
@@ -77,13 +74,15 @@ export function getRabbitMQConfig(
       RascalQueues.NotificationsSettings,
       RascalQueues.ContestWorkerPolicy,
       RascalQueues.ContestProjection,
+      RascalQueues.DiscordBotPolicy,
     ]);
     copyConfigs(allBindings, vhostConfig.bindings, [
       RascalBindings.ChainEvent,
       RascalBindings.NotificationsProvider,
       RascalBindings.NotificationsSettings,
       RascalBindings.ContestWorkerPolicy,
       RascalBindings.ContestProjection,
+      RascalBindings.DiscordBotPolicy,
     ]);
     copyConfigs(allPublications, vhostConfig.publications, [
       RascalPublications.MessageRelayer,
@@ -94,20 +93,7 @@ export function getRabbitMQConfig(
       RascalSubscriptions.NotificationsSettings,
       RascalSubscriptions.ContestWorkerPolicy,
       RascalSubscriptions.ContestProjection,
-    ]);
-  } else if (service === RascalConfigServices.DiscobotService) {
-    copyConfigs(allExchanges, vhostConfig.exchanges, [
-      RascalExchanges.Discobot,
-    ]);
-    copyConfigs(allQueues, vhostConfig.queues, [RascalQueues.DiscordListener]);
-    copyConfigs(allBindings, vhostConfig.bindings, [
-      RascalBindings.DiscordListener,
-    ]);
-    copyConfigs(allPublications, vhostConfig.publications, [
-      RascalPublications.DiscordListener,
-    ]);
-    copyConfigs(allSubscriptions, vhostConfig.subscriptions, [
-      RascalSubscriptions.DiscordListener,
+      RascalSubscriptions.DiscordBotPolicy,
     ]);
   }
 

libs/adapters/src/rabbitmq/types.ts

@@ -6,13 +6,12 @@ import {
 } from '@hicommonwealth/core';
 
 export enum RascalPublications {
-  DiscordListener = BrokerPublications.DiscordListener,
   MessageRelayer = BrokerPublications.MessageRelayer,
 }
 
 // SnapshotListener and ChainEvent subscriptions will eventually be replaced by NotificationsProvider
 export enum RascalSubscriptions {
-  DiscordListener = BrokerSubscriptions.DiscordListener,
+  DiscordBotPolicy = BrokerSubscriptions.DiscordBotPolicy,
   ChainEvent = BrokerSubscriptions.ChainEvent,
   NotificationsProvider = BrokerSubscriptions.NotificationsProvider,
   NotificationsSettings = BrokerSubscriptions.NotificationsSettings,
@@ -22,13 +21,12 @@ export enum RascalSubscriptions {
 
 export enum RascalExchanges {
   DeadLetter = 'DeadLetterExchange',
-  Discobot = 'DiscobotExchange',
   MessageRelayer = 'MessageRelayerExchange',
 }
 
 export enum RascalQueues {
   DeadLetter = 'DeadLetterQueue',
-  DiscordListener = 'DiscordMessageQueueV2',
+  DiscordBotPolicy = 'DiscordBotPolicy',
   ChainEvent = 'ChainEventQueue',
   NotificationsProvider = 'NotificationsProviderQueue',
   NotificationsSettings = 'NotificationsSettingsQueue',
@@ -40,7 +38,7 @@ export enum RascalBindings {
   NotificationsProvider = 'NotificationsProvider',
   NotificationsSettings = 'NotificationsSettings',
   DeadLetter = 'DeadLetterBinding',
-  DiscordListener = 'DiscordMessageBinding',
+  DiscordBotPolicy = 'DiscordBotPolicy',
   ChainEvent = 'ChainEventBinding',
   ContestWorkerPolicy = 'ContestWorkerPolicy',
   ContestProjection = 'ContestProjection',
@@ -56,8 +54,15 @@ export enum RascalRoutingKeys {
 
   NotificationsSettingsPreferencesUpdated = EventNames.SubscriptionPreferencesUpdated,
 
+  DiscordThreadCreated = EventNames.DiscordThreadCreated,
+  DiscordThreadBodyUpdated = EventNames.DiscordThreadBodyUpdated,
+  DiscordThreadTitleUpdated = EventNames.DiscordThreadTitleUpdated,
+  DiscordThreadDeleted = EventNames.DiscordThreadDeleted,
+  DiscordThreadCommentCreated = EventNames.DiscordThreadCommentCreated,
+  DiscordThreadCommentUpdated = EventNames.DiscordThreadCommentUpdated,
+  DiscordThreadCommentDeleted = EventNames.DiscordThreadCommentDeleted,
+
   DeadLetter = 'DeadLetter',
-  DiscordListener = EventNames.DiscordMessageCreated,
   ChainEvent = EventNames.ChainEventCreated,
 
   ContestWorkerPolicyThreadCreated = `${EventNames.ThreadCreated}.${RoutingKeyTags.Contest}.#`,

libs/adapters/src/utils/startHealthCheckLoop.ts

@@ -7,7 +7,6 @@ export enum ServiceKey {
   CommonwealthConsumer = 'commonwealth-consumer',
   MessageRelayer = 'message-relayer',
   DiscordBotListener = 'discord-bot-listener',
-  DiscordBotConsumer = 'discord-bot-consumer',
   ChainEventsApp = 'chain-events-app',
   ChainEventsConsumer = 'chain-events-consumer',
   ChainEventsSubscriber = 'chain-events-subscriber',

libs/adapters/test/rabbitmq/rabbitMQAdapter.spec.ts

@@ -137,9 +137,10 @@ describe('RabbitMQ', () => {
       expect(res).to.be.false;
     });
 
-    test('should return false if the topic is not included in the current instance', async () => {
+    // we only have 1 app with RabbitMQ so this test no longer applies
+    test.skip('should return false if the topic is not included in the current instance', async () => {
       const res = await rmqAdapter.subscribe(
-        BrokerSubscriptions.DiscordListener,
+        BrokerSubscriptions.DiscordBotPolicy,
         Snapshot(),
       );
       expect(res).to.be.false;