Sirbot

Author: ovv

Diff for: .env.sample

@@ -11,6 +11,8 @@ RUN apk add --no-cache tzdata gcc g++ make postgresql-dev build-base && \
     echo "UTC" >> /etc/timezone && \
     apk del tzdata
 
+RUN apk add --no-cache libffi-dev git
+
 COPY requirements requirements
 RUN pip install -r requirements/development.txt
 

Diff for: Dockerfile

@@ -12,6 +12,7 @@ services:
       DATABASE_URL: "postgresql://${USER}:@postgresql:5432/pyslackers_dev"
       SLACK_INVITE_TOKEN: "${SLACK_INVITE_TOKEN}"
       SLACK_TOKEN: "${SLACK_TOKEN}"
+      SLACK_SIGNING_SECRET: "${SLACK_SIGNING_SECRET}"
     ports:
       - "8000:8000"
     volumes:

Diff for: docker-compose.yml

@@ -0,0 +1,92 @@
+"""empty message
+
+Revision ID: 5b03cd946e60
+Revises: 3df042324a1f
+Create Date: 2020-03-01 20:33:03.352360+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "5b03cd946e60"
+down_revision = "3df042324a1f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "slack_channels",
+        sa.Column("id", sa.Text(), nullable=False),
+        sa.Column("name", sa.Text(), nullable=True),
+        sa.Column("created", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("archived", sa.Boolean(), nullable=True),
+        sa.Column("members", sa.Integer(), nullable=True),
+        sa.Column("topic", sa.Text(), nullable=True),
+        sa.Column("purpose", sa.Text(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+    )
+    op.create_index("ix_slack_channels_id", "slack_channels", ["id"], unique=False)
+    op.create_index("ix_slack_channels_name", "slack_channels", ["name"], unique=False)
+
+    op.create_table(
+        "slack_messages",
+        sa.Column("id", sa.Text(), nullable=False),
+        sa.Column("send_at", sa.DateTime(), nullable=True),
+        sa.Column("user", sa.Text(), nullable=True),
+        sa.Column("channel", sa.Text(), nullable=True),
+        sa.Column("message", sa.Text(), nullable=True),
+        sa.Column("raw", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_slack_messages_channel", "slack_messages", ["channel", "send_at"], unique=False
+    )
+    op.create_index("ix_slack_messages_user", "slack_messages", ["user", "send_at"], unique=False)
+    op.create_index(
+        "ix_slack_messages_user_channel",
+        "slack_messages",
+        ["user", "channel", "send_at"],
+        unique=False,
+    )
+
+    op.create_table(
+        "slack_users",
+        sa.Column("id", sa.Text(), nullable=False),
+        sa.Column("name", sa.Text(), nullable=True),
+        sa.Column("deleted", sa.Boolean(), nullable=True),
+        sa.Column("admin", sa.Boolean(), nullable=True),
+        sa.Column("bot", sa.Boolean(), nullable=True),
+        sa.Column("timezone", sa.Text(), nullable=True),
+        sa.Column("first_seen", sa.DateTime(timezone=True), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+    )
+    op.create_index("ix_slack_users_admin", "slack_users", ["id", "admin"], unique=False)
+    op.create_index("ix_slack_users_id", "slack_users", ["id"], unique=False)
+    op.create_index("ix_slack_users_name", "slack_users", ["id", "name"], unique=False)
+    op.create_index("ix_slack_users_timezone", "slack_users", ["id", "timezone"], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("ix_slack_users_timezone", table_name="slack_users")
+    op.drop_index("ix_slack_users_name", table_name="slack_users")
+    op.drop_index("ix_slack_users_id", table_name="slack_users")
+    op.drop_index("ix_slack_users_admin", table_name="slack_users")
+    op.drop_table("slack_users")
+
+    op.drop_index("ix_slack_messages_user_channel", table_name="slack_messages")
+    op.drop_index("ix_slack_messages_user", table_name="slack_messages")
+    op.drop_index("ix_slack_messages_channel", table_name="slack_messages")
+    op.drop_table("slack_messages")
+
+    op.drop_index("ix_slack_channels_name", table_name="slack_channels")
+    op.drop_index("ix_slack_channels_id", table_name="slack_channels")
+    op.drop_table("slack_channels")
+    # ### end Alembic commands ###

Diff for: migrations/versions/2020-03-01T20-33-03Z_5b03cd946e60_.py

@@ -9,7 +9,14 @@
 from sentry_sdk.integrations.logging import LoggingIntegration
 from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 
-from .contexts import apscheduler, client_session, redis_pool, postgresql_pool, slack_client
+from .contexts import (
+    apscheduler,
+    client_session,
+    redis_pool,
+    postgresql_pool,
+    slack_client,
+    background_jobs,
+)
 from .middleware import request_context_middleware
 from . import settings, website, sirbot
 
@@ -58,7 +65,9 @@ async def app_factory(*args) -> web.Application:  # pylint: disable=unused-argum
         SLACK_TOKEN=settings.SLACK_TOKEN,
     )
 
-    app.cleanup_ctx.extend([apscheduler, client_session, redis_pool, postgresql_pool, slack_client])
+    app.cleanup_ctx.extend(
+        [apscheduler, client_session, redis_pool, postgresql_pool, slack_client, background_jobs]
+    )
 
     app.router.add_get("/", index)
 

Diff for: pyslackersweb/__init__.py

@@ -1,14 +1,25 @@
-from typing import AsyncGenerator
-
 import json
+import datetime
+import logging
+
+from typing import AsyncGenerator
 
 import aioredis
 import asyncpgsa
+import asyncpg.pool
+import sqlalchemy as sa
 
 from asyncpgsa.connection import get_dialect
 from aiohttp import ClientSession, web
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from slack.io.aiohttp import SlackAPI
+from sqlalchemy import select
+
+from pyslackersweb.util.log import ContextAwareLoggerAdapter
+from . import tasks, models
+
+
+logger = ContextAwareLoggerAdapter(logging.getLogger(__name__))
 
 
 def _register_in_app(app: web.Application, name: str, item) -> None:
@@ -58,3 +69,41 @@ async def slack_client(app: web.Application) -> AsyncGenerator[None, None]:
     _register_in_app(app, "slack_client_legacy", slack_legacy)
 
     yield
+
+
+async def background_jobs(app: web.Application) -> AsyncGenerator[None, None]:
+    scheduler = app["scheduler"]
+    pg: asyncpg.pool.Pool = app["pg"]
+    slack_client_: SlackAPI = app["slack_client"]
+
+    next_run_time = None
+    if await _is_empty_table(pg, models.SlackUsers.c.id):
+        next_run_time = datetime.datetime.now() + datetime.timedelta(minutes=1)
+
+    scheduler.add_job(
+        tasks.sync_slack_users,
+        "cron",
+        minute=0,
+        args=(slack_client_, pg),
+        next_run_time=next_run_time,
+    )
+
+    next_run_time = None
+    if await _is_empty_table(pg, models.SlackChannels.c.id):
+        next_run_time = datetime.datetime.now() + datetime.timedelta(minutes=1)
+
+    scheduler.add_job(
+        tasks.sync_slack_channels,
+        "cron",
+        minute=15,
+        args=(slack_client_, pg),
+        next_run_time=next_run_time,
+    )
+
+    yield
+
+
+async def _is_empty_table(pg: asyncpg.pool.Pool, column: sa.Column) -> bool:
+    async with pg.acquire() as conn:
+        result = await conn.fetchval(select([column]).limit(1))
+        return result is None

Diff for: pyslackersweb/contexts.py

@@ -29,4 +29,35 @@ class Source(Enum):
         nullable=False,
     ),
     sa.Index("ix_domain_blocked", "domain", "blocked"),
+    sa.Index("ix_domains_domain", "domain"),
+)
+
+SlackChannels = sa.Table(
+    "slack_channels",
+    metadata,
+    sa.Column("id", sa.Text, primary_key=True),
+    sa.Column("name", sa.Text, unique=True),
+    sa.Column("created", sa.DateTime(timezone=True)),
+    sa.Column("archived", sa.Boolean),
+    sa.Column("members", sa.Integer),
+    sa.Column("topic", sa.Text),
+    sa.Column("purpose", sa.Text),
+    sa.Index("ix_slack_channels_id", "id"),
+    sa.Index("ix_slack_channels_name", "name"),
+)
+
+SlackUsers = sa.Table(
+    "slack_users",
+    metadata,
+    sa.Column("id", sa.Text, primary_key=True),
+    sa.Column("name", sa.Text, unique=True),
+    sa.Column("deleted", sa.Boolean),
+    sa.Column("admin", sa.Boolean),
+    sa.Column("bot", sa.Boolean),
+    sa.Column("timezone", sa.Text),
+    sa.Column("first_seen", sa.DateTime(timezone=True), default=datetime.now),
+    sa.Index("ix_slack_users_id", "id"),
+    sa.Index("ix_slack_users_name", "id", "name"),
+    sa.Index("ix_slack_users_admin", "id", "admin"),
+    sa.Index("ix_slack_users_timezone", "id", "timezone"),
 )

Diff for: pyslackersweb/models.py

@@ -1,6 +1,10 @@
+import logging
+
 from aiohttp import web
 
-from .views import routes
+from pyslackersweb.sirbot.views import readthedocs, slack
+
+logger = logging.getLogger(__name__)
 
 
 async def app_factory() -> web.Application:
@@ -11,5 +15,6 @@ async def app_factory() -> web.Application:
         scheduler=None,  # populated via parent app signal
     )
 
-    sirbot.add_routes(routes)
+    readthedocs.add_routes(sirbot.router)
+    slack.add_routes(sirbot.router)
     return sirbot

Diff for: pyslackersweb/sirbot/__init__.py

@@ -0,0 +1,14 @@
+import logging
+
+import asyncpg
+import sqlalchemy as sa
+
+from pyslackersweb import models
+from pyslackersweb.util.log import ContextAwareLoggerAdapter
+
+
+logger = ContextAwareLoggerAdapter(logging.getLogger(__name__))
+
+
+async def is_admin(conn: asyncpg.connection.Connection, user: str) -> bool:
+    return await conn.fetchval(sa.select([models.SlackUsers.c.admin]).where(id=user))

Diff for: pyslackersweb/sirbot/database.py

@@ -0,0 +1,44 @@
+import datetime
+import dataclasses
+
+from typing import Optional
+from decimal import Decimal
+
+import sqlalchemy as sa
+
+from sqlalchemy.dialects.postgresql import JSONB
+
+from pyslackersweb.models import metadata
+
+
+@dataclasses.dataclass(frozen=True)
+class StockQuote:
+    # pylint: disable=too-many-instance-attributes
+
+    symbol: str
+    company: str
+    price: Decimal
+    change: Decimal
+    change_percent: Decimal
+    market_open: Decimal
+    market_close: Decimal
+    high: Decimal
+    low: Decimal
+    volume: Decimal
+    time: datetime.datetime
+    logo: Optional[str] = None
+
+
+SlackMessage = sa.Table(
+    "slack_messages",
+    metadata,
+    sa.Column("id", sa.Text, primary_key=True),
+    sa.Column("send_at", sa.DateTime),
+    sa.Column("user", sa.Text),
+    sa.Column("channel", sa.Text),
+    sa.Column("message", sa.Text),
+    sa.Column("raw", JSONB),
+    sa.Index("ix_slack_messages_user", "user", "send_at"),
+    sa.Index("ix_slack_messages_channel", "channel", "send_at"),
+    sa.Index("ix_slack_messages_user_channel", "user", "channel", "send_at"),
+)

Diff for: pyslackersweb/sirbot/models.py

@@ -1,8 +1,16 @@
 import os
 
-# production
+IS_PRODUCTION = os.environ.get("PLATFORM_BRANCH") == "master"
+
+# production settings
 READTHEDOCS_NOTIFICATION_CHANNEL = "community_projects"
+SLACK_TEAM_ID = os.environ.get("SLACK_TEAM_ID")
+SLACK_SIGNING_SECRET = os.environ.get("SLACK_SIGNING_SECRET", "")
+SLACK_ADMIN_CHANNEL = os.environ.get("SLACK_ADMIN_CHANNEL", "")
+SLACK_INTRODUCTION_CHANNEL = "introductions"
 
-# Development
-if os.environ.get("PLATFORM_BRANCH") != "master":
+# Development settings
+if not IS_PRODUCTION:
     READTHEDOCS_NOTIFICATION_CHANNEL = "general"
+    SLACK_ADMIN_CHANNEL = "CJ1BWMBDX"  # general
+    SLACK_INTRODUCTION_CHANNEL = "general"