增加celery架构和common部分

Author: git

common/__init__.py

@@ -0,0 +1,38 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+########################################
+#
+#    公共模块
+#
+########################################
+import os
+import yaml
+import redis
+
+
+# 项目目录
+PROJECT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+
+# 获取配置文件信息
+with open(os.path.join(PROJECT_DIR, 'config', 'config.yaml')) as fd:
+    config = yaml.load(fd)
+
+redis_client = redis.Redis(host=config['redis']['host'],
+                           port=config['redis']['port'],
+                           db=config['redis']['db'],
+                           password=config['redis']['password']
+                           )
+
+pool = None
+def get_redis_db():
+    global pool
+    if not pool:
+        pool = redis.ConnectionPool(host=config['redis']['host'],
+                                                    port=config['redis']['port'],
+                                                    db=config['redis']['db'],
+                                                    password=config['redis']['password'])
+    return redis.Redis(connection_pool=pool)
+redisdb = get_redis_db()
+
+

common/base.py

@@ -0,0 +1,37 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+############################################
+#
+#  日志模块
+#
+############################################
+import os
+import sys
+import logging
+
+DEFAULT_LOG_NAME = 'access.log'
+
+def logger_init(name=None, level=logging.DEBUG):
+    if not name:
+        name = DEFAULT_LOG_NAME
+    filepath = os.path.join(os.path.dirname(__file__), os.pardir, 'log', name)
+    logger = logging.getLogger()
+    logger.setLevel(level)
+    ch = logging.FileHandler(filepath)
+    formatter = logging.Formatter('[%(asctime)s] [%(filename)s:%(lineno)d] [%(levelname)s] %(message)s',
+                                  datefmt='%Y-%m-%d %H:%M:%S')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+    return logger
+
+def simple_log_init(level=logging.DEBUG):
+    log_format = '[%(asctime)s] [%(filename)s:%(lineno)d] [%(levelname)s] %(message)s'
+    logging.basicConfig(stream=sys.stdout, level=level, format=log_format)
+    return logging
+
+logger = logger_init()
+
+if __name__ == '__main__':
+    logger = logger_init('hello.log')
+    logger.debug('hello world')

common/log.py

@@ -0,0 +1,15 @@
+mysql:
+  host: '127.0.0.1'
+  user: root
+  port: 3306
+  password: '123456'
+  db: db_test
+  charset: utf8
+
+redis:
+  host: '127.0.0.1'
+  port: 6379
+  db: 1
+  password: ''
+  prefix: 'bd_test_'
+

config/config.yaml

@@ -0,0 +1,3 @@
+#!/bin/sh
+apt install -y redis-server
+pip install -r requirements.txt

install.sh

@@ -0,0 +1,6 @@
+[program:celery]
+command=/root/venv3/bin/celery -A mcelery worker -B -l info
+directory=/root/learn-python
+stdout_logfile=/root/learn-python/log/celery.log
+autorestart=true
+redirect_stderr=true

mcelery/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+# -*- coding:utf-8 -*-
+
+######################################
+#   worker
+#
+######################################
+
+from celery import Celery
+app = Celery('mcelery', include=['mcelery.tasks'])
+app.config_from_object('mcelery.config')
+
+if __name__ == '__main__':
+    app.start()

mcelery/celery.conf

@@ -0,0 +1,28 @@
+#!/usr/bin/python
+# -*- coding:utf-8 -*-
+
+#####################################
+# 配置celery的broker backend
+#
+######################################
+from datetime import timedelta
+from celery.schedules import crontab
+
+CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/5'
+BROKER_URL = 'redis://127.0.0.1:6379/6'
+
+# 定时
+CELERY_TIMEZONE = 'Asia/Shanghai'
+CELERYBEAT_SCHEDULE = {
+    'add-every-30-seconds': {
+         'task': 'mcelery.tasks.add',
+         'schedule': timedelta(seconds=5),
+         'args': (16, 16)
+    },
+    # Executes every Monday morning at 7:30 A.M
+    'add-every-monday-morning': {
+        'task': 'mcelery.tasks.add',
+        'schedule': crontab(hour=7, minute=30, day_of_week=1),
+        'args': (5, 5),
+    },
+}

mcelery/celery.py

@@ -0,0 +1,25 @@
+# 安装
+pip2 install supervisor
+
+# 配置文件
+echo_supervisord_conf > /etc/supervisord.conf
+mkdir /etc/supervisor.d
+在 supervisord.conf最后添加：
+[include]
+files = supervisor.d/*.conf
+
+# 拷贝config文件
+cp mcelery/celery.conf /etc/supervisor.d/
+
+# 启动supervisor服务（如果已启动勿理会）
+supervisord
+
+# 变更配置文件之后,要加载最新的配置文件
+supervisorctl update
+
+# 查看celery状态，是否正常启动
+supervisorctl status
+
+# 如果有必要，重启celery
+supervisorctl restart celery
+

mcelery/config.py

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+# -*- coding:utf-8 -*-
+
+###################################
+#   任务
+#
+####################################
+from mcelery.celery import app
+
+@app.task
+def add(x, y):
+    return x + y

mcelery/supervisor.md

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+# -*- coding:utf-8 -*-
+
+# 触发任务
+# 在mcelery同级目录执行 python -m mcelery.trigger
+
+import time
+from mcelery.tasks import add
+
+
+result = add.delay(4, 4)
+while not result.ready():
+    time.sleep(1)
+print ('task done: {0}'.format(result.get()))

mcelery/tasks.py

@@ -0,0 +1,6 @@
+celery==4.2.1
+numpy==1.16.0
+openpyxl==2.5.14
+pandas==0.24.0
+PyYAML==3.13
+redis==3.1.0

mcelery/trigger.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+