AccessLogAnalyzer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AccessLogAnalyzer.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "do3HqxsYCvth"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/KoukiNishihara/AccessLogAnalyzer/blob/master/AccessLogAnalyzer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ISY5bwOWCntD",
        "colab_type": "text"
      },
      "source": [
        "# git clone していない場合の実行\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "964fc1a4-f229-47b0-a00f-8cfb752365e2",
        "id": "h-0DalYeEszs",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!git clone https://github.com/KoukiNishihara/AccessLogAnalyzer.git"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'AccessLogAnalyzer' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pj0KzoRFboXC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd AccessLogAnalyzer/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wgMkTF_fAxxk",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "# git clone 済みの場合の実行\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RB5lsyDsG9ky",
        "colab_type": "code",
        "outputId": "db21a710-06f8-4e3c-83b2-01dea4129b1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "!pip3 install apache-log-parser"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: apache-log-parser in /usr/local/lib/python3.6/dist-packages (1.7.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from apache-log-parser) (1.12.0)\n",
            "Requirement already satisfied: user-agents in /usr/local/lib/python3.6/dist-packages (from apache-log-parser) (2.0)\n",
            "Requirement already satisfied: ua-parser>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from user-agents->apache-log-parser) (0.8.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R3iZNUHLaMlR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import datetime\n",
        "import apache_log_parser"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ff-BCA55FKy9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#これを実行するとプログラムが始まる\n",
        "def Main():\n",
        "    #諸々の初期化\n",
        "    filename=['access_log'] #ファイル名\n",
        "    host_dict={}        #ホスト名とアクセス数を格納するもの\n",
        "    time_list=[]        #単位時間あたりのアクセス数\n",
        "    line_parser = apache_log_parser.make_parser(\"%h %l %u %t \\\"%r\\\" %>s %b \\\"%{Referer}i\\\" \\\"%{User-Agent}i\\\"\")\n",
        "    file_num=0          #解析したファイル数\n",
        "    time_range=[]       #解析期間\n",
        "\n",
        "    host_dict,time_list,time_range,file_num=Analyze(filename,host_dict,time_list,time_range,file_num)   #ログの解析\n",
        "    Output(host_dict,time_list,time_range,file_num)     #出力"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tbxOWfiJLxYC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#入力を促し，ログの解析をする\n",
        "def Analyze(filename,host_dict,time_list,time_range,file_num):\n",
        "    time_range=input('ログを収集する期間を指定してください.指定がない場合はそのままEnterをおしてください\\n例）2000/11/06 2017/11/07\\n').split()\n",
        "\n",
        "    # 0時から23時までを0で初期化\n",
        "    for i in range(0,24):\n",
        "        time_list.append(0)\n",
        "\n",
        "    file_num=FileRead(filename,host_dict,time_list,time_range)\n",
        "    print()\n",
        "\n",
        "    filename=input('\"access_log\"以外に解析したいファイルがあればファイル名を入力してください．指定がない場合はそのままEnterをおしてください.\\n例）ファイル名\\n例）ファイル名1　ファイル名2　・・・　ファイル名n\\n').split()\n",
        "\n",
        "    file_num+=FileRead(filename,host_dict,time_list,time_range)\n",
        "    return  host_dict,time_list,time_range,file_num"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "drinOVTuKNGQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#ファイル読み込み\n",
        "def FileRead(filenames=['access_log'],host_dict={},time_list=[],time_range=[]):\n",
        "    file_num=0\n",
        "    if(len(time_range)==0 or len(time_range)==2):   #解析期間の指定がない場合，もしくは初めの期間と終わりの期間が指定された場合\n",
        "\n",
        "        if(len(time_range)==2):         #初めの期間と終わりの期間が指定された場合\n",
        "            start=time_range[0]\n",
        "            end=time_range[1]\n",
        "\n",
        "        for name in filenames:  \n",
        "            file_num+=1             #解析するファイル数のカウント\n",
        "\n",
        "            with open(name,'r') as fh:      #ファイル読み込み\n",
        "                line =fh.readline()         #一行づつ読み込む\n",
        "\n",
        "                while line:\n",
        "                    log_line_data = line_parser(line)\n",
        "                    date=log_line_data[\"time_received_datetimeobj\"].strftime(\"%Y/%m/%d\")\n",
        "\n",
        "                    if((len(time_range)==0) or (start<=date and end>=date)):    #解析期間内である場合\n",
        "                        CountHost(log_line_data,host_dict)      #host名とアクセス数をカウントする\n",
        "                        CountPerHour(log_line_data,time_list)   #単位時間あたりのアクセス数をカウント\n",
        "                    line =fh.readline()\n",
        "    return file_num"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q0nJXpORgEsk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#　hostのアクセス数をカウント\n",
        "def CountHost(log_line_data,host_dict):\n",
        "\n",
        "    host_name=log_line_data[\"remote_host\"]\n",
        "\n",
        "    if host_name not in host_dict.keys():\n",
        "        i=1\n",
        "        update_data = {host_name:i}\n",
        "        host_dict.update(update_data)\n",
        "\n",
        "    else:\n",
        "        host_dict[host_name]+=1\n",
        "\n",
        "    return host_dict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JLDTd0OOhZKm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 単位時間あたりのカウント\n",
        "def CountPerHour(log_line_data,time_list):\n",
        "\n",
        "    time_log=log_line_data['time_received_datetimeobj'].hour\n",
        "    time_list[time_log]+=1\n",
        "\n",
        "    return host_dict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Gl6P7EJyTSR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#解析したものを出力\n",
        "def Output(host_dict,time_list,time_range,file_num):\n",
        "    print(\"解析したファイルは\",file_num,\"個です\")\n",
        "\n",
        "    if(len(host_dict)==0):\n",
        "        print(\"アクセスはありませんでした\")\n",
        "    else :\n",
        "        if(len(time_range)==2):\n",
        "            print(\"解析期間は\",time_range[0],\"から\",time_range[1],\"です.\")\n",
        "        elif(len(time_range)==0):\n",
        "                print(\"解析期間は全期間です.\")\n",
        "\n",
        "    print()\n",
        "    for k,v in sorted(host_dict.items(),key=lambda x:x[1],reverse=True):\n",
        "        print(\"host:\",k,\"count:\",v)\n",
        "    print(\"------------------------------\")\n",
        "\n",
        "    i=0\n",
        "    for n in time_list:\n",
        "        print(\"hour:\",i,\"cout:\",n)\n",
        "        i+=1    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D5ixA7xJQyv9",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "*   解析期間を指定せず他のファイルを指定してしなかった場合\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "21ikEkMbIa7t",
        "colab_type": "code",
        "outputId": "d911e341-3f93-4571-d614-f3fef9d2822f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 680
        }
      },
      "source": [
        "Main()"
      ],
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ログを収集する期間を指定してください.指定がない場合はそのままEnterをおしてください\n",
            "例）2000/11/06 2017/11/07\n",
            "\n",
            "\n",
            "\"access_log\"以外に解析したいファイルがあればファイル名を入力してください．指定がない場合はそのままEnterをおしてください.\n",
            "例）ファイル名\n",
            "例）ファイル名1　ファイル名2　・・・　ファイル名n\n",
            "\n",
            "解析したファイルは 1 個です\n",
            "解析期間は全期間です.\n",
            "\n",
            "host: 10.2.3.4 count: 3\n",
            "host: 10.2.3.5 count: 1\n",
            "host: 10.2.3.6 count: 1\n",
            "------------------------------\n",
            "hour: 0 cout: 2\n",
            "hour: 1 cout: 0\n",
            "hour: 2 cout: 0\n",
            "hour: 3 cout: 1\n",
            "hour: 4 cout: 0\n",
            "hour: 5 cout: 0\n",
            "hour: 6 cout: 0\n",
            "hour: 7 cout: 0\n",
            "hour: 8 cout: 0\n",
            "hour: 9 cout: 0\n",
            "hour: 10 cout: 1\n",
            "hour: 11 cout: 0\n",
            "hour: 12 cout: 0\n",
            "hour: 13 cout: 0\n",
            "hour: 14 cout: 0\n",
            "hour: 15 cout: 0\n",
            "hour: 16 cout: 0\n",
            "hour: 17 cout: 0\n",
            "hour: 18 cout: 0\n",
            "hour: 19 cout: 0\n",
            "hour: 20 cout: 1\n",
            "hour: 21 cout: 0\n",
            "hour: 22 cout: 0\n",
            "hour: 23 cout: 0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fiPSuo3dRIzS",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "*  解析期間を指定せず，他のファイル指定した場合\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "0638d68e-58c8-48cf-e0cc-68752e238879",
        "id": "4B1FW6YRQfud",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        }
      },
      "source": [
        "Main()"
      ],
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ログを収集する期間を指定してください.指定がない場合はそのままEnterをおしてください\n",
            "例）2000/11/06 2017/11/07\n",
            "\n",
            "\n",
            "\"access_log\"以外に解析したいファイルがあればファイル名を入力してください．指定がない場合はそのままEnterをおしてください.\n",
            "例）ファイル名\n",
            "例）ファイル名1　ファイル名2　・・・　ファイル名n\n",
            "other_log\n",
            "解析したファイルは 2 個です\n",
            "解析期間は全期間です.\n",
            "\n",
            "host: 10.2.3.4 count: 3\n",
            "host: xxx.xxx.xxx.xxx count: 2\n",
            "host: 10.2.3.5 count: 1\n",
            "host: 10.2.3.6 count: 1\n",
            "------------------------------\n",
            "hour: 0 cout: 2\n",
            "hour: 1 cout: 0\n",
            "hour: 2 cout: 0\n",
            "hour: 3 cout: 1\n",
            "hour: 4 cout: 0\n",
            "hour: 5 cout: 0\n",
            "hour: 6 cout: 0\n",
            "hour: 7 cout: 0\n",
            "hour: 8 cout: 0\n",
            "hour: 9 cout: 0\n",
            "hour: 10 cout: 1\n",
            "hour: 11 cout: 0\n",
            "hour: 12 cout: 0\n",
            "hour: 13 cout: 0\n",
            "hour: 14 cout: 0\n",
            "hour: 15 cout: 0\n",
            "hour: 16 cout: 0\n",
            "hour: 17 cout: 0\n",
            "hour: 18 cout: 0\n",
            "hour: 19 cout: 0\n",
            "hour: 20 cout: 1\n",
            "hour: 21 cout: 0\n",
            "hour: 22 cout: 0\n",
            "hour: 23 cout: 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hWrUUgjGRf4V",
        "colab_type": "text"
      },
      "source": [
        "* 期間指定を指定して，他のファイルを指定した場合\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "c42a023f-a3f7-4e52-fe87-b954b8fe8fbe",
        "id": "p6MCpAUhQpaz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 680
        }
      },
      "source": [
        "Main()"
      ],
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ログを収集する期間を指定してください.指定がない場合はそのままEnterをおしてください\n",
            "例）2000/11/06 2017/11/07\n",
            "2000/11/06 2017/11/07\n",
            "\n",
            "\"access_log\"以外に解析したいファイルがあればファイル名を入力してください．指定がない場合はそのままEnterをおしてください.\n",
            "例）ファイル名\n",
            "例）ファイル名1　ファイル名2　・・・　ファイル名n\n",
            "other_log\n",
            "解析したファイルは 2 個です\n",
            "解析期間は 2000/11/06 から 2017/11/07 です.\n",
            "\n",
            "host: 10.2.3.4 count: 3\n",
            "host: 10.2.3.5 count: 1\n",
            "host: 10.2.3.6 count: 1\n",
            "------------------------------\n",
            "hour: 0 cout: 2\n",
            "hour: 1 cout: 0\n",
            "hour: 2 cout: 0\n",
            "hour: 3 cout: 1\n",
            "hour: 4 cout: 0\n",
            "hour: 5 cout: 0\n",
            "hour: 6 cout: 0\n",
            "hour: 7 cout: 0\n",
            "hour: 8 cout: 0\n",
            "hour: 9 cout: 0\n",
            "hour: 10 cout: 1\n",
            "hour: 11 cout: 0\n",
            "hour: 12 cout: 0\n",
            "hour: 13 cout: 0\n",
            "hour: 14 cout: 0\n",
            "hour: 15 cout: 0\n",
            "hour: 16 cout: 0\n",
            "hour: 17 cout: 0\n",
            "hour: 18 cout: 0\n",
            "hour: 19 cout: 0\n",
            "hour: 20 cout: 1\n",
            "hour: 21 cout: 0\n",
            "hour: 22 cout: 0\n",
            "hour: 23 cout: 0\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}