首页 > 代码库 > python网站访问统计

python网站访问统计

#!/usr/local/bin/python3
# coding:utf-8

# ====================================================
# Author: chang - EMail:changbo@hmg100.com
# Last modified: 2017-5-13
# Filename: accesstimes.py
# Description: real time analysis nginx log,base time, os, re, pymysql, Thread 
# blog:http://www.cnblogs.com/changbo
# ====================================================

"""
需求:每隔1分钟读取nginx日志文件
notice:
模拟日志切割过程中初始化脚本参数
cp access.log access2017xxxx.log  && echo > access.log && echo ‘0‘> offset1.txt
"""


import time
import os
import re
import pymysql
from threading import Thread
import logging
# from django.db import connection

logging.basicConfig(level=logging.DEBUG,
                    format=%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s,
                    datefmt=%a, %d %b %Y %H:%M:%S,
                    filename=accesstimes.log,
                    filemode=w)

keypage = [/sys/get_user.do]
keyIP = [127.0.0.1, 119.28.61.122]
engdate = {Jan: 1, Feb: 2, Mar: 3, Apr: 4, May: 5, Jun: 6, Jul: 7, Aug: 8, Sept: 9,
           Oct: 10, Nov: 11, Dec: 12}


def dateformat(nginxdate):
    day = (nginxdate.split([)[1]).split(/)[0]
    month = engdate[(nginxdate.split([)[1]).split(/)[1]]
    year = (nginxdate.split([)[1]).split(/)[2]

    return year + - + month + - + day


# write log offset
def writeoffset(number):
    with open(offset1.txt, w+) as f3:
        f3.write(number)
        f3.flush()


# get log offset
def getoffset():
    with open(offset1.txt) as f2:
        offset = f2.readline()
        return offset

db = pymysql.connect("xxx.xxx.xxx.xxx", "xxx", "xxxxxx", "yunwei")
cursor = db.cursor()
# cleantable = ‘TRUNCATE abnormal‘


def handleline(logline):
    susptmp = logline.split(" ")
    if len(susptmp) > 2:
        if susptmp[0] not in keyIP:
            del susptmp[1:3]
            if len(susptmp) > 2:
                ip = susptmp[0]
                time1 = (susptmp[1].split(:, 1))[1]
                dated = dateformat((susptmp[1].split(:, 1))[0])

                # print(ip + ‘---‘, time1 + ‘---‘, date + ‘---‘, user + ‘---‘, passd + ‘---‘)
                sql = "INSERT INTO reqinfo(ip, timedd, datedd) VALUES(‘%s‘, ‘%s‘, ‘%s‘)" % (ip, time1, dated)
                try:
                    cursor.execute(sql)
                    db.commit()
                    logging.debug(Insert success!)
                except Exception as e:
                    logging.debug(e)


# online analysis log
def analysislog():
    with open(access.log) as f1:
        while True:
            # get offset
            lastoffset = getoffset()
            # jump the Specify log line
            f1.seek(int(lastoffset))
            # 获取该行偏移量
            where = f1.tell()
            line = f1.readline()
            writeoffset(str(where))
            if not line:
                time.sleep(10)
                f1.seek(where)
            else:
                # 处理该行,并获取改行的偏移量且写入文件
                handleline(line)
                nowoffset = f1.tell()
                writeoffset(str(nowoffset))

if __name__ == __main__:
    if not os.path.exists(offset1.txt):
        with open("offset1.txt", w) as f:
            f.write(0)

    t1 = Thread(target=analysislog)
    t1.start()

 

#!/usr/local/bin/python3
# coding:utf-8

# ====================================================
# Author: chang - EMail:changbo@hmg100.com
# Last modified: 2017-5-13
# Filename: reqcounts.py
# Description: real time analysis nginx log,pymysql, Thread, logging
# blog:http://www.cnblogs.com/changbo
# ====================================================


import pymysql
import logging
from threading import Thread

logging.basicConfig(level=logging.DEBUG,
                    format=%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s,
                    datefmt=%a, %d %b %Y %H:%M:%S,
                    filename=reqcounts.log,
                    filemode=w)


db = pymysql.connect("xxx.xxxx.xxx.xxx", "xxxx", "xxxxx", "xxxxx")
cursor = db.cursor()
# cleantable = ‘TRUNCATE tablename‘


def analysisdb():
    listtime = []
    listip = []
    listdated = []

    sql3 = SELECT ip, timedd, datedd FROM reqinfo
    cursor.execute(sql3)
    results = cursor.fetchall()
    for row in results:
        listtime.append(row[1])
        listip.append(row[0])
        listdated.append(row[2])

        try:
            # 统计1分钟内页面访问次数
            sql1 = "SELECT count(*) from reqinfo where timedd=‘%s‘ and ip=‘%s‘ and datedd=‘%s‘" % (
                listtime[0], listip[0], listdated[0])
            sql4 = "DELETE from reqinfo where timedd=‘%s‘ and ip=‘%s‘ and datedd=‘%s‘" % (
                listtime[0], listip[0], listdated[0])
            sql5 = "DELETE FROM reqcounts WHERE timesddd=0"
            cursor.execute(sql1)
            datad = cursor.fetchone()
            sql2 = "INSERT INTO reqcounts(ip, timesddd, timeddd, dateddd) VALUES(‘%s‘ , ‘%s‘ , ‘%s‘, ‘%s‘)" % (
            listip[0], datad[0], listtime[0], listdated[0])
            cursor.execute(sql2)
            db.commit()
            logging.debug(-----Insert success -------)
            # delete already insert data of requinfo
            cursor.execute(sql4)
            db.commit()
            del listtime[0]
            del listip[0]
            del listdated[0]
            cursor.execute(sql5)
            db.commit()
        except Exception as e:
            logging.debug(e)

if __name__ == __main__:
        t2 = Thread(target=analysisdb)
        t2.start()

 

html 页面

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    {% load static %}
    <link rel="stylesheet"  type="text/css" href="{% static "css/style.css" %}" />
    <title>访问统计</title>
<script language="javascript" type="text/javascript">
window.onload = function(){
    var oTable = document.getElementById("bbsTab");
    for(var i=0;i<oTable.rows.length;i++){
        oTable.rows[i].cells[0].innerHTML = (i+1);
        if(i%2==0)    //偶数行
            oTable.rows[i].className = "ys01";
    }
}
</script>
</head>
<body>

<div id="container">
<table class="zebra">
<caption>IP/s 并发量统计</caption>
<thead>
<tr>
    <th>序号</th>
    <th>ip</th>
    <th>登录次数</th>
    <th>时间</th>
    <th>日期</th>
</tr>
</thead>
<tbody id="bbsTab">
{% for reqinfocount in reqinfocounts %}
    <tr>
    <td></td>
    <td>{{ reqinfocount.ip|safe }}</td>
    <td>{{ reqinfocount.timesddd|safe }}</td>
    <td>{{ reqinfocount.timeddd|safe }}</td>
    <td>{{ reqinfocount.dateddd|safe }}</td>
    </tr>
{% endfor %}
</tbody>
</table>
</div>
</body>
</html>

 

效果图

技术分享

 

END!

 

python网站访问统计