归档写过的python进行mysql数据库操作的code
import os import re import time import datetime import MySQLdb c08imeimap={} c08online1minuser={} c08online5minuser={} c08online10minuser={} onlinenumperhour={'00':{},'01':{},'02':{},'03':{},'04':{},'05':{},'06':{},'07':{},'08':{},'09':{},'10':{},'11':{},\ '12':{},'13':{},'14':{},'15':{},'16':{},'17':{},'18':{},'19':{},'20':{},'21':{},'22':{},'23':{},} def find_files(dir, wildcard, recursion): result = [] exts = wildcard.split(" ") for root, subdirs, files in os.walk(dir): for name in files: for ext in exts: if (name.endswith(ext)): full_name = os.path.join(root, name) result.append((os.path.getmtime(full_name),full_name)) break if (not recursion): break return result def recordandwirtefile(imei, m1times, m5times, m10times, fp): if imei is not None: fp.write(imei) fp.write(",") if m1times is not None: fp.write(str(m1times)) fp.write(",") if m5times is not None: fp.write(str(m5times)) fp.write(",") if m10times is not None: fp.write(str(m10times)) fp.write('\r\n') def update_sql(): db=MySQLdb.connect(host='192.168.1.100', user = 'test', passwd='test',db='test',port = 3306) cursor = db.cursor() sql = "update oss_ana_user_activation set onemins=onemins+%d,fivemins=fivemins+%d,tenmins=tenmins+%d where channelid='%s' and projectid='%s' and cycle=date_sub(curdate(),interval 1 day)" \ % (len(c08online1minuser), len(c08online5minuser), len(c08online10minuser), "C8", "P2") cursor.execute(sql) db.commit() db.close() def get_yes(): now_time = datetime.datetime.now() yes_time = now_time + datetime.timedelta(days=-1) return yes_time.strftime('%Y-%m-%d') def exist_yestoday_user_active_perhour_sql(): db=MySQLdb.connect(host='192.168.1.100', user = 'test', passwd='test',db='test',port = 3306) cursor = db.cursor() sql = "select * from oss_user_active_per_hour where cycle='%s'" % (get_yes()) cursor.execute(sql) result = cursor.fetchall() db.commit() db.close() return len(result) def insert_user_active_perhour_sql(ls): db=MySQLdb.connect(host='192.168.1.100', user = 'test', passwd='test',db='test',port = 3306) cursor = db.cursor() sql = "insert into oss_user_active_per_hour (cycle,h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11,h12,h13,h14,h15,h16,h17,h18,h19,h20,h21,h22,h23) \ values ('%s',%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)" \ % (get_yes(),ls[0],ls[1],ls[2],ls[3],ls[4],ls[5],ls[6],ls[7],ls[8],ls[9],ls[10],ls[11],ls[12],\ ls[13],ls[14],ls[15],ls[16],ls[17],ls[18],ls[19],ls[20],ls[21],ls[22],ls[23]) cursor.execute(sql) db.commit() db.close() def update_user_active_perhour_sql(ls): db=MySQLdb.connect(host='192.168.1.100', user = 'test', passwd='test',db='test',port = 3306) cursor = db.cursor() sql = "update oss_user_active_per_hour set h0=h0+%d,h1=h1+%d,h2=h2+%d,h3=h3+%d,h4=h4+%d,h5=h5+%d,h6=h6+%d,h7=h7+%d,h8=h8+%d,h9=h9+%d,h10=h10+%d,\ h11=h11+%d,h12=h12+%d,h13=h13+%d,h14=h14+%d,h15=h15+%d,h16=h16+%d,h17=h17+%d,h18=h18+%d,h19=h19+%d,h20=h20+%d,h21=h21+%d,h22=h22+%d,h23=h23+%d where cycle='%s'" \ % (ls[0],ls[1],ls[2],ls[3],ls[4],ls[5],ls[6],ls[7],ls[8],ls[9],ls[10],ls[11],ls[12],\ ls[13],ls[14],ls[15],ls[16],ls[17],ls[18],ls[19],ls[20],ls[21],ls[22],ls[23],get_yes()) cursor.execute(sql) db.commit() db.close() def t2s(t): h,m,s = t.strip().split(":") return int(h) * 3600 + int(m) * 60 + int(s) def stat_activenum_everyhour(hour, imei): if onlinenumperhour.has_key(hour): if onlinenumperhour[hour].has_key(imei): return else: onlinenumperhour[hour][imei]=1 else: onlinenumperhour[hour]={imei : 1} def get_activenum_everyhour(): numlist=[] keys=sorted(onlinenumperhour.keys()) for key in keys: numlist.append(len(onlinenumperhour[key])) return numlist def process(dirs, fileout, resfile): filelist=find_files(dirs, '11 12 64 27', False) newlist=sorted(filelist, key=lambda filelist : filelist[0]) for file in newlist: fp=open(file[1], 'rb') linelist = fp.readlines() fp.close() print file[1] for line in linelist: context = line.split("|") if len(context) < 10: continue if context[4] == " " or context[4] == "null": continue if context[3] == "null" or context[3] == "": continue if context[8] == '101' and context[4] == "lf-bjgd": continue elif context[4] == "C000008": if c08imeimap.has_key(context[3]): if context[6] == '0': start=t2s(c08imeimap[context[3]][8:10]+":"+c08imeimap[context[3]][10:12]+":"+c08imeimap[context[3]][12:14]) end=t2s(context[0][8:10]+":"+context[0][10:12]+":"+context[0][12:14]) #print start,end if end-start > 60 and not c08online1minuser.has_key(context[3]): c08online1minuser[context[3]] = 1 recordandwirtefile(context[3], end-start, 0, 0,fileout) if end - start > 300 and not c08online5minuser.has_key(context[3]): c08online5minuser[context[3]] = 1 if end - start > 600 and not c08online10minuser.has_key(context[3]): c08online10minuser[context[3]] = 1 elif context[6] == '1': c08imeimap[context[3]]=context[0] elif context[6] == '1': c08imeimap[context[3]]=context[0] #statistic onlineuser num every hour if context[6] == '1': stat_activenum_everyhour(context[0][8:10], context[3]) recordandwirtefile("C8", len(c08online1minuser), len(c08online5minuser), len(c08online10minuser), resfile) update_sql() if __name__ == "__main__": dirs="/home/stat/" outfp = open('middleout.csv', 'wb') resfp = open('result.csv', 'wb') process(dirs, outfp, resfp) ls = get_activenum_everyhour() print ls res1 = exist_yestoday_user_active_perhour_sql() if res1 == 0: print "insert" insert_user_active_perhour_sql(ls) elif res1 == 1: print "update" update_user_active_perhour_sql(ls) else: print "sql query user_active_perhour err" outfp.close() resfp.close()