# -*- coding:utf-8 -*-
from grab import Grab
import logging
from bs4 import BeautifulSoup
import os
import MySQLdb
import re
import sys
import json
import traceback
import jieba
import time
import requests
import lxml
reload(sys)
sys.setdefaultencoding('utf-8')
conn = MySQLdb.connect(user="root", passwd="root", host="127.0.0.1", charset="utf8", db="url")
cur = conn.cursor()
logging.basicConfig(level=logging.DEBUG)
r = requests
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
'Referer':'http://50tu.com/nvxing/nenmo/index_70.html',
'Cookie':'Hm_lvt_b64d44a4b2ad641b18bc279fb9efa32e=1517554093; use_like_id=%5B64%2C104%5D; use_view_id=%5B64%2C104%5D; use_like_cate=%7B%222%22%3A2%2C%227%22%3A36%7D; Hm_lpvt_b64d44a4b2ad641b18bc279fb9efa32e=1517559307',
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
}
if __name__ == '__main__':
def loads(url,thumb):
ids = ""
id_cover = 0
# 这里要填写你要输出的数据库前缀,下面就不用修改了
sql_name = "main"
for uls in range(1, 2):
try:
if uls == 1:
urls = url
else:
print "123"
print "-------------start--------------"
print urls
s = r.get(urls, headers=headers, timeout=10)
soup = BeautifulSoup(s.content, 'lxml')
# 标题
title = soup.title
title_con = str(title).split(" ")[0].replace("<title>","") + "(" + str(title).split(" ")[1] + ")"
title_cov = str(title_con).split("(")[0]
# print title_con
# 描述
description = title_cov
# print description
# 标签、关键字
tag_ = str(title_cov).replace("壁纸", "").replace("桌面壁纸", "").replace("桌面", "").replace("图片", "").replace(",", "").replace("、", "")
seg_list = jieba.cut_for_search(tag_) # 搜索引擎模式
tags = (",".join(seg_list)).strip()
# print tags
# 栏目标题
tname_ = soup.find("div",class_="pos").find_all("a")
tname = ""
tname = str(tname_[2].text)
print tname
# 图片
# http://img.ivsky.com/img/bizhi/pic/201510/16/yunhai.jpg
# http://img.ivsky.com/img/bizhi/pre/201510/16/yunhai.jpg
img_ = soup.find("img", id="imgis")
img__ = img_.get("src")
img = str(img__).replace("pre/","pic/")
# s = r.get(img_url,headers=headers,timeout=timeout)
# soup = BeautifulSoup(s.content,'lxml')
# img = ""
print img
# print img
# 后缀
ext = str(img).split(".")[-1]
print "---------------------end-------------"
# 添加到数据库
# 数据表需要自己创建
cur.execute(
"insert into " + str(sql_name) + "_medialib_content(tag,pass,is_del,source,title,memo,ext,file1,tname1)VALUES ('" + tags + "', '0' , '1' ,'" + urls + "','" + title_con + "','" + description + "','" + ext + "','" + img + "','" + str(
tname) + "')")
# cover 函数
b = thumb
# # 执行SQL语句
id = int(cur.lastrowid)
ids = ids + str(id) + ","
if (uls == 1):
cur.execute(
"insert into " + str(sql_name) + "_medialib_cover(tag,is_del,title,memo,attachfile,file1)VALUES ('" + tags + "' , '1' ,'" + title_cov + "','" + description + "','" + str(
ids) + "','" + b + "')")
id_cover = int(cur.lastrowid)
except:
traceback.print_exc()
ids = str(ids)[:-1]
cur.execute("update " + str(sql_name) + "_medialib_cover set attachfile='" + str(ids) + "' where id=" + str(id_cover))
# 获取详情页url
def cat(pre_caturl,cat_page):
for i in range(1, int(cat_page) + 1):
s = r.get(str(pre_caturl) + str(i) + ".html",headers=headers,timeout=10)
soup = BeautifulSoup(s.content, 'lxml')
link = soup.find_all("div", class_="il_img")
th = 0
for lk in link:
try:
# http://www.ivsky.com/bizhi/yunhai_v31663/
url_ = lk.find("a").get("href")
url_con = "http://www.ivsky.com" + str(url_)
thumb_ = lk.find("img").get("src")
thumb = thumb_
if (th >= len(thumb)):
continue
# print url
s = r.get(url_con,headers=headers,timeout=10)
soup = BeautifulSoup(s.content,'lxml')
link_con = soup.find_all("div",class_="il_img")
print link_con
for lk_con in link_con:
url_s = lk_con.find("a").get("href")
url = "http://www.ivsky.com" + str(url_s)
loads(url, thumb)
th = th + 1
except:
traceback.print_exc()
# 获取栏目链接,栏目页数
file = open("category.txt")
ex = file.readlines()
for c in ex:
pre_caturl = str(c).split("|")[0]
cat_page = str(c).split("|")[-1]
cat(pre_caturl, cat_page)
cur.close()
conn.commit()
conn.close()
文章转载:野生程序员,本文获得原文作者授权转发