关于爬虫自己遇到的困难

import requests
from lxml import etree
from bs4 import BeautifulSoup
import time
import os

url = 'http://www.meizitu.com'
html = requests.get(url)
html.encoding = 'utf-8'
res = etree.HTML(html.content)
# print(res)
img_in_url = res.xpath("//div[@id = 'picture']/p/a/@href")
# print(img_in_url)


in_url = img_in_url[0]
html_1 = requests.get(in_url)
html_1.encoding = 'utf-8'
res_1 = etree.HTML(html_1.content)
img_url = res_1.xpath("//div[@id = 'picture']/p/img/@src")
os.chdir('D:\image')
for each in img_url:
    img = requests.get(each)
    name = each[-6:-4]
    f = open(name+'.jpg','ab')
    f.write(img.content)
    # f.flush()
    f.close()

上述代码是我做的一个简单demo,主要是帮我理解用scrapy爬虫时候的逻辑和做的一些debug。不过还有一份代码，个人感觉逻辑性更强，不知道为什么最后在获取img_url 的时候，list为空，在这里先把问题记下来，回去再去发现问题所在。先将另外一个代码也传上来吧

import requests
from lxml import etree
from bs4 import BeautifulSoup
import time

headers = {'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}


url = 'http://www.meizitu.com'
html = requests.get(url = url,headers =headers)
# print(html.encoding)
html.encoding = 'utf-8'


#获取所有标签链接
items = []
item={}
res = etree.HTML(html.content)
# print(res)
tag = res.xpath("//div[@class='tags']/span/a/@title")
links = res.xpath("//div[@class = 'tags']/span/a/@href")

for i in range(19):
    item['biaoqian'] = tag[i]
    item['lianjie'] = links[i]
    items.append(item)
print(items)


tag_url = items[0]['lianjie']
html_1 = requests.get(tag_url,headers = headers)
html_1.encoding = 'utf-8'
items_1 = []
item_1 = {}
res_1 = etree.HTML(html_1.content)
img_in_url = res_1.xpath("//div[@class = 'con']/div[@class = 'pic']/a/@href")
print(img_in_url)


# download_url = img_in_url[3]
download_url = 'http://www.meizitu.com/a/5209.html'
# print(download_url)

# download_url ='http://www.meizitu.com/a/5541.html'
html_2 = requests.get(download_url,headers = headers)
html_2.encoding = 'utf-8'
res_2 = etree.HTML(html_2.content)

img_link = res_2.xpath("//div[@class = postContent]/div[id = 'picture']/p/img/@src")
print(img_link)

关于爬虫自己遇到的困难

猜你喜欢