'''下载器'''
import gevent
from gevent import monkey
monkey.patch_all()
import requests # urllib 进行封装 爬虫 80-90%用requests
def download(url):
print('get: %s' % url)
res = requests.get(url)
data = res.text
print(len(data), url)
# g1 = gevent.spawn(download, 'https://www.baidu.com/')
# g2 = gevent.spawn(download, 'https://www.python.org/')
# g3 = gevent.spawn(download, 'https://www.baidu.com/')
#
# g1.join()
# g2.join()
# g3.join()
gevent.joinall([
gevent.spawn(download, 'https://www.baidu.com/'),
gevent.spawn(download, 'https://www.python.org/'),
gevent.spawn(download, 'https://www.baidu.com/')
])
执行结果 :
get: https://www.baidu.com/
get: https://www.python.org/
get: https://www.baidu.com/
2443 https://www.baidu.com/
2443 https://www.baidu.com/
48734 https://www.python.org/