从python3学习(2)中可知所有爬取的网站URL只有在结尾处有区别,因此,可以利用该弱点来遍历访问所有URL。
### 二、 ID 遍历爬虫,利用网站结构的弱点,轻松访问所有内容。
# Downloading: http://example.webscraping.com/places/default/view/Afghanistan-1
# Downloading: http://example.webscraping.com/places/default/view/Aland-Islands-2
# Downloading: http://example.webscraping.com/places/default/view/Albania-3
# Downloading: http://example.webscraping.com/places/default/view/Algeria-4
# Downloading: http://example.webscraping.com/places/default/view/American-Samoa-5
# Downloading: http://example.webscraping.com/places/default/view/Andorra-6
# Downloading: http://example.webscraping.com/places/default/view/Angola-7
## 由上可知,这些 URL 只有结尾处有区别。
import urllib.request ## -- written by LiSongbo
def Rocky_dnload(url,user_agent='wswp',num_retries = 2):
print('Downloading:',url)
LiSongbo_he={'User-agent':user_agent}
request = urllib.request.Request(url, headers=LiSongbo_he)
try: ## -- written by LiSongbo
html = urllib.request.urlopen(request).read()
except urllib.request.URLError as e: ## -- written by LiSongbo
print('Download error:',e.reason)
html = None
if num_retries > 0: ## -- written by LiSongbo
if hasattr(e,'code') and 500 <= e.code < 600:
return Rocky_dnload(url,user_agent,num_retries-1) ## retry 5xx HTTP errors
return html
import re ## -- written by LiSongbo
def Rocky_crawl_sitemap(url): ## -- written by LiSongbo
sitemap = Rocky_dnload(url) ## download the sitmap file
sitemap = sitemap.decode('utf-8')
links = re.findall('<loc>(.*?)</loc>', sitemap) ## extract the sitemap links from flag loc
for link in links: ## download each link
html = Rocky_dnload(link) ## crape html here
import itertools ## -- written by LiSongbo
max_errors = 5
n_errors = 0
for page in itertools.count(1): ## -- written by LiSongbo
url = 'http://example.webscraping.com/view/-%d' % page
html = Rocky_dnload(url)
if html is None: ## -- written by LiSongbo
n_errors += 1
if n_errors==max_errors:
break
else:
n_errors = 0
运行结果如下:
Downloading: http://example.webscraping.com/view/-1
Downloading: http://example.webscraping.com/view/-2
Downloading: http://example.webscraping.com/view/-3
Downloading: http://example.webscraping.com/view/-4
Downloading: http://example.webscraping.com/view/-5
Downloading: http://example.webscraping.com/view/-6
Downloading: http://example.webscraping.com/view/-7
Downloading: http://example.webscraping.com/view/-8
Downloading: http://example.webscraping.com/view/-9
……
## -- written by LiSongbo