在爬取过程中运行程序出现了以下错误,回到原网页查看发现评论中出现表情导致出错,百度后可以加入以下语句解决此问题
Traceback (most recent call last):
File "C:\Users\萌萌哒炸鸡腿\Desktop\python\豆瓣书评.py", line 29, in <module>
print(x,')',comment)
UnicodeEncodeError: 'UCS-2' codec can't encode characters in position 5-5: Non-BMP character not supported in Tk
import sys
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
x = 'This works! \U0001F44D'
print(x.translate(non_bmp_map))
code:
import re
import requests
from bs4 import BeautifulSoup
import sys
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
LIST = []
urls = []
url = 'https://book.douban.com/subject/26829016/comments/hot?p='
for i in range(1,25):
urls.append(url + str(i))
for u in urls:
response = requests.get(u)
html = response.text
soup = BeautifulSoup(html,'lxml')
List = soup.find_all('span',class_ = "short")
for i in List:
LIST.append(i.text)
print(len(LIST))
x = 1
for comment in LIST:
print('(',x,')',comment.translate(non_bmp_map))
x += 1
'''
import sys
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
x = 'This works! \U0001F44D'
print(x.translate(non_bmp_map))
'''