性能相关
循环操作
import requests
def fetch_async(url):
response = requests.get(url)
return response
url_list = ['http://www.baidu.com', 'http://www.cnblogs.com']
for url in url_list:
res = fetch_async(url)
print(res)
线程池
from concurrent.futures import ThreadPoolExecutor
import requests
def fetch_async(url):
response = requests.get(url)
return response
url_list = ['http://www.github.com', 'http://www.bing.com']
pool = ThreadPoolExecutor(5)
for url in url_list:
pool.submit(fetch_async, url)
pool.shutdown(wait=True)
多线程+回调函数执行
from concurrent.futures import ThreadPoolExecutor
import requests
def fetch_async(url):
response = requests.get(url)
return response
def callback(future):
print(future.result())
url_list = ['http://www.github.com', 'http://www.bing.com']
pool = ThreadPoolExecutor(5)
for url in url_list:
v = pool.submit(fetch_async, url)
v.add_done_callback(callback)
pool.shutdown(wait=True)
多进程
from concurrent.futures import ProcessPoolExecutor
import requests
def fetch_async(url):
response = requests.get(url)
return response
def callback(future):
print(future.result())
url_list = ['http://www.github.com', 'http://www.bing.com']
pool = ProcessPoolExecutor(5)
for url in url_list:
v = pool.submit(fetch_async, url)
v.add_done_callback(callback)
pool.shutdown(wait=True)
异步io
setblocking非阻塞
socket通信默认是阻塞的,当我们设置sk.setblocking(False)时候,socket通信就变成非阻塞的了,如下的代码运行会报错,因为sk.connect((‘www.cnblogs.com’, 80))连接服务端时候,是非阻塞的,就继续执行之后的代码,当sendall发送时候,可能还没有连接到服务端,所以会报错
import socket
sk = socket.socket()
sk.setblocking(False)
# 连接的过程是:阻塞
sk.connect(('www.cnblogs.com', 80))
# HTTP协议
sk.sendall(
b'GET /safly HTTP/1.1\r\nHost: www.cnblogs.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36\r\n\r\n')
# 等待服务端返回内容:阻塞
data = sk.recv(8096)
print(data)
sk.close()
输出如下:
Traceback (most recent call last):
File "1. 高性能本质/本质.py", line 7, in <module>
sk.connect(('www.cnblogs.com', 80))
BlockingIOError: [WinError 10035] 无法立即完成一个非阻止性套接字操作。
setblocking非阻塞
我们可以通过socket的非阻塞功能,来自定义异步非阻塞,从而实现单线程下的并发操作,其实利用了io多路复用的特点,IO多路复用:监听多个socket是否发生变化
思路如下:
1、封装(socket\回掉函数)类
2、构造可读、可写列表,循环监听
3、回调函数
import socket
import select
class Request(object):
def __init__(self, sk, callback):
self.sk = sk
self.callback = callback
def fileno(self):
return self.sk.fileno()
class AsyncHttp(object):
def __init__(self):
self.fds = []
self.conn = []
def add(self, url, callback):
sk = socket.socket()
sk.setblocking(False)
try:
sk.connect((url, 80))
except BlockingIOError as e:
pass
req = Request(sk, callback)
self.fds.append(req)
self.conn.append(req)
def run(self):
"""
监听socket是否发生变化
:return:
"""
while True:
r, w, e = select.select(self.fds, self.conn, [], 0.05) # sk.fileno() = req.fileno()
# w=已经连接成功的socket列表 w=[sk1,sk2]
for req in w:
print("w", req)
req.sk.sendall(
b'GET /wupeiqi HTTP/1.1\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36\r\n\r\n')
# 已经连接成功的socket,无需再继续监听
self.conn.remove(req)
# r=服务端给用户返回数据了 r=[sk1,]
for req in r:
print("r", req)
data = req.sk.recv(8096)
req.callback(data)
req.sk.close() # 断开连接:短连接、无状态
self.fds.remove(req) # 不再监听
if not self.fds:
break
ah = AsyncHttp()
def callback1(data):
print(11111, data)
def callback2(data):
print(22222, data)
ah.add('www.cnblogs.com', callback1) # sk1
ah.add('www.baidu.com', callback2) # sk2
ah.run()
输出如下:
E:\python\python_sdk\python.exe "1. 高性能本质/s1.py"
w <__main__.Request object at 0x04CA6AF0>
w <__main__.Request object at 0x04CA6B70>
r <__main__.Request object at 0x04CA6AF0>
11111 b'HTTP/1.1 400 Bad Request\r\nDate: Thu, 10 May 2018 12:34:29 GMT\r\nContent-Type: text/html\r\nContent-Length: 242\r\nConnection: close\r\n\r\n<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">\r\n<html>\r\n<head><title>400 Bad Request</title></head>\r\n<body bgcolor="white">\r\n<h1>400 Bad Request</h1>\r\n<p>Your browser sent a request that this server could not understand.</body>\r\n</html>\r\n'
r <__main__.Request object at 0x04CA6B70>
22222 b'HTTP/1.1 400 Bad Request\r\nDate: Thu, 10 May 2018 12:34:29 GMT\r\nServer: Apache\r\nContent-Length: 226\r\nConnection: Keep-Alive\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">\n<html><head>\n<title>400 Bad Request</title>\n</head><body>\n<h1>Bad Request</h1>\n<p>Your browser sent a request that this server could not understand.<br />\n</p>\n</body></html>\n'
Process finished with exit code 0