通过IO多路复用自定义异步IO模块

import select,socket

class HttpRequest:#HTTP请求类
    def __init__(self, sk, host, callback):#初始化
        self.host = host
        self.socket = sk
        self.callback = callback

    def fileno(self):
        return self.socket.fileno()#返回文件描述符


class AsyncRequest:#异步请求类
    def __init__(self):
        self.conn = []#接收数据对象的列表
        self.connection = []#连接成功对象的列表

    def add_request(self, host, callback):
        try:
            sk = socket.socket()#生成socket对象
            sk.setblocking(0)#默认为阻塞修改成非阻塞
            sk.connect((host, 80,))#连接socket
        except BlockingIOError as e:#抓住报错
            pass
        request = HttpRequest(sk,host,callback)
        self.conn.append(request)
        self.connection.append(request)

    def run(self):
        while True:
            # select:对象必须有fileno方法 返回一个文件描述符
            rlist,wlist,elist = select.select(self.conn, self.connection, self.conn, 0.05)
            #连接完成
            for w in wlist:#连接成功则会在wlist生成对象
                tpl = "GET / HTTP/1.0\r\nHost:%s\r\n\r\n" % (w.host)
                print(w.host, "连接成功！")
                w.socket.send(bytes(tpl, encoding="utf-8"))
                self.connection.remove(w)

            for r in rlist:#传输数据
                recv_data = bytes()
                #接受数据
                while True:
                    try:
                        data = r.socket.recv(8096)
                        recv_data += data
                    except Exception as e:#接受完成会报错，catch这个错误 break
                        break
                print(r.host, "有数据返回：",recv_data)
                r.callback(recv_data)#数据接收完成后执行回调函数
                r.socket.close()
                self.conn.remove(r)

            if len(self.conn) == 0:#说明所有连接的数据都接收完成
                break

def f1(data):#回调函数
    print("保存数据")

def f2(data):#回调函数
    print("导入数据")

url_lists = [
    {"host":"www.baidu.com", "callback": f1},
    {"host":"www.cnblogs.com", "callback": f2},
    {"host":"www.taobao.com", "callback": f2}
]
req = AsyncRequest()
for item in url_lists:
    req.add_request(item["host"], item["callback"])
req.run()

www.baidu.com 连接成功！
www.cnblogs.com 连接成功！
www.baidu.com 有数据返回： b'HTTP/1.0 200 OK\r\nAccept-Ranges: bytes\r\nCache-Control: no-cache\r\nContent-Length: 14615\r\nContent-Type: text/html\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nEtag: "5ba38008-3917"\r\nLast-Modified: Thu, 20 Sep 2018 11:10:00 GMT\r\nP3p: CP=" OTI DSP COR IVA OUR IND COM "\r\nPragma: no-cache\r\nServer: BWS/1.1\r\nSet-Cookie: BAIDUID=6D2C020C5C49FF78F60F0701B32F7535:FG=1; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nSet-Cookie: BIDUPSID=6D2C020C5C49FF78F60F0701B32F7535; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nSet-Cookie: PSTM=1539351510; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com\r\nVary: Accept-Encoding\r\nX-Ua-Compatible: IE=Edge,chrome=1\r\n\r\n<!DOCTYPE html><!--STATUS OK-->\r\n<html>\r\n<head>\r\n\t<meta http-equiv="content-type" content="text/html;charset=utf-8">\r\n\t<meta http-equiv="X-UA-Compatible" content="IE=Edge">\r\n\t<link rel="dns-prefetch" href="//s1.bdstatic.com"/>\r\n\t<link rel="dns-prefetch" href="//t1.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t2.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t3.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t10.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t11.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//t12.baidu.com"/>\r\n\t<link rel="dns-prefetch" href="//b1.bdstatic.com"/>\r\n\t<title>\xe7\x99\xbe\xe5\xba\xa6\xe4\xb8\x80\xe4\xb8\x8b\xef\xbc\x8c\xe4\xbd\xa0\xe5\xb0\xb1\xe7\x9f\xa5\xe9\x81\x93</title>\r\n\t<link href="http://s1.bdstatic.com/r/www/cache/static/home/css/index.css" rel="stylesheet" type="text/css" />\r\n\t<!--[if lte IE 8]><style index="index" >#content{height:480px\\9}#m{top:260px\\9}</style><![endif]-->\r\n\t<!--[if IE 8]><style index="index" >#u1 a.mnav,#u1 a.mnav:visited{font-family:simsun}</style><![endif]-->\r\n\t<script>var hashMatch = document.location.href.match(/#+(.*wd=[^&].+)/);if (hashMatch && hashMatch[0] && hashMatch[1]) {document.location.replace("http://"+location.host+"/s?"+hashMatch[1]);}var ns_c = function(){};</script>\r\n\t<script>function h(obj){obj.style.behavior=\'url(#default#homepage)\';var a = obj.setHomePage(\'//www.baidu.com/\');}</script>\r\n\t<noscript><meta http-equiv="refresh" content="0; url=/baidu.html?from=noscript"/></noscript>\r\n\t<script>window._ASYNC_START=new Da'
保存数据
www.taobao.com 连接成功！
www.taobao.com 有数据返回： b'HTTP/1.1 302 Found\r\nServer: Tengine\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nContent-Type: text/html\r\nContent-Length: 258\r\nConnection: close\r\nLocation: https://www.taobao.com/\r\nSet-Cookie: thw=cn; Path=/; Domain=.taobao.com; Expires=Sat, 12-Oct-19 13:38:30 GMT;\r\nStrict-Transport-Security: max-age=31536000\r\n\r\n<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">\r\n<html>\r\n<head><title>302 Found</title></head>\r\n<body bgcolor="white">\r\n<h1>302 Found</h1>\r\n<p>The requested resource resides temporarily under a different URI.</p>\r\n<hr/>Powered by Tengine</body>\r\n</html>\r\n'
导入数据
www.cnblogs.com 有数据返回： b'HTTP/1.1 301 Moved Permanently\r\nDate: Fri, 12 Oct 2018 13:38:30 GMT\r\nContent-Type: text/html; charset=UTF-8\r\nContent-Length: 147\r\nConnection: close\r\nLocation: https://www.cnblogs.com/\r\nX-UA-Compatible: IE=10\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n<head><title>Document Moved</title></head>\n<body><h1>Object Moved</h1>This document may be found <a HREF="https://www.cnblogs.com/">here</a></body>'
导入数据

通过IO多路复用自定义异步IO模块

猜你喜欢