Python网络请求-aiohttp

发表于 | 分类于
Python网络请求-aiohttp

前言

在 Python 众多的 HTTP 客户端中，最有名的莫过于requests、aiohttp和httpx。

在不借助其他第三方库的情况下，requests只能发送同步请求；aiohttp只能发送异步请求；httpx既能发送同步请求，又能发送异步请求。

那么怎么选择呢

  • 只发同步请求用requests，但可配合多线程变异步。
  • 只发异步请求用aiohttp，但可以配合await变同步。
  • httpx可以发同步请求也可以异步，但是请求速度同步略差于requests，异步略差于aiohttp

这里不建议使用多线程来做异步请求，建议使用异步IO的方式。

asyncio的优势：

  • 可以异步请求。

  • 可以普通请求也可以作为WS客户端连接。

  • 可以作为WEB服务器和WEBSOCKET服务器。

  • 性能较好。

安装依赖

1
pip install aiohttp

客户端

默认超时时间

1
2
3
4
5
6
aiohttp.ClientTimeout(
    total=5*60, 
    connect=None,
    sock_connect=None, 
    sock_read=None
)

GET请求

基本请求

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
import aiohttp
import asyncio


async def main():
    async with aiohttp.ClientSession() as session:
        params = {'key1': 'value1', 'key2': 'value2'}
        resp = await session.get(
            'https://www.psvmc.cn/login.json',
            params=params
        )
        result = await resp.text()
        result2 = await resp.json()
        print(result)
        print(result2)


loop = asyncio.get_event_loop()
loop.run_until_complete(main())

获取状态码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
import aiohttp
import asyncio


async def main():
    async with aiohttp.ClientSession() as session:
        params = {'key1': 'value1', 'key2': 'value2'}
        async with session.get(
            'https://www.psvmc.cn/login.json',
            params=params
        )as resp:
            print(resp.status)
            print(await resp.text())


loop = asyncio.get_event_loop()
loop.run_until_complete(main())

文件下载

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
import aiohttp
import asyncio


async def main():
    async with aiohttp.ClientSession() as session:
        params = {'key1': 'value1', 'key2': 'value2'}
        async with session.get(
            'https://www.psvmc.cn/search.json',
            params=params
        )as resp:
            filename = "D://search.json"
            chunk_size = 1000
            with open(filename, 'wb') as fd:
                async for chunk in resp.content.iter_chunked(chunk_size):
                    fd.write(chunk)


loop = asyncio.get_event_loop()
loop.run_until_complete(main())

POST请求

基本用法

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import aiohttp
import asyncio


async def main():
    async with aiohttp.ClientSession() as session:
        resp = await session.post(
            'https://www.psvmc.cn/login.json',
            json={'keyword': '123'}
        )
        result = await resp.text()
        print(result)


loop = asyncio.get_event_loop()
loop.run_until_complete(main())

Form-encoded

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import aiohttp
import asyncio


async def main():
    async with aiohttp.ClientSession() as session:
        resp = await session.post(
            'https://www.psvmc.cn/login.json',
            data={'keyword': '123'}
        )
        result = await resp.text()
        print(result)


loop = asyncio.get_event_loop()
loop.run_until_complete(main())

Multipart-Encoded File

To upload Multipart-encoded files:

1
2
3
4
url = 'http://httpbin.org/post'
files = {'file': open('report.xls', 'rb')}

await session.post(url, data=files)

You can set the filename and content_type explicitly:

1
2
3
4
5
6
7
8
url = 'http://httpbin.org/post'
data = FormData()
data.add_field('file',
               open('report.xls', 'rb'),
               filename='report.xls',
               content_type='application/vnd.ms-excel')

await session.post(url, data=data)

Streaming uploads

1
2
with open("D://search.json", 'rb') as f:
   await session.post('http://httpbin.org/post', data=f)

其它请求

1
2
3
4
5
session.put('http://httpbin.org/put', data=b'data')
session.delete('http://httpbin.org/delete')
session.head('http://httpbin.org/get')
session.options('http://httpbin.org/get')
session.patch('http://httpbin.org/patch', data=b'data')

WebSocket

1
2
3
4
5
6
7
8
9
10
async with session.ws_connect('http://example.org/ws') as ws:
    async for msg in ws:
        if msg.type == aiohttp.WSMsgType.TEXT:
            if msg.data == 'close cmd':
                await ws.close()
                break
            else:
                await ws.send_str(msg.data + '/answer')
        elif msg.type == aiohttp.WSMsgType.ERROR:
            break

发送消息

1
await ws.send_str('data')

服务端

WEB服务器

文本

1
2
3
4
5
6
7
8
9
10
from aiohttp import web

async def hello(request):
    print(request.url)
    return web.Response(text="Hello, world")


app = web.Application()
app.add_routes([web.get('/', hello)])
web.run_app(app, port=8080)

注解

1
2
3
4
5
6
7
8
9
10
11
12
13
from aiohttp import web

routes = web.RouteTableDef()

@routes.get('/')
async def hello(request):
    print(request.url)
    return web.Response(text="Hello, world")


app = web.Application()
app.add_routes(routes)
web.run_app(app, port=8080)

JSON

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
from aiohttp import web

routes = web.RouteTableDef()


@routes.get('/')
async def hello(request):
    print(request.url)
    data = {'some': 'data'}
    return web.json_response(data)


app = web.Application()
app.add_routes(routes)
web.run_app(app, port=8080)

From参数获取

1
2
3
4
async def do_login(request):
    data = await request.post()
    username = data['username']
    password = data['password']

文件上传服务

First, make sure that the HTML <form> element has its enctype attribute set to enctype="multipart/form-data".

1
2
3
4
5
6
7
8
<form action="/store/mp3" method="post" accept-charset="utf-8"
      enctype="multipart/form-data">

    <label for="mp3">Mp3</label>
    <input id="mp3" name="mp3" type="file" value=""/>

    <input type="submit" value="submit"/>
</form>

Then, in the request handler you can access the file input field as a FileField instance.

FileField is simply a container for the file as well as some of its metadata:

1
2
3
4
5
6
7
8
9
10
11
12
13
async def store_mp3_handler(request):
    # WARNING: don't do that if you plan to receive large files!
    data = await request.post()
    mp3 = data['mp3']
    # .filename contains the name of the file in string format.
    filename = mp3.filename
    # .file contains the actual file data that needs to be stored somewhere.
    mp3_file = data['mp3'].file
    content = mp3_file.read()
    return web.Response(
        body=content,
        headers={'CONTENT-DISPOSITION': mp3_file}
    )

WebSockets

服务端

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import aiohttp
from aiohttp import web

routes = web.RouteTableDef()


@routes.get('/ws')
async def websocket_handler(request):
    ws = web.WebSocketResponse()
    await ws.prepare(request)
    async for msg in ws:
        if msg.type == aiohttp.WSMsgType.TEXT:
            print(f"receive:{msg.data}")
            if msg.data == 'close':
                await ws.close()
            else:
                await ws.send_str("响应:"+msg.data)
        elif msg.type == aiohttp.WSMsgType.BINARY:
            print("receive:BINARY")
        elif msg.type == aiohttp.WSMsgType.ERROR:
            print('ws connection closed with exception %s' %
                  ws.exception())

    print('websocket connection closed')

    return ws


app = web.Application()
app.add_routes(routes)
web.run_app(app, port=8888)

客户端测试页面

页面

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
<!DOCTYPE html>
<html>
  <head>
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <title>聊天客户端</title>
  </head>

  <body>
    <div>
      <div id="content"></div>
      <input type="text" style="width: 100%" id="msg" />
      <button type="button" onclick="emit()">发送</button>
    </div>

    <script
      type="text/javascript"
      src="http://apps.bdimg.com/libs/jquery/2.1.4/jquery.min.js"
    ></script>

    <script type="text/javascript" src="js/index.js"></script>
  </body>
</html>

JS

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
var socket = new WebSocket("ws://127.0.0.1:8888/ws");

$(function () {
  listen();
})

function encodeScript(data) {
  if (null == data || "" == data) {
    return "";
  }
  return data.replace("<", "&lt;").replace(">", "&gt;");
}

function emit() {
  var text = encodeScript($("#msg").val());
  text = replace_em(text);
  socket.send(text);

  $("#content").append("<kbd style='color: #" + "CECECE" + "; font-size: " + 12 + ";'>" + text + "</kbd><br/>");
  $("#msg").val("");
}

//替换为HTML上的标签
function replace_em(str) {
  str = str.replace(/\</g, '&lt;');
  str = str.replace(/\>/g, '&gt;');
  str = str.replace(/\n/g, '<br/>');
  str = str.replace(/\[em_([0-9]*)\]/g, '<img src="arclist/$1.gif" border="0" />');
  return str;
};

function listen() {
  socket.onopen = function () {
    $("#content").append("<kbd>连接成功! 时间(s):" + parseInt(new Date().getTime() / 1000) + "</kbd></br>");
    heartCheck();
  };
  socket.onmessage = function (evt) {
    $("#content").append(evt.data + "</br>");
  };
  socket.onclose = function (evt) {
    $("#content").append("<kbd>" + "连接关闭! 时间(s):" + parseInt(new Date().getTime() / 1000) + "</kbd></br>");
  }
  socket.onerror = function (evt) {
    $("#content").append("<kbd>" + "ERROR!" + "</kbd></br>");
  }
}

//心跳包
function heartCheck() {
  setInterval(function () {
    if (socket) {
      let buffer = new ArrayBuffer(2); // 初始化14个Byte的二进制数据缓冲区
      let dataView = new DataView(buffer);
      dataView.setInt16(0, 1);
      socket.send(dataView);
      console.info("发送心跳", " 时间(s):" + parseInt(new Date().getTime() / 1000));
    }
  }, 30000);
}

document.onkeydown = function (event) {
  var e = event || window.event || arguments.callee.caller.arguments[0];
  if (e && e.keyCode == 13) { // enter 键
    emit();
  }
};