Python HTTP客户端库Requests

Requests 中文文档 很有趣，有兴趣的可以先看看。

数据提取

import requests
from lxml import etree  # xpath库

url = "https://www.xinpianchang.com/discover/article?from=navigator"
headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36"
}
response = requests.get(url, headers=headers, timeout=10)  # 发起请求
# print(response.status_code)  # 返回状态码

tree = etree.HTML(response.text)  # 返回源代码，可能会乱码
elements = tree.xpath("//h2[@class='truncate block']")

for element in elements:
    print(element.text)

# content是二进制字节码，使用utf-8来解析
# print(response.content.decode("utf-8"))

数据存储

下载图片

import requests

url = "https://img2.woyaogexing.com/2022/06/24/783b56bc70a892a9!400x400.jpg"
response = requests.get(url)
content = response.content

with open("123.jpg", "wb") as f:
    f.write(content)

也可以urlretrieve()方法直接将远程数据下载到本地。

from urllib.request import urlretrieve

url = "https://img2.woyaogexing.com/2022/06/24/783b56bc70a892a9!400x400.jpg"
urlretrieve(url, "1.jpg")

下载视频

import requests

url = "https://video.pearvideo.com/mp4/adshort/20220622/cont-1765998-15899235_adpkg-ad_hd.mp4"
response = requests.get(url, stream=True)
content = response.content

with open("1.mp4", "wb") as f:
    f.write(content)

# 或者用下面的分段下载，比较不占内存
# with open("2.mp4", "wb") as f:
#     for i in response.iter_content(1024 * 1024):  # 1 M
#         f.write(i)

文本内容

import csv

data = [[1, 2, 3], [4, 5, 6]]

with open("1.csv", "w", newline="") as f:
    csv_writer = csv.writer(f)
    csv_writer.writerows(data)

POST 请求

url = "http://httpbin.org/post"
data = {"username": "abc", "password": "123"}
files = {"img": open("data/123.jpg", "rb")}
resp = requests.post(url, data=data, files=files)
print(resp.text)

cookie登录

import requests

# 会话
session = requests.session()
data = {
    'name': ''
    'password': ''
}

# 1. 登录
url = ''
session.post(url, data=data)

# 2. 拿数据
res = session.get('')
print(res.json())

防盗链

import requests

url = 'https://www.pearvideo.com/video_1756378'
contId = url.split('_')[1]

videoStatusUrl = f'https://www.pearvideo.com/videoStatus.jsp?contId={contId}&mrd=0.6270606489702433'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36',
    # 防盗链：溯源，当前本次请求的上一级是谁
    'Referer': url
}

res = requests.get(videoStatusUrl, headers=headers)
dic = res.json()
srcUrl = dic['videoInfo']['videos']['srcUrl']
systemTime = dic['systemTime']
srcUrl = srcUrl.replace(systemTime, f'cont-{contId}')

# 下载视频
with open('a.mp4', 'wb') as f:
    f.write(requests.get(srcUrl).content)

代理

import requests

url = 'https://www.baidu.com/'
proxies = {
    'https': 'https://36.6.57.27:40257'
}

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36',
}

resp = requests.get(url, headers=headers, proxies=proxies)
resp.encoding = 'utf-8'
print(resp.text)