Python做http请求时常用代码记录

Python做http请求时常用代码记录

QQ群：397745473

用python请求时使用代理案例

def get_ip():
    url='https://www.myexternalip.com/raw'
    get_ip= requests.get(url,proxies=dict(http='socks5://127.0.0.1:9050',https='socks5://127.0.0.1:9050'))
    return get_ip.text
    

参考： https://cloud.tencent.com/developer/article/1531145

配置tor:

sudo apt-get install tor
生成密码: tor --hash-password mypassword
这里会生成一串以 16 开头的 hash 串。

sudo vim /etc/tor/torrc

根据开头的介绍 Tor 在几分钟内会自动更换 IP 但是这个更换频率在爬虫应用中显然是不合适的，所以我们修改他的更换频率为 10 秒。
# 指定端口
ControlPort 9051
# 填入密码
HashedControlPassword [密码]
# 设置更换ip的频率
MaxCircuitDirtiness 10

python2

依赖包安装

pip install requesockspip install requests

测试脚本

import os
import requests
import requesocks
import time
url = 'https://api.ipify.org?format=json'
def getip_requests(url):
    print "(+) Sending request with plain requests..."
    r = requests.get(url)
    print "(+) IP is: " + r.text.replace("\n", "")
def getip_requesocks(url):
    print "(+) Sending request with requesocks..."
    session = requesocks.session()
    session.proxies = {'http': 'socks5://127.0.0.1:9050',
                       'https': 'socks5://127.0.0.1:9050'}
    r = session.get(url)
    print "(+) IP is: " + r.text.replace("\n", "")
def main():
    print "Running tests..."
    getip_requests(url)
    while True:
        getip_requesocks(url)
        time.sleep(10);
if __name__ == "__main__":
    main()
    

Python3

依赖包

pip3 install pysocks
pip3 install requests

测试脚本

import requests
import time
import sys
import socket
import socks


def getip_requests(url):
    print("(+) Sending request with plain requests...")
    r = requests.get(url)
    print("(+) IP is: " + r.text.replace("\n", ""))


def main():
    print("Running tests...")
    url = 'https://api.ipify.org?format=json'
    socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", 9050)
    socket.socket = socks.socksocket
    while True:
        getip_requests(url)
        time.sleep(10);


if __name__ == '__main__':
    main()

QQ群：397745473