用于网页采集的免费代理

curl -sL https://raw.githubusercontent.com/xyzs996/free-proxy-health-list/main/all.txt -o proxies.txt

import itertools, requests

proxies = [line.strip() for line in open("proxies.txt") if line.strip()]
pool = itertools.cycle(proxies)

def fetch(url):
    for _ in range(10):
        proxy = next(pool)
        try:
            return requests.get(
                url,
                proxies={"http": f"http://{proxy}", "https": f"http://{proxy}"},
                timeout=8,
            )
        except requests.RequestException:
            continue
    raise RuntimeError("no working proxy")