搭建轮换代理池

import collections, itertools, requests

proxies = [l.strip() for l in open("proxies.txt") if l.strip()]
failures = collections.Counter()
pool = itertools.cycle(proxies)

def get(url, attempts=10):
    for _ in range(attempts):
        proxy = next(pool)
        if failures[proxy] >= 3:
            continue
        try:
            return requests.get(
                url,
                proxies={"http": f"http://{proxy}", "https": f"http://{proxy}"},
                timeout=8,
            )
        except requests.RequestException:
            failures[proxy] += 1
    raise RuntimeError("pool exhausted")