|
|
|
@ -24,12 +24,9 @@ headers = { |
|
|
|
|
lock = threading.Lock() # 多线程全局资源锁 |
|
|
|
|
total = 1 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
address = '127.0.0.1:1080' |
|
|
|
|
address = '127.0.0.1:8118' |
|
|
|
|
# proxies = {"http": "socks5://{0}".format(address), "https": "socks5://{0}".format(address)} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proxies={"http": address,"https": address} |
|
|
|
|
proxies = {"http": "http://{0}".format(address), "https": "https://{0}".format(address)} |
|
|
|
|
|
|
|
|
|
def get_cookies(): |
|
|
|
|
_cookies = {} |
|
|
|
@ -41,7 +38,11 @@ def get_cookies(): |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def getReq(url, **kwargs): |
|
|
|
|
return requests.get(url, headers=headers, cookies=get_cookies(), proxies=proxies, **kwargs) |
|
|
|
|
try: |
|
|
|
|
return requests.get(url, headers=headers, cookies=get_cookies(), proxies=proxies, **kwargs) |
|
|
|
|
except Exception as e: |
|
|
|
|
logger.error(e) |
|
|
|
|
raise e |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def crawl(url): |
|
|
|
|