diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index f7138c64..aa0500ac 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,7 +1,7 @@ -from flask import Flask, g +from flask import Flask, g, request from proxypool.storages.redis import RedisClient -from proxypool.setting import API_HOST, API_PORT, API_THREADED, IS_DEV - +from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV +import functools __all__ = ['app'] @@ -10,6 +10,25 @@ app.debug = True +def auth_required(func): + @functools.wraps(func) + def decorator(*args, **kwargs): + # conditional decorator, when setting API_KEY is set, otherwise just ignore this decorator + if API_KEY == "": + return func(*args, **kwargs) + if request.headers.get('API-KEY', None) is not None: + api_key = request.headers.get('API-KEY') + else: + return {"message": "Please provide an API key in header"}, 400 + # Check if API key is correct and valid + if request.method == "GET" and api_key == API_KEY: + return func(*args, **kwargs) + else: + return {"message": "The provided API key is not valid"}, 403 + + return decorator + + def get_conn(): """ get redis client object @@ -21,6 +40,7 @@ def get_conn(): @app.route('/') +@auth_required def index(): """ get home page, you can define your own templates @@ -30,6 +50,7 @@ def index(): @app.route('/random') +@auth_required def get_proxy(): """ get a random proxy @@ -40,6 +61,7 @@ def get_proxy(): @app.route('/all') +@auth_required def get_proxy_all(): """ get a random proxy @@ -56,6 +78,7 @@ def get_proxy_all(): @app.route('/count') +@auth_required def get_count(): """ get the count of proxies diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index f002056a..353332ac 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -3,11 +3,11 @@ from loguru import logger from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient -from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS +from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ + TEST_DONT_SET_MAX_SCORE from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError - EXCEPTIONS = ( ClientProxyConnectionError, ConnectionRefusedError, @@ -23,14 +23,14 @@ class Tester(object): """ tester for testing proxies in queue """ - + def __init__(self): """ init redis """ self.redis = RedisClient() self.loop = asyncio.get_event_loop() - + async def test(self, proxy: Proxy): """ test single proxy @@ -55,15 +55,18 @@ async def test(self, proxy: Proxy): async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, allow_redirects=False) as response: if response.status in TEST_VALID_STATUS: - self.redis.max(proxy) - logger.debug(f'proxy {proxy.string()} is valid, set max score') + if TEST_DONT_SET_MAX_SCORE: + logger.debug(f'proxy {proxy.string()} is valid, remain current score') + else: + self.redis.max(proxy) + logger.debug(f'proxy {proxy.string()} is valid, set max score') else: self.redis.decrease(proxy) logger.debug(f'proxy {proxy.string()} is invalid, decrease score') except EXCEPTIONS: self.redis.decrease(proxy) logger.debug(f'proxy {proxy.string()} is invalid, decrease score') - + @logger.catch def run(self): """ @@ -84,14 +87,15 @@ def run(self): if not cursor: break + def run_tester(): host = '96.113.165.182' port = '3128' tasks = [tester.test(Proxy(host=host, port=port))] tester.loop.run_until_complete(asyncio.wait(tasks)) + if __name__ == '__main__': tester = Tester() tester.run() # run_tester() - diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index f29b5655..a2d18abe 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -92,7 +92,7 @@ def run_server(self): logger.error("unsupported APP_PROD_METHOD") return else: - app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) + app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED, use_reloader=False) def run(self): global tester_process, getter_process, server_process diff --git a/proxypool/setting.py b/proxypool/setting.py index e68f45ca..495ad55f 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -53,9 +53,9 @@ 'REDIS_KEY', 'proxies:universal')) # definition of proxy scores -PROXY_SCORE_MAX = 100 -PROXY_SCORE_MIN = 0 -PROXY_SCORE_INIT = 10 +PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100) +PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0) +PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10) # definition of proxy number PROXY_NUMBER_MAX = 50000 @@ -77,11 +77,17 @@ # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', # }) TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302]) +# whether to set max score when one proxy is tested valid +TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False) # definition of api API_HOST = env.str('API_HOST', '0.0.0.0') API_PORT = env.int('API_PORT', 5555) API_THREADED = env.bool('API_THREADED', True) +# add an api key to get proxy +# need a header of `API-KEY` in get request to pass the authenticate +# API_KEY='', do not need `API-KEY` header +API_KEY = env.str('API_KEY', '') # flags of enable ENABLE_TESTER = env.bool('ENABLE_TESTER', True)