Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 0a6d861

Browse files
authored
#140 及修改可环境变量配置的参数 (#202)
* 1、设置scheduler.py中dev模式的flask run,不进行启动时的自动重新加载reload * 1、修改PROXY_SCORE_MAX,PROXY_SCORE_MIN,PROXY_SCORE_INIT三项配置,为可环境变量配置 2、添加可环境变量配置项TEST_DONT_SET_MAX_SCORE,允许设置当tester检测到某个proxy可用时,只是保持原score,而不将其score设置成max。 * 增加获取proxy接口的认证header, API-KEY。可配置,默认不需要
1 parent a0edcb0 commit 0a6d861

File tree

4 files changed

+48
-15
lines changed

4 files changed

+48
-15
lines changed

‎proxypool/processors/server.py

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
from flask import Flask, g
1+
from flask import Flask, g, request
22
from proxypool.storages.redis import RedisClient
3-
from proxypool.setting import API_HOST, API_PORT, API_THREADED, IS_DEV
4-
3+
from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV
4+
importfunctools
55

66
__all__ = ['app']
77

@@ -10,6 +10,25 @@
1010
app.debug = True
1111

1212

13+
def auth_required(func):
14+
@functools.wraps(func)
15+
def decorator(*args, **kwargs):
16+
# conditional decorator, when setting API_KEY is set, otherwise just ignore this decorator
17+
if API_KEY == "":
18+
return func(*args, **kwargs)
19+
if request.headers.get('API-KEY', None) is not None:
20+
api_key = request.headers.get('API-KEY')
21+
else:
22+
return {"message": "Please provide an API key in header"}, 400
23+
# Check if API key is correct and valid
24+
if request.method == "GET" and api_key == API_KEY:
25+
return func(*args, **kwargs)
26+
else:
27+
return {"message": "The provided API key is not valid"}, 403
28+
29+
return decorator
30+
31+
1332
def get_conn():
1433
"""
1534
get redis client object
@@ -21,6 +40,7 @@ def get_conn():
2140

2241

2342
@app.route('/')
43+
@auth_required
2444
def index():
2545
"""
2646
get home page, you can define your own templates
@@ -30,6 +50,7 @@ def index():
3050

3151

3252
@app.route('/random')
53+
@auth_required
3354
def get_proxy():
3455
"""
3556
get a random proxy
@@ -40,6 +61,7 @@ def get_proxy():
4061

4162

4263
@app.route('/all')
64+
@auth_required
4365
def get_proxy_all():
4466
"""
4567
get a random proxy
@@ -56,6 +78,7 @@ def get_proxy_all():
5678

5779

5880
@app.route('/count')
81+
@auth_required
5982
def get_count():
6083
"""
6184
get the count of proxies

‎proxypool/processors/tester.py

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
from loguru import logger
44
from proxypool.schemas import Proxy
55
from proxypool.storages.redis import RedisClient
6-
from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS
6+
from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \
7+
TEST_DONT_SET_MAX_SCORE
78
from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError
89
from asyncio import TimeoutError
910

10-
1111
EXCEPTIONS = (
1212
ClientProxyConnectionError,
1313
ConnectionRefusedError,
@@ -23,14 +23,14 @@ class Tester(object):
2323
"""
2424
tester for testing proxies in queue
2525
"""
26-
26+
2727
def __init__(self):
2828
"""
2929
init redis
3030
"""
3131
self.redis = RedisClient()
3232
self.loop = asyncio.get_event_loop()
33-
33+
3434
async def test(self, proxy: Proxy):
3535
"""
3636
test single proxy
@@ -55,15 +55,18 @@ async def test(self, proxy: Proxy):
5555
async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT,
5656
allow_redirects=False) as response:
5757
if response.status in TEST_VALID_STATUS:
58-
self.redis.max(proxy)
59-
logger.debug(f'proxy {proxy.string()} is valid, set max score')
58+
if TEST_DONT_SET_MAX_SCORE:
59+
logger.debug(f'proxy {proxy.string()} is valid, remain current score')
60+
else:
61+
self.redis.max(proxy)
62+
logger.debug(f'proxy {proxy.string()} is valid, set max score')
6063
else:
6164
self.redis.decrease(proxy)
6265
logger.debug(f'proxy {proxy.string()} is invalid, decrease score')
6366
except EXCEPTIONS:
6467
self.redis.decrease(proxy)
6568
logger.debug(f'proxy {proxy.string()} is invalid, decrease score')
66-
69+
6770
@logger.catch
6871
def run(self):
6972
"""
@@ -84,14 +87,15 @@ def run(self):
8487
if not cursor:
8588
break
8689

90+
8791
def run_tester():
8892
host = '96.113.165.182'
8993
port = '3128'
9094
tasks = [tester.test(Proxy(host=host, port=port))]
9195
tester.loop.run_until_complete(asyncio.wait(tasks))
9296

97+
9398
if __name__ == '__main__':
9499
tester = Tester()
95100
tester.run()
96101
# run_tester()
97-

‎proxypool/scheduler.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def run_server(self):
9292
logger.error("unsupported APP_PROD_METHOD")
9393
return
9494
else:
95-
app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
95+
app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED, use_reloader=False)
9696

9797
def run(self):
9898
global tester_process, getter_process, server_process

‎proxypool/setting.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,9 @@
5353
'REDIS_KEY', 'proxies:universal'))
5454

5555
# definition of proxy scores
56-
PROXY_SCORE_MAX = 100
57-
PROXY_SCORE_MIN = 0
58-
PROXY_SCORE_INIT = 10
56+
PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100)
57+
PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0)
58+
PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10)
5959

6060
# definition of proxy number
6161
PROXY_NUMBER_MAX = 50000
@@ -77,11 +77,17 @@
7777
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
7878
# })
7979
TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302])
80+
# whether to set max score when one proxy is tested valid
81+
TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False)
8082

8183
# definition of api
8284
API_HOST = env.str('API_HOST', '0.0.0.0')
8385
API_PORT = env.int('API_PORT', 5555)
8486
API_THREADED = env.bool('API_THREADED', True)
87+
# add an api key to get proxy
88+
# need a header of `API-KEY` in get request to pass the authenticate
89+
# API_KEY='', do not need `API-KEY` header
90+
API_KEY = env.str('API_KEY', '')
8591

8692
# flags of enable
8793
ENABLE_TESTER = env.bool('ENABLE_TESTER', True)

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /