Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 8a21396

Browse files
author
Germey
committed
add env prefix
1 parent 08385f6 commit 8a21396

File tree

4 files changed

+58
-38
lines changed

4 files changed

+58
-38
lines changed

‎proxypool/scheduler.py

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ class Scheduler():
1818
"""
1919
scheduler
2020
"""
21-
21+
2222
def run_tester(self, cycle=CYCLE_TESTER):
2323
"""
2424
run tester
@@ -33,7 +33,7 @@ def run_tester(self, cycle=CYCLE_TESTER):
3333
tester.run()
3434
loop += 1
3535
time.sleep(cycle)
36-
36+
3737
def run_getter(self, cycle=CYCLE_GETTER):
3838
"""
3939
run getter
@@ -48,7 +48,7 @@ def run_getter(self, cycle=CYCLE_GETTER):
4848
getter.run()
4949
loop += 1
5050
time.sleep(cycle)
51-
51+
5252
def run_server(self):
5353
"""
5454
run server for api
@@ -57,42 +57,48 @@ def run_server(self):
5757
logger.info('server not enabled, exit')
5858
return
5959
app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
60-
60+
6161
def run(self):
6262
global tester_process, getter_process, server_process
6363
try:
6464
logger.info('starting proxypool...')
6565
if ENABLE_TESTER:
66-
tester_process = multiprocessing.Process(target=self.run_tester)
66+
tester_process = multiprocessing.Process(
67+
target=self.run_tester)
6768
logger.info(f'starting tester, pid {tester_process.pid}...')
6869
tester_process.start()
69-
70+
7071
if ENABLE_GETTER:
71-
getter_process = multiprocessing.Process(target=self.run_getter)
72+
getter_process = multiprocessing.Process(
73+
target=self.run_getter)
7274
logger.info(f'starting getter, pid{getter_process.pid}...')
7375
getter_process.start()
74-
76+
7577
if ENABLE_SERVER:
76-
server_process = multiprocessing.Process(target=self.run_server)
78+
server_process = multiprocessing.Process(
79+
target=self.run_server)
7780
logger.info(f'starting server, pid{server_process.pid}...')
7881
server_process.start()
79-
80-
tester_process.join()
81-
getter_process.join()
82-
server_process.join()
82+
83+
tester_processandtester_process.join()
84+
getter_processandgetter_process.join()
85+
server_processandserver_process.join()
8386
except KeyboardInterrupt:
8487
logger.info('received keyboard interrupt signal')
85-
tester_process.terminate()
86-
getter_process.terminate()
87-
server_process.terminate()
88+
tester_processandtester_process.terminate()
89+
getter_processandgetter_process.terminate()
90+
server_processandserver_process.terminate()
8891
finally:
8992
# must call join method before calling is_alive
90-
tester_process.join()
91-
getter_process.join()
92-
server_process.join()
93-
logger.info(f'tester is {"alive" if tester_process.is_alive() else "dead"}')
94-
logger.info(f'getter is {"alive" if getter_process.is_alive() else "dead"}')
95-
logger.info(f'server is {"alive" if server_process.is_alive() else "dead"}')
93+
tester_process and tester_process.join()
94+
getter_process and getter_process.join()
95+
server_process and server_process.join()
96+
logger.info(
97+
f'tester is {"alive" if tester_process.is_alive() else "dead"}')
98+
logger.info(
99+
f'getter is {"alive" if getter_process.is_alive() else "dead"}')
100+
logger.info(
101+
f'server is {"alive" if server_process.is_alive() else "dead"}')
96102
logger.info('proxy terminated')
97103

98104

‎proxypool/setting.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,19 @@
2424
APP_TEST = IS_TEST = APP_ENV == TEST_MODE
2525

2626
# redis host
27-
REDIS_HOST = env.str('REDIS_HOST', '127.0.0.1')
27+
REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST',
28+
env.str('REDIS_HOST', '127.0.0.1'))
2829
# redis port
29-
REDIS_PORT = env.int('REDIS_PORT', 6379)
30+
REDIS_PORT = env.int('PROXYPOOL_REDIS_PORT', env.int('REDIS_PORT', 6379))
3031
# redis password, if no password, set it to None
31-
REDIS_PASSWORD = env.str('REDIS_PASSWORD', None)
32+
REDIS_PASSWORD = env.str('PROXYPOOL_REDIS_PASSWORD',
33+
env.str('REDIS_PASSWORD', None))
3234
# redis db, if no choice, set it to 0
33-
REDIS_DB = env.int('REDIS_DB', 0)
34-
# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0
35-
REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None)
36-
37-
if REDIS_CONNECTION_STRING:
38-
REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB = parse_redis_connection_string(REDIS_CONNECTION_STRING)
35+
REDIS_DB = env.int('PROXYPOOL_REDIS_DB', env.int('REDIS_DB', 0))
36+
# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0,
37+
# please refer to https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
38+
REDIS_CONNECTION_STRING = env.str(
39+
'PROXYPOOL_REDIS_CONNECTION_STRING', env.str('REDIS_CONNECTION_STRING', None))
3940

4041
# redis hash table key name
4142
REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal')
@@ -78,4 +79,3 @@
7879

7980
# logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days')
8081
# logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week')
81-

‎proxypool/storages/redis.py

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import redis
22
from proxypool.exceptions import PoolEmptyException
33
from proxypool.schemas.proxy import Proxy
4-
from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \
4+
from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \
55
PROXY_SCORE_INIT
66
from random import choice
77
from typing import List
@@ -18,14 +18,21 @@ class RedisClient(object):
1818
redis connection client of proxypool
1919
"""
2020

21-
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, **kwargs):
21+
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB,
22+
connection_string=REDIS_CONNECTION_STRING, **kwargs):
2223
"""
2324
init redis client
2425
:param host: redis host
2526
:param port: redis port
2627
:param password: redis password
28+
:param connection_string: redis connection_string
2729
"""
28-
self.db = redis.StrictRedis(host=host, port=port, password=password, db=db, decode_responses=True, **kwargs)
30+
# if set connection_string, just use it
31+
if connection_string:
32+
self.db = redis.StrictRedis.from_url(connection_string)
33+
else:
34+
self.db = redis.StrictRedis(
35+
host=host, port=port, password=password, db=db, decode_responses=True, **kwargs)
2936

3037
def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int:
3138
"""
@@ -51,11 +58,13 @@ def random(self) -> Proxy:
5158
:return: proxy, like 8.8.8.8:8
5259
"""
5360
# try to get proxy with max score
54-
proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX , PROXY_SCORE_MAX)
61+
proxies = self.db.zrangebyscore(
62+
REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX)
5563
if len(proxies):
5664
return convert_proxy_or_proxies(choice(proxies))
5765
# else get proxy by rank
58-
proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN , PROXY_SCORE_MAX)
66+
proxies = self.db.zrevrange(
67+
REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)
5968
if len(proxies):
6069
return convert_proxy_or_proxies(choice(proxies))
6170
# else raise error
@@ -125,4 +134,3 @@ def batch(self, cursor, count) -> List[Proxy]:
125134
conn = RedisClient()
126135
result = conn.random()
127136
print(result)
128-

‎proxypool/utils/proxy.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33

44
def is_valid_proxy(data):
5+
"""
6+
check this string is within proxy format
7+
"""
58
if data.__contains__(':'):
69
ip = data.split(':')[0]
710
port = data.split(':')[1]
@@ -11,6 +14,9 @@ def is_valid_proxy(data):
1114

1215

1316
def is_ip_valid(ip):
17+
"""
18+
check this string is within ip format
19+
"""
1420
a = ip.split('.')
1521
if len(a) != 4:
1622
return False

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /