Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 0494975

Browse files
GermeyGermey
and
Germey
authored
Add prefix for redis env (#137)
* add env prefix * add prefix Co-authored-by: Germey <qicu@microsoft.com>
1 parent 642a595 commit 0494975

File tree

7 files changed

+62
-56
lines changed

7 files changed

+62
-56
lines changed

‎build.yaml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ services:
55
container_name: redis4proxypool
66
ports:
77
- "6374:6379"
8-
# restart: always
98
proxypool:
109
build: .
1110
image: "germey/proxypool:master"
@@ -16,4 +15,4 @@ services:
1615
# volumes:
1716
# - proxypool/crawlers/private:/app/proxypool/crawlers/private
1817
environment:
19-
REDIS_HOST: redis4proxypool
18+
PROXYPOOL_REDIS_CONNECTION_STRING: redis://@redis4proxypool:6379/0

‎docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,4 @@ services:
1414
# volumes:
1515
# - proxypool/crawlers/private:/app/proxypool/crawlers/private
1616
environment:
17-
REDIS_HOST: redis4proxypool
17+
PROXYPOOL_REDIS_HOST: redis4proxypool

‎proxypool/scheduler.py

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ class Scheduler():
1818
"""
1919
scheduler
2020
"""
21-
21+
2222
def run_tester(self, cycle=CYCLE_TESTER):
2323
"""
2424
run tester
@@ -33,7 +33,7 @@ def run_tester(self, cycle=CYCLE_TESTER):
3333
tester.run()
3434
loop += 1
3535
time.sleep(cycle)
36-
36+
3737
def run_getter(self, cycle=CYCLE_GETTER):
3838
"""
3939
run getter
@@ -48,7 +48,7 @@ def run_getter(self, cycle=CYCLE_GETTER):
4848
getter.run()
4949
loop += 1
5050
time.sleep(cycle)
51-
51+
5252
def run_server(self):
5353
"""
5454
run server for api
@@ -57,42 +57,48 @@ def run_server(self):
5757
logger.info('server not enabled, exit')
5858
return
5959
app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
60-
60+
6161
def run(self):
6262
global tester_process, getter_process, server_process
6363
try:
6464
logger.info('starting proxypool...')
6565
if ENABLE_TESTER:
66-
tester_process = multiprocessing.Process(target=self.run_tester)
66+
tester_process = multiprocessing.Process(
67+
target=self.run_tester)
6768
logger.info(f'starting tester, pid {tester_process.pid}...')
6869
tester_process.start()
69-
70+
7071
if ENABLE_GETTER:
71-
getter_process = multiprocessing.Process(target=self.run_getter)
72+
getter_process = multiprocessing.Process(
73+
target=self.run_getter)
7274
logger.info(f'starting getter, pid{getter_process.pid}...')
7375
getter_process.start()
74-
76+
7577
if ENABLE_SERVER:
76-
server_process = multiprocessing.Process(target=self.run_server)
78+
server_process = multiprocessing.Process(
79+
target=self.run_server)
7780
logger.info(f'starting server, pid{server_process.pid}...')
7881
server_process.start()
79-
80-
tester_process.join()
81-
getter_process.join()
82-
server_process.join()
82+
83+
tester_processandtester_process.join()
84+
getter_processandgetter_process.join()
85+
server_processandserver_process.join()
8386
except KeyboardInterrupt:
8487
logger.info('received keyboard interrupt signal')
85-
tester_process.terminate()
86-
getter_process.terminate()
87-
server_process.terminate()
88+
tester_processandtester_process.terminate()
89+
getter_processandgetter_process.terminate()
90+
server_processandserver_process.terminate()
8891
finally:
8992
# must call join method before calling is_alive
90-
tester_process.join()
91-
getter_process.join()
92-
server_process.join()
93-
logger.info(f'tester is {"alive" if tester_process.is_alive() else "dead"}')
94-
logger.info(f'getter is {"alive" if getter_process.is_alive() else "dead"}')
95-
logger.info(f'server is {"alive" if server_process.is_alive() else "dead"}')
93+
tester_process and tester_process.join()
94+
getter_process and getter_process.join()
95+
server_process and server_process.join()
96+
logger.info(
97+
f'tester is {"alive" if tester_process.is_alive() else "dead"}')
98+
logger.info(
99+
f'getter is {"alive" if getter_process.is_alive() else "dead"}')
100+
logger.info(
101+
f'server is {"alive" if server_process.is_alive() else "dead"}')
96102
logger.info('proxy terminated')
97103

98104

‎proxypool/setting.py

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
from os.path import dirname, abspath, join
33
from environs import Env
44
from loguru import logger
5-
from proxypool.utils.parse import parse_redis_connection_string
65

76

87
env = Env()
@@ -24,21 +23,23 @@
2423
APP_TEST = IS_TEST = APP_ENV == TEST_MODE
2524

2625
# redis host
27-
REDIS_HOST = env.str('REDIS_HOST', '127.0.0.1')
26+
REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST',
27+
env.str('REDIS_HOST', '127.0.0.1'))
2828
# redis port
29-
REDIS_PORT = env.int('REDIS_PORT', 6379)
29+
REDIS_PORT = env.int('PROXYPOOL_REDIS_PORT', env.int('REDIS_PORT', 6379))
3030
# redis password, if no password, set it to None
31-
REDIS_PASSWORD = env.str('REDIS_PASSWORD', None)
31+
REDIS_PASSWORD = env.str('PROXYPOOL_REDIS_PASSWORD',
32+
env.str('REDIS_PASSWORD', None))
3233
# redis db, if no choice, set it to 0
33-
REDIS_DB = env.int('REDIS_DB', 0)
34-
# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0
35-
REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None)
36-
37-
if REDIS_CONNECTION_STRING:
38-
REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB = parse_redis_connection_string(REDIS_CONNECTION_STRING)
34+
REDIS_DB = env.int('PROXYPOOL_REDIS_DB', env.int('REDIS_DB', 0))
35+
# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0,
36+
# please refer to https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
37+
REDIS_CONNECTION_STRING = env.str(
38+
'PROXYPOOL_REDIS_CONNECTION_STRING', env.str('REDIS_CONNECTION_STRING', None))
3939

4040
# redis hash table key name
41-
REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal')
41+
REDIS_KEY = env.str('PROXYPOOL_REDIS_KEY', env.str(
42+
'REDIS_KEY', 'proxies:universal'))
4243

4344
# definition of proxy scores
4445
PROXY_SCORE_MAX = 100
@@ -78,4 +79,3 @@
7879

7980
# logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days')
8081
# logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week')
81-

‎proxypool/storages/redis.py

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import redis
22
from proxypool.exceptions import PoolEmptyException
33
from proxypool.schemas.proxy import Proxy
4-
from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \
4+
from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \
55
PROXY_SCORE_INIT
66
from random import choice
77
from typing import List
@@ -18,14 +18,21 @@ class RedisClient(object):
1818
redis connection client of proxypool
1919
"""
2020

21-
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, **kwargs):
21+
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB,
22+
connection_string=REDIS_CONNECTION_STRING, **kwargs):
2223
"""
2324
init redis client
2425
:param host: redis host
2526
:param port: redis port
2627
:param password: redis password
28+
:param connection_string: redis connection_string
2729
"""
28-
self.db = redis.StrictRedis(host=host, port=port, password=password, db=db, decode_responses=True, **kwargs)
30+
# if set connection_string, just use it
31+
if connection_string:
32+
self.db = redis.StrictRedis.from_url(connection_string)
33+
else:
34+
self.db = redis.StrictRedis(
35+
host=host, port=port, password=password, db=db, decode_responses=True, **kwargs)
2936

3037
def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int:
3138
"""
@@ -51,11 +58,13 @@ def random(self) -> Proxy:
5158
:return: proxy, like 8.8.8.8:8
5259
"""
5360
# try to get proxy with max score
54-
proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX , PROXY_SCORE_MAX)
61+
proxies = self.db.zrangebyscore(
62+
REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX)
5563
if len(proxies):
5664
return convert_proxy_or_proxies(choice(proxies))
5765
# else get proxy by rank
58-
proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN , PROXY_SCORE_MAX)
66+
proxies = self.db.zrevrange(
67+
REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)
5968
if len(proxies):
6069
return convert_proxy_or_proxies(choice(proxies))
6170
# else raise error
@@ -125,4 +134,3 @@ def batch(self, cursor, count) -> List[Proxy]:
125134
conn = RedisClient()
126135
result = conn.random()
127136
print(result)
128-

‎proxypool/utils/parse.py

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎proxypool/utils/proxy.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33

44
def is_valid_proxy(data):
5+
"""
6+
check this string is within proxy format
7+
"""
58
if data.__contains__(':'):
69
ip = data.split(':')[0]
710
port = data.split(':')[1]
@@ -11,6 +14,9 @@ def is_valid_proxy(data):
1114

1215

1316
def is_ip_valid(ip):
17+
"""
18+
check this string is within ip format
19+
"""
1420
a = ip.split('.')
1521
if len(a) != 4:
1622
return False

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /