Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 7c77ad0

Browse files
committed
validate testing ip
1 parent 78b3244 commit 7c77ad0

File tree

4 files changed

+29
-44
lines changed

4 files changed

+29
-44
lines changed

‎README.md‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,6 @@ proxypool | 2020年02月19日 17:09:46,596 INFO success: tester entered RUNNING stat
7474

7575
这时候访问 [http://localhost:5555/random](http://localhost:5555/random) 即可获取一个随机可用代理。
7676

77-
当然你也可以选择自己 Build,直接运行如下命令即可:
78-
79-
```
80-
docker-compose -f build.yaml up
81-
```
82-
8377
如果下载速度特别慢,可以自行修改 Dockerfile,修改:
8478

8579
```diff
@@ -347,11 +341,6 @@ class Daili66Crawler(BaseCrawler):
347341
348342
本项目提供了 Kubernetes 部署脚本,如需部署到 Kubernetes,请参考 [kubernetes](./kubernetes)。
349343
350-
## 待开发
351-
352-
- [ ] 前端页面管理
353-
- [ ] 使用情况统计分析
354-
355344
如有一起开发的兴趣可以在 Issue 留言,非常感谢!
356345
357346
## LICENSE

‎build.yaml‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎docker-compose.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,15 @@ services:
33
redis4proxypool:
44
image: redis:alpine
55
container_name: redis4proxypool
6-
# ports:
7-
# - "6374:6379"
86
proxypool:
7+
build: .
98
image: "germey/proxypool:master"
109
container_name: proxypool
1110
ports:
1211
- "5555:5555"
1312
restart: always
1413
# volumes:
15-
# - proxypool/crawlers/private:/app/proxypool/crawlers/private
14+
# - proxypool/crawlers/private:~/proxypool/crawlers/private
1615
environment:
1716
PROXYPOOL_REDIS_HOST: redis4proxypool
1817

‎proxypool/processors/tester.py‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -45,27 +45,33 @@ async def test(self, proxy: Proxy):
4545
logger.debug(f'testing {proxy.string()}')
4646
# if TEST_ANONYMOUS is True, make sure that
4747
# the proxy has the effect of hiding the real IP
48+
# logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}')
4849
if TEST_ANONYMOUS:
4950
url = 'https://httpbin.org/ip'
5051
async with session.get(url, timeout=TEST_TIMEOUT) as response:
5152
resp_json = await response.json()
5253
origin_ip = resp_json['origin']
54+
# logger.debug(f'origin ip is {origin_ip}')
5355
async with session.get(url, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT) as response:
5456
resp_json = await response.json()
5557
anonymous_ip = resp_json['origin']
58+
logger.debug(f'anonymous ip is {anonymous_ip}')
5659
assert origin_ip != anonymous_ip
5760
assert proxy.host == anonymous_ip
5861
async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT,
5962
allow_redirects=False) as response:
6063
if response.status in TEST_VALID_STATUS:
6164
if TEST_DONT_SET_MAX_SCORE:
62-
logger.debug(f'proxy {proxy.string()} is valid, remain current score')
65+
logger.debug(
66+
f'proxy {proxy.string()} is valid, remain current score')
6367
else:
6468
self.redis.max(proxy)
65-
logger.debug(f'proxy {proxy.string()} is valid, set max score')
69+
logger.debug(
70+
f'proxy {proxy.string()} is valid, set max score')
6671
else:
6772
self.redis.decrease(proxy)
68-
logger.debug(f'proxy {proxy.string()} is invalid, decrease score')
73+
logger.debug(
74+
f'proxy {proxy.string()} is invalid, decrease score')
6975
# if independent tester class found, create new set of storage and do the extra test
7076
for tester in self.testers:
7177
key = tester.key
@@ -82,18 +88,25 @@ async def test(self, proxy: Proxy):
8288
is_valid = await tester.parse(resp_text, test_url, proxy.string())
8389
if is_valid:
8490
if tester.test_dont_set_max_score:
85-
logger.info(f'key[{key}] proxy {proxy.string()} is valid, remain current score')
91+
logger.info(
92+
f'key[{key}] proxy {proxy.string()} is valid, remain current score')
8693
else:
87-
self.redis.max(proxy, key, tester.proxy_score_max)
88-
logger.info(f'key[{key}] proxy {proxy.string()} is valid, set max score')
94+
self.redis.max(
95+
proxy, key, tester.proxy_score_max)
96+
logger.info(
97+
f'key[{key}] proxy {proxy.string()} is valid, set max score')
8998
else:
90-
self.redis.decrease(proxy, tester.key, tester.proxy_score_min)
91-
logger.info(f'key[{key}] proxy {proxy.string()} is invalid, decrease score')
99+
self.redis.decrease(
100+
proxy, tester.key, tester.proxy_score_min)
101+
logger.info(
102+
f'key[{key}] proxy {proxy.string()} is invalid, decrease score')
92103

93104
except EXCEPTIONS:
94105
self.redis.decrease(proxy)
95-
[self.redis.decrease(proxy, tester.key, tester.proxy_score_min) for tester in self.testers]
96-
logger.debug(f'proxy {proxy.string()} is invalid, decrease score')
106+
[self.redis.decrease(proxy, tester.key, tester.proxy_score_min)
107+
for tester in self.testers]
108+
logger.debug(
109+
f'proxy {proxy.string()} is invalid, decrease score')
97110

98111
@logger.catch
99112
def run(self):
@@ -107,10 +120,12 @@ def run(self):
107120
logger.debug(f'{count} proxies to test')
108121
cursor = 0
109122
while True:
110-
logger.debug(f'testing proxies use cursor {cursor}, count {TEST_BATCH}')
123+
logger.debug(
124+
f'testing proxies use cursor {cursor}, count {TEST_BATCH}')
111125
cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH)
112126
if proxies:
113-
tasks = [self.loop.create_task(self.test(proxy)) for proxy in proxies]
127+
tasks = [self.loop.create_task(
128+
self.test(proxy)) for proxy in proxies]
114129
self.loop.run_until_complete(asyncio.wait(tasks))
115130
if not cursor:
116131
break

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /