Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 84eae85

Browse files
committed
feat: add funcy and some update
Change-Id: I29e5e1b012a122e4244e90d2a4582430946edff8
1 parent c3e4787 commit 84eae85

15 files changed

+732
-19
lines changed

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,12 @@ python的强大之处有很大的一方面在于它有各种各样非常强大
315315

316316
## [tqdm](content/tqdm.md)
317317

318+
## [schematics](content/schematics.md)
319+
320+
## [pyautogui](content/pyautogui.md)
321+
322+
## [funcy](content/funcy.md)
323+
318324
## [tools](content/tools.md)
319325

320326
## [Other_thing](content/other_thing.md)

‎code/funcy_cache_demo.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# -*- coding: utf-8 -*-
2+
import time
3+
import funcy as fc
4+
5+
from functools import wraps
6+
7+
8+
def cache(func):
9+
local_cache = {}
10+
11+
@wraps(func)
12+
def wrapper(*args, **kwargs):
13+
key = "{}:{}".format(";".join(map(str, args)),
14+
";".join(["{}:{}".format(*map(str, item)) for item in sorted(kwargs.items())]))
15+
if key in local_cache:
16+
return local_cache[key]
17+
else:
18+
value = func(*args, **kwargs)
19+
local_cache[key] = value
20+
return value
21+
22+
return wrapper
23+
24+
25+
# @cache
26+
@fc.memoize
27+
def get_timestamp(x):
28+
return int(time.time())
29+
30+
31+
if __name__ == '__main__':
32+
print(get_timestamp(1))
33+
time.sleep(1)
34+
print(get_timestamp(1))
35+
time.sleep(1)
36+
print(get_timestamp(1))
37+
time.sleep(2)
38+
print(get_timestamp(2))
39+
time.sleep(1)
40+
print(get_timestamp(2))
41+
print(get_timestamp.memory)

‎code/funcy_demo.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# -*- coding: utf-8 -*-
2+
import time
3+
import funcy as fc
4+
5+
from functools import wraps
6+
from compiler.ast import flatten as ast_flatten
7+
8+
9+
def flatten(elements):
10+
return [_ for item in elements for _ in flatten(item)] if isinstance(elements, list) else [elements]
11+
12+
13+
@fc.once
14+
def call_once():
15+
print("only once called")
16+
17+
18+
@fc.once
19+
def call_once_with_args(x):
20+
print("only once with args called")
21+
22+
23+
if __name__ == '__main__':
24+
a = [1, 2, [3, 4, [5, 6], [7, 8]], [9, 10]]
25+
print(fc.flatten(a))
26+
print(ast_flatten(a))
27+
print(flatten(a))
28+
29+
call_once()
30+
call_once()
31+
call_once_with_args(1)
32+
call_once_with_args(2)

‎code/funcy_flatten_demo.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# -*- coding: utf-8 -*-
2+
3+
import funcy as fc
4+
# only support in python2
5+
from compiler.ast import flatten as ast_flatten
6+
7+
8+
def flatten(elements):
9+
result = []
10+
for item in elements:
11+
if isinstance(item, list):
12+
result.extend(flatten(item))
13+
else:
14+
result.append(item)
15+
return result
16+
17+
18+
def old_flatten(elements):
19+
return [_ for item in elements for _ in old_flatten(item)] if isinstance(elements, list) else [elements]
20+
# result = []
21+
# for item in elements:
22+
# if isinstance(item, list):
23+
# item = old_flatten(item)
24+
# else:
25+
# item = [item]
26+
# result.extend(item)
27+
# return result
28+
29+
30+
if __name__ == '__main__':
31+
a = [1, 2, [3, 4, [5, 6], [7, 8]], [9, 10]]
32+
print(fc.flatten(a))
33+
print(ast_flatten(a))
34+
print(flatten(a))

‎code/funcy_list_demo.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# -*- coding: utf-8 -*-
2+
import funcy as fc
3+
4+
if __name__ == '__main__':
5+
a = [1,3,4,5,2,5,1,5,2,5,1]
6+
print(fc.distinct(a))
7+
print(set(a))
8+
9+
# 按个数分组,舍弃多余的元素
10+
print(fc.partition(2, range(10)))
11+
print(fc.partition(3, range(10)))
12+
# 按个数分组,多余的元素单列
13+
print(fc.chunks(2, range(10)))
14+
print(fc.chunks(3, range(10)))
15+
16+
# 此处不能用 lstrip 或者 rstrip, 因为会将输入字符串当成字符数组
17+
print("open_api_enforce_interface".lstrip("open_api"))
18+
print("open_api_enforce_interface".rstrip("_interface"))
19+
print(fc.cut_prefix("open_api_test_interface", "open_api"))
20+
print(fc.cut_suffix("open_api_test_interface", "interface"))

‎code/funcy_once_demo.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# -*- coding: utf-8 -*-
2+
import funcy as fc
3+
4+
5+
@fc.once
6+
def call_once():
7+
print("only once called")
8+
9+
10+
@fc.once
11+
def call_once_with_args(x):
12+
print("only once with args called")
13+
14+
15+
if __name__ == '__main__':
16+
call_once()
17+
call_once()
18+
call_once_with_args(1)
19+
call_once_with_args(2)

‎code/funcy_retry_demo.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# -*- coding: utf-8 -*-
2+
import time
3+
import funcy as fc
4+
5+
from functools import wraps
6+
7+
# def retry(times=3):
8+
# def inner(func):
9+
# @wraps(func)
10+
# def wrapper(*args, **kwargs):
11+
# for i in range(times):
12+
# try:
13+
# return func(*args, **kwargs)
14+
# except Exception as e:
15+
# if i == times-1:
16+
# raise e
17+
# print("occur error:%r" % e)
18+
#
19+
# return wrapper
20+
# return inner
21+
22+
# def retry(times=3):
23+
# def inner(func):
24+
# @wraps(func)
25+
# def wrapper(*args, **kwargs):
26+
# try:
27+
# return func(*args, **kwargs)
28+
# except Exception as e:
29+
# if times <= 1:
30+
# raise e
31+
# print("occur error:%r" % e)
32+
# return retry(times-1)(func)(*args, **kwargs)
33+
#
34+
# return wrapper
35+
#
36+
# return inner
37+
38+
39+
def retry(times=3):
40+
def inner(func):
41+
@wraps(func)
42+
def wrapper(*args, **kwargs):
43+
count = 0
44+
while count < times:
45+
try:
46+
return func(*args, **kwargs)
47+
except Exception as e:
48+
print("occur error:%r" % e)
49+
count += 1
50+
raise
51+
return wrapper
52+
53+
return inner
54+
55+
56+
@fc.retry(5)
57+
# @retry(3)
58+
def raise_exception():
59+
time.sleep(1)
60+
return 1 / 0
61+
62+
63+
if __name__ == '__main__':
64+
print(raise_exception())

‎content/concurrent.md‎

Lines changed: 92 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,27 @@ python 中的异步库,集成 threading 和 multiprocessing 两个库。
44

55
可以使用 `ThreadPoolExecutor` 和 `ProcessPoolExecutor` 来做线程和进程
66

7-
8-
```
7+
### 简单使用
8+
9+
这个库使用很简单,常见方法也即那么几种
10+
> [concurrent.futures --- 启动并行任务](https://docs.python.org/zh-cn/3/library/concurrent.futures.html)
11+
12+
- concurrent.futures.Executor|ThreadPoolExecutor|ProcessPoolExecutor
13+
- submit
14+
- map
15+
- shutdown
16+
- concurrent.futures.Future
17+
- cancel
18+
- cancel
19+
- cancel
20+
- done
21+
- cancel
22+
- exception
23+
- add_done_callback
24+
- concurrent.futures.wait
25+
- concurrent.futures.as_completed
26+
27+
```python
928
# -*- coding: utf-8 -*-
1029

1130
import time
@@ -76,12 +95,78 @@ if __name__ == '__main__':
7695

7796
```
7897

79-
参考链接
98+
### 动态添加任务
99+
100+
```python
101+
# -*- coding: utf-8 -*-
102+
import time
103+
import random
104+
import traceback
80105

81-
[使用Python进行并发编程-PoolExecutor篇](http://www.dongwm.com/archives/%E4%BD%BF%E7%94%A8Python%E8%BF%9B%E8%A1%8C%E5%B9%B6%E5%8F%91%E7%BC%96%E7%A8%8B-PoolExecutor%E7%AF%87/)
106+
from concurrent import futures
107+
from concurrent.futures import ThreadPoolExecutor
108+
109+
result = []
110+
111+
112+
def handle(num, pool):
113+
try:
114+
print("thread for:%d" % num)
115+
sleep_time = random.random()
116+
time.sleep(sleep_time)
117+
if sleep_time < 0.95:
118+
print("thread pool add one")
119+
print("before pool size:%d" % pool._work_queue.qsize())
120+
pool.submit(handle, num*10, pool)
121+
# result.append(pool.submit(handle, 111, pool))
122+
print("after pool size:%d" % pool._work_queue.qsize())
123+
except Exception as e:
124+
# 问题在这里:
125+
# py2: cannot schedule new futures after shutdown
126+
# py3: cannot schedule new futures after interpreter shutdown
127+
print(repr(e))
128+
traceback.print_stack()
129+
130+
131+
if __name__ == '__main__':
132+
# 使用 map 的时候需要注意,其实 pool 会自动帮你把参数 zip 一下,先合并再分别映射
133+
# 其实最后不用调用 shutdown 也会自动等待所有任务结束,然后主进程结束
134+
thread_pool = ThreadPoolExecutor(30)
135+
# map_result = thread_pool.map(handle, range(2), [thread_pool] * 2)
136+
# result.extend(list(map_result))
137+
# 实际上,使用 submit 多个参数的时候也需要注意,参数直接摆开,和 threading 不一样
138+
for i in range(4):
139+
result.append(thread_pool.submit(handle, i, thread_pool))
140+
print("start threading")
141+
# 在没有 shutdown 之后为什么还会自动结束呢?谁在
142+
# thread_pool.shutdown()
143+
while result:
144+
print("before result length:%d" % len(result))
145+
for f in futures.as_completed(result):
146+
result.remove(f)
147+
# 此处还是有风险,如果这里结束了,全都结束了之后,还是会自动调用 shutdown
148+
# 其实还有待提交的请求,这样就尴尬了。
149+
# 但是加等待时间也不能够
150+
if len(result) == 0:
151+
time.sleep(2)
152+
print("after result length:%d" % len(result))
153+
# 还是用 lock 锁定一个计数器比较现实,计数器归零再结束。
154+
155+
# futures.wait(result)
156+
# thread_pool.shutdown()
157+
# 不需要 as_completed 也不需要手动 wait,实际上在 shutdown 的时候其实就是会等待结束的
158+
# 如果使用上下文管理器的话会自动 shutdown
159+
# 问题是在子线程中不能再往线程中加任务了,有点问题
160+
# with ThreadPoolExecutor(30) as thread_pool:
161+
# result = thread_pool.map(handle, range(2), [thread_pool] * 2)
162+
# print("start threading")
163+
# thread_pool.submit(handle, 111, thread_pool)
82164

83-
[使用Python的 concurrent.futures 模块](https://python-parallel-programmning-cookbook.readthedocs.io/zh_CN/latest/chapter4/02_Using_the_concurrent.futures_Python_modules.html)
165+
```
84166

85-
[python并发库:concurrent.futures的使用](https://blog.csdn.net/drdairen/article/details/69487643)
167+
参考链接
86168

87-
[python并发 1:使用 futures 处理并发](https://segmentfault.com/a/1190000009819359)
169+
[使用Python进行并发编程-PoolExecutor篇](http://www.dongwm.com/archives/%E4%BD%BF%E7%94%A8Python%E8%BF%9B%E8%A1%8C%E5%B9%B6%E5%8F%91%E7%BC%96%E7%A8%8B-PoolExecutor%E7%AF%87/)
170+
[使用Python的 concurrent.futures 模块](https://python-parallel-programmning-cookbook.readthedocs.io/zh_CN/latest/chapter4/02_Using_the_concurrent.futures_Python_modules.html)
171+
[python并发库:concurrent.futures的使用](https://blog.csdn.net/drdairen/article/details/69487643)
172+
[python并发 1:使用 futures 处理并发](https://segmentfault.com/a/1190000009819359)

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /