Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 5d2c665

Browse files
authored
Merge pull request #8 from adhorn/failurerate
adding possibility for a controlled failure rate
2 parents 567a0bf + 19c4708 commit 5d2c665

File tree

3 files changed

+38
-29
lines changed

3 files changed

+38
-29
lines changed

‎README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,16 @@
1010
* Using for SSM Parameter Store to control the experiment using ```isEnabled```
1111
* Per Lambda function injection control using Environment variable (```FAILURE_INJECTION_PARAM```) (thanks to Gunnar Grosh)
1212
* Support for Serverless Framework using ```sls deploy``` (thanks to Gunnar Grosh)
13+
* Support for adding rate of failure using ```rate```. (Default rate = 1)
1314

1415
### Parameter Store Object
1516
```json
1617
{
1718
"delay": 200,
1819
"isEnabled": true,
1920
"error_code": 404,
20-
"exception_msg": "I FAILED"
21+
"exception_msg": "I FAILED",
22+
"rate": 0.5
2123
}
2224
```
2325
Deploy the chaos config in paramater store.

‎python/chaos_lib.py‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,9 @@ def get_config(config_key):
2727
param = SSMParameter(os.environ['FAILURE_INJECTION_PARAM'])
2828
try:
2929
value = json.loads(param.value)
30-
isEnabled = value["isEnabled"]
31-
if not isEnabled:
30+
if not value["isEnabled"]:
3231
return 0
33-
key_ = value[config_key]
34-
return key_
32+
return value[config_key], value.get('rate', 1)
3533
except InvalidParameterError as e:
3634
# key does not exist in SSM
3735
raise InvalidParameterError("{} does not exist in SSM".format(e))
@@ -42,18 +40,14 @@ def get_config(config_key):
4240

4341
def corrupt_delay(func):
4442
def latency(*args, **kw):
45-
delay = get_config('delay')
46-
print(delay)
43+
delay, rate = get_config('delay')
44+
print("delay: {0}, rate: {1}".format(delay, rate))
4745
start = time.time()
48-
# if delay exist, delaying with that value
49-
if delay > 0:
50-
time.sleep(delay / 1000.0)
51-
# if delay = -1 make random delays
52-
elif delay < 0:
53-
# add latency approx 50% of the time
54-
if random.random() > 0.5:
55-
# random sleep time between 1 and 10 seconds
56-
time.sleep(random.randint(1, 10))
46+
# if delay and rate exist, delaying with that value at that rate
47+
if delay > 0 and rate >= 0:
48+
# add latency approx rate% of the time
49+
if random.random() <= rate:
50+
time.sleep(delay / 1000.0)
5751

5852
result = func(*args, **kw)
5953
end = time.time()
@@ -69,19 +63,27 @@ def latency(*args, **kw):
6963
def corrupt_expection(func):
7064
def wrapper(*args, **kwargs):
7165
result = func(*args, **kwargs)
72-
exception_msg = get_config('exception_msg')
73-
print("exception_msg from config {}".format(exception_msg))
74-
print("corrupting now")
75-
raise Exception(exception_msg)
66+
exception_msg, rate = get_config('exception_msg')
67+
print("exception_msg from config {0} with a rate of {1}".format(exception_msg, rate))
68+
# add injection approx rate% of the time
69+
if random.random() <= rate:
70+
print("corrupting now")
71+
raise Exception(exception_msg)
72+
else:
73+
return result
7674
return wrapper
7775

7876

7977
def corrupt_statuscode(func):
8078
def wrapper(*args, **kwargs):
8179
result = func(*args, **kwargs)
82-
error_code = get_config('error_code')
83-
print("Error from config {}".format(error_code))
84-
print("corrupting now")
85-
result['statusCode'] = error_code
86-
return result
80+
error_code, rate = get_config('error_code')
81+
print("Error from config {0} at a rate of {1}".format(error_code, rate))
82+
# add injection approx rate% of the time
83+
if random.random() <= rate:
84+
print("corrupting now")
85+
result['statusCode'] = error_code
86+
return result
87+
else:
88+
return result
8789
return wrapper

‎python/test.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class TestStringMethods(unittest.TestCase):
2323
def setUp(self):
2424
os.environ['FAILURE_INJECTION_PARAM'] = 'test.config'
2525
client.put_parameter(
26-
Value="{ \"delay\": 200, \"isEnabled\": true, \"error_code\": 404, \"exception_msg\": \"I FAILED\"}",
26+
Value="{ \"delay\": 200, \"isEnabled\": true, \"error_code\": 404, \"exception_msg\": \"I FAILED\", \"rate\": 0.5 }",
2727
Name='test.config',
2828
Type='String',
2929
Overwrite=True
@@ -35,18 +35,23 @@ def tearDown(self):
3535

3636
@ignore_warnings
3737
def test_get_config(self):
38-
isEnabled = get_config('isEnabled')
38+
isEnabled, rate = get_config('isEnabled')
3939
self.assertEqual(isEnabled, True or False)
40+
self.assertEqual(rate, 0.5)
4041

4142
@ignore_warnings
4243
def test_get_config_delay(self):
43-
delay = get_config('delay')
44+
delay, rate = get_config('delay')
4445
self.assertEqual(delay, 200)
46+
self.assertEqual(rate, 0.5)
47+
4548

4649
@ignore_warnings
4750
def test_get_config_error_code(self):
48-
delay = get_config('error_code')
51+
delay, rate = get_config('error_code')
4952
self.assertEqual(delay, 404)
53+
self.assertEqual(rate, 0.5)
54+
5055

5156
@ignore_warnings
5257
def test_get_config_bad_key(self):

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /