Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 4d143ce

Browse files
Add core function with pyLambda structure
1 parent 2e1f9c4 commit 4d143ce

File tree

2 files changed

+334
-0
lines changed

2 files changed

+334
-0
lines changed

‎pylambda/monitor_lambda/event.json

Whitespace-only changes.

‎pylambda/monitor_lambda/service.py

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
# -*- coding: utf-8 -*-
2+
from __future__ import division
3+
import boto3
4+
import re
5+
import requests
6+
import json
7+
from botocore.exceptions import ClientError
8+
import config
9+
10+
11+
def dynamodb_scan(dynamo_client, table_name):
12+
table = dynamo_client.Table(table_name)
13+
try:
14+
response = table.scan()
15+
except ClientError as e:
16+
print(e.response['Error']['Message'])
17+
else:
18+
if 'Items' not in response:
19+
return None
20+
else:
21+
item = response['Items']
22+
return item
23+
return None
24+
25+
26+
def dynamodb_insert_data(dynamodb, table_name, content_arr):
27+
result = False
28+
table = dynamodb.Table(table_name)
29+
try:
30+
insert_result = table.put_item(
31+
Item=content_arr
32+
)
33+
if insert_result['ResponseMetadata']['HTTPStatusCode'] == 200:
34+
result = True
35+
except ClientError as e:
36+
print(e.response['Error']['Message'])
37+
38+
return result
39+
40+
41+
'''
42+
dynamodb = boto3.resource("dynamodb",
43+
aws_access_key_id=config.aws_access_key_id,
44+
aws_secret_access_key=config.aws_secret_access_key,
45+
region_name=config.aws_region)
46+
content_arr = {
47+
"monitor_id": "TEST2",
48+
"alert": False,
49+
"warning": False
50+
}
51+
52+
insert_result = dynamodb_insert_data(dynamodb, config.table_name_current, content_arr)
53+
print insert_result
54+
'''
55+
56+
57+
def dynamodb_read_pkey(dynamo_client, table_name, key, index):
58+
table = dynamo_client.Table(table_name)
59+
try:
60+
response = table.get_item(
61+
Key={
62+
key: str(index)
63+
}
64+
)
65+
except ClientError as e:
66+
print(e.response['Error']['Message'])
67+
else:
68+
if 'Item' not in response:
69+
return None
70+
else:
71+
item = response['Item']
72+
return item
73+
return None
74+
75+
76+
'''
77+
dynamo_client = boto3.resource("dynamodb",
78+
aws_access_key_id=config.aws_access_key_id,
79+
aws_secret_access_key=config.aws_secret_access_key,
80+
region_name=config.aws_region)
81+
82+
83+
item = dynamodb_read_pkey(dynamo_client, config.table_name_current, 'monitor_id', 'Cyclops-Beta-SignIn_FailedRate')
84+
print item
85+
'''
86+
87+
88+
def dynamodb_delete_item(dynamo_client, table_name, key, value):
89+
table = dynamo_client.Table(table_name)
90+
try:
91+
table.delete_item(
92+
Key={
93+
key: value
94+
}
95+
)
96+
except ClientError as e:
97+
return False
98+
else:
99+
return True
100+
101+
102+
'''
103+
dynamo_client = boto3.resource("dynamodb",
104+
aws_access_key_id=config.aws_access_key_id,
105+
aws_secret_access_key=config.aws_secret_access_key,
106+
region_name=config.aws_region)
107+
delete_result = dynamodb_delete_item(dynamo_client, config.table_name_current, 'monitor_id', 'TEST2')
108+
print delete_result
109+
'''
110+
111+
112+
def get_log_hit(elk_conn, json_data):
113+
r = requests.post(elk_conn + "?size=0", data=json_data)
114+
json_res = json.loads(r.text)
115+
return json_res['hits']['total']
116+
117+
118+
def get_log_dict_value(elk_conn, json_data, value_param):
119+
120+
r = requests.post(elk_conn + "?size=0", data=json_data)
121+
json_res = json.loads(r.text)
122+
123+
dict_list = value_param.split(".")
124+
125+
str_key = ""
126+
for key in dict_list:
127+
str_key += '[\'' + key + '\']'
128+
129+
return eval("json_res" + str_key)
130+
131+
132+
def slack_notify(slack_webhook, post_data):
133+
r = requests.post(slack_webhook, data=post_data)
134+
if r == '200':
135+
return True
136+
else:
137+
return False
138+
139+
#post_data = {"text": "Test"}
140+
#slack_notify("http://SLACK_WEB_HOOK_URL", json.dumps(post_data))
141+
142+
143+
def monitor_current_update(monitor_id, alarm_level, alarm_bool):
144+
dynamo_client = boto3.resource("dynamodb",
145+
aws_access_key_id=config.aws_access_key_id,
146+
aws_secret_access_key=config.aws_secret_access_key,
147+
region_name=config.aws_region)
148+
table = dynamo_client .Table(config.table_name_current)
149+
try:
150+
table.update_item(
151+
Key={
152+
'monitor_id': monitor_id
153+
},
154+
UpdateExpression="set " + alarm_level + " = :p",
155+
ExpressionAttributeValues={
156+
':p': alarm_bool
157+
},
158+
ReturnValues="UPDATED_NEW"
159+
)
160+
except ClientError:
161+
return False
162+
else:
163+
return True
164+
165+
166+
def check_monitor_current_status(monitor_id, alarm_level):
167+
dynamo_client = boto3.resource("dynamodb",
168+
aws_access_key_id=config.aws_access_key_id,
169+
aws_secret_access_key=config.aws_secret_access_key,
170+
region_name=config.aws_region)
171+
172+
item = dynamodb_read_pkey(dynamo_client, config.table_name_current, 'monitor_id', monitor_id)
173+
if item is not None:
174+
return item[alarm_level]
175+
else:
176+
return None
177+
178+
179+
#r = check_monitor_current_status('TEST2', 'warning')
180+
#print r
181+
182+
183+
def handle_monitor_current(monitor_id, alarm_level, alarm_bool):
184+
alarm_status = "NoChange" # NoChange; Recovered; Start
185+
dynamodb = boto3.resource("dynamodb",
186+
aws_access_key_id=config.aws_access_key_id,
187+
aws_secret_access_key=config.aws_secret_access_key,
188+
region_name=config.aws_region)
189+
190+
current_status = check_monitor_current_status(monitor_id, alarm_level)
191+
if current_status is True:
192+
if alarm_bool is False:
193+
monitor_current_update(monitor_id, alarm_level, alarm_bool)
194+
alarm_status = "Recovered"
195+
elif current_status is False:
196+
if alarm_bool is True:
197+
monitor_current_update(monitor_id, alarm_level, alarm_bool)
198+
alarm_status = "Start"
199+
elif current_status is None:
200+
if alarm_level == 'warning':
201+
content_arr = {
202+
"monitor_id": monitor_id,
203+
'warning': alarm_bool,
204+
'alert': False
205+
}
206+
elif alarm_level == 'alert':
207+
content_arr = {
208+
"monitor_id": monitor_id,
209+
'warning': False,
210+
'alert': alarm_bool
211+
}
212+
dynamodb_insert_data(dynamodb, config.table_name_current, content_arr)
213+
if alarm_bool is True:
214+
alarm_status = "Start"
215+
216+
return alarm_status
217+
218+
219+
# NoChange; Recovered; Start
220+
#alarm_status = handle_monitor_current("TEST3", "warning", True)
221+
#print alarm_status
222+
223+
224+
def handle_notify(monitor_id, cal_result, config_alarms, config_notify, alarm_status_warning, alarm_status_alert):
225+
send_notify = ""
226+
#print "[bool_warning]" + str(bool_warning) + "[alarm_status_warning]" + alarm_status_warning + "[bool_alert]" + str(bool_alert) + "[alarm_status_alert]" + alarm_status_alert
227+
# alarm_status: NoChange; Recovered; Start
228+
# Warning
229+
if alarm_status_warning != 'NoChange':
230+
send_notify = "warning"
231+
# Slack Notify
232+
if 'slack' in config_notify:
233+
slack_webhook = config_notify['slack']
234+
warning_data = {"text": "[" + alarm_status_warning + "]" + "[WARN] " + monitor_id + ": " + str(cal_result) + config_alarms['warning']}
235+
slack_notify(slack_webhook, json.dumps(warning_data))
236+
237+
# Alert
238+
if alarm_status_alert != 'NoChange':
239+
send_notify += "alert"
240+
# Slack Notify
241+
if 'slack' in config_notify:
242+
slack_webhook = config_notify['slack']
243+
alert_data = {"text": "[" + alarm_status_alert + "]" + "[ALERT] " + monitor_id + ": " + str(cal_result) + config_alarms['alert']}
244+
slack_notify(slack_webhook, json.dumps(alert_data))
245+
246+
return send_notify
247+
248+
249+
def handler(event, context):
250+
dynamo_client = boto3.resource("dynamodb",
251+
aws_access_key_id=config.aws_access_key_id,
252+
aws_secret_access_key=config.aws_secret_access_key,
253+
region_name=config.aws_region)
254+
items = dynamodb_scan(dynamo_client, config.table_name_config)
255+
return_list = []
256+
257+
for i in range(len(items)):
258+
enable = items[i]['enable']
259+
if enable:
260+
monitor_id = items[i]['monitor_id']
261+
262+
elk_env = items[i]['elk_env']
263+
if elk_env == 'staging':
264+
elk_conn = config.elk_stg
265+
elif elk_env == 'production':
266+
elk_conn = config.elk_prod
267+
268+
formula = items[i]['formula']
269+
270+
# Replace ${} params
271+
pattern = re.compile(r'\$\{([A-Za-z0-9_.]+)\}')
272+
params = re.findall(pattern, formula)
273+
dict_log_count = {}
274+
for param in params:
275+
param_log_count = get_log_hit(elk_conn, json.dumps(items[i]['parameters'][param]))
276+
dict_log_count.update({"${"+param+"}": param_log_count})
277+
278+
# Replace %{} params
279+
pattern_2 = re.compile(r'%\{([A-Za-z0-9_.]+)\}')
280+
params_2 = re.findall(pattern_2, formula)
281+
dict_log_value = {}
282+
for param in params_2:
283+
param_log_value = get_log_dict_value(elk_conn, json.dumps(items[i]['parameters'][param]), param)
284+
dict_log_value.update({"%{"+param+"}": param_log_value})
285+
286+
# Replace formula for calculation
287+
formula_replaced = formula
288+
289+
for key, value in dict_log_count.iteritems():
290+
formula_replaced = formula_replaced.replace(key, str(value))
291+
292+
for key, value in dict_log_value.iteritems():
293+
formula_replaced = formula_replaced.replace(key, str(value))
294+
295+
try:
296+
#cal_result = float(eval(formula_replaced))
297+
cal_result = eval(formula_replaced)
298+
299+
except ZeroDivisionError:
300+
raise Exception("Division by zero")
301+
302+
config_alarms = items[i]['alarms']
303+
if 'warning' in config_alarms:
304+
bool_warning = eval(str(cal_result) + config_alarms['warning'])
305+
alarm_status_warning = handle_monitor_current(monitor_id, "warning", bool_warning)
306+
307+
if 'alert' in config_alarms:
308+
bool_alert = eval(str(cal_result) + config_alarms['alert'])
309+
alarm_status_alert = handle_monitor_current(monitor_id, "alert", bool_alert)
310+
311+
config_notify = items[i]['notify']
312+
send_notify = handle_notify(monitor_id, cal_result, config_alarms, config_notify, alarm_status_warning, alarm_status_alert)
313+
314+
monitor_arr = {
315+
"monitor_id": monitor_id,
316+
"elk_env": elk_env,
317+
"elk_conn": elk_conn,
318+
"dict_log_count": dict_log_count,
319+
"formula": formula,
320+
"formula_replaced": formula_replaced,
321+
"cal_result": cal_result,
322+
"config_alarms": config_alarms,
323+
"bool_warning": str(bool_warning),
324+
"alarm_status_warning": alarm_status_warning,
325+
"bool_alert": str(bool_alert),
326+
"alarm_status_alert": alarm_status_alert,
327+
"send_notify": send_notify
328+
}
329+
return_list.append(monitor_arr)
330+
331+
return return_list
332+
333+
#result = handler(None, None)
334+
#print json.dumps(result)

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /