Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit b934acd

Browse files
authored
[WIP] add config, sampling rate support, and bump up version (#14)
* add sampling rate and config support - added initialization code to improve the performance, and refactor the sampling to central api - code cleanup - version bump
1 parent a05fbda commit b934acd

File tree

5 files changed

+155
-57
lines changed

5 files changed

+155
-57
lines changed

‎.gitignore

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ celerybeat-schedule
8787
.env
8888

8989
# virtualenv
90-
.venv/
91-
venv/
90+
.venv*/
91+
venv*/
9292
ENV/
9393

9494
# Spyder project settings

‎moesif_aws_lambda/global_variable.py

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
"""This module is to declare global objects."""
2+
from datetime import datetime
3+
from moesifpythonrequest.app_config.app_config import AppConfig
4+
from moesifapi.moesif_api_client import *
5+
import os
6+
7+
# MoesifAPI Client
8+
global api_client
9+
api_client = None
10+
11+
# App Config class
12+
global app_config
13+
app_config = None
14+
15+
# App Config
16+
global config
17+
config = None
18+
19+
# App Config sampling percentage
20+
global sampling_percentage
21+
sampling_percentage = 100
22+
23+
# App Config eTag
24+
global config_etag
25+
config_etag = None
26+
27+
# App Config last updated time
28+
global last_updated_time
29+
last_updated_time = datetime.utcnow()
30+
31+
# the delta time for refreshing config, if the last update time is greater than this
32+
global refresh_config_time_seconds
33+
refresh_config_time_seconds = 2 * 60
34+
35+
# initialize the config first time on cold start.
36+
print("[moesif] start init - config")
37+
38+
# Initialize the client
39+
if os.environ.get("MOESIF_APPLICATION_ID"):
40+
api_client = MoesifAPIClient(os.environ["MOESIF_APPLICATION_ID"]).api
41+
else:
42+
raise Exception('Moesif Application ID is required in settings')
43+
44+
app_config = AppConfig()
45+
config = app_config.get_config(api_client, True)
46+
47+
print("[moesif] end init - config")

‎moesif_aws_lambda/middleware.py

Lines changed: 103 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,18 @@
77
from .client_ip import ClientIp
88
from .update_companies import Company
99
from .update_users import User
10+
from . import global_variable as gv
11+
1012
from datetime import *
1113
import base64
1214
import json
1315
import os
1416
from pprint import pprint
1517
import base64
18+
19+
import random
20+
import math
21+
1622
try:
1723
from urllib import urlencode
1824
except ImportError:
@@ -24,32 +30,40 @@
2430
def get_time_took_in_ms(start_time, end_time):
2531
return (end_time - start_time).total_seconds() * 1000
2632

33+
2734
def start_capture_outgoing(moesif_options):
2835
try:
2936
if moesif_options.get('DEBUG', False):
30-
print('Start capturing outgoing requests')
37+
print('[moesif] Start capturing outgoing requests')
38+
3139
# Start capturing outgoing requests
3240
moesif_options['APPLICATION_ID'] = os.environ["MOESIF_APPLICATION_ID"]
3341
StartCapture().start_capture_outgoing(moesif_options)
42+
43+
if moesif_options.get('DEBUG', False):
44+
print("[moesif] end capturing moesif options")
3445
except Exception as e:
3546
print('Error while starting to capture the outgoing events')
3647
print(e)
48+
return
49+
3750

3851
# Initialized the client
39-
if os.environ["MOESIF_APPLICATION_ID"]:
40-
api_client = MoesifAPIClient(os.environ["MOESIF_APPLICATION_ID"]).api
41-
else:
42-
raise Exception('Moesif Application ID is required in settings')
52+
api_client = gv.api_client
53+
4354

4455
def update_user(user_profile, moesif_options):
4556
User().update_user(user_profile, api_client, moesif_options)
4657

58+
4759
def update_users_batch(user_profiles, moesif_options):
4860
User().update_users_batch(user_profiles, api_client, moesif_options)
4961

62+
5063
def update_company(company_profile, moesif_options):
5164
Company().update_company(company_profile, api_client, moesif_options)
5265

66+
5367
def update_companies_batch(companies_profiles, moesif_options):
5468
Company().update_companies_batch(companies_profiles, api_client, moesif_options)
5569

@@ -70,12 +84,9 @@ def __init__(self, handler):
7084
self.event = None
7185
self.context = None
7286
self.payload_version = None
73-
74-
# Intialized the client
75-
if os.environ.get("MOESIF_APPLICATION_ID"):
76-
self.api_client = MoesifAPIClient(os.environ["MOESIF_APPLICATION_ID"]).api
77-
else:
78-
raise Exception('Moesif Application ID is required in settings')
87+
88+
# Set the client
89+
self.api_client = api_client
7990

8091
def clear_state(self):
8192
"""Function to clear state of local variable"""
@@ -108,16 +119,16 @@ def get_user_id(self, event, context):
108119
username = rc_identity_id
109120
except:
110121
if self.DEBUG:
111-
print("MOESIF can not fetch apiKey from cognitoIdentityId event, setting userId to None.")
122+
print("[moesif] can not fetch apiKey from cognitoIdentityId event, setting userId to None.")
112123
except Exception as e:
113124
if self.DEBUG:
114-
print("MOESIF can not execute identify_user function, please check moesif settings.")
125+
print("[moesif] can not execute identify_user function, please check moesif settings.")
115126
print(e)
116127
end_time_get_user_id = datetime.utcnow()
117128
if self.DEBUG:
118129
print("[moesif] Time took in fetching user id in millisecond - " + str(get_time_took_in_ms(start_time_get_user_id, end_time_get_user_id)))
119130
return username
120-
131+
121132
def get_company_id(self, event, context):
122133
"""Function to fetch CompanyId"""
123134
start_time_get_company_id = datetime.utcnow()
@@ -128,23 +139,23 @@ def get_company_id(self, event, context):
128139
company_id = identify_company(event, context)
129140
except Exception as e:
130141
if self.DEBUG:
131-
print("MOESIF can not execute identify_company function, please check moesif settings.")
142+
print("[moesif] can not execute identify_company function, please check moesif settings.")
132143
print(e)
133144
end_time_get_company_id = datetime.utcnow()
134145
if self.DEBUG:
135146
print("[moesif] Time took in fetching company id in millisecond - " + str(get_time_took_in_ms(start_time_get_company_id, end_time_get_company_id)))
136147
return company_id
137-
148+
138149
def build_uri(self, event, payload_format_version_1_0):
139150

140-
uri = event['headers'].get('X-Forwarded-Proto', event['headers'].get('x-forwarded-proto', 'http')) + '://' + event['headers'].get('Host', event['headers'].get('host', 'localhost'))
141-
151+
uri = event['headers'].get('X-Forwarded-Proto', event['headers'].get('x-forwarded-proto', 'http')) + '://' + event['headers'].get('Host', event['headers'].get('host', 'localhost'))
152+
142153
if payload_format_version_1_0:
143154
uri = uri + event.get('path', '/')
144155
if event.get('multiValueQueryStringParameters', {}):
145-
uri = uri + '?' + urlencode(event['multiValueQueryStringParameters'], doseq=True)
156+
uri = uri + '?' + urlencode(event['multiValueQueryStringParameters'], doseq=True)
146157
elif event.get('queryStringParameters', {}):
147-
uri = uri + '?' + urlencode(event['queryStringParameters'])
158+
uri = uri + '?' + urlencode(event['queryStringParameters'])
148159
else:
149160
uri = uri + event.get('rawPath', '/')
150161
if event.get('rawQueryString', {}):
@@ -211,7 +222,7 @@ def before(self, event, context):
211222
else:
212223
request_verb = event.get('requestContext', {}).get('http', {}).get('method')
213224
if request_verb is None:
214-
print('MOESIF: [before] AWS Lambda trigger must be a Load Balancer or API Gateway See https://docs.aws.amazon.com/lambda/latest/dg/services-alb.html or https://docs.aws.amazon.com/lambda/latest/dg/with-on-demand-https.html.')
225+
print('[moesif] : [before] AWS Lambda trigger must be a Load Balancer or API Gateway See https://docs.aws.amazon.com/lambda/latest/dg/services-alb.html or https://docs.aws.amazon.com/lambda/latest/dg/with-on-demand-https.html.')
215226
self.event = None
216227
self.context = None
217228
self.payload_version = None
@@ -224,23 +235,23 @@ def before(self, event, context):
224235
req_headers = APIHelper.json_deserialize(event['headers'])
225236
except Exception as e:
226237
if self.DEBUG:
227-
print('MOESIF Error while fetching request headers')
238+
print('[moesif] Error while fetching request headers')
228239
print(e)
229240

230241
# Request Time
231242
if self.is_payload_format_version_1_0(self.payload_version):
232243
epoch = event and event.get('requestContext', {}).get('requestTimeEpoch')
233244
else:
234245
epoch = event and event.get('requestContext', {}).get('timeEpoch')
235-
if epoch is not None:
246+
if epoch is not None:
236247
# Dividing by 1000 to convert from ms to seconds and `.0` to preserve millisecond precision
237248
request_time = datetime.utcfromtimestamp(epoch/1000.0)
238249
else:
239250
request_time = datetime.utcnow()
240-
251+
241252
# Request Body
242253
req_body, req_transfer_encoding = self.process_body(event)
243-
254+
244255
# Metadata
245256
start_time_get_metadata = datetime.utcnow()
246257
try:
@@ -258,15 +269,15 @@ def before(self, event, context):
258269
}
259270
except:
260271
if self.DEBUG:
261-
print("MOESIF can not fetch default function_name and request_context from aws context, setting metadata to None.")
272+
print("[moesif] can not fetch default function_name and request_context from aws context, setting metadata to None.")
262273
except Exception as e:
263274
if self.DEBUG:
264-
print("MOESIF can not execute GET_METADATA function, please check moesif settings.")
275+
print("[moesif] can not execute GET_METADATA function, please check moesif settings.")
265276
print(e)
266277
end_time_get_metadata = datetime.utcnow()
267278
if self.DEBUG:
268279
print("[moesif] Time took in fetching metadata in millisecond - " + str(get_time_took_in_ms(start_time_get_metadata, end_time_get_metadata)))
269-
280+
270281
# User Id
271282
start_time_identify_user = datetime.utcnow()
272283
self.user_id = self.get_user_id(event, context)
@@ -294,10 +305,10 @@ def before(self, event, context):
294305
self.session_token = rc_api_key
295306
except KeyError:
296307
if self.DEBUG:
297-
print("MOESIF can not fetch apiKey from aws event, setting session_token to None.")
308+
print("[moesif] can not fetch apiKey from aws event, setting session_token to None.")
298309
except Exception as e:
299310
if self.DEBUG:
300-
print("MOESIF can not execute GET_SESSION_TOKEN function, please check moesif settings.")
311+
print("[moesif] can not execute GET_SESSION_TOKEN function, please check moesif settings.")
301312
print(e)
302313

303314
# Api Version
@@ -312,12 +323,12 @@ def before(self, event, context):
312323
api_version = context.function_version
313324
except KeyError:
314325
if self.DEBUG:
315-
print("MOESIF can not fetch default function_version from aws context, setting api_version to None.")
326+
print("[moesif] can not fetch default function_version from aws context, setting api_version to None.")
316327
except Exception as e:
317328
if self.DEBUG:
318-
print("MOESIF can not execute GET_API_VERSION function, please check moesif settings.")
329+
print("[moesif] can not execute GET_API_VERSION function, please check moesif settings.")
319330
print(e)
320-
331+
321332
# IpAddress
322333
if self.is_payload_format_version_1_0(self.payload_version):
323334
ip_address = event.get('requestContext', {}).get('identity', {}).get('sourceIp', None)
@@ -339,11 +350,12 @@ def before(self, event, context):
339350
print("[moesif] Time took before the handler is invoked in millisecond - " + str(get_time_took_in_ms(start_time_before_handler_function, end_time_before_handler_function)))
340351
# Return event, context
341352
return event, context
342-
353+
343354
def after(self, retval):
344355
"""This function runs after the handler is invoked, is passed the response and must return an response too."""
345-
356+
346357
start_time_after_handler_function = datetime.utcnow()
358+
event_send = None
347359
if self.event is not None:
348360
# Response body
349361
resp_body, resp_transfer_encoding = self.process_body(retval)
@@ -362,27 +374,27 @@ def after(self, retval):
362374
company_id = self.company_id,
363375
session_token = self.session_token,
364376
metadata = self.metadata)
365-
377+
366378
# Mask Event Model
367379
try:
368380
mask_event_model = self.moesif_options.get('MASK_EVENT_MODEL', None)
369381
if mask_event_model is not None:
370382
event_model = mask_event_model(event_model)
371-
except:
383+
exceptExceptionase:
372384
if self.DEBUG:
373-
print("MOESIF Can not execute MASK_EVENT_MODEL function. Please check moesif settings.")
385+
print("[moesif] Can not execute MASK_EVENT_MODEL function. Please check moesif settings.", e)
374386

375387
# Skip Event
376388
try:
377389
skip_event = self.moesif_options.get('SKIP', None)
378390
if skip_event is not None:
379391
if skip_event(self.event, self.context):
380392
if self.DEBUG:
381-
print('MOESIF Skip sending event to Moesif')
393+
print('[moesif] Skip sending event to Moesif')
382394
return retval
383-
except:
395+
exceptExceptionase:
384396
if self.DEBUG:
385-
print("MOESIF Having difficulty executing skip_event function. Please check moesif settings.")
397+
print("[moesif] Having difficulty executing skip_event function. Please check moesif settings.", e)
386398

387399
# Add direction field
388400
event_model.direction = "Incoming"
@@ -391,17 +403,56 @@ def after(self, retval):
391403
if self.DEBUG:
392404
print('Moesif Event Model:')
393405
print(json.dumps(self.event))
394-
395-
if self.DEBUG:
396-
start_time_sending_event_w_rsp = datetime.utcnow()
397-
event_send = self.api_client.create_event(event_model)
398-
end_time_sending_event_w_rsp = datetime.utcnow()
399-
if self.DEBUG:
400-
print("[moesif] Time took in sending event to moesif in millisecond - " + str(get_time_took_in_ms(start_time_sending_event_w_rsp, end_time_sending_event_w_rsp)))
401-
print('[moesif] Event Sent successfully ' + str(event_send))
402-
else:
403-
self.api_client.create_event(event_model)
404-
406+
407+
# Sampling Rate
408+
try:
409+
random_percentage = random.random() * 100
410+
gv.sampling_percentage = gv.app_config.get_sampling_percentage(
411+
event_model,
412+
gv.config,
413+
self.user_id,
414+
self.company_id,
415+
)
416+
417+
if gv.sampling_percentage >= random_percentage:
418+
event_model.weight = 1 if gv.sampling_percentage == 0 else math.floor(
419+
100 / gv.sampling_percentage)
420+
421+
if self.DEBUG:
422+
start_time_sending_event_w_rsp = datetime.utcnow()
423+
event_send = self.api_client.create_event(event_model)
424+
end_time_sending_event_w_rsp = datetime.utcnow()
425+
print("[moesif] sampling_percentage" + str(
426+
gv.sampling_percentage) + " and random percentage: " + str(random_percentage))
427+
print("[moesif] Time took in sending event to moesif in millisecond - " + str(
428+
get_time_took_in_ms(start_time_sending_event_w_rsp, end_time_sending_event_w_rsp)))
429+
print('[moesif] Event Sent successfully ' + str(event_send))
430+
431+
else:
432+
if datetime.utcnow() > gv.last_updated_time + timedelta(seconds=gv.refresh_config_time_seconds):
433+
event_send = self.api_client.create_event(event_model)
434+
else:
435+
self.api_client.create_event(event_model)
436+
437+
try:
438+
# Check if we need to update config
439+
new_config_etag = event_send['x-moesif-config-etag']
440+
if gv.config_etag is None or (gv.config_etag != new_config_etag):
441+
gv.config_etag = new_config_etag
442+
gv.config = gv.app_config.get_config(self.api_client, self.DEBUG)
443+
except (KeyError, TypeError, ValueError) as ex:
444+
# ignore the error because "event_send" is not set in non-blocking call
445+
pass
446+
finally:
447+
gv.last_updated_time = datetime.utcnow()
448+
449+
else:
450+
if self.DEBUG:
451+
print("Skipped Event due to sampling percentage: " + str(
452+
gv.sampling_percentage) + " and random percentage: " + str(random_percentage))
453+
except Exception as ex:
454+
print("[moesif] Error when fetching sampling rate from app config", ex)
455+
405456
end_time_after_handler_function = datetime.utcnow()
406457
if self.DEBUG:
407458
print("[moesif] Time took after the handler is invoked in millisecond - " + str(get_time_took_in_ms(start_time_after_handler_function, end_time_after_handler_function)))

‎requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
moesifapi==1.4.0
22
lambda_decorators==0.3.0
3-
moesifpythonrequest==0.2.0
3+
moesifpythonrequest==0.3.0

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /