-
Couldn't load subscription status.
- Fork 757
Psycopg2Instrumentor Django not collect database queries and hits #2989
-
Hello, recently I included the automatic opentelemetry libraries on my Django project but we cannot see the database queries.
I use one Postgres database with psycopg2-binary==2.9.3.
tracking.py
from opentelemetry import trace from opentelemetry.instrumentation.django import DjangoInstrumentor from opentelemetry.instrumentation.logging import LoggingInstrumentor from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor from opentelemetry.instrumentation.redis import RedisInstrumentor from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.urllib import URLLibInstrumentor from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter def instrument_app(): provider = TracerProvider() trace.set_tracer_provider(provider) trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) LoggingInstrumentor().instrument(trace_provider=provider, set_logging_format=True) DjangoInstrumentor().instrument(trace_provider=provider, is_sql_commentor_enabled=True) Psycopg2Instrumentor().instrument(trace_provider=provider, skip_dep_check=True, enable_commenter=True) URLLibInstrumentor().instrument(trace_provider=provider) URLLib3Instrumentor().instrument(trace_provider=provider) RequestsInstrumentor().instrument(trace_provider=provider) RedisInstrumentor().instrument(trace_provider=provider)
manage.py
#!/usr/bin/env python """Django's command-line utility for administrative tasks.""" import os import sys from tracing import instrument_app def main(): os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_template.settings") instrument_app() try: from django.core.management import execute_from_command_line except ImportError as exc: raise ImportError( "Couldn't import Django. Are you sure it's installed and " "available on your PYTHONPATH environment variable? Did you " "forget to activate a virtual environment?" ) from exc execute_from_command_line(sys.argv) if __name__ == "__main__": main()
.env
...
##################
#### OPENTELEMETRY
OTEL_SERVICE_NAME=giza-necropolis
OTEL_RESOURCE_ATTRIBUTES=service.name=giza-necropolis
OTEL_LOG_LEVEL=debug
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces
OTEL_PYTHON_DJANGO_INSTRUMENT=True
pip freeze
asgiref==3.5.1
async-timeout==4.0.2
attrs==21.4.0
autoflake==1.7.7
autowrapt==1.0
backoff==2.2.1
black==22.10.0
certifi==2021年10月8日
cfgv==3.3.1
charset-normalizer==2.0.12
click==8.1.3
commonmark==0.9.1
coverage==6.5.0
Deprecated==1.2.13
distlib==0.3.6
Django==4.0.4
django-cors-headers==3.11.0
django-debug-toolbar==3.7.0
django-health-check==3.16.5
django-stomp==5.0.0
django-stomp-debug-callback==1.0.0
djangorestframework==3.13.1
docopt==0.6.2
drf-link-navigation-pagination==0.1.0
drf-spectacular==0.22.1
exceptiongroup==1.0.0rc9
execnet==1.9.0
filelock==3.8.0
flake8==5.0.4
flake8-bugbear==22.10.25
gevent==21.12.0
googleapis-common-protos==1.56.4
greenlet==1.1.2
grpcio==1.50.0
gunicorn==20.1.0
http-utils==0.1.3
icdiff==2.0.5
identify==2.5.7
idna==3.3
inflection==0.5.1
iniconfig==1.1.1
isort==5.10.1
jsonschema==4.5.1
mccabe==0.7.0
mypy==0.982
mypy-extensions==0.4.3
nodeenv==1.7.0
oneagent-sdk==1.4.0.20210127.165413
opentelemetry-api==1.13.0
opentelemetry-distro==0.34b0
opentelemetry-exporter-otlp==1.13.0
opentelemetry-exporter-otlp-proto-grpc==1.13.0
opentelemetry-exporter-otlp-proto-http==1.13.0
opentelemetry-instrumentation==0.34b0
opentelemetry-instrumentation-asgi==0.34b0
opentelemetry-instrumentation-aws-lambda==0.34b0
opentelemetry-instrumentation-dbapi==0.34b0
opentelemetry-instrumentation-django==0.34b0
opentelemetry-instrumentation-grpc==0.34b0
opentelemetry-instrumentation-logging==0.34b0
opentelemetry-instrumentation-psycopg2==0.34b0
opentelemetry-instrumentation-redis==0.34b0
opentelemetry-instrumentation-requests==0.34b0
opentelemetry-instrumentation-sqlite3==0.34b0
opentelemetry-instrumentation-urllib==0.34b0
opentelemetry-instrumentation-urllib3==0.34b0
opentelemetry-instrumentation-wsgi==0.34b0
opentelemetry-propagator-aws-xray==1.0.1
opentelemetry-propagator-jaeger==1.13.0
opentelemetry-proto==1.13.0
opentelemetry-sdk==1.13.0
opentelemetry-semantic-conventions==0.34b0
opentelemetry-util-http==0.34b0
packaging==21.3
pathspec==0.10.1
platformdirs==2.5.2
pluggy==1.0.0
pprintpp==0.4.0
pre-commit==2.20.0
protobuf==3.20.3
psycopg2-binary==2.9.3
pycodestyle==2.9.1
pyflakes==2.5.0
Pygments==2.13.0
pyparsing==3.0.9
pyrsistent==0.18.1
pytest==7.2.0
pytest-clarity==1.0.1
pytest-cov==4.0.0
pytest-django==4.5.2
pytest-env==0.8.1
pytest-icdiff==0.6
pytest-mock==3.10.0
pytest-xdist==3.0.2
python-json-logger==2.0.2
pytz==2022.1
PyYAML==6.0
redis==4.3.4
request-id-django-log==0.1.1
requests==2.27.1
rich==12.6.0
six==1.16.0
sqlparse==0.4.2
stomp.py==7.0.0
tenacity==8.0.1
toml==0.10.2
tomli==2.0.1
types-requests==2.28.11.2
types-urllib3==1.26.25.1
typing_extensions==4.4.0
uptrace==1.13.0
uritemplate==4.1.1
urllib3==1.26.9
virtualenv==20.16.5
wrapt==1.14.1
zope.event==4.5.0
zope.interface==5.4.0
Example testing traced view
import logging from django.core.cache import cache from drf_spectacular.utils import extend_schema from drf_spectacular.utils import extend_schema_view from http_utils.session import request_session from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView from django_template.apps.example.api.openapi import standard_responses from django_template.apps.example.api.v1.serializers import UserAttributesSerializer from django_template.apps.example.models import AuditAction _logger = logging.getLogger(__name__) @extend_schema_view(get=extend_schema(tags=["tag1", "tag2"]), post=extend_schema(tags=["tag1"])) class UserManagementAttributesAPIView(APIView): """ View with responsibility to manipulate UserAttributes data. """ @extend_schema( responses={200: UserAttributesSerializer} | standard_responses, ) def post(self, request): user_id = "xxxxxxxx" _logger.debug("The following user is trying to refresh his attributes: %s", user_id) serializer = UserAttributesSerializer(data=request.data) serializer.is_valid(raise_exception=True) # testing logging trace _logger.info("What I received: %s", serializer.validated_data) # testing db trace AuditAction(user_id=user_id, action=UserManagementAttributesAPIView.post.__name__, success=True).save() # testing get trace with request_session() as request_http_util: request_http_util.get('https://murilove.free.beeceptor.com/test') # testing redis trace cache.get('fake') return Response(serializer.data, status=status.HTTP_200_OK) @extend_schema( responses={200: UserAttributesSerializer} | standard_responses, ) def get(self, request): user_id = "Salted User has been logged" _logger.debug("The following user is trying to retrieve his attributes: %s", user_id) serializer = UserAttributesSerializer( { "full_name": "Carl Edward Sagan", "given_name": "Carl", "family_name": "Sagan", "user_metadata": { "city": "santo andré", "state": "alagoas", "birthday": "23-06-1989", "gender": "male", }, } ) return Response(serializer.data, status=status.HTTP_200_OK)
Looking at jaeger example we can see the:
- get external request
- get redis key
but we cannot see the database statement
Screen Shot 2022年10月26日 at 13 50 41
Do we have any lessons learned that can help me understand why Postgres is not working? Any configuration variables? Or could it be a problem?
Beta Was this translation helpful? Give feedback.
All reactions
-
👍 11
lol Finally working....
What was cheating me was another instrumentation package called autodynatrace
This package provides instrumentation for APM Dynatrace, with it installed the project had a wrapper overrided from psycopg2 opentelemetry wrapper cursor with what this library uses.
In the end... uninstalling the library, I was able to see the instrumentation made by opentelemetry in jaeger õ/
Lesson learned:
- Don't use two instrumentation libraries in the project.
- The psycopg2 library works with psycopg2-binary as well as long as the skip_dep_check=True flag is used as an instrument parameter.
how was it in the project:
tracing.py
import logging from opentelemetry import trace from o...
Replies: 2 comments 1 reply
-
The Psycopg2Instrumentor instruments the psycopg2 library not the psycopg2-binary. Prior discussion and proposed solutions open-telemetry/opentelemetry-python-contrib#610
Beta Was this translation helpful? Give feedback.
All reactions
-
looking the instrument code we had the flag skip_dep_check to turn-off the requirements to run instrumentation code.
I'm using this flag and when I use a breakpoint I see that it goes through the line of code and triggers the wrapper.
Assuming that psycopg2 and psycopg2-binary use the psycopg2 import shouldn't the instrumentation lib work normally?
Beta Was this translation helpful? Give feedback.
All reactions
-
👍 1
-
lol Finally working....
What was cheating me was another instrumentation package called autodynatrace
Screen Shot 2022年10月28日 at 16 15 06
This package provides instrumentation for APM Dynatrace, with it installed the project had a wrapper overrided from psycopg2 opentelemetry wrapper cursor with what this library uses.
In the end... uninstalling the library, I was able to see the instrumentation made by opentelemetry in jaeger õ/
Screen Shot 2022年10月28日 at 16 15 58
Lesson learned:
- Don't use two instrumentation libraries in the project.
- The psycopg2 library works with psycopg2-binary as well as long as the skip_dep_check=True flag is used as an instrument parameter.
how was it in the project:
tracing.py
import logging from opentelemetry import trace from opentelemetry.instrumentation.django import DjangoInstrumentor from opentelemetry.instrumentation.logging import LoggingInstrumentor from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor from opentelemetry.instrumentation.redis import RedisInstrumentor from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.urllib import URLLibInstrumentor from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter def request_hook(span, request): pass # print("TESTING request_hook") def response_hook(span, request, response): pass # print("TESTING response_hook") def log_hook(span, record): pass # print("LOOG HOOK") def instrument_app(): provider = TracerProvider() trace.set_tracer_provider(provider) trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) DjangoInstrumentor().instrument(tracer_provider=provider, is_sql_commentor_enabled=True, request_hook=request_hook, response_hook=response_hook) LoggingInstrumentor().instrument(tracer_provider=provider, log_hook=log_hook) Psycopg2Instrumentor().instrument(tracer_provider=provider, skip_dep_check=True, enable_commenter=True) URLLibInstrumentor().instrument(tracer_provider=provider) URLLib3Instrumentor().instrument(tracer_provider=provider) RequestsInstrumentor().instrument(tracer_provider=provider) RedisInstrumentor().instrument(tracer_provider=provider)
manage.py
#!/usr/bin/env python """Django's command-line utility for administrative tasks.""" import os import sys from opentelemetry.tracing import instrument_app def main(): os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_template.settings") instrument_app() try: from django.core.management import execute_from_command_line except ImportError as exc: raise ImportError( "Couldn't import Django. Are you sure it's installed and " "available on your PYTHONPATH environment variable? Did you " "forget to activate a virtual environment?" ) from exc execute_from_command_line(sys.argv) if __name__ == "__main__": main()
example view
import logging from django.core.cache import cache from drf_spectacular.utils import extend_schema from drf_spectacular.utils import extend_schema_view from http_utils.session import request_session from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView from django_template.apps.example.api.openapi import standard_responses from django_template.apps.example.api.v1.serializers import UserAttributesSerializer from django_template.apps.example.models import AuditAction _logger = logging.getLogger(__name__) @extend_schema_view(get=extend_schema(tags=["tag1", "tag2"]), post=extend_schema(tags=["tag1"])) class UserManagementAttributesAPIView(APIView): """ View with responsibility to manipulate UserAttributes data. """ @extend_schema( responses={200: UserAttributesSerializer} | standard_responses, ) def post(self, request): user_id = "123456qwerty" _logger.debug("The following user is trying to refresh his attributes: %s", user_id) serializer = UserAttributesSerializer(data=request.data) serializer.is_valid(raise_exception=True) # testing logging trace _logger.info("What I received: %s", serializer.validated_data) # testing db trace AuditAction(user_id=user_id, action=UserManagementAttributesAPIView.post.__name__, success=True).save() # # testing get trace with request_session() as request_http_util: request_http_util.get('https://murilove.free.beeceptor.com/test') # testing redis trace cache.get('fake') return Response(serializer.data, status=status.HTTP_200_OK) @extend_schema( responses={200: UserAttributesSerializer} | standard_responses, ) def get(self, request): user_id = "Salted User has been logged" _logger.debug("The following user is trying to retrieve his attributes: %s", user_id) serializer = UserAttributesSerializer( { "full_name": "Carl Edward Sagan", "given_name": "Carl", "family_name": "Sagan", "user_metadata": { "city": "santo andré", "state": "alagoas", "birthday": "23-06-1989", "gender": "male", }, } ) return Response(serializer.data, status=status.HTTP_200_OK)
pip freeze
root@56f2c8b54f95:/app# pip freeze asgiref==3.5.1 async-timeout==4.0.2 attrs==21.4.0 autoflake==1.4 backoff==2.2.1 black==22.3.0 CacheControl==0.12.11 cachy==0.3.0 certifi==2021年10月8日 cffi==1.15.1 cfgv==3.3.1 charset-normalizer==2.0.12 cleo==1.0.0a5 click==8.1.3 commonmark==0.9.1 coverage==6.4 crashtest==0.3.1 cryptography==38.0.1 Deprecated==1.2.13 distlib==0.3.4 Django==4.0.4 django-cors-headers==3.11.0 django-debug-toolbar==3.4.0 django-health-check==3.16.5 django-stomp==5.0.0 django-stomp-debug-callback==1.0.0 djangorestframework==3.13.1 docopt==0.6.2 drf-link-navigation-pagination==0.1.0 drf-spectacular==0.22.1 dulwich==0.20.46 execnet==1.9.0 filelock==3.6.0 flake8==4.0.1 flake8-bugbear==22.4.25 gevent==21.12.0 googleapis-common-protos==1.56.2 greenlet==1.1.2 grpcio==1.50.0 gunicorn==20.1.0 html5lib==1.1 http-utils==0.1.3 icdiff==2.0.5 identify==2.5.0 idna==3.3 inflection==0.5.1 iniconfig==1.1.1 isort==5.10.1 jaraco.classes==3.2.3 jeepney==0.8.0 jsonschema==4.5.1 keyring==23.9.3 lockfile==0.12.2 mccabe==0.6.1 more-itertools==9.0.0 msgpack==1.0.4 mypy==0.950 mypy-extensions==0.4.3 nodeenv==1.6.0 opentelemetry-api==1.13.0 opentelemetry-exporter-otlp==1.13.0 opentelemetry-exporter-otlp-proto-grpc==1.13.0 opentelemetry-exporter-otlp-proto-http==1.13.0 opentelemetry-instrumentation==0.34b0 opentelemetry-instrumentation-dbapi==0.34b0 opentelemetry-instrumentation-django==0.34b0 opentelemetry-instrumentation-logging==0.34b0 opentelemetry-instrumentation-psycopg2==0.34b0 opentelemetry-instrumentation-redis==0.34b0 opentelemetry-instrumentation-requests==0.34b0 opentelemetry-instrumentation-urllib==0.34b0 opentelemetry-instrumentation-urllib3==0.34b0 opentelemetry-instrumentation-wsgi==0.34b0 opentelemetry-proto==1.13.0 opentelemetry-sdk==1.13.0 opentelemetry-semantic-conventions==0.34b0 opentelemetry-util-http==0.34b0 packaging==21.3 pathspec==0.9.0 pexpect==4.8.0 pkginfo==1.8.3 platformdirs==2.5.2 pluggy==1.0.0 poetry==1.2.2 poetry-core==1.3.2 poetry-plugin-export==1.1.2 pprintpp==0.4.0 pre-commit==2.18.1 protobuf==3.20.3 psycopg2-binary==2.9.3 ptyprocess==0.7.0 py==1.11.0 pycodestyle==2.8.0 pycparser==2.21 pyflakes==2.4.0 Pygments==2.12.0 pylev==1.4.0 pyparsing==3.0.8 pyrsistent==0.18.1 pytest==7.1.2 pytest-clarity==1.0.1 pytest-cov==3.0.0 pytest-django==4.5.2 pytest-env==0.6.2 pytest-forked==1.4.0 pytest-icdiff==0.5 pytest-mock==3.7.0 pytest-xdist==2.5.0 python-json-logger==2.0.2 pytz==2022.1 PyYAML==6.0 redis==4.3.4 request-id-django-log==0.1.1 requests==2.27.1 requests-toolbelt==0.9.1 rich==12.3.0 SecretStorage==3.3.3 shellingham==1.5.0 six==1.16.0 sqlparse==0.4.2 stomp.py==7.0.0 tenacity==8.0.1 toml==0.10.2 tomli==2.0.1 tomlkit==0.11.5 types-requests==2.27.25 types-urllib3==1.26.14 typing_extensions==4.2.0 uritemplate==4.1.1 urllib3==1.26.9 virtualenv==20.14.1 webencodings==0.5.1 wrapt==1.14.1 zope.event==4.5.0 zope.interface==5.4.0
Beta Was this translation helpful? Give feedback.
All reactions
-
👍 1