Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Psycopg2Instrumentor Django not collect database queries and hits #2989

Discussion options

Hello, recently I included the automatic opentelemetry libraries on my Django project but we cannot see the database queries.
I use one Postgres database with psycopg2-binary==2.9.3.

tracking.py

from opentelemetry import trace
from opentelemetry.instrumentation.django import DjangoInstrumentor
from opentelemetry.instrumentation.logging import LoggingInstrumentor
from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.urllib import URLLibInstrumentor
from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
def instrument_app():
 provider = TracerProvider()
 trace.set_tracer_provider(provider)
 trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
 trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
 LoggingInstrumentor().instrument(trace_provider=provider, set_logging_format=True)
 DjangoInstrumentor().instrument(trace_provider=provider, is_sql_commentor_enabled=True)
 Psycopg2Instrumentor().instrument(trace_provider=provider, skip_dep_check=True, enable_commenter=True)
 URLLibInstrumentor().instrument(trace_provider=provider)
 URLLib3Instrumentor().instrument(trace_provider=provider)
 RequestsInstrumentor().instrument(trace_provider=provider)
 RedisInstrumentor().instrument(trace_provider=provider)

manage.py

#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
from tracing import instrument_app
def main():
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_template.settings")
 instrument_app()
 try:
 from django.core.management import execute_from_command_line
 except ImportError as exc:
 raise ImportError(
 "Couldn't import Django. Are you sure it's installed and "
 "available on your PYTHONPATH environment variable? Did you "
 "forget to activate a virtual environment?"
 ) from exc
 execute_from_command_line(sys.argv)
if __name__ == "__main__":
 main()

.env

...
##################
#### OPENTELEMETRY
OTEL_SERVICE_NAME=giza-necropolis
OTEL_RESOURCE_ATTRIBUTES=service.name=giza-necropolis
OTEL_LOG_LEVEL=debug
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces
OTEL_PYTHON_DJANGO_INSTRUMENT=True

pip freeze

asgiref==3.5.1
async-timeout==4.0.2
attrs==21.4.0
autoflake==1.7.7
autowrapt==1.0
backoff==2.2.1
black==22.10.0
certifi==2021年10月8日
cfgv==3.3.1
charset-normalizer==2.0.12
click==8.1.3
commonmark==0.9.1
coverage==6.5.0
Deprecated==1.2.13
distlib==0.3.6
Django==4.0.4
django-cors-headers==3.11.0
django-debug-toolbar==3.7.0
django-health-check==3.16.5
django-stomp==5.0.0
django-stomp-debug-callback==1.0.0
djangorestframework==3.13.1
docopt==0.6.2
drf-link-navigation-pagination==0.1.0
drf-spectacular==0.22.1
exceptiongroup==1.0.0rc9
execnet==1.9.0
filelock==3.8.0
flake8==5.0.4
flake8-bugbear==22.10.25
gevent==21.12.0
googleapis-common-protos==1.56.4
greenlet==1.1.2
grpcio==1.50.0
gunicorn==20.1.0
http-utils==0.1.3
icdiff==2.0.5
identify==2.5.7
idna==3.3
inflection==0.5.1
iniconfig==1.1.1
isort==5.10.1
jsonschema==4.5.1
mccabe==0.7.0
mypy==0.982
mypy-extensions==0.4.3
nodeenv==1.7.0
oneagent-sdk==1.4.0.20210127.165413
opentelemetry-api==1.13.0
opentelemetry-distro==0.34b0
opentelemetry-exporter-otlp==1.13.0
opentelemetry-exporter-otlp-proto-grpc==1.13.0
opentelemetry-exporter-otlp-proto-http==1.13.0
opentelemetry-instrumentation==0.34b0
opentelemetry-instrumentation-asgi==0.34b0
opentelemetry-instrumentation-aws-lambda==0.34b0
opentelemetry-instrumentation-dbapi==0.34b0
opentelemetry-instrumentation-django==0.34b0
opentelemetry-instrumentation-grpc==0.34b0
opentelemetry-instrumentation-logging==0.34b0
opentelemetry-instrumentation-psycopg2==0.34b0
opentelemetry-instrumentation-redis==0.34b0
opentelemetry-instrumentation-requests==0.34b0
opentelemetry-instrumentation-sqlite3==0.34b0
opentelemetry-instrumentation-urllib==0.34b0
opentelemetry-instrumentation-urllib3==0.34b0
opentelemetry-instrumentation-wsgi==0.34b0
opentelemetry-propagator-aws-xray==1.0.1
opentelemetry-propagator-jaeger==1.13.0
opentelemetry-proto==1.13.0
opentelemetry-sdk==1.13.0
opentelemetry-semantic-conventions==0.34b0
opentelemetry-util-http==0.34b0
packaging==21.3
pathspec==0.10.1
platformdirs==2.5.2
pluggy==1.0.0
pprintpp==0.4.0
pre-commit==2.20.0
protobuf==3.20.3
psycopg2-binary==2.9.3
pycodestyle==2.9.1
pyflakes==2.5.0
Pygments==2.13.0
pyparsing==3.0.9
pyrsistent==0.18.1
pytest==7.2.0
pytest-clarity==1.0.1
pytest-cov==4.0.0
pytest-django==4.5.2
pytest-env==0.8.1
pytest-icdiff==0.6
pytest-mock==3.10.0
pytest-xdist==3.0.2
python-json-logger==2.0.2
pytz==2022.1
PyYAML==6.0
redis==4.3.4
request-id-django-log==0.1.1
requests==2.27.1
rich==12.6.0
six==1.16.0
sqlparse==0.4.2
stomp.py==7.0.0
tenacity==8.0.1
toml==0.10.2
tomli==2.0.1
types-requests==2.28.11.2
types-urllib3==1.26.25.1
typing_extensions==4.4.0
uptrace==1.13.0
uritemplate==4.1.1
urllib3==1.26.9
virtualenv==20.16.5
wrapt==1.14.1
zope.event==4.5.0
zope.interface==5.4.0

Example testing traced view

import logging
from django.core.cache import cache
from drf_spectacular.utils import extend_schema
from drf_spectacular.utils import extend_schema_view
from http_utils.session import request_session
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView
from django_template.apps.example.api.openapi import standard_responses
from django_template.apps.example.api.v1.serializers import UserAttributesSerializer
from django_template.apps.example.models import AuditAction
_logger = logging.getLogger(__name__)
@extend_schema_view(get=extend_schema(tags=["tag1", "tag2"]), post=extend_schema(tags=["tag1"]))
class UserManagementAttributesAPIView(APIView):
 """
 View with responsibility to manipulate UserAttributes data.
 """
 @extend_schema(
 responses={200: UserAttributesSerializer} | standard_responses,
 )
 def post(self, request):
 user_id = "xxxxxxxx"
 _logger.debug("The following user is trying to refresh his attributes: %s", user_id)
 serializer = UserAttributesSerializer(data=request.data)
 serializer.is_valid(raise_exception=True)
 # testing logging trace
 _logger.info("What I received: %s", serializer.validated_data)
 # testing db trace
 AuditAction(user_id=user_id, action=UserManagementAttributesAPIView.post.__name__, success=True).save()
 # testing get trace
 with request_session() as request_http_util:
 request_http_util.get('https://murilove.free.beeceptor.com/test')
 # testing redis trace
 cache.get('fake')
 return Response(serializer.data, status=status.HTTP_200_OK)
 @extend_schema(
 responses={200: UserAttributesSerializer} | standard_responses,
 )
 def get(self, request):
 user_id = "Salted User has been logged"
 _logger.debug("The following user is trying to retrieve his attributes: %s", user_id)
 serializer = UserAttributesSerializer(
 {
 "full_name": "Carl Edward Sagan",
 "given_name": "Carl",
 "family_name": "Sagan",
 "user_metadata": {
 "city": "santo andré",
 "state": "alagoas",
 "birthday": "23-06-1989",
 "gender": "male",
 },
 }
 )
 return Response(serializer.data, status=status.HTTP_200_OK)

Looking at jaeger example we can see the:

  • get external request
  • get redis key

but we cannot see the database statement

Screen Shot 2022年10月26日 at 13 50 41

Do we have any lessons learned that can help me understand why Postgres is not working? Any configuration variables? Or could it be a problem?

You must be logged in to vote

lol Finally working....

What was cheating me was another instrumentation package called autodynatrace

This package provides instrumentation for APM Dynatrace, with it installed the project had a wrapper overrided from psycopg2 opentelemetry wrapper cursor with what this library uses.

In the end... uninstalling the library, I was able to see the instrumentation made by opentelemetry in jaeger õ/

Lesson learned:

  • Don't use two instrumentation libraries in the project.
  • The psycopg2 library works with psycopg2-binary as well as long as the skip_dep_check=True flag is used as an instrument parameter.

how was it in the project:
tracing.py

import logging
from opentelemetry import trace
from o...

Replies: 2 comments 1 reply

Comment options

The Psycopg2Instrumentor instruments the psycopg2 library not the psycopg2-binary. Prior discussion and proposed solutions open-telemetry/opentelemetry-python-contrib#610

You must be logged in to vote
1 reply
Comment options

looking the instrument code we had the flag skip_dep_check to turn-off the requirements to run instrumentation code.

I'm using this flag and when I use a breakpoint I see that it goes through the line of code and triggers the wrapper.

Assuming that psycopg2 and psycopg2-binary use the psycopg2 import shouldn't the instrumentation lib work normally?

Comment options

lol Finally working....

What was cheating me was another instrumentation package called autodynatrace
Screen Shot 2022年10月28日 at 16 15 06

This package provides instrumentation for APM Dynatrace, with it installed the project had a wrapper overrided from psycopg2 opentelemetry wrapper cursor with what this library uses.

In the end... uninstalling the library, I was able to see the instrumentation made by opentelemetry in jaeger õ/

Screen Shot 2022年10月28日 at 16 15 58

Lesson learned:

  • Don't use two instrumentation libraries in the project.
  • The psycopg2 library works with psycopg2-binary as well as long as the skip_dep_check=True flag is used as an instrument parameter.

how was it in the project:
tracing.py

import logging
from opentelemetry import trace
from opentelemetry.instrumentation.django import DjangoInstrumentor
from opentelemetry.instrumentation.logging import LoggingInstrumentor
from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.urllib import URLLibInstrumentor
from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
def request_hook(span, request):
 pass
 # print("TESTING request_hook")
def response_hook(span, request, response):
 pass
 # print("TESTING response_hook")
def log_hook(span, record):
 pass
 # print("LOOG HOOK")
def instrument_app():
 provider = TracerProvider()
 trace.set_tracer_provider(provider)
 trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
 trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
 DjangoInstrumentor().instrument(tracer_provider=provider, is_sql_commentor_enabled=True, request_hook=request_hook, response_hook=response_hook)
 LoggingInstrumentor().instrument(tracer_provider=provider, log_hook=log_hook)
 Psycopg2Instrumentor().instrument(tracer_provider=provider, skip_dep_check=True, enable_commenter=True)
 URLLibInstrumentor().instrument(tracer_provider=provider)
 URLLib3Instrumentor().instrument(tracer_provider=provider)
 RequestsInstrumentor().instrument(tracer_provider=provider)
 RedisInstrumentor().instrument(tracer_provider=provider)

manage.py

#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
from opentelemetry.tracing import instrument_app
def main():
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_template.settings")
 instrument_app()
 try:
 from django.core.management import execute_from_command_line
 except ImportError as exc:
 raise ImportError(
 "Couldn't import Django. Are you sure it's installed and "
 "available on your PYTHONPATH environment variable? Did you "
 "forget to activate a virtual environment?"
 ) from exc
 execute_from_command_line(sys.argv)
if __name__ == "__main__":
 main()

example view

import logging
from django.core.cache import cache
from drf_spectacular.utils import extend_schema
from drf_spectacular.utils import extend_schema_view
from http_utils.session import request_session
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView
from django_template.apps.example.api.openapi import standard_responses
from django_template.apps.example.api.v1.serializers import UserAttributesSerializer
from django_template.apps.example.models import AuditAction
_logger = logging.getLogger(__name__)
@extend_schema_view(get=extend_schema(tags=["tag1", "tag2"]), post=extend_schema(tags=["tag1"]))
class UserManagementAttributesAPIView(APIView):
 """
 View with responsibility to manipulate UserAttributes data.
 """
 @extend_schema(
 responses={200: UserAttributesSerializer} | standard_responses,
 )
 def post(self, request):
 user_id = "123456qwerty"
 _logger.debug("The following user is trying to refresh his attributes: %s", user_id)
 serializer = UserAttributesSerializer(data=request.data)
 serializer.is_valid(raise_exception=True)
 # testing logging trace
 _logger.info("What I received: %s", serializer.validated_data)
 # testing db trace
 AuditAction(user_id=user_id, action=UserManagementAttributesAPIView.post.__name__, success=True).save()
 # # testing get trace
 with request_session() as request_http_util:
 request_http_util.get('https://murilove.free.beeceptor.com/test')
 # testing redis trace
 cache.get('fake')
 return Response(serializer.data, status=status.HTTP_200_OK)
 @extend_schema(
 responses={200: UserAttributesSerializer} | standard_responses,
 )
 def get(self, request):
 user_id = "Salted User has been logged"
 _logger.debug("The following user is trying to retrieve his attributes: %s", user_id)
 serializer = UserAttributesSerializer(
 {
 "full_name": "Carl Edward Sagan",
 "given_name": "Carl",
 "family_name": "Sagan",
 "user_metadata": {
 "city": "santo andré",
 "state": "alagoas",
 "birthday": "23-06-1989",
 "gender": "male",
 },
 }
 )
 return Response(serializer.data, status=status.HTTP_200_OK)

pip freeze

root@56f2c8b54f95:/app# pip freeze
asgiref==3.5.1
async-timeout==4.0.2
attrs==21.4.0
autoflake==1.4
backoff==2.2.1
black==22.3.0
CacheControl==0.12.11
cachy==0.3.0
certifi==2021年10月8日
cffi==1.15.1
cfgv==3.3.1
charset-normalizer==2.0.12
cleo==1.0.0a5
click==8.1.3
commonmark==0.9.1
coverage==6.4
crashtest==0.3.1
cryptography==38.0.1
Deprecated==1.2.13
distlib==0.3.4
Django==4.0.4
django-cors-headers==3.11.0
django-debug-toolbar==3.4.0
django-health-check==3.16.5
django-stomp==5.0.0
django-stomp-debug-callback==1.0.0
djangorestframework==3.13.1
docopt==0.6.2
drf-link-navigation-pagination==0.1.0
drf-spectacular==0.22.1
dulwich==0.20.46
execnet==1.9.0
filelock==3.6.0
flake8==4.0.1
flake8-bugbear==22.4.25
gevent==21.12.0
googleapis-common-protos==1.56.2
greenlet==1.1.2
grpcio==1.50.0
gunicorn==20.1.0
html5lib==1.1
http-utils==0.1.3
icdiff==2.0.5
identify==2.5.0
idna==3.3
inflection==0.5.1
iniconfig==1.1.1
isort==5.10.1
jaraco.classes==3.2.3
jeepney==0.8.0
jsonschema==4.5.1
keyring==23.9.3
lockfile==0.12.2
mccabe==0.6.1
more-itertools==9.0.0
msgpack==1.0.4
mypy==0.950
mypy-extensions==0.4.3
nodeenv==1.6.0
opentelemetry-api==1.13.0
opentelemetry-exporter-otlp==1.13.0
opentelemetry-exporter-otlp-proto-grpc==1.13.0
opentelemetry-exporter-otlp-proto-http==1.13.0
opentelemetry-instrumentation==0.34b0
opentelemetry-instrumentation-dbapi==0.34b0
opentelemetry-instrumentation-django==0.34b0
opentelemetry-instrumentation-logging==0.34b0
opentelemetry-instrumentation-psycopg2==0.34b0
opentelemetry-instrumentation-redis==0.34b0
opentelemetry-instrumentation-requests==0.34b0
opentelemetry-instrumentation-urllib==0.34b0
opentelemetry-instrumentation-urllib3==0.34b0
opentelemetry-instrumentation-wsgi==0.34b0
opentelemetry-proto==1.13.0
opentelemetry-sdk==1.13.0
opentelemetry-semantic-conventions==0.34b0
opentelemetry-util-http==0.34b0
packaging==21.3
pathspec==0.9.0
pexpect==4.8.0
pkginfo==1.8.3
platformdirs==2.5.2
pluggy==1.0.0
poetry==1.2.2
poetry-core==1.3.2
poetry-plugin-export==1.1.2
pprintpp==0.4.0
pre-commit==2.18.1
protobuf==3.20.3
psycopg2-binary==2.9.3
ptyprocess==0.7.0
py==1.11.0
pycodestyle==2.8.0
pycparser==2.21
pyflakes==2.4.0
Pygments==2.12.0
pylev==1.4.0
pyparsing==3.0.8
pyrsistent==0.18.1
pytest==7.1.2
pytest-clarity==1.0.1
pytest-cov==3.0.0
pytest-django==4.5.2
pytest-env==0.6.2
pytest-forked==1.4.0
pytest-icdiff==0.5
pytest-mock==3.7.0
pytest-xdist==2.5.0
python-json-logger==2.0.2
pytz==2022.1
PyYAML==6.0
redis==4.3.4
request-id-django-log==0.1.1
requests==2.27.1
requests-toolbelt==0.9.1
rich==12.3.0
SecretStorage==3.3.3
shellingham==1.5.0
six==1.16.0
sqlparse==0.4.2
stomp.py==7.0.0
tenacity==8.0.1
toml==0.10.2
tomli==2.0.1
tomlkit==0.11.5
types-requests==2.27.25
types-urllib3==1.26.14
typing_extensions==4.2.0
uritemplate==4.1.1
urllib3==1.26.9
virtualenv==20.14.1
webencodings==0.5.1
wrapt==1.14.1
zope.event==4.5.0
zope.interface==5.4.0
You must be logged in to vote
0 replies
Answer selected by MatheusGeiger
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Category
Q&A
Labels
None yet

AltStyle によって変換されたページ (->オリジナル) /