Merge branch 'master' into sergio.prada/inject-trace-data-in-lambda-invocations

* master:
  ci: introduce riot, use it to run tracer tests (#1737)
  fix(django): add check for instrument view mro (#1744)
  fix: snapshot decorator for pytest fixtures (#1743)
  tests(logging): don't test warn (#1738)
  feat(flask): store request headers (#1735)
  ci: fix deploy_master (#1739)

Author: Czechh

.circleci/config.yml

@@ -55,8 +55,8 @@ commands:
     steps:
       - setup_tox
       - run: |
-          apt-get update
-          apt-get install -y --no-install-recommends libenchant-dev
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libenchant-dev
       - run: pip install awscli
       - run: tox -e docs
       - run:
@@ -106,6 +106,30 @@ commands:
           paths:
             - ".tox"
 
+  run_test:
+    description: "Run tests matching a pattern"
+    parameters:
+      pattern:
+        type: string
+        default: ""
+      wait:
+        type: string
+        default: ""
+    steps:
+      - attach_workspace:
+          at: .
+      - run: pip3 install riot
+      - when:
+          condition:
+            << parameters.wait >>
+          steps:
+            - run:
+                name: "Waiting for << parameters.wait >>"
+                command: pip install tox && tox -e 'wait' << parameters.wait >>
+      - run:
+          name: "Running tests matching '<< parameters.pattern >>'"
+          command: "riot -v run -s '<< parameters.pattern >>'"
+
   run_tox_scenario:
     description: "Run scripts/run-tox-scenario with setup, caching and persistence"
     parameters:
@@ -231,6 +255,19 @@ jobs:
       - setup_tox
       - run: tox -e flake8
 
+  build_base:
+    executor: ddtrace_dev
+    steps:
+      - checkout
+      - run: pip3 install riot
+      - run:
+          name: "Generate base virtual environments."
+          command: "riot -v generate"
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "."
+
   build-docker-ci-image:
     executor: cimg_base
     steps:
@@ -292,8 +329,8 @@ jobs:
   tracer:
     executor: ddtrace_dev
     steps:
-      - run_tox_scenario:
-          pattern: '^py..-tracer'
+      - run_test:
+          pattern: "tracer"
 
   opentracer:
     executor: ddtrace_dev
@@ -444,15 +481,15 @@ jobs:
           pattern: '^falcon_contrib'
 
   django:
-    executor: ddtrace_dev
-    docker:
-      - image: *ddtrace_dev_image
-      - image: *redis_image
-      - image: *memcached_image
-      - *datadog_agent
+    <<: *machine_executor
     steps:
-      - run_tox_scenario:
-          pattern: '^django_contrib'
+      - checkout
+      - run: SNAPSHOT_CI=1 docker-compose up -d memcached redis testagent
+      - run:
+          command: docker-compose logs -f
+          background: true
+      - run:
+          command: ./scripts/ddtest scripts/run-tox-scenario '^django_contrib-'
 
   djangorestframework:
     executor: ddtrace_dev
@@ -803,6 +840,7 @@ requires_pre_test: &requires_pre_test
     - black
     - flake8
     - ccheck
+    - build_base
 
 requires_tests: &requires_tests
   - build_docs
@@ -893,6 +931,7 @@ workflows:
       - black
       - flake8
       - ccheck
+      - build_base
 
       # Test building the package
       - test_build_alpine: *requires_pre_test
@@ -963,7 +1002,6 @@ workflows:
       - sqlite3: *requires_pre_test
       - test_logging: *requires_pre_test
       - tornado: *requires_pre_test
-      # tracer
       - tracer: *requires_pre_test
       - vertica: *requires_pre_test
       - build-docker-ci-image: *requires_pre_test

.github/PULL_REQUEST_TEMPLATE/integration.md

@@ -27,3 +27,4 @@ This PR adds support for [`<integration>`](<!--link to relevant integration docs
 - [ ] 500-level responses are tagged as errors
 - [ ] Async (if applicable)
   - [ ] Span parenting behaves as expected.
+- [ ] Request headers specified in the config are stored

Dockerfile

@@ -63,6 +63,6 @@ RUN \
   && pyenv global 3.8.6 2.7.17 3.5.10 3.6.12 3.7.9 3.9.0 \
   && pip install --upgrade pip
 
-RUN pip install tox
+RUN pip install tox riot
 
 CMD ["/bin/bash"]

ddtrace/contrib/django/__init__.py

@@ -252,9 +252,10 @@
 with require_modules(required_modules) as missing_modules:
     if not missing_modules:
         from .middleware import TraceMiddleware
+        from . import patch as _patch
         from .patch import patch, unpatch
 
-        __all__ = ["patch", "unpatch", "TraceMiddleware"]
+        __all__ = ["patch", "unpatch", "TraceMiddleware", "_patch"]
 
 
 # define the Django app configuration

ddtrace/contrib/django/patch.py

@@ -465,13 +465,11 @@ def instrument_view(django, view):
 
     We want to wrap all lifecycle/http method functions for every class in the MRO for this view
     """
-    if isfunction(view):
-        return _instrument_view(django, view)
+    if hasattr(view, "__mro__"):
+        for cls in reversed(getmro(view)):
+            _instrument_view(django, cls)
 
-    for cls in reversed(getmro(view)):
-        _instrument_view(django, cls)
-
-    return view
+    return _instrument_view(django, view)
 
 
 def _instrument_view(django, view):

ddtrace/contrib/flask/middleware.py

@@ -3,6 +3,8 @@
 from ...internal.logger import get_logger
 from ...propagation.http import HTTPPropagator
 from ...utils.deprecation import deprecated
+from ddtrace.http import store_request_headers
+from ddtrace import config
 
 import flask.templating
 from flask import g, request, signals
@@ -155,6 +157,7 @@ def _finish_span(self, span, exception=None):
         endpoint = ''
         url = ''
         if request:
+            store_request_headers(request.headers, span, config.flask)
             method = request.method
             endpoint = request.endpoint or code
             url = request.base_url or ''

releasenotes/notes/django-view-mro-check-782374fb9c427305.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Check Django view before instrumenting MRO

releasenotes/notes/feature-flask-4bdb2b839fada0fb.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Store request headers in Flask integration.

riotfile.py

@@ -0,0 +1,28 @@
+from riot import Suite, Case
+
+global_deps = [
+    "mock",
+    "pytest<4",
+    "pytest-benchmark",
+]
+
+global_env = [("PYTEST_ADDOPTS", "--color=yes")]
+
+suites = [
+    Suite(
+        name="tracer",
+        command="pytest tests/tracer/",
+        cases=[
+            Case(
+                pys=[
+                    2.7,
+                    3.5,
+                    3.6,
+                    3.7,
+                    3.8,
+                ],
+                pkgs=[("msgpack", [""])],
+            ),
+        ],
+    ),
+]

tests/__init__.py

@@ -1,15 +1,15 @@
 import contextlib
-import functools
 import inspect
 import os
 import sys
 from contextlib import contextmanager
 
 import pytest
+from ddtrace.vendor import wrapt
 
 import ddtrace
 from ddtrace import Tracer, Span
-from ddtrace.compat import httplib
+from ddtrace.compat import httplib, to_unicode
 from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.encoding import JSONEncoder
 from ddtrace.ext import http
@@ -815,54 +815,55 @@ def snapshot(ignores=None, tracer=ddtrace.tracer):
     """
     ignores = ignores or []
 
-    def dec(f):
-        @functools.wraps(f)
-        def wrapper(*args, **kwargs):
-            if len(args) > 1:
-                self = args[0]
-                clsname = self.__class__.__name__
-            else:
-                clsname = ""
+    @wrapt.decorator
+    def wrapper(wrapped, instance, args, kwargs):
+        if len(args) > 1:
+            self = args[0]
+            clsname = self.__class__.__name__
+        else:
+            clsname = ""
 
-            module = inspect.getmodule(f)
+        module = inspect.getmodule(wrapped)
 
-            # Use the fully qualified function name as a unique test token to
-            # identify the snapshot.
-            token = "{}{}{}.{}".format(module.__name__, "." if clsname else "", clsname, f.__name__)
+        # Use the fully qualified function name as a unique test token to
+        # identify the snapshot.
+        token = "{}{}{}.{}".format(module.__name__, "." if clsname else "", clsname, wrapped.__name__)
 
-            conn = httplib.HTTPConnection(tracer.writer.api.hostname, tracer.writer.api.port)
+        conn = httplib.HTTPConnection(tracer.writer.api.hostname, tracer.writer.api.port)
+        try:
+            # clear queue in case traces have been generated before test case is
+            # itself run
+            tracer.writer.flush_queue()
+
+            # Signal the start of this test case to the test agent.
             try:
-                # Signal the start of this test case to the test agent.
-                try:
-                    conn.request("GET", "/test/start?token=%s" % token)
-                except Exception as e:
-                    pytest.fail("Could not connect to test agent: %s" % str(e), pytrace=False)
-
-                r = conn.getresponse()
-                if r.status != 200:
-                    # The test agent returns nice error messages we can forward to the user.
-                    raise SnapshotFailed(r.read().decode())
-
-                # Run the test.
-                ret = f(*args, **kwargs)
-
-                # Flush out any remnant traces.
-                tracer.writer.flush_queue()
-
-                # Query for the results of the test.
-                conn = httplib.HTTPConnection(tracer.writer.api.hostname, tracer.writer.api.port)
-                conn.request("GET", "/test/snapshot?ignores=%s&token=%s" % (",".join(ignores), token))
-                r = conn.getresponse()
-                if r.status != 200:
-                    raise SnapshotFailed(r.read().decode())
-                return ret
-            except SnapshotFailed as e:
-                # Fail the test if a failure has occurred and print out the
-                # message we got from the test agent.
-                pytest.fail(str(e), pytrace=False)
-            finally:
-                conn.close()
-
-        return wrapper
-
-    return dec
+                conn.request("GET", "/test/start?token=%s" % token)
+            except Exception as e:
+                pytest.fail("Could not connect to test agent: %s" % str(e), pytrace=False)
+
+            r = conn.getresponse()
+            if r.status != 200:
+                # The test agent returns nice error messages we can forward to the user.
+                raise SnapshotFailed(r.read())
+
+            # Run the test.
+            ret = wrapped(*args, **kwargs)
+
+            # Flush out any remnant traces.
+            tracer.writer.flush_queue()
+
+            # Query for the results of the test.
+            conn = httplib.HTTPConnection(tracer.writer.api.hostname, tracer.writer.api.port)
+            conn.request("GET", "/test/snapshot?ignores=%s&token=%s" % (",".join(ignores), token))
+            r = conn.getresponse()
+            if r.status != 200:
+                raise SnapshotFailed(r.read())
+            return ret
+        except SnapshotFailed as e:
+            # Fail the test if a failure has occurred and print out the
+            # message we got from the test agent.
+            pytest.fail(to_unicode(e.args[0]), pytrace=False)
+        finally:
+            conn.close()
+
+    return wrapper

tests/contrib/django/conftest.py

@@ -24,29 +24,24 @@ def pytest_configure():
     django.setup()
 
 
-@pytest.fixture(autouse=True)
-def patch_django(tracer):
+@pytest.fixture
+def tracer():
+    tracer = DummyTracer()
     # Patch Django and override tracer to be our test tracer
     pin = Pin.get_from(django)
     original_tracer = pin.tracer
     Pin.override(django, tracer=tracer)
 
     # Yield to our test
-    yield
+    yield tracer
+    tracer.writer.pop()
 
     # Reset the tracer pinned to Django and unpatch
     # DEV: unable to properly unpatch and reload django app with each test
     # unpatch()
     Pin.override(django, tracer=original_tracer)
 
 
-@pytest.fixture
-def tracer():
-    tracer = DummyTracer()
-    yield tracer
-    tracer.writer.pop()
-
-
 @pytest.fixture
 def test_spans(tracer):
     container = TracerSpanContainer(tracer)