feat: Add datadog_diagnostics plugin app (#722)

See #692

Testing setup:
https://2u-internal.atlassian.net/wiki/spaces/ENG/pages/1173618788/Running+Datadog+in+devstack

And then in lms-shell:

```
make requirements
pip install ddtrace
pip install -e /edx/src/archexp/
./wrap-datadog.sh ./server.sh
```

Expect to see this log message:
`Attached MissingSpanProccessor for Datadog diagnostics`

NOTE: This prints "Spans created = 0; spans finished = 0" in devstack when
shut down with ctrl-c, but not when restarted due to autoreload (where it
prints correct info). Something is initializing Django twice and one span
processor is getting span info while the other is printing at shutdown.
There's more to debug here, but it seems stable enough to least try
deploying it.

CHANGELOG.rst

@@ -14,6 +14,12 @@ Change Log
 Unreleased
 ~~~~~~~~~~
 
+[3.4.0] - 2024-07-10
+~~~~~~~~~~~~~~~~~~~~
+Added
+-----
+* Added ``datadog_diagnostics`` plugin app
+
 [3.3.2] - 2024-04-19
 ~~~~~~~~~~~~~~~~~~~~
 Changed

edx_arch_experiments/__init__.py

@@ -2,4 +2,4 @@
 A plugin to include applications under development by the architecture team at 2U.
 """
 
-__version__ = '3.3.2'
+__version__ = '3.4.0'

edx_arch_experiments/datadog_diagnostics/README.rst

@@ -0,0 +1,13 @@
+Datadog Diagnostics
+###################
+
+When installed in the LMS as a plugin app, the ``datadog_diagnostics`` app adds additional logging for debugging our Datadog integration.
+
+This is intended as a temporary situation while we debug the `trace concatenation issue <https://github.com/edx/edx-arch-experiments/issues/692>`_.
+
+Usage
+*****
+
+In LMS:
+
+- Install ``edx-arch-experiments`` as a dependency

edx_arch_experiments/datadog_diagnostics/apps.py

@@ -0,0 +1,50 @@
+"""
+App for emitting additional diagnostic information for the Datadog integration.
+"""
+
+import logging
+
+from django.apps import AppConfig
+
+log = logging.getLogger(__name__)
+
+
+class MissingSpanProcessor:
+    """Datadog span processor that logs unfinished spans at shutdown."""
+    spans_started = 0
+    spans_finished = 0
+    open_spans = {}
+
+    def on_span_start(self, span):
+        self.spans_started += 1
+        self.open_spans[span.span_id] = span
+
+    def on_span_finish(self, span):
+        self.spans_finished += 1
+        del self.open_spans[span.span_id]
+
+    def shutdown(self, _timeout):
+        log.info(f"Spans created = {self.spans_started}; spans finished = {self.spans_finished}")
+        for span in self.open_spans.values():
+            log.error(f"Span created but not finished: {span._pprint()}")  # pylint: disable=protected-access
+
+
+class DatadogDiagnostics(AppConfig):
+    """
+    Django application to log diagnostic information for Datadog.
+    """
+    name = 'edx_arch_experiments.datadog_diagnostics'
+
+    # Mark this as a plugin app
+    plugin_app = {}
+
+    def ready(self):
+        try:
+            from ddtrace import tracer  # pylint: disable=import-outside-toplevel
+            tracer._span_processors.append(MissingSpanProcessor())  # pylint: disable=protected-access
+            log.info("Attached MissingSpanProcessor for Datadog diagnostics")
+        except ImportError:
+            log.warning(
+                "Unable to attach MissingSpanProcessor for Datadog diagnostics"
+                " -- ddtrace module not found."
+            )

setup.py

@@ -164,6 +164,7 @@ def is_requirement(line):
             "arch_experiments = edx_arch_experiments.apps:EdxArchExperimentsConfig",
             "config_watcher = edx_arch_experiments.config_watcher.apps:ConfigWatcher",
             "codejail_service = edx_arch_experiments.codejail_service.apps:CodejailService",
+            "datadog_diagnostics = edx_arch_experiments.datadog_diagnostics.apps:DatadogDiagnostics",
         ],
         "cms.djangoapp": [
             "config_watcher = edx_arch_experiments.config_watcher.apps:ConfigWatcher",