[ADD] runbot_merge: sentry instrumentation

Currently sentry is only hooked from the outside, which doesn't
necessarily provide sufficiently actionable information.

Add some a few hooks to (try and) report odoo / mergebot metadata:

- add the user to WSGI transactions
- add a transaction (with users) around crons
- add the webhook event info to webhook requests
- add a few spans to the long-running crons, when they cover multiple
  units per iteration (e.g. a span per branch being staged)

Closes #544

forwardport/models/forwardport.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 import logging
 import pathlib
+
+import sentry_sdk
+
 import resource
 import subprocess
 import uuid
@@ -28,7 +31,8 @@ class Queue:
     def _process(self):
         for b in self.search(self._search_domain(), order='create_date, id', limit=self.limit):
             try:
-                b._process_item()
+                with sentry_sdk.start_span(description=self._name):
+                    b._process_item()
                 b.unlink()
                 self.env.cr.commit()
             except Exception:
@@ -68,6 +72,7 @@ class ForwardPortTasks(models.Model, Queue):
 
     def _process_item(self):
         batch = self.batch_id
+        sentry_sdk.set_tag('forward-porting', batch.prs.mapped('display_name'))
         newbatch = batch.prs._port_forward()
 
         if newbatch:
@@ -109,6 +114,7 @@ class UpdateQueue(models.Model, Queue):
 
     def _process_item(self):
         previous = self.new_root
+        sentry_sdk.set_tag("update-root", self.new_root.display_name)
         with ExitStack() as s:
             for child in self.new_root._iter_descendants():
                 self.env.cr.execute("""

runbot_merge/controllers/__init__.py

@@ -3,6 +3,7 @@ import hmac
 import logging
 import json
 
+import sentry_sdk
 import werkzeug.exceptions
 
 from odoo.http import Controller, request, route
@@ -18,6 +19,13 @@ class MergebotController(Controller):
     def index(self):
         req = request.httprequest
         event = req.headers['X-Github-Event']
+        with sentry_sdk.configure_scope() as scope:
+            if scope.transaction:
+                # only in 1.8.0 (or at least 1.7.2
+                if hasattr(scope, 'set_transaction_name'):
+                    scope.set_transaction_name(f"webhook {event}")
+                else: # but our servers use 1.4.3
+                    scope.transaction = f"webhook {event}"
 
         github._gh.info(self._format(req))
 
@@ -39,6 +47,7 @@ class MergebotController(Controller):
                              req.headers.get('X-Hub-Signature'))
                 return werkzeug.exceptions.Forbidden()
 
+        sentry_sdk.set_context('webhook', request.jsonrequest)
         return c(env, request.jsonrequest)
 
     def _format(self, request):

runbot_merge/models/project.py

@@ -1,6 +1,8 @@
 import logging
 import re
 
+import sentry_sdk
+
 from odoo import models, fields
 
 _logger = logging.getLogger(__name__)
@@ -69,7 +71,9 @@ class Project(models.Model):
             ('staging_enabled', '=', True),
         ]):
             try:
-                with self.env.cr.savepoint():
+                with self.env.cr.savepoint(), \
+                    sentry_sdk.start_span(description=f'create staging {branch.name}') as span:
+                    span.set_tag('branch', branch.name)
                     branch.try_staging()
             except Exception:
                 _logger.exception("Failed to create staging for branch %r", branch.name)

runbot_merge/models/pull_requests.py

@@ -19,6 +19,7 @@ from itertools import takewhile
 from typing import Optional
 
 import requests
+import sentry_sdk
 import werkzeug
 from werkzeug.datastructures import Headers
 
@@ -2031,7 +2032,10 @@ class Stagings(models.Model):
             FOR UPDATE
             ''', [tuple(self.mapped('batch_ids.prs.id'))])
             try:
-                self._safety_dance(gh, staging_heads)
+                with sentry_sdk.start_span(description="merge staging") as span:
+                    span.set_tag("staging", self.id)
+                    span.set_tag("branch", self.target.name)
+                    self._safety_dance(gh, staging_heads)
             except exceptions.FastForwardError as e:
                 logger.warning(
                     "Could not fast-forward successful staging on %s:%s",

runbot_merge/sentry.py

@@ -6,7 +6,10 @@ from sentry_sdk.integrations.logging import LoggingIntegration
 from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
 
 from odoo import http
-from runbot_merge.exceptions import FastForwardError, Mismatch, MergeError, Unmergeable
+from odoo.addons.base.models.ir_cron import ir_cron
+from odoo.http import WebRequest
+
+from .exceptions import FastForwardError, Mismatch, MergeError, Unmergeable
 
 
 def delegate(self, attr):
@@ -41,8 +44,41 @@ def setup_sentry(dsn):
             LoggingIntegration(level=logging.INFO, event_level=logging.WARNING),
         ],
         before_send=event_filter,
+        # apparently not in my version of the sdk
+        # functions_to_trace = []
     )
     http.root = SentryWsgiMiddleware(http.root)
+    instrument_odoo()
+
+def instrument_odoo():
+    """Monkeypatches odoo core to copy odoo metadata into sentry for more
+    informative events
+    """
+    # add user to wsgi request context
+    old_call_function = WebRequest._call_function
+    def _call_function(self, *args, **kwargs):
+        if self.uid:
+            sentry_sdk.set_user({
+                'id': self.uid,
+                'email': self.env.user.email,
+                'username': self.env.user.login,
+            })
+        else:
+            sentry_sdk.set_user({'username': '<public>'})
+        return old_call_function(self, *args, **kwargs)
+    WebRequest._call_function = _call_function
+
+    # create transaction for tracking crons, add user to that
+    old_callback = ir_cron._callback
+    def _callback(self, cron_name, server_action_id, job_id):
+        sentry_sdk.start_transaction(name=f"cron {cron_name}")
+        sentry_sdk.set_user({
+            'id': self.env.user.id,
+            'email': self.env.user.email,
+            'username': self.env.user.login,
+        })
+        return old_callback(self, cron_name, server_action_id, job_id)
+    ir_cron._callback = _callback
 
 dummy_record = logging.LogRecord(name="", level=logging.NOTSET, pathname='', lineno=0, msg='', args=(), exc_info=None)
 # mapping of exception types to predicates, if the predicate returns `True` the