# -*- coding: utf-8 -*- from __future__ import annotations import base64 import collections import colorsys import datetime import hashlib import io import json import logging import pathlib from dataclasses import dataclass from email.utils import formatdate from enum import Flag, auto from functools import cached_property from itertools import chain, product from math import ceil from typing import Tuple, cast, Mapping, Optional, List import markdown import markupsafe import werkzeug.exceptions import werkzeug.wrappers from PIL import Image, ImageDraw, ImageFont from odoo.http import Controller, route, request from odoo.tools import file_open HORIZONTAL_PADDING = 20 VERTICAL_PADDING = 5 _logger = logging.getLogger(__name__) LIMIT = 20 class MergebotDashboard(Controller): @route('/runbot_merge', auth="public", type="http", website=True, sitemap=True) def dashboard(self): projects = request.env['runbot_merge.project'].with_context(active_test=False).sudo().search([]) stagings = { branch: projects.env['runbot_merge.stagings'].search([ ('target', '=', branch.id)], order='staged_at desc', limit=6) for project in projects for branch in project.branch_ids if branch.active } prefetch_set = list({ id for stagings in stagings.values() for id in stagings.ids }) for st in stagings.values(): st._prefetch_ids = prefetch_set return request.render('runbot_merge.dashboard', { 'projects': projects, 'stagings_map': stagings, }) @route('/runbot_merge/', auth='public', type='http', website=True, sitemap=False) def stagings(self, branch_id, until=None, state=''): branch = request.env['runbot_merge.branch'].browse(branch_id).sudo().exists() if not branch: raise werkzeug.exceptions.NotFound() staging_domain = [('target', '=', branch.id)] if until: staging_domain.append(('staged_at', '<=', until)) if state: staging_domain.append(('state', '=', state)) stagings = request.env['runbot_merge.stagings'].with_context(active_test=False).sudo().search(staging_domain, order='staged_at desc', limit=LIMIT + 1) return request.render('runbot_merge.branch_stagings', { 'branch': branch, 'stagings': stagings[:LIMIT], 'until': until, 'state': state, 'next': stagings[-1].staged_at if len(stagings) > LIMIT else None, }) def _entries(self): changelog = pathlib.Path(__file__).parent.parent / 'changelog' if changelog.is_dir(): return [ (d.name, [f.read_text(encoding='utf-8') for f in d.iterdir() if f.is_file()]) for d in changelog.iterdir() ] return [] def entries(self, item_converter): entries = collections.OrderedDict() for key, items in sorted(self._entries(), reverse=True): entries.setdefault(key, []).extend(map(item_converter, items)) return entries @route('/runbot_merge/changelog', auth='public', type='http', website=True, sitemap=True) def changelog(self): md = markdown.Markdown(extensions=['nl2br'], output_format='html5') entries = self.entries(lambda t: markupsafe.Markup(md.convert(t))) return request.render('runbot_merge.changelog', { 'entries': entries, }) @route('///pull/', auth='public', type='http', website=True, sitemap=False) def pr(self, org, repo, pr, png): pr_id = request.env['runbot_merge.pull_requests'].sudo().search([ ('repository.name', '=', f'{org}/{repo}'), ('number', '=', int(pr)), ]) if not pr_id: raise werkzeug.exceptions.NotFound() if not pr_id.repository.group_id <= request.env.user.groups_id: _logger.warning( "Access error: %s (%s) tried to access %s but lacks access", request.env.user.login, request.env.user.name, pr_id.display_name, ) raise werkzeug.exceptions.NotFound() if png: return raster_render(pr_id) st = {} if pr_id.statuses: # normalise `statuses` to map to a dict st = { k: {'state': v} if isinstance(v, str) else v for k, v in json.loads(pr_id.statuses_full).items() } return request.render('runbot_merge.view_pull_request', { 'pr': pr_id, 'merged_head': json.loads(pr_id.commits_map).get(''), 'statuses': st }) def raster_render(pr): default_headers = { 'Content-Type': 'image/png', 'Last-Modified': formatdate(), # - anyone can cache the image, so public # - crons run about every minute so that's how long a request is fresh # - if the mergebot can't be contacted, allow using the stale response (no must-revalidate) # - intermediate caches can recompress the PNG if they want (pillow is not a very good PNG generator) # - the response is mutable even during freshness, technically (as there # is no guarantee the freshness window lines up with the cron, plus # some events are not cron-based) # - maybe don't allow serving the stale image *while* revalidating? # - allow serving a stale image for a day if the server returns 500 'Cache-Control': 'public, max-age=60, stale-if-error=86400', } if if_none_match := request.httprequest.headers.get('If-None-Match'): # just copy the existing value out if we received any default_headers['ETag'] = if_none_match # weak validation: check the latest modification date of all objects involved project, repos, branches, genealogy = pr.env.ref('runbot_merge.dashboard-pre')\ ._run_action_code_multi({'pr': pr}) # last-modified should be in RFC2822 format, which is what # email.utils.formatdate does (sadly takes a timestamp but...) last_modified = formatdate(max(( o.write_date or datetime.datetime.min for o in chain( project, repos, branches, genealogy, genealogy.all_prs | pr, ) )).timestamp()) # The (304) response must not contain a body and must include the headers # that would have been sent in an equivalent 200 OK response headers = {**default_headers, 'Last-Modified': last_modified} if request.httprequest.headers.get('If-Modified-Since') == last_modified: return werkzeug.wrappers.Response(status=304, headers=headers) batches = pr.env.ref('runbot_merge.dashboard-prep')._run_action_code_multi({ 'pr': pr, 'repos': repos, 'branches': branches, 'genealogy': genealogy, }) etag = hashlib.sha256(f"(P){pr.id},{pr.repository.id},{pr.target.id},{pr.batch_id.blocked}".encode()) # repos and branches should be in a consistent order so can just hash that etag.update(''.join(f'(R){r.name}' for r in repos).encode()) etag.update(''.join(f'(T){b.name},{b.active}' for b in branches).encode()) # and product of deterministic iterations should be deterministic for r, b in product(repos, branches): ps = batches[r, b] etag.update(f"(B){ps['state']},{ps['detached']},{ps['active']}".encode()) etag.update(''.join( f"(PS){p['label']},{p['closed']},{p['number']},{p['checked']},{p['reviewed']},{p['attached']},{p['pr'].staging_id.id}" for p in ps['prs'] ).encode()) etag = headers['ETag'] = base64.b32encode(etag.digest()).decode() if if_none_match == etag: return werkzeug.wrappers.Response(status=304, headers=headers) if not pr.batch_id.target: im = render_inconsistent_batch(pr.batch_id) else: im = render_full_table(pr, branches, repos, batches) buffer = io.BytesIO() im.save(buffer, 'png', optimize=True) return werkzeug.wrappers.Response(buffer.getvalue(), headers=headers) class Decoration(Flag): STRIKETHROUGH = auto() @dataclass(frozen=True) class Text: content: str font: ImageFont.FreeTypeFont color: Color decoration: Decoration = Decoration(0) @cached_property def width(self) -> int: return ceil(self.font.getlength(self.content)) @property def height(self) -> int: return sum(self.font.getmetrics()) def draw(self, image: ImageDraw.ImageDraw, left: int, top: int): image.text((left, top), self.content, fill=self.color, font=self.font) if Decoration.STRIKETHROUGH in self.decoration: x1, _, x2, _ = self.font.getbbox(self.content) _, y1, _, y2 = self.font.getbbox('x') # put the strikethrough line about 1/3rd down the x (default seems # to be a bit above halfway down but that's ugly with numbers which # is most of our stuff) y = top + y1 + (y2 - y1) / 3 image.line([(left + x1, y), (left + x2, y)], self.color) @dataclass(frozen=True) class Checkbox: checked: Optional[bool] font: ImageFont.FreeTypeFont color: Color success: Color error: Color @cached_property def width(self) -> int: return ceil(max( self.font.getlength(BOX_EMPTY), self.font.getlength(CHECK_MARK), self.font.getlength(CROSS), )) @property def height(self): return sum(self.font.getmetrics()) def draw(self, image: ImageDraw.ImageDraw, left: int, top: int): image.text((left, top+5), BOX_EMPTY, fill=self.color, font=self.font) if self.checked is True: image.text((left, top+4), CHECK_MARK, fill=self.success, font=self.font) elif self.checked is False: image.text((left, top+4), CROSS, fill=self.error, font=self.font) @dataclass(frozen=True) class Line: spans: List[Text | Checkbox | Lines] @property def width(self) -> int: return sum(s.width for s in self.spans) @property def height(self) -> int: return max(s.height for s in self.spans) if self.spans else 0 def draw(self, image: ImageDraw.ImageDraw, left: int, top: int): for span in self.spans: span.draw(image, left, top) left += span.width @dataclass(frozen=True) class Lines: lines: List[Line] @property def width(self) -> int: return max(l.width for l in self.lines) @property def height(self) -> int: return sum(l.height for l in self.lines) def draw(self, image: ImageDraw.ImageDraw, left: int, top: int): for line in self.lines: line.draw(image, left, top) top += line.height @dataclass(frozen=True) class Cell: content: Lines | Line | Text background: Color = (255, 255, 255) attached: bool = True @cached_property def width(self) -> int: return self.content.width + 2 * HORIZONTAL_PADDING @cached_property def height(self) -> int: return self.content.height + 2 * VERTICAL_PADDING def render_full_table(pr, branches, repos, batches): with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Regular.ttf', 'rb') as f: font = ImageFont.truetype(f, size=16, layout_engine=0) f.seek(0) supfont = ImageFont.truetype(f, size=13, layout_engine=0) with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Bold.ttf', 'rb') as f: bold = ImageFont.truetype(f, size=16, layout_engine=0) with file_open('web/static/src/libs/fontawesome/fonts/fontawesome-webfont.ttf', 'rb') as f: icons = ImageFont.truetype(f, size=16, layout_engine=0) rowheights = collections.defaultdict(int) colwidths = collections.defaultdict(int) cells = {} for b in chain([None], branches): for r in chain([None], repos): opacity = 1.0 if b is None or b.active else 0.5 current_row = b == pr.target background = BG['info'] if current_row or r == pr.repository else BG[None] if b is None: # first row cell = Cell(Text("" if r is None else r.name, bold, TEXT), background) elif r is None: # first column cell = Cell(Text(b.name, font, blend(TEXT, opacity, over=background)), background) elif current_row: ps = batches[r, b] bgcolor = lighten(BG[ps['state']], by=-0.05) if pr in ps['pr_ids'] else BG[ps['state']] background = blend(bgcolor, opacity, over=background) foreground = blend((39, 110, 114), opacity, over=background) success = blend(SUCCESS, opacity, over=background) error = blend(ERROR, opacity, over=background) boxes = { False: Checkbox(False, icons, foreground, success, error), True: Checkbox(True, icons, foreground, success, error), None: Checkbox(None, icons, foreground, success, error), } prs = [] attached = True for p in ps['prs']: pr = p['pr'] attached = attached and p['attached'] if pr.staging_id: sub = ": is staged" elif pr.error: sub = ": staging failed" else: sub = "" lines = [ Line([Text( f"#{p['number']}{sub}", font, foreground, decoration=Decoration.STRIKETHROUGH if p['closed'] else Decoration(0), )]), ] # no need for details if closed or in error if pr.state not in ('merged', 'closed', 'error') and not pr.staging_id: if pr.draft: lines.append(Line([boxes[False], Text("is in draft", font, error)])) lines.extend([ Line([ boxes[bool(pr.squash or pr.merge_method)], Text( "merge method: {}".format('single' if pr.squash else (pr.merge_method or 'missing')), font, foreground if pr.squash or pr.merge_method else error, ), ]), Line([ boxes[bool(pr.reviewed_by)], Text( "Reviewed" if pr.reviewed_by else "Not Reviewed", font, foreground if pr.reviewed_by else error, ) ]), Line([ boxes[pr.batch_id.skipchecks or pr.status == 'success'], Text("CI", font, foreground if pr.batch_id.skipchecks or pr.status == 'success' else error), ]), ]) if not pr.batch_id.skipchecks: statuses = json.loads(pr.statuses_full) for ci in pr.repository.status_ids._for_pr(pr): st = (statuses.get(ci.context.strip()) or {'state': 'pending'})['state'] color = foreground if st in ('error', 'failure'): color = error box = boxes[False] elif st == 'success': box = boxes[True] else: box = boxes[None] lines.append(Line([ Text(" - ", font, color), box, Text(f"{ci.repo_id.name}: {ci.context}", font, color) ])) prs.append(Lines(lines)) cell = Cell(Line(prs), background, attached) else: ps = batches[r, b] bgcolor = lighten(BG[ps['state']], by=-0.05) if pr in ps['pr_ids'] else BG[ps['state']] background = blend(bgcolor, opacity, over=background) foreground = blend((39, 110, 114), opacity, over=background) line = [] attached = True for p in ps['prs']: line.append(Text( f"#{p['number']}", font, foreground, decoration=Decoration.STRIKETHROUGH if p['closed'] else Decoration(0), )) attached = attached and p['attached'] for attribute in filter(None, [ 'error' if p['pr'].error else '', '' if p['checked'] else 'missing statuses', '' if p['reviewed'] else 'missing r+', '' if p['attached'] else 'detached', 'staged' if p['pr'].staging_id else 'ready' if p['pr']._ready else '' ]): color = SUCCESS if attribute in ('staged', 'ready') else ERROR line.append(Text(f' {attribute}', supfont, blend(color, opacity, over=background))) line.append(Text(" ", font, foreground)) cell = Cell(Line(line), background, attached) cells[r, b] = cell rowheights[b] = max(rowheights[b], cell.height) colwidths[r] = max(colwidths[r], cell.width) im = Image.new("RGB", (sum(colwidths.values()), sum(rowheights.values())), "white") # no need to set the font here because every text element has its own draw = ImageDraw.Draw(im, 'RGB') top = 0 for b in chain([None], branches): left = 0 for r in chain([None], repos): cell = cells[r, b] # for a given cell, we first print the background, then the text, then # the borders # need to subtract 1 because pillow uses inclusive rect coordinates right = left + colwidths[r] - 1 bottom = top + rowheights[b] - 1 draw.rectangle( (left, top, right, bottom), cell.background, ) # draw content adding padding cell.content.draw(draw, left=left + HORIZONTAL_PADDING, top=top + VERTICAL_PADDING) # draw bottom-right border draw.line([ (left, bottom), (right, bottom), (right, top), ], fill=(172, 176, 170)) if not cell.attached: # overdraw previous cell's bottom border draw.line([(left, top-1), (right-1, top-1)], fill=ERROR) left += colwidths[r] top += rowheights[b] return im def render_inconsistent_batch(batch): """If a batch has inconsistent targets, just point out the inconsistency by listing the PR and targets """ with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Regular.ttf', 'rb') as f: font = ImageFont.truetype(f, size=16, layout_engine=0) im = Image.new("RGB", (4000, 4000), color=BG['danger']) w = h = 0 def draw(label, draw=ImageDraw.Draw(im)): nonlocal w, h draw.text((0, h), label, fill=blend(ERROR, 1.0, over=BG['danger']), font=font) _, _, ww, hh = font.getbbox(label) w = max(w, ww) h += hh draw(" Inconsistent targets:") for p in batch.prs: draw(f" • {p.display_name} has target '{p.target.name}'") draw(" To resolve, either retarget or close the mis-targeted pull request(s).") return im.crop((0, 0, w+10, h+5)) Color = Tuple[int, int, int] TEXT: Color = (102, 102, 102) ERROR: Color = (220, 53, 69) SUCCESS: Color = (40, 167, 69) BG: Mapping[str | None, Color] = collections.defaultdict(lambda: (255, 255, 255), { 'info': (217, 237, 247), 'success': (223, 240, 216), 'warning': (252, 248, 227), 'danger': (242, 222, 222), }) CHECK_MARK = "\uf00c" CROSS = "\uf00d" BOX_EMPTY = "\uf096" def blend_single(c: int, over: int, opacity: float) -> int: return round(over * (1 - opacity) + c * opacity) def blend(color: Color, opacity: float, *, over: Color = (255, 255, 255)) -> Color: assert 0.0 <= opacity <= 1.0 return ( blend_single(color[0], over[0], opacity), blend_single(color[1], over[1], opacity), blend_single(color[2], over[2], opacity), ) def lighten(color: Color, *, by: float) -> Color: # colorsys uses values in the range [0, 1] rather than pillow/CSS-style [0, 225] r, g, b = tuple(c / 255 for c in color) hue, lightness, saturation = colorsys.rgb_to_hls(r, g, b) # by% of the way between value and 1.0 if by >= 0: lightness += (1.0 - lightness) * by # -by% of the way between 0 and value else:lightness *= (1.0 + by) return cast(Color, tuple( round(c * 255) for c in colorsys.hls_to_rgb(hue, lightness, saturation) ))