runbot/runbot/models/commit.py

import datetime
import subprocess

from ..common import os, RunbotException, make_github_session
import glob
import shutil

from odoo import models, fields, api, registry
from odoo.tools import file_open
import logging

_logger = logging.getLogger(__name__)


class Commit(models.Model):
    _name = 'runbot.commit'
    _description = "Commit"

    _sql_constraints = [
        (
            "commit_unique",
            "unique (name, repo_id, rebase_on_id)",
            "Commit must be unique to ensure correct duplicate matching",
        )
    ]
    name = fields.Char('SHA')
    tree_hash = fields.Char('Tree hash', readonly=True)
    repo_id = fields.Many2one('runbot.repo', string='Repo group')
    date = fields.Datetime('Commit date')
    author = fields.Char('Author')
    author_email = fields.Char('Author Email')
    committer = fields.Char('Committer')
    committer_email = fields.Char('Committer Email')
    subject = fields.Text('Subject')
    dname = fields.Char('Display name', compute='_compute_dname')
    rebase_on_id = fields.Many2one('runbot.commit', 'Rebase on commit')

    @api.model_create_multi
    def create(self, vals_list):
        for vals in vals_list:
            if 'date' not in vals:
                vals['date'] = fields.Datetime.now()
        return super().create(vals_list)

    def _get_commit_infos(self, sha, repo):
        fields = ['date', 'author', 'author_email', 'committer', 'committer_email', 'subject', 'tree_hash']
        pretty_format = '%x00'.join(['%ct', '%an', '%ae', '%cn', '%ce', '%s', '%T'])
        vals = {}
        try:
            vals = dict(zip(fields, repo._git(['show', '-s', f'--pretty=format:{pretty_format}', sha]).split('\x00')))
            vals['date'] = datetime.datetime.fromtimestamp(int(vals['date']))
        except subprocess.CalledProcessError as e:
            _logger.warning('git show failed with message %s', e.output.decode())
        return vals

    def _get(self, name, repo_id, vals=None, rebase_on_id=False):
        commit = self.search([('name', '=', name), ('repo_id', '=', repo_id), ('rebase_on_id', '=', rebase_on_id)])
        if not commit:
            if not vals:
                repo = self.env['runbot.repo'].browse(repo_id)
                vals = self._get_commit_infos(name, repo)
            commit = self.env['runbot.commit'].create({**vals, 'name': name, 'repo_id': repo_id, 'rebase_on_id': rebase_on_id})
        return commit

    def _rebase_on(self, commit):
        if self == commit:
            return self
        return self._get(self.name, self.repo_id.id, self.read()[0], commit.id)

    def _get_available_modules(self):
        for manifest_file_name in self.repo_id.manifest_files.split(','):  # '__manifest__.py' '__openerp__.py'
            for addons_path in (self.repo_id.addons_paths or '').split(','):  # '' 'addons' 'odoo/addons'
                sep = os.path.join(addons_path, '*')
                for manifest_path in glob.glob(self._source_path(sep, manifest_file_name)):
                    module = os.path.basename(os.path.dirname(manifest_path))
                    yield (addons_path, module, manifest_file_name)

    def _list_files(self, patterns):
        #example: git ls-files --with-tree=abcf390f90dbdd39fd61abc53f8516e7278e0931 ':(glob)addons/*/*.py' ':(glob)odoo/addons/*/*.py'
        # note that glob is needed to avoid the star matching **
        self.ensure_one()
        return self.repo_id._git(['ls-files', '--with-tree', self.name, *patterns]).split('\n')

    def _list_available_modules(self):
        # beta version, may replace _get_available_modules latter
        addons_paths = (self.repo_id.addons_paths or '').split(',')
        patterns = []
        for manifest_file_name in self.repo_id.manifest_files.split(','):  # '__manifest__.py' '__openerp__.py'
            for addon_path in addons_paths:
                addon_path = addon_path or '.'
                patterns.append(f':(glob){addon_path}/*/{manifest_file_name}')
        for file_path in self._list_files(patterns):
            if file_path:
                elems = file_path.rsplit('/', 2)
                if len(elems) == 3:
                    addons_path, module, manifest_file_name = elems
                else:
                    addons_path = ''
                    module, manifest_file_name = elems
                yield (addons_path, module, manifest_file_name)


    def _export(self, build):
        """Export a git repo into a sources"""
        #  TODO add automated tests
        self.ensure_one()
        if not self.env['runbot.commit.export'].search([('build_id', '=', build.id), ('commit_id', '=', self.id)]):
            self.env['runbot.commit.export'].create({'commit_id': self.id, 'build_id': build.id})
        export_path = self._source_path()

        if os.path.isdir(export_path):
            _logger.info('git export: exporting to %s (already exists)', export_path)
            return export_path


        _logger.info('git export: exporting to %s (new)', export_path)
        os.makedirs(export_path)

        self.repo_id._fetch(self.name)
        export_sha = self.name
        if self.rebase_on_id:
            export_sha = self.rebase_on_id.name
            self.rebase_on_id.repo_id._fetch(export_sha)

        p1 = subprocess.Popen(['git', '--git-dir=%s' % self.repo_id.path, 'archive', export_sha], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        p2 = subprocess.Popen(['tar', '--mtime', self.date.strftime('%Y-%m-%d %H:%M:%S'), '-xC', export_path], stdin=p1.stdout, stdout=subprocess.PIPE)
        p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
        (_, err) = p2.communicate()
        p1.poll()  # fill the returncode
        if p1.returncode:
            _logger.info("git export: removing corrupted export %r", export_path)
            shutil.rmtree(export_path)
            raise RunbotException("Git archive failed for %s with error code %s. (%s)" % (self.name, p1.returncode, p1.stderr.read().decode()))
        if err:
            _logger.info("git export: removing corrupted export %r", export_path)
            shutil.rmtree(export_path)
            raise RunbotException("Export for %s failed. (%s)" % (self.name, err))

        if self.rebase_on_id:
            # we could be smart here and detect if merge_base == commit, in witch case checkouting base_commit is enough. Since we don't have this info
            # and we are exporting in a custom folder anyway, lets
            _logger.info('Applying patch for %s', self.name)
            p1 = subprocess.Popen(['git', '--git-dir=%s' % self.repo_id.path, 'diff', '%s...%s' % (export_sha, self.name)], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
            p2 = subprocess.Popen(['patch', '-p0', '-d', export_path], stdin=p1.stdout, stdout=subprocess.PIPE)
            p1.stdout.close()
            (message, err) = p2.communicate()
            p1.poll()
            if err:
                shutil.rmtree(export_path)
                raise RunbotException("Apply patch failed for %s...%s. (%s)" % (export_sha, self.name, err))
            if p1.returncode or p2.returncode:
                shutil.rmtree(export_path)
                raise RunbotException("Apply patch failed for %s...%s with error code %s+%s. (%s)" % (export_sha, self.name, p1.returncode, p2.returncode, message))

        return export_path

    def _read_source(self, file, mode='r'):
        file_path = self._source_path(file)
        try:
            with file_open(file_path, mode) as f:
                return f.read()
        except:
            return False

    def _source_path(self, *paths):
        export_name = self.name
        if self.rebase_on_id:
            export_name = '%s_%s' % (self.name, self.rebase_on_id.name)
        return self.repo_id._source_path(export_name, *paths)

    @api.depends('name', 'repo_id.name')
    def _compute_dname(self):
        for commit in self:
            commit.dname = '%s:%s' % (commit.repo_id.name, commit.name[:8])

    def _github_status(self, build, context, state, target_url, description=None):
        self.ensure_one()
        Status = self.env['runbot.commit.status']
        last_status = Status.search([('commit_id', '=', self.id), ('context', '=', context)], order='id desc', limit=1)
        if last_status and last_status.state == state:
            _logger.info('Skipping already sent status %s:%s for %s', context, state, self.name)
            return
        last_status = Status.create({
            'build_id': build.id if build else False,
            'commit_id': self.id,
            'context': context,
            'state': state,
            'target_url': target_url,
            'description': description or context,
            'to_process': True,
        })


class CommitLink(models.Model):
    _name = 'runbot.commit.link'
    _description = "Build commit"

    commit_id = fields.Many2one('runbot.commit', 'Commit', required=True, index=True)
    # Link info
    match_type = fields.Selection([('new', 'New head of branch'), ('head', 'Head of branch'), ('base_head', 'Found on base branch'), ('base_match', 'Found on base branch')])  # HEAD, DEFAULT
    branch_id = fields.Many2one('runbot.branch', string='Found in branch')  # Shouldn't be use for anything else than display

    base_commit_id = fields.Many2one('runbot.commit', 'Base head commit', index=True)
    merge_base_commit_id = fields.Many2one('runbot.commit', 'Merge Base commit', index=True)
    base_behind = fields.Integer('# commits behind base')
    base_ahead = fields.Integer('# commits ahead base')
    file_changed = fields.Integer('# file changed')
    diff_add = fields.Integer('# line added')
    diff_remove = fields.Integer('# line removed')


class CommitStatus(models.Model):
    _name = 'runbot.commit.status'
    _description = 'Commit status'
    _order = 'id desc'

    commit_id = fields.Many2one('runbot.commit', string='Commit', required=True, index=True)
    context = fields.Char('Context', required=True)
    state = fields.Char('State', required=True, copy=True)
    build_id = fields.Many2one('runbot.build', string='Build', index=True)
    target_url = fields.Char('Url')
    description = fields.Char('Description')
    sent_date = fields.Datetime('Sent Date')
    to_process = fields.Boolean('Status was not processed yet', index=True)

    def _send_to_process(self):
        commits_status = self.search([('to_process', '=', True)], order='create_date DESC, id DESC')
        if commits_status:
            _logger.info('Sending %s commit status', len(commits_status))
            commits_status._send()

    def _send(self):
        session_cache = {}
        processed = set()
        for commit_status in self.sorted(lambda cs: (cs.create_date, cs.id), reverse=True): # ensure most recent are processed first
            commit_status.to_process = False
            # only send the last status for each commit+context
            key = (commit_status.context, commit_status.commit_id.name)
            if key not in processed:
                processed.add(key)
                status = {
                    'context': commit_status.context,
                    'state': commit_status.state,
                    'target_url': commit_status.target_url,
                    'description': commit_status.description,
                }
                for remote in commit_status.commit_id.repo_id.remote_ids.filtered('send_status'):
                    if not remote.token:
                        _logger.warning('No token on remote %s, skipping status', remote.mapped("name"))
                    else:
                        if remote.token not in session_cache:
                            session_cache[remote.token] = make_github_session(remote.token)
                        session = session_cache[remote.token]
                        _logger.info(
                            "github updating %s status %s to %s in repo %s",
                            status['context'], commit_status.commit_id.name, status['state'], remote.name)
                        remote._github('/repos/:owner/:repo/statuses/%s' % commit_status.commit_id.name,
                            status,
                            ignore_errors=True,
                            session=session
                        )
                commit_status.sent_date = fields.Datetime.now()
            else:
                _logger.info('Skipping outdated status for %s %s', commit_status.context, commit_status.commit_id.name)



class CommitExport(models.Model):
    _name = 'runbot.commit.export'
    _description = 'Commit export'

    build_id = fields.Many2one('runbot.build', index=True)
    commit_id = fields.Many2one('runbot.commit')

    host = fields.Char(related='build_id.host', store=True)