From nobody Tue May 13 19:32:24 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886273827332.18972804526254; Wed, 13 Jun 2018 03:37:53 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx07.intmail.prod.int.phx2.redhat.com [10.5.11.22]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id AE382307D85D; Wed, 13 Jun 2018 10:37:52 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 9C94F1001938; Wed, 13 Jun 2018 10:37:52 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 8C9854BB78; Wed, 13 Jun 2018 10:37:52 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbpp1009764 for ; Wed, 13 Jun 2018 06:37:51 -0400 Received: by smtp.corp.redhat.com (Postfix) id ECB8920244E0; Wed, 13 Jun 2018 10:37:50 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id 3591C2017D0F for ; Wed, 13 Jun 2018 10:37:50 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:39 +0200 Message-Id: <20180613103744.30891-6-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 05/10] git: switch to Result model X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.84 on 10.5.11.22 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.48]); Wed, 13 Jun 2018 10:37:52 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" Operate on a Result named 'git' instead of using multiple properties. Only the hook has to convert the Result to namedtuple format, until the REST API and web views are converted to access the database directly. --- api/blobs.py | 8 ++ api/migrations/0028_populate_git_results.py | 104 ++++++++++++++++ api/migrations/__init__.py | 49 ++++++++ mods/git.py | 124 +++++++++----------- tests/test_git.py | 34 +++--- 5 files changed, 235 insertions(+), 84 deletions(-) create mode 100644 api/migrations/0028_populate_git_results.py diff --git a/api/blobs.py b/api/blobs.py index b1c7d34..20d4567 100644 --- a/api/blobs.py +++ b/api/blobs.py @@ -35,3 +35,11 @@ def load_blob_json(name): except json.decoder.JSONDecodeError as e: logging.error('Failed to load blob %s: %s' %(name, e)) return None + +def delete_blob(name): + fn =3D os.path.join(settings.DATA_DIR, "blob", name + ".xz") + try: + os.remove(fn) + except FileNotFoundError: + pass + diff --git a/api/migrations/0028_populate_git_results.py b/api/migrations/0= 028_populate_git_results.py new file mode 100644 index 0000000..482641a --- /dev/null +++ b/api/migrations/0028_populate_git_results.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations +from django.db.models import Count +from api.migrations import get_property, set_property, delete_property_blob + +import datetime +import lzma + +# For Result's constant status values +import api.models + +def result_from_properties(apps, schema_editor): + # We can't import the models directly as they may be a newer + # version than this migration expects. We use the historical version. + # The code is mostly based on the implementation of the old + # "rest_results_hook" method in GitModule, which used to build + # a Result namedtuple from the properties + Message =3D apps.get_model('api', 'Message') + MessageProperty =3D apps.get_model('api', 'MessageProperty') + MessageResult =3D apps.get_model('api', 'MessageResult') + LogEntry =3D apps.get_model('api', 'LogEntry') + messages =3D Message.objects.filter(properties__name__startswith=3D'gi= t.').distinct() + for m in messages: + need_apply =3D get_property(MessageProperty, 'git.need-apply', mes= sage=3Dm) + if need_apply is None: + continue + log =3D get_property(MessageProperty, "git.apply-log", message=3Dm) + r =3D MessageResult(name=3D'git', message=3Dm) + if log: + log_xz =3D lzma.compress(log.encode("utf-8")) + log_entry =3D LogEntry(data_xz=3Dlog_xz) + log_entry.save() + r.log_entry =3D log_entry + if get_property(MessageProperty, "git.apply-failed", message= =3Dm): + r.status =3D api.models.Result.FAILURE + else: + git_repo =3D get_property(MessageProperty, "git.repo", mes= sage=3Dm) + git_tag =3D get_property(MessageProperty, "git.tag", messa= ge=3Dm) + git_url =3D get_property(MessageProperty, "git.url", messa= ge=3Dm) + git_base =3D get_property(MessageProperty, "git.base", mes= sage=3Dm) + data =3D {} + if git_repo and git_tag: + data['repo'] =3D git_repo + data['tag'] =3D 'refs/tags/' + git_tag + if git_url: + data['url'] =3D git_url + if git_base: + data['base'] =3D git_base + r.data =3D data + r.status =3D api.models.Result.SUCCESS + else: + status =3D api.models.Result.PENDING + r.last_update =3D datetime.datetime.utcnow() + r.save() + messages =3D Message.objects.filter(properties__name=3D'git.apply-log'= , properties__blob=3DTrue) + for m in messages: + delete_property_blob(MessageProperty, "git.apply-log", message=3Dm) + MessageProperty.objects.filter(name__startswith=3D'git.').delete() + +def result_to_properties(apps, schema_editor): + # We can't import the models directly as they may be a newer + # version than this migration expects. We use the historical version. + Message =3D apps.get_model('api', 'Message') + MessageProperty =3D apps.get_model('api', 'MessageProperty') + MessageResult =3D apps.get_model('api', 'MessageResult') + LogEntry =3D apps.get_model('api', 'LogEntry') + messages =3D Message.objects.filter(results__name=3D'git') + for m in messages: + r =3D MessageResult.objects.get(name=3D'git', message=3Dm) + if not r: + continue + if r.status =3D=3D api.models.Result.PENDING: + set_property(MessageProperty, 'git.need-apply', True, message= =3Dm) + else: + log =3D lzma.decompress(r.log_entry.data_xz).decode("utf-8") + set_property(MessageProperty, 'git.need-apply', False, message= =3Dm) + set_property(MessageProperty, 'git.apply-log', log, message=3D= m) + if r.status =3D=3D api.models.Result.FAILURE: + set_property(MessageProperty, "git.apply-failed", True, me= ssage=3Dm) + else: + set_property(MessageProperty, "git.apply-failed", False, m= essage=3Dm) + if 'repo' in r.data: + set_property(MessageProperty, "git.repo", r.data['repo= '], message=3Dm) + if 'tag' in r.data: + set_property(MessageProperty, "git.tag", r.data['repo'= ][len('refs/tags/'):], message=3Dm) + if 'url' in r.data: + set_property(MessageProperty, "git.url", r.data['url']= , message=3Dm) + if 'base' in r.data: + set_property(MessageProperty, "git.base", r.data['base= '], message=3Dm) + MessageResult.objects.filter(message=3Dm, name=3D'git').delete() + +class Migration(migrations.Migration): + + dependencies =3D [ + ('api', '0027_auto_20180521_0152'), + ] + + operations =3D [ + migrations.RunPython(result_from_properties, + reverse_code=3Dresult_to_properties), + ] diff --git a/api/migrations/__init__.py b/api/migrations/__init__.py index e69de29..f4b7eb7 100644 --- a/api/migrations/__init__.py +++ b/api/migrations/__init__.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright 2018 Red Hat, Inc. +# +# Authors: +# Paolo Bonzini +# +# This work is licensed under the MIT License. Please see the LICENSE fil= e or +# http://opensource.org/licenses/MIT. + + +import json +from api import blobs + +def load_blob_json_safe(name): + try: + return json.loads(blobs.load_blob(name)) + except Exception as e: + return 'Failed to load blob %s: %s' % (name, e) + +def get_property_raw(model, name, **kwargs): + try: + mp =3D model.objects.get(name=3Dname, **kwargs) + return mp + except model.DoesNotExist: + return None + +def load_property(mp): + if mp.blob: + return load_blob_json_safe(mp.value) + else: + return json.loads(mp.value) + +def get_property(model, name, **kwargs): + mp =3D get_property_raw(model, name, **kwargs) + return load_property(mp) if mp else None + +def delete_property_blob(model, name, **kwargs): + mp =3D get_property_raw(model, name, **kwargs) + if mp.blob: + blobs.delete_blob(mp.value) + +def set_property(model, name, value, **kwargs): + if value is not None: + value =3D json.dumps(value) + mp, created =3D model.objects.get_or_create(name=3Dname, **kwargs) + mp.value =3D value + mp.blob =3D False + mp.save() diff --git a/mods/git.py b/mods/git.py index fadce4c..b8c2ff0 100644 --- a/mods/git.py +++ b/mods/git.py @@ -18,7 +18,7 @@ from django.core.exceptions import PermissionDenied from django.utils.html import format_html from mod import PatchewModule from event import declare_event, register_handler, emit_event -from api.models import Message, MessageProperty, Result, ResultTuple +from api.models import Message, MessageProperty, Project, Result, ResultTu= ple from api.rest import PluginMethodField from api.views import APILoginRequiredView, prepare_series from patchew.logviewer import LogView @@ -26,16 +26,24 @@ from schema import * =20 _instance =3D None =20 +def _get_git_result(msg): + try: + return msg.results.get(name=3D"git") + except: + return None +Message.git_result =3D property(_get_git_result) + + class GitLogViewer(LogView): def content(self, request, **kwargs): series =3D kwargs['series'] obj =3D Message.objects.find_series(series) if not obj: raise Http404("Object not found: " + series) - log =3D obj.get_property("git.apply-log") - if log is None: + r =3D obj.git_result + if r is None or not r.is_completed(): raise Http404("Git apply log not found") - return log + return r.log =20 =20 class GitModule(PatchewModule): @@ -66,9 +74,16 @@ class GitModule(PatchewModule): register_handler("SeriesComplete", self.on_series_update) register_handler("TagsUpdate", self.on_series_update) =20 + def mark_as_pending_apply(self, series): + r =3D series.git_result or series.create_result(name=3D'git') + r.log =3D None + r.status =3D Result.PENDING + r.data =3D {} + r.save() + def on_series_update(self, event, series, **params): if series.is_complete: - series.set_property("git.need-apply", True) + self.mark_as_pending_apply(series) =20 def get_project_config(self, project, what): return project.get_property("git." + what) @@ -114,50 +129,20 @@ class GitModule(PatchewModule): =20 def get_based_on(self, message, request, format): git_base =3D self.get_base(message) - if not git_base: - return None - - return { - "repo": git_base.get_property("git.repo"), - "tag": 'refs/tags/' + git_base.get_property("git.tag") - } - + return git_base.data if git_base else None =20 def rest_series_fields_hook(self, request, fields, detailed): fields['based_on'] =3D PluginMethodField(obj=3Dself, required=3DFa= lse) =20 def rest_results_hook(self, obj, results, detailed=3DFalse): - if not isinstance(obj, Message): - return - log =3D obj.get_property("git.apply-log") - data =3D None - if log: - if obj.get_property("git.apply-failed"): - status =3D Result.FAILURE - else: - git_repo =3D obj.get_property("git.repo") - git_tag =3D obj.get_property("git.tag") - git_url =3D obj.get_property("git.url") - git_base =3D obj.get_property("git.base") - data =3D {} - if git_repo and git_tag: - data['repo'] =3D git_repo - data['tag'] =3D 'refs/tags/' + git_tag - if git_url: - data['url'] =3D git_url - if git_base: - data['base'] =3D git_base - status =3D Result.SUCCESS - else: - status =3D Result.PENDING - results.append(ResultTuple(name=3D'git', obj=3Dobj, status=3Dstatu= s, - log=3Dlog, data=3Ddata, renderer=3Dself)) + Result.get_result_tuples(obj, "git", results) =20 def prepare_message_hook(self, request, message, detailed): if not message.is_series_head: return - if message.get_property("git.apply-log"): - if message.get_property("git.apply-failed"): + r =3D message.git_result + if r and r.is_completed(): + if r.is_failure(): title =3D "Failed in applying to current master" message.status_tags.append({ "title": title, @@ -165,10 +150,10 @@ class GitModule(PatchewModule): "char": "G", }) else: - git_url =3D message.get_property("git.url") - git_repo =3D message.get_property("git.repo") - git_tag =3D message.get_property("git.tag") + git_url =3D r.data.get('url') if git_url: + git_repo =3D r.data['repo'] + git_tag =3D r.data['tag'] message.status_tags.append({ "url": git_url, "title": format_html("Applied as tag {} in repo {}= ", git_tag, git_repo), @@ -181,9 +166,7 @@ class GitModule(PatchewModule): "type": "info", "char": "G", }) - if request.user.is_authenticated: - if message.get_property("git.apply-failed") !=3D None or \ - message.get_property("git.need-apply") =3D=3D None: + if request.user.is_authenticated: url =3D reverse("git_reset", kwargs=3D{"series": message.message_id}) message.extra_ops.append({"url": url, @@ -238,7 +221,8 @@ class GitModule(PatchewModule): filter(project=3Dseries.project, message_id=3Dbase_id)= .first() if not base: return None - return base if base.get_property("git.repo") else None + r =3D base.git_result + return r if r and r.data.get("repo") else None =20 def prepare_series_hook(self, request, series, response): po =3D series.project @@ -247,8 +231,8 @@ class GitModule(PatchewModule): response[prop] =3D po.get_property(prop) base =3D self.get_base(series) if base: - response["git.repo"] =3D base.get_property("git.repo") - response["git.base"] =3D base.get_property("git.tag") + response["git.repo"] =3D base.data["repo"] + response["git.base"] =3D base.data["tag"] =20 def _poll_project(self, po): repo, branch =3D self._get_project_repo_and_branch(po) @@ -268,10 +252,7 @@ class GitModule(PatchewModule): obj =3D Message.objects.find_series(series) if not obj: raise Http404("Not found: " + series) - for p in obj.get_properties(): - if p.startswith("git.") and p !=3D "git.need-apply": - obj.set_property(p, None) - obj.set_property("git.need-apply", True) + self.mark_as_pending_apply(obj) return HttpResponseRedirect(request.META.get('HTTP_REFERER')) =20 def www_url_hook(self, urlpatterns): @@ -287,11 +268,9 @@ class ApplierGetView(APILoginRequiredView): allowed_groups =3D ["importers"] =20 def handle(self, request): - mp =3D MessageProperty.objects.filter(name=3D"git.need-apply", - value=3D'true', - message__is_complete=3DTrue).f= irst() - if mp: - return prepare_series(request, mp.message) + m =3D Message.objects.filter(results__name=3D"git", results__statu= s=3D"pending").first() + if m: + return prepare_series(request, m) =20 class ApplierReportView(APILoginRequiredView): name =3D "applier-report" @@ -299,12 +278,23 @@ class ApplierReportView(APILoginRequiredView): =20 def handle(self, request, project, message_id, tag, url, base, repo, failed, log): - s =3D Message.objects.series_heads().get(project__name=3Dproject, - message_id=3Dmessage_id) - s.set_property("git.tag", tag) - s.set_property("git.url", url) - s.set_property("git.base", base) - s.set_property("git.repo", repo) - s.set_property("git.apply-failed", failed) - s.set_property("git.apply-log", log) - s.set_property("git.need-apply", False) + p =3D Project.objects.get(name=3Dproject) + r =3D Message.objects.series_heads().get(project=3Dp, + message_id=3Dmessage_id).gi= t_result + r.log =3D log + if failed: + r.status =3D Result.FAILURE + else: + data =3D {} + data['repo'] =3D repo + data['tag'] =3D 'refs/tags/' + tag + if url: + data['url'] =3D url + elif url_template and tag: + url_template =3D p.get_property("git.url_template") + data['url'] =3D url_template.replace("%t", tag) + if base: + data['base'] =3D base + r.data =3D data + r.status =3D Result.SUCCESS + r.save() diff --git a/tests/test_git.py b/tests/test_git.py index d64f901..509949a 100755 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -14,7 +14,7 @@ sys.path.append(os.path.dirname(__file__)) from patchewtest import PatchewTestCase, main import shutil import subprocess -from api.models import Message +from api.models import Message, Result =20 class GitTest(PatchewTestCase): =20 @@ -34,37 +34,37 @@ class GitTest(PatchewTestCase): def do_apply(self): self.cli(["apply", "--applier-mode"]) for s in Message.objects.series_heads(): - self.assertFalse(s.get_property("git.need-apply")) + self.assertNotEqual(s.git_result.status, Result.PENDING) =20 def test_need_apply(self): self.cli_import("0001-simple-patch.mbox.gz") s =3D Message.objects.series_heads()[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.need-apply"), True) + self.assertEqual(s.git_result.status, Result.PENDING) self.do_apply() =20 def test_need_apply_multiple(self): self.cli_import("0004-multiple-patch-reviewed.mbox.gz") s =3D Message.objects.series_heads()[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.need-apply"), True) + self.assertEqual(s.git_result.status, Result.PENDING) self.do_apply() =20 def test_need_apply_incomplete(self): self.cli_import("0012-incomplete-series.mbox.gz") s =3D Message.objects.series_heads()[0] self.assertEqual(s.is_complete, False) - self.assertEqual(s.get_property("git.need-apply"), None) + self.assertEqual(s.git_result is None, True) =20 def test_apply(self): self.cli_import("0013-foo-patch.mbox.gz") self.do_apply() s =3D Message.objects.series_heads()[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.repo"), self.repo) - self.assertEqual(s.get_property("git.tag"), - "patchew/20160628014747.20971-1-famz@redhat.com") - self.assertEqual(s.get_property("git.url"), + self.assertEqual(s.git_result.data['repo'], self.repo) + self.assertEqual(s.git_result.data['tag'], + "refs/tags/patchew/20160628014747.20971-1-famz@re= dhat.com") + self.assertEqual(s.git_result.data['url'], self.repo + " patchew/20160628014747.20971-1-famz= @redhat.com") =20 def test_apply_with_base(self): @@ -74,10 +74,10 @@ class GitTest(PatchewTestCase): self.do_apply() s =3D Message.objects.series_heads().filter(message_id=3D"20160628= 014747.20971-2-famz@redhat.com")[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.repo"), self.repo) - self.assertEqual(s.get_property("git.tag"), - "patchew/20160628014747.20971-2-famz@redhat.com") - self.assertEqual(s.get_property("git.url"), + self.assertEqual(s.git_result.data['repo'], self.repo) + self.assertEqual(s.git_result.data['tag'], + "refs/tags/patchew/20160628014747.20971-2-famz@re= dhat.com") + self.assertEqual(s.git_result.data['url'], self.repo + " patchew/20160628014747.20971-2-famz= @redhat.com") =20 def test_apply_with_base_and_brackets(self): @@ -87,10 +87,10 @@ class GitTest(PatchewTestCase): self.do_apply() s =3D Message.objects.series_heads().filter(message_id=3D"20160628= 014747.20971-2-famz@redhat.com")[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.repo"), self.repo) - self.assertEqual(s.get_property("git.tag"), - "patchew/20160628014747.20971-2-famz@redhat.com") - self.assertEqual(s.get_property("git.url"), + self.assertEqual(s.git_result.data['repo'], self.repo) + self.assertEqual(s.git_result.data['tag'], + "refs/tags/patchew/20160628014747.20971-2-famz@re= dhat.com") + self.assertEqual(s.git_result.data['url'], self.repo + " patchew/20160628014747.20971-2-famz= @redhat.com") =20 def test_rest_need_apply(self): --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel