The huge logs are not stored anymore as properties, this means that the property
tables can just use JSONField and the blob field can be removed.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 api/blobs.py                              |  7 ----
 api/migrations/0030_deblob_properties.py  | 34 ++++++++++++++++++
 api/migrations/0031_auto_20180520_1654.py | 34 ++++++++++++++++++
 api/migrations/__init__.py                |  7 ++--
 api/models.py                             | 42 ++++-------------------
 5 files changed, 79 insertions(+), 45 deletions(-)
 create mode 100644 api/migrations/0030_deblob_properties.py
 create mode 100644 api/migrations/0031_auto_20180520_1654.py

diff --git a/api/blobs.py b/api/blobs.py
index 20d4567..4927952 100644
--- a/api/blobs.py
+++ b/api/blobs.py
@@ -29,13 +29,6 @@ def load_blob(name):
     fn = os.path.join(settings.DATA_DIR, "blob", name + ".xz")
     return lzma.open(fn, 'r').read().decode("utf-8")
 
-def load_blob_json(name):
-    try:
-        return json.loads(load_blob(name))
-    except json.decoder.JSONDecodeError as e:
-        logging.error('Failed to load blob %s: %s' %(name, e))
-        return None
-
 def delete_blob(name):
     fn = os.path.join(settings.DATA_DIR, "blob", name + ".xz")
     try:
diff --git a/api/migrations/0030_deblob_properties.py b/api/migrations/0030_deblob_properties.py
new file mode 100644
index 0000000..801f7a2
--- /dev/null
+++ b/api/migrations/0030_deblob_properties.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations
+from django.db.models import Count
+
+import json
+from api import blobs
+
+def deblob_properties(apps, schema_editor):
+    def do_deblob_properties(model):
+        objects = model.objects.filter(blob=True)
+        for obj in objects:
+            obj.blob = False
+            if obj.value is not None:
+                obj.value = blobs.load_blob(obj.value)
+            obj.save()
+
+    # We can't import the models directly as they may be a newer
+    # version than this migration expects. We use the historical version.
+    do_deblob_properties(apps.get_model('api', 'MessageProperty'))
+    do_deblob_properties(apps.get_model('api', 'ProjectProperty'))
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0029_populate_testing_results'),
+    ]
+
+    operations = [
+        migrations.RunPython(deblob_properties,
+                             reverse_code=migrations.RunPython.noop),
+    ]
diff --git a/api/migrations/0031_auto_20180520_1654.py b/api/migrations/0031_auto_20180520_1654.py
new file mode 100644
index 0000000..edb0c4f
--- /dev/null
+++ b/api/migrations/0031_auto_20180520_1654.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.12 on 2018-05-20 16:54
+from __future__ import unicode_literals
+
+from django.db import migrations
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0030_deblob_properties'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='messageproperty',
+            name='blob',
+        ),
+        migrations.RemoveField(
+            model_name='projectproperty',
+            name='blob',
+        ),
+        migrations.AlterField(
+            model_name='messageproperty',
+            name='value',
+            field=jsonfield.fields.JSONField(),
+        ),
+        migrations.AlterField(
+            model_name='projectproperty',
+            name='value',
+            field=jsonfield.fields.JSONField(),
+        ),
+    ]
diff --git a/api/migrations/__init__.py b/api/migrations/__init__.py
index f4b7eb7..510b26a 100644
--- a/api/migrations/__init__.py
+++ b/api/migrations/__init__.py
@@ -26,7 +26,7 @@ def get_property_raw(model, name, **kwargs):
         return None
 
 def load_property(mp):
-    if mp.blob:
+    if hasattr(mp, 'blob') and mp.blob:
         return load_blob_json_safe(mp.value)
     else:
         return json.loads(mp.value)
@@ -37,7 +37,7 @@ def get_property(model, name, **kwargs):
 
 def delete_property_blob(model, name, **kwargs):
     mp = get_property_raw(model, name, **kwargs)
-    if mp.blob:
+    if hasattr(mp, 'blob') and mp.blob:
         blobs.delete_blob(mp.value)
 
 def set_property(model, name, value, **kwargs):
@@ -45,5 +45,6 @@ def set_property(model, name, value, **kwargs):
         value = json.dumps(value)
     mp, created = model.objects.get_or_create(name=name, **kwargs)
     mp.value = value
-    mp.blob = False
+    if hasattr(mp, 'blob'):
+        mp.blob = False
     mp.save()
diff --git a/api/models.py b/api/models.py
index 3ca87a6..c8a5396 100644
--- a/api/models.py
+++ b/api/models.py
@@ -10,7 +10,6 @@
 
 
 from collections import namedtuple
-import json
 import datetime
 import re
 
@@ -23,8 +22,9 @@ import lzma
 
 from mbox import MboxMessage
 from event import emit_event, declare_event
-from .blobs import save_blob, load_blob, load_blob_json
+from .blobs import save_blob, load_blob
 import mod
+import lzma
 
 class LogEntry(models.Model):
     data_xz = models.BinaryField()
@@ -171,37 +171,23 @@ class Project(models.Model):
     def get_property(self, prop, default=None):
         a = ProjectProperty.objects.filter(project=self, name=prop).first()
         if a:
-            if a.blob:
-                return load_blob_json(a.value)
-            else:
-                return json.loads(a.value)
+            return a.value
         else:
             return default
 
     def get_properties(self):
         r = {}
         for m in ProjectProperty.objects.filter(project=self):
-            if m.blob:
-                r[m.name] = load_blob_json(m.value)
-            else:
-                r[m.name] = json.loads(m.value)
+            r[m.name] = m.value
         return r
 
     def _do_set_property(self, prop, value):
         if value == None:
             ProjectProperty.objects.filter(project=self, name=prop).delete()
             return
-        # TODO: drop old blob
-        json_data = json.dumps(value)
-        blob = len(json_data) > 1024
-        if blob:
-            value = save_blob(json_data)
-        else:
-            value = json.dumps(value)
         pp, created = ProjectProperty.objects.get_or_create(project=self,
                                                             name=prop)
         pp.value = value
-        pp.blob = blob
         pp.save()
 
     def set_property(self, prop, value):
@@ -298,8 +284,7 @@ class ProjectResult(Result):
 class ProjectProperty(models.Model):
     project = models.ForeignKey('Project', on_delete=models.CASCADE)
     name = models.CharField(max_length=1024, db_index=True)
-    value = models.CharField(max_length=1024)
-    blob = models.BooleanField(blank=True, default=False)
+    value = jsonfield.JSONField()
 
     class Meta:
         unique_together = ('project', 'name',)
@@ -562,10 +547,7 @@ class Message(models.Model):
             all_props = self.properties.all()
         r = {}
         for m in all_props:
-            if m.blob:
-                r[m.name] = load_blob_json(m.value)
-            else:
-                r[m.name] = json.loads(m.value)
+            r[m.name] = m.value
         self._properties = r
         return r
 
@@ -573,17 +555,9 @@ class Message(models.Model):
         if value == None:
             MessageProperty.objects.filter(message=self, name=prop).delete()
             return
-        json_data = json.dumps(value)
-        blob = len(json_data) > 1024
         mp, created = MessageProperty.objects.get_or_create(message=self,
                                                             name=prop)
-        # TODO: drop old blob
-        if blob:
-            value = save_blob(json_data)
-        else:
-            value = json_data
         mp.value = value
-        mp.blob = blob
         mp.save()
         # Invalidate cache
         self._properties = None
@@ -724,9 +698,7 @@ class MessageProperty(models.Model):
     message = models.ForeignKey('Message', on_delete=models.CASCADE,
                                 related_name='properties')
     name = models.CharField(max_length=256)
-    # JSON encoded value
-    value = models.CharField(max_length=1024)
-    blob = models.BooleanField(blank=True, default=False)
+    value = jsonfield.JSONField()
 
     def __str__(self):
         if len(self.value) > 30:
-- 
2.17.0


_______________________________________________
Patchew-devel mailing list
Patchew-devel@redhat.com
https://www.redhat.com/mailman/listinfo/patchew-devel