https://bugzilla.redhat.com/show_bug.cgi?id=916061

If the QEMU version running is new enough (based on QEMU_CAPS_QUERY_DUMP)
we can add a 'detach' boolean to the dump-guest-memory command in order
to tell QEMU to run in a thread. Then, use the qemuDumpWaitForCompletion
in order to 'watch' the dump and save the stats into the jobInfo data
so that tools (such as virsh) can pick up and display the completion
percentage. The processing is similar to qemuMigrationWaitForCompletion
at least with respect to calling the query-dump in a while loop that
gets a micro sleep in order for qemuDomainGetJobInfo to be able to
pull out the changed migration stats values.

As an added benefit, while the domain is being dumped, we don't lock
out other commands.

Signed-off-by: John Ferlan <jferlan@redhat.com>
---
 src/qemu/qemu_driver.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 80 insertions(+), 4 deletions(-)

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 066c778726..bcbacf6549 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -3750,6 +3750,73 @@ qemuDomainManagedSaveRemove(virDomainPtr dom, unsigned int flags)
 }
 
 
+/**
+ * qemuDumpWaitForCompletion:
+ * @driver: qemu driver data
+ * @vm: domain object
+ * @asyncJob: async job id
+ *
+ * If the query dump capability exists, then it's possible to start a
+ * guest memory dump operation using a thread via a 'detach' qualifier
+ * to the dump guest memory command. This allows the async check if the
+ * dump is done.
+ *
+ * Returns 0 on success, -1 on failure
+ */
+static int
+qemuDumpWaitForCompletion(virQEMUDriverPtr driver,
+                          virDomainObjPtr vm,
+                          qemuDomainAsyncJob asyncJob)
+{
+    qemuDomainObjPrivatePtr priv = vm->privateData;
+    qemuDomainJobInfoPtr jobInfo = priv->job.current;
+    qemuMonitorDumpStats stats;
+    struct timespec ts = { .tv_sec = 0, .tv_nsec = 50 * 1000 * 1000ull };
+    int rv;
+
+    do {
+        if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
+            return -1;
+
+        rv = qemuMonitorQueryDump(priv->mon, &stats);
+
+        if (qemuDomainObjExitMonitor(driver, vm) < 0 || rv < 0)
+            return -1;
+
+        /* Save the stats in the migration stats so that a GetJobInfo
+         * will be able to compute the completion percentage */
+        jobInfo->stats.ram_total = stats.total;
+        jobInfo->stats.ram_remaining = stats.total - stats.completed;
+        jobInfo->stats.ram_transferred = stats.completed;
+        switch (stats.status) {
+        case QEMU_MONITOR_DUMP_STATUS_NONE:
+        case QEMU_MONITOR_DUMP_STATUS_FAILED:
+        case QEMU_MONITOR_DUMP_STATUS_LAST:
+            virReportError(VIR_ERR_OPERATION_FAILED,
+                           _("dump query failed, status=%d"), stats.status);
+            return -1;
+            break;
+
+        case QEMU_MONITOR_DUMP_STATUS_ACTIVE:
+            jobInfo->status = QEMU_DOMAIN_JOB_STATUS_ACTIVE;
+            VIR_DEBUG("dump active, bytes written='%llu' remaining='%llu'",
+                      stats.completed, stats.total - stats.completed);
+            break;
+
+        case QEMU_MONITOR_DUMP_STATUS_COMPLETED:
+            jobInfo->status = QEMU_DOMAIN_JOB_STATUS_COMPLETED;
+            VIR_DEBUG("dump completed, bytes written='%llu'", stats.completed);
+            break;
+        }
+        virObjectUnlock(vm);
+        nanosleep(&ts, NULL);
+        virObjectLock(vm);
+    } while (stats.status == QEMU_MONITOR_DUMP_STATUS_ACTIVE);
+
+    return 0;
+}
+
+
 static int
 qemuDumpToFd(virQEMUDriverPtr driver,
              virDomainObjPtr vm,
@@ -3758,6 +3825,7 @@ qemuDumpToFd(virQEMUDriverPtr driver,
              const char *dumpformat)
 {
     qemuDomainObjPrivatePtr priv = vm->privateData;
+    bool detach = false;
     int ret = -1;
 
     if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DUMP_GUEST_MEMORY)) {
@@ -3766,10 +3834,13 @@ qemuDumpToFd(virQEMUDriverPtr driver,
         return -1;
     }
 
+    detach = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_QUERY_DUMP);
+
     if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, fd) < 0)
         return -1;
 
-    VIR_FREE(priv->job.current);
+    if (!detach)
+        VIR_FREE(priv->job.current);
     priv->job.dump_memory_only = true;
 
     if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0)
@@ -3784,15 +3855,20 @@ qemuDumpToFd(virQEMUDriverPtr driver,
                              "for this QEMU binary"),
                            dumpformat);
             ret = -1;
+            ignore_value(qemuDomainObjExitMonitor(driver, vm));
             goto cleanup;
         }
     }
 
-    ret = qemuMonitorDumpToFd(priv->mon, fd, dumpformat, false);
+    ret = qemuMonitorDumpToFd(priv->mon, fd, dumpformat, detach);
 
- cleanup:
-    ignore_value(qemuDomainObjExitMonitor(driver, vm));
+    if ((qemuDomainObjExitMonitor(driver, vm) < 0) || ret < 0)
+        goto cleanup;
 
+    if (detach)
+        ret = qemuDumpWaitForCompletion(driver, vm, asyncJob);
+
+ cleanup:
     return ret;
 }
 
-- 
2.13.6

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

On Fri, Nov 17, 2017 at 18:17:31 -0500, John Ferlan wrote:
> https://bugzilla.redhat.com/show_bug.cgi?id=916061
> 
> If the QEMU version running is new enough (based on QEMU_CAPS_QUERY_DUMP)
> we can add a 'detach' boolean to the dump-guest-memory command in order
> to tell QEMU to run in a thread. Then, use the qemuDumpWaitForCompletion
> in order to 'watch' the dump and save the stats into the jobInfo data
> so that tools (such as virsh) can pick up and display the completion
> percentage. The processing is similar to qemuMigrationWaitForCompletion
> at least with respect to calling the query-dump in a while loop that
> gets a micro sleep in order for qemuDomainGetJobInfo to be able to
> pull out the changed migration stats values.

Looks like the code was inspired by our old migration flow with a lot of
copy&paste work. However, I don't think this is necessary. Does QEMU
provide any event when a detached dump completes? If so, we should use
it. If QEMU does not provide the event, I think we should only add
support for detached dump once the event is implemented in QEMU. In
other words, we should never poll for dump progress every 50ms.

Jirka

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

On 11/20/2017 03:53 AM, Jiri Denemark wrote:
> On Fri, Nov 17, 2017 at 18:17:31 -0500, John Ferlan wrote:
>> https://bugzilla.redhat.com/show_bug.cgi?id=916061
>>
>> If the QEMU version running is new enough (based on QEMU_CAPS_QUERY_DUMP)
>> we can add a 'detach' boolean to the dump-guest-memory command in order
>> to tell QEMU to run in a thread. Then, use the qemuDumpWaitForCompletion
>> in order to 'watch' the dump and save the stats into the jobInfo data
>> so that tools (such as virsh) can pick up and display the completion
>> percentage. The processing is similar to qemuMigrationWaitForCompletion
>> at least with respect to calling the query-dump in a while loop that
>> gets a micro sleep in order for qemuDomainGetJobInfo to be able to
>> pull out the changed migration stats values.
> 
> Looks like the code was inspired by our old migration flow with a lot of
> copy&paste work. However, I don't think this is necessary. Does QEMU
> provide any event when a detached dump completes? If so, we should use
> it. If QEMU does not provide the event, I think we should only add
> support for detached dump once the event is implemented in QEMU. In
> other words, we should never poll for dump progress every 50ms.
> 
> Jirka
> 

Based on the output of the related QEMU bz there is an event:


 {"execute": "dump-guest-memory", "arguments": { "detach": true,
"paging": false, "protocol": "file:/home/dump.normal"}}
{"timestamp": {"seconds": 1489391067, "microseconds": 147976}, "event":
"STOP"}
{"return": {}}
{"timestamp": {"seconds": 1489391068, "microseconds": 219429}, "event":
"DUMP_COMPLETED", "data": {"result": {"total": 2164457472, "status":
"completed", "completed": 2164457472}}}
{"timestamp": {"seconds": 1489391068, "microseconds": 219876}, "event":
"RESUME"}

 {"execute": "query-dump"}
{"return": {"total": 2164457472, "status": "completed", "completed":
2164457472}}


Yet more code to go learn I guess.

John

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list