From bd4982a6c6f1b133aff38873bc3b580af15cd334 Mon Sep 17 00:00:00 2001 From: Anthony PERARD Date: Wed, 3 Oct 2012 13:45:24 +0000 Subject: [PATCH 1/7] xen: Fix, no unplug of pt device by platform device. The Xen platform device will unplug any NICs if requested by the guest (PVonHVM) including a NIC that would have been passthrough. This patch makes sure that a passthrough device will not be unplug. Reported-by: "Zhang, Yang Z" Signed-off-by: Anthony PERARD Signed-off-by: Stefano Stabellini --- hw/xen_platform.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hw/xen_platform.c b/hw/xen_platform.c index 0d6c2ff8c3..956dbfeae4 100644 --- a/hw/xen_platform.c +++ b/hw/xen_platform.c @@ -85,8 +85,10 @@ static void log_writeb(PCIXenPlatformState *s, char val) static void unplug_nic(PCIBus *b, PCIDevice *d, void *o) { + /* We have to ignore passthrough devices */ if (pci_get_word(d->config + PCI_CLASS_DEVICE) == - PCI_CLASS_NETWORK_ETHERNET) { + PCI_CLASS_NETWORK_ETHERNET + && strcmp(d->name, "xen-pci-passthrough") != 0) { qdev_free(&d->qdev); } } @@ -98,8 +100,10 @@ static void pci_unplug_nics(PCIBus *bus) static void unplug_disks(PCIBus *b, PCIDevice *d, void *o) { + /* We have to ignore passthrough devices */ if (pci_get_word(d->config + PCI_CLASS_DEVICE) == - PCI_CLASS_STORAGE_IDE) { + PCI_CLASS_STORAGE_IDE + && strcmp(d->name, "xen-pci-passthrough") != 0) { qdev_unplug(&(d->qdev), NULL); } } From aabc8530c7ba2be89e21463f051056ad7c255e6e Mon Sep 17 00:00:00 2001 From: Xudong Hao Date: Wed, 3 Oct 2012 13:46:23 +0000 Subject: [PATCH 2/7] qemu/xen: Add 64 bits big bar support on qemu Currently it is assumed PCI device BAR access < 4G memory. If there is such a device whose BAR size is larger than 4G, it must access > 4G memory address. This patch enable the 64bits big BAR support on qemu. Signed-off-by: Xudong Hao Signed-off-by: Xiantao Zhang Signed-off-by: Stefano Stabellini --- hw/xen_pt.c | 7 +++++-- hw/xen_pt_config_init.c | 39 ++++++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/hw/xen_pt.c b/hw/xen_pt.c index 307119a12f..838bcea4d6 100644 --- a/hw/xen_pt.c +++ b/hw/xen_pt.c @@ -410,14 +410,17 @@ static int xen_pt_register_regions(XenPCIPassthroughState *s) if (r->type & XEN_HOST_PCI_REGION_TYPE_PREFETCH) { type |= PCI_BASE_ADDRESS_MEM_PREFETCH; } + if (r->type & XEN_HOST_PCI_REGION_TYPE_MEM_64) { + type |= PCI_BASE_ADDRESS_MEM_TYPE_64; + } } memory_region_init_io(&s->bar[i], &ops, &s->dev, "xen-pci-pt-bar", r->size); pci_register_bar(&s->dev, i, type, &s->bar[i]); - XEN_PT_LOG(&s->dev, "IO region %i registered (size=0x%08"PRIx64 - " base_addr=0x%08"PRIx64" type: %#x)\n", + XEN_PT_LOG(&s->dev, "IO region %i registered (size=0x%lx"PRIx64 + " base_addr=0x%lx"PRIx64" type: %#x)\n", i, r->size, r->base_addr, type); } diff --git a/hw/xen_pt_config_init.c b/hw/xen_pt_config_init.c index e524a4094d..0a5f82cb80 100644 --- a/hw/xen_pt_config_init.c +++ b/hw/xen_pt_config_init.c @@ -342,6 +342,23 @@ static int xen_pt_cmd_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry, #define XEN_PT_BAR_IO_RO_MASK 0x00000003 /* BAR ReadOnly mask(I/O) */ #define XEN_PT_BAR_IO_EMU_MASK 0xFFFFFFFC /* BAR emul mask(I/O) */ +static bool is_64bit_bar(PCIIORegion *r) +{ + return !!(r->type & PCI_BASE_ADDRESS_MEM_TYPE_64); +} + +static uint64_t xen_pt_get_bar_size(PCIIORegion *r) +{ + if (is_64bit_bar(r)) { + uint64_t size64; + size64 = (r + 1)->size; + size64 <<= 32; + size64 += r->size; + return size64; + } + return r->size; +} + static XenPTBarFlag xen_pt_bar_reg_parse(XenPCIPassthroughState *s, XenPTRegInfo *reg) { @@ -366,7 +383,7 @@ static XenPTBarFlag xen_pt_bar_reg_parse(XenPCIPassthroughState *s, /* check unused BAR */ r = &d->io_regions[index]; - if (r->size == 0) { + if (!xen_pt_get_bar_size(r)) { return XEN_PT_BAR_FLAG_UNUSED; } @@ -481,7 +498,12 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry, switch (s->bases[index].bar_flag) { case XEN_PT_BAR_FLAG_MEM: bar_emu_mask = XEN_PT_BAR_MEM_EMU_MASK; - bar_ro_mask = XEN_PT_BAR_MEM_RO_MASK | (r_size - 1); + if (!r_size) { + /* low 32 bits mask for 64 bit bars */ + bar_ro_mask = XEN_PT_BAR_ALLF; + } else { + bar_ro_mask = XEN_PT_BAR_MEM_RO_MASK | (r_size - 1); + } break; case XEN_PT_BAR_FLAG_IO: bar_emu_mask = XEN_PT_BAR_IO_EMU_MASK; @@ -489,7 +511,7 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry, break; case XEN_PT_BAR_FLAG_UPPER: bar_emu_mask = XEN_PT_BAR_ALLF; - bar_ro_mask = 0; /* all upper 32bit are R/W */ + bar_ro_mask = r_size ? r_size - 1 : 0; break; default: break; @@ -501,22 +523,13 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry, /* check whether we need to update the virtual region address or not */ switch (s->bases[index].bar_flag) { + case XEN_PT_BAR_FLAG_UPPER: case XEN_PT_BAR_FLAG_MEM: /* nothing to do */ break; case XEN_PT_BAR_FLAG_IO: /* nothing to do */ break; - case XEN_PT_BAR_FLAG_UPPER: - if (cfg_entry->data) { - if (cfg_entry->data != (XEN_PT_BAR_ALLF & ~bar_ro_mask)) { - XEN_PT_WARN(d, "Guest attempt to set high MMIO Base Address. " - "Ignore mapping. " - "(offset: 0x%02x, high address: 0x%08x)\n", - reg->offset, cfg_entry->data); - } - } - break; default: break; } From 39f42439d0629d3921629dc4b38e68df8f2f7b83 Mon Sep 17 00:00:00 2001 From: Anthony PERARD Date: Wed, 3 Oct 2012 13:48:19 +0000 Subject: [PATCH 3/7] QMP, Introduce xen-set-global-dirty-log command. This command is used during a migration of a guest under Xen. It calls memory_global_dirty_log_start or memory_global_dirty_log_stop according to the argument pass to the command. Signed-off-by: Anthony PERARD Reviewed-by: Luiz Capitulino --- qapi-schema.json | 13 +++++++++++++ qmp-commands.hx | 24 ++++++++++++++++++++++++ xen-all.c | 15 +++++++++++++++ xen-stub.c | 5 +++++ 4 files changed, 57 insertions(+) diff --git a/qapi-schema.json b/qapi-schema.json index 14e44199b7..4a4a850f5f 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -1955,6 +1955,19 @@ ## { 'command': 'xen-save-devices-state', 'data': {'filename': 'str'} } +## +# @xen-set-global-dirty-log +# +# Enable or disable the global dirty log mode. +# +# @enable: true to enable, false to disable. +# +# Returns: nothing +# +# Since: 1.3 +## +{ 'command': 'xen-set-global-dirty-log', 'data': { 'enable': 'bool' } } + ## # @device_del: # diff --git a/qmp-commands.hx b/qmp-commands.hx index 6e21ddba61..662b7cf32c 100644 --- a/qmp-commands.hx +++ b/qmp-commands.hx @@ -490,6 +490,30 @@ Example: "arguments": { "filename": "/tmp/save" } } <- { "return": {} } +EQMP + + { + .name = "xen-set-global-dirty-log", + .args_type = "enable:b", + .mhandler.cmd_new = qmp_marshal_input_xen_set_global_dirty_log, + }, + +SQMP +xen-set-global-dirty-log +------- + +Enable or disable the global dirty log mode. + +Arguments: + +- "enable": Enable it or disable it. + +Example: + +-> { "execute": "xen-set-global-dirty-log", + "arguments": { "enable": true } } +<- { "return": {} } + EQMP { diff --git a/xen-all.c b/xen-all.c index f76b051eee..f75ae9fd28 100644 --- a/xen-all.c +++ b/xen-all.c @@ -14,6 +14,7 @@ #include "hw/pc.h" #include "hw/xen_common.h" #include "hw/xen_backend.h" +#include "qmp-commands.h" #include "range.h" #include "xen-mapcache.h" @@ -36,6 +37,7 @@ static MemoryRegion ram_memory, ram_640k, ram_lo, ram_hi; static MemoryRegion *framebuffer; +static bool xen_in_migration; /* Compatibility with older version */ #if __XEN_LATEST_INTERFACE_VERSION__ < 0x0003020a @@ -552,10 +554,14 @@ static void xen_log_sync(MemoryListener *listener, MemoryRegionSection *section) static void xen_log_global_start(MemoryListener *listener) { + if (xen_enabled()) { + xen_in_migration = true; + } } static void xen_log_global_stop(MemoryListener *listener) { + xen_in_migration = false; } static void xen_eventfd_add(MemoryListener *listener, @@ -588,6 +594,15 @@ static MemoryListener xen_memory_listener = { .priority = 10, }; +void qmp_xen_set_global_dirty_log(bool enable, Error **errp) +{ + if (enable) { + memory_global_dirty_log_start(); + } else { + memory_global_dirty_log_stop(); + } +} + /* VCPU Operations, MMIO, IO ring ... */ static void xen_reset_vcpu(void *opaque) diff --git a/xen-stub.c b/xen-stub.c index 8ff2b79ac6..5e66ba8f04 100644 --- a/xen-stub.c +++ b/xen-stub.c @@ -11,6 +11,7 @@ #include "qemu-common.h" #include "hw/xen.h" #include "memory.h" +#include "qmp-commands.h" void xenstore_store_pv_console_info(int i, CharDriverState *chr) { @@ -54,3 +55,7 @@ int xen_init(void) void xen_register_framebuffer(MemoryRegion *mr) { } + +void qmp_xen_set_global_dirty_log(bool enable, Error **errp) +{ +} From 910b38e4dc4c37683c8b821e75a7f4cf095e4b21 Mon Sep 17 00:00:00 2001 From: Anthony PERARD Date: Wed, 3 Oct 2012 13:48:45 +0000 Subject: [PATCH 4/7] xen: Introduce xen_modified_memory. This function is to be used during live migration. Every write access to the guest memory should call this funcion so the Xen tools knows which pages are dirty. Signed-off-by: Anthony PERARD Acked-by: Stefano Stabellini --- hw/xen.h | 1 + xen-all.c | 21 +++++++++++++++++++++ xen-stub.c | 4 ++++ 3 files changed, 26 insertions(+) diff --git a/hw/xen.h b/hw/xen.h index e5926b7b8a..d14e92d5aa 100644 --- a/hw/xen.h +++ b/hw/xen.h @@ -48,6 +48,7 @@ void xenstore_store_pv_console_info(int i, struct CharDriverState *chr); struct MemoryRegion; void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, struct MemoryRegion *mr); +void xen_modified_memory(ram_addr_t start, ram_addr_t length); #endif struct MemoryRegion; diff --git a/xen-all.c b/xen-all.c index f75ae9fd28..b11542c3e7 100644 --- a/xen-all.c +++ b/xen-all.c @@ -1228,3 +1228,24 @@ void xen_shutdown_fatal_error(const char *fmt, ...) /* destroy the domain */ qemu_system_shutdown_request(); } + +void xen_modified_memory(ram_addr_t start, ram_addr_t length) +{ + if (unlikely(xen_in_migration)) { + int rc; + ram_addr_t start_pfn, nb_pages; + + if (length == 0) { + length = TARGET_PAGE_SIZE; + } + start_pfn = start >> TARGET_PAGE_BITS; + nb_pages = ((start + length + TARGET_PAGE_SIZE - 1) >> TARGET_PAGE_BITS) + - start_pfn; + rc = xc_hvm_modified_memory(xen_xc, xen_domid, start_pfn, nb_pages); + if (rc) { + fprintf(stderr, + "%s failed for "RAM_ADDR_FMT" ("RAM_ADDR_FMT"): %i, %s\n", + __func__, start, nb_pages, rc, strerror(-rc)); + } + } +} diff --git a/xen-stub.c b/xen-stub.c index 5e66ba8f04..921439263a 100644 --- a/xen-stub.c +++ b/xen-stub.c @@ -59,3 +59,7 @@ void xen_register_framebuffer(MemoryRegion *mr) void qmp_xen_set_global_dirty_log(bool enable, Error **errp) { } + +void xen_modified_memory(ram_addr_t start, ram_addr_t length) +{ +} From 51d7a9eb2b64e787c90bea1027308087eac22065 Mon Sep 17 00:00:00 2001 From: Anthony PERARD Date: Wed, 3 Oct 2012 13:49:05 +0000 Subject: [PATCH 5/7] exec: Introduce helper to set dirty flags. This new helper/hook is used in the next patch to add an extra call in a single place. Signed-off-by: Anthony PERARD Reviewed-by: Avi Kivity --- exec.c | 52 +++++++++++++++++----------------------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/exec.c b/exec.c index bb6aa4a070..366684cd78 100644 --- a/exec.c +++ b/exec.c @@ -3417,6 +3417,18 @@ int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr, } #else + +static void invalidate_and_set_dirty(target_phys_addr_t addr, + target_phys_addr_t length) +{ + if (!cpu_physical_memory_is_dirty(addr)) { + /* invalidate code */ + tb_invalidate_phys_page_range(addr, addr + length, 0); + /* set dirty bit */ + cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG)); + } +} + void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, int len, int is_write) { @@ -3462,13 +3474,7 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, /* RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); - if (!cpu_physical_memory_is_dirty(addr1)) { - /* invalidate code */ - tb_invalidate_phys_page_range(addr1, addr1 + l, 0); - /* set dirty bit */ - cpu_physical_memory_set_dirty_flags( - addr1, (0xff & ~CODE_DIRTY_FLAG)); - } + invalidate_and_set_dirty(addr1, l); qemu_put_ram_ptr(ptr); } } else { @@ -3534,13 +3540,7 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr, /* ROM/RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); - if (!cpu_physical_memory_is_dirty(addr1)) { - /* invalidate code */ - tb_invalidate_phys_page_range(addr1, addr1 + l, 0); - /* set dirty bit */ - cpu_physical_memory_set_dirty_flags( - addr1, (0xff & ~CODE_DIRTY_FLAG)); - } + invalidate_and_set_dirty(addr1, l); qemu_put_ram_ptr(ptr); } len -= l; @@ -3666,13 +3666,7 @@ void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len, l = TARGET_PAGE_SIZE; if (l > access_len) l = access_len; - if (!cpu_physical_memory_is_dirty(addr1)) { - /* invalidate code */ - tb_invalidate_phys_page_range(addr1, addr1 + l, 0); - /* set dirty bit */ - cpu_physical_memory_set_dirty_flags( - addr1, (0xff & ~CODE_DIRTY_FLAG)); - } + invalidate_and_set_dirty(addr1, l); addr1 += l; access_len -= l; } @@ -3978,13 +3972,7 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, stl_p(ptr, val); break; } - if (!cpu_physical_memory_is_dirty(addr1)) { - /* invalidate code */ - tb_invalidate_phys_page_range(addr1, addr1 + 4, 0); - /* set dirty bit */ - cpu_physical_memory_set_dirty_flags(addr1, - (0xff & ~CODE_DIRTY_FLAG)); - } + invalidate_and_set_dirty(addr1, 4); } } @@ -4051,13 +4039,7 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, stw_p(ptr, val); break; } - if (!cpu_physical_memory_is_dirty(addr1)) { - /* invalidate code */ - tb_invalidate_phys_page_range(addr1, addr1 + 2, 0); - /* set dirty bit */ - cpu_physical_memory_set_dirty_flags(addr1, - (0xff & ~CODE_DIRTY_FLAG)); - } + invalidate_and_set_dirty(addr1, 2); } } From e226939de5814527a21396903b08c3d0ed989558 Mon Sep 17 00:00:00 2001 From: Anthony PERARD Date: Wed, 3 Oct 2012 13:49:22 +0000 Subject: [PATCH 6/7] exec, memory: Call to xen_modified_memory. This patch add some calls to xen_modified_memory to notify Xen about dirtybits during migration. Signed-off-by: Anthony PERARD Reviewed-by: Avi Kivity --- exec-obsolete.h | 2 ++ exec.c | 1 + 2 files changed, 3 insertions(+) diff --git a/exec-obsolete.h b/exec-obsolete.h index c09925610d..286e2f75e1 100644 --- a/exec-obsolete.h +++ b/exec-obsolete.h @@ -24,6 +24,7 @@ #endif #ifndef CONFIG_USER_ONLY +#include "hw/xen.h" ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, MemoryRegion *mr); @@ -111,6 +112,7 @@ static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, for (addr = start; addr < end; addr += TARGET_PAGE_SIZE) { cpu_physical_memory_set_dirty_flags(addr, dirty_flags); } + xen_modified_memory(addr, length); } static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start, diff --git a/exec.c b/exec.c index 366684cd78..1114a0932b 100644 --- a/exec.c +++ b/exec.c @@ -3427,6 +3427,7 @@ static void invalidate_and_set_dirty(target_phys_addr_t addr, /* set dirty bit */ cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG)); } + xen_modified_memory(addr, length); } void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, From 8aba7dc02d5660df7e7d8651304b3079908358be Mon Sep 17 00:00:00 2001 From: Anthony PERARD Date: Wed, 3 Oct 2012 13:49:40 +0000 Subject: [PATCH 7/7] xen: Set the vram dirty when an error occur. If the call to xc_hvm_track_dirty_vram() fails, then we set dirtybit on all the video ram. This case happens during migration. Signed-off-by: Anthony PERARD Acked-by: Stefano Stabellini --- xen-all.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xen-all.c b/xen-all.c index b11542c3e7..e6308be23a 100644 --- a/xen-all.c +++ b/xen-all.c @@ -507,7 +507,8 @@ static void xen_sync_dirty_bitmap(XenIOState *state, bitmap); if (rc < 0) { if (rc != -ENODATA) { - fprintf(stderr, "xen: track_dirty_vram failed (0x" TARGET_FMT_plx + memory_region_set_dirty(framebuffer, 0, size); + DPRINTF("xen: track_dirty_vram failed (0x" TARGET_FMT_plx ", 0x" TARGET_FMT_plx "): %s\n", start_addr, start_addr + size, strerror(-rc)); }