From c02b37814cf6962b3320ab34e1b3a26a2323772f Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 4 Oct 2018 12:08:46 -0400 Subject: [PATCH 1/7] tests: Add migration test for aarch64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds migration test support for aarch64. The test code, which implements the same functionality as x86, is booted as a kernel in qemu. Here are the design choices we make for aarch64: * We choose this -kernel approach because aarch64 QEMU doesn't provide a built-in fw like x86 does. So instead of relying on a boot loader, we use -kernel approach for aarch64. * The serial output is sent to PL011 directly. * The physical memory base for mach-virt machine is 0x40000000. We change the start_address and end_address for aarch64. In addition to providing the binary, this patch also includes the source code and the build script in tests/migration/aarch64. So users can change the source and/or re-compile the binary as they wish. Reviewed-by: Juan Quintela Reviewed-by: Andrew Jones Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Wei Huang Message-Id: <1538669326-28135-1-git-send-email-wei@redhat.com> Signed-off-by: Dr. David Alan Gilbert --- tests/Makefile.include | 1 + tests/migration-test.c | 27 ++++++++-- tests/migration/Makefile | 2 +- tests/migration/aarch64/Makefile | 18 +++++++ tests/migration/aarch64/a-b-kernel.S | 75 ++++++++++++++++++++++++++++ tests/migration/aarch64/a-b-kernel.h | 18 +++++++ tests/migration/migration-test.h | 9 ++++ 7 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 tests/migration/aarch64/Makefile create mode 100644 tests/migration/aarch64/a-b-kernel.S create mode 100644 tests/migration/aarch64/a-b-kernel.h diff --git a/tests/Makefile.include b/tests/Makefile.include index 8264af64a8..9324a8be6b 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -402,6 +402,7 @@ check-qtest-arm-y += tests/hexloader-test$(EXESUF) check-qtest-aarch64-y = tests/numa-test$(EXESUF) check-qtest-aarch64-$(CONFIG_SDHCI) += tests/sdhci-test$(EXESUF) check-qtest-aarch64-y += tests/boot-serial-test$(EXESUF) +check-qtest-aarch64-y += tests/migration-test$(EXESUF) check-qtest-microblazeel-y = $(check-qtest-microblaze-y) diff --git a/tests/migration-test.c b/tests/migration-test.c index 20f38f1930..5bdc0bd519 100644 --- a/tests/migration-test.c +++ b/tests/migration-test.c @@ -86,12 +86,13 @@ static const char *tmpfs; * repeatedly. It outputs a 'B' at a fixed rate while it's still running. */ #include "tests/migration/i386/a-b-bootblock.h" +#include "tests/migration/aarch64/a-b-kernel.h" -static void init_bootfile_x86(const char *bootpath) +static void init_bootfile(const char *bootpath, void *content) { FILE *bootfile = fopen(bootpath, "wb"); - g_assert_cmpint(fwrite(x86_bootsect, 512, 1, bootfile), ==, 1); + g_assert_cmpint(fwrite(content, 512, 1, bootfile), ==, 1); fclose(bootfile); } @@ -428,7 +429,7 @@ static int test_migrate_start(QTestState **from, QTestState **to, got_stop = false; if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) { - init_bootfile_x86(bootpath); + init_bootfile(bootpath, x86_bootsect); cmd_src = g_strdup_printf("-machine accel=%s -m 150M" " -name source,debug-threads=on" " -serial file:%s/src_serial" @@ -459,6 +460,24 @@ static int test_migrate_start(QTestState **from, QTestState **to, start_address = PPC_TEST_MEM_START; end_address = PPC_TEST_MEM_END; + } else if (strcmp(arch, "aarch64") == 0) { + init_bootfile(bootpath, aarch64_kernel); + cmd_src = g_strdup_printf("-machine virt,accel=%s,gic-version=max " + "-name vmsource,debug-threads=on -cpu max " + "-m 150M -serial file:%s/src_serial " + "-kernel %s ", + accel, tmpfs, bootpath); + cmd_dst = g_strdup_printf("-machine virt,accel=%s,gic-version=max " + "-name vmdest,debug-threads=on -cpu max " + "-m 150M -serial file:%s/dest_serial " + "-kernel %s " + "-incoming %s ", + accel, tmpfs, bootpath, uri); + + start_address = ARM_TEST_MEM_START; + end_address = ARM_TEST_MEM_END; + + g_assert(sizeof(aarch64_kernel) <= ARM_TEST_MAX_KERNEL_SIZE); } else { g_assert_not_reached(); } @@ -545,7 +564,7 @@ static void test_deprecated(void) { QTestState *from; - from = qtest_start(""); + from = qtest_start("-machine none"); deprecated_set_downtime(from, 0.12345); deprecated_set_speed(from, 12345); diff --git a/tests/migration/Makefile b/tests/migration/Makefile index dc3b551976..91237a84d9 100644 --- a/tests/migration/Makefile +++ b/tests/migration/Makefile @@ -5,7 +5,7 @@ # See the COPYING file in the top-level directory. # -TARGET_LIST = i386 +TARGET_LIST = i386 aarch64 SRC_PATH = ../.. diff --git a/tests/migration/aarch64/Makefile b/tests/migration/aarch64/Makefile new file mode 100644 index 0000000000..9c4fa18e76 --- /dev/null +++ b/tests/migration/aarch64/Makefile @@ -0,0 +1,18 @@ +# To specify cross compiler prefix, use CROSS_PREFIX= +# $ make CROSS_PREFIX=aarch64-linux-gnu- + +.PHONY: all clean +all: a-b-kernel.h + +a-b-kernel.h: aarch64.kernel + echo "$$__note" > $@ + xxd -i $< | sed -e 's/.*int.*//' >> $@ + +aarch64.kernel: aarch64.elf + $(CROSS_PREFIX)objcopy -O binary $< $@ + +aarch64.elf: a-b-kernel.S + $(CROSS_PREFIX)gcc -o $@ -nostdlib -Wl,--build-id=none $< + +clean: + $(RM) *.kernel *.elf diff --git a/tests/migration/aarch64/a-b-kernel.S b/tests/migration/aarch64/a-b-kernel.S new file mode 100644 index 0000000000..0225945348 --- /dev/null +++ b/tests/migration/aarch64/a-b-kernel.S @@ -0,0 +1,75 @@ +# +# Copyright (c) 2018 Red Hat, Inc. and/or its affiliates +# +# Author: +# Wei Huang +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. +# +# Note: Please make sure the compiler compiles the assembly code below with +# pc-relative address. Also the branch instructions should use relative +# addresses only. + +#include "../migration-test.h" + +.section .text + + .globl _start + +_start: + /* disable MMU to use phys mem address */ + mrs x0, sctlr_el1 + bic x0, x0, #(1<<0) + msr sctlr_el1, x0 + isb + + /* traverse test memory region */ + mov x0, #ARM_TEST_MEM_START + mov x1, #ARM_TEST_MEM_END + + /* output char 'A' to PL011 */ + mov w3, 'A' + mov x2, #ARM_MACH_VIRT_UART + strb w3, [x2] + + /* clean up memory */ + mov w3, #0 + mov x4, x0 +clean: + strb w3, [x4] + add x4, x4, #TEST_MEM_PAGE_SIZE + cmp x4, x1 + ble clean + + /* w5 keeps a counter so we can limit the output speed */ + mov w5, #0 + + /* main body */ +mainloop: + mov x4, x0 + +innerloop: + /* increment the first byte of each page by 1 */ + ldrb w3, [x4] + add w3, w3, #1 + and w3, w3, #0xff + strb w3, [x4] + + /* make sure QEMU user space can see consistent data as MMU is off */ + dc civac, x4 + + add x4, x4, #TEST_MEM_PAGE_SIZE + cmp x4, x1 + blt innerloop + + add w5, w5, #1 + and w5, w5, #0xff + cmp w5, #0 + bne mainloop + + /* output char 'B' to PL011 */ + mov w3, 'B' + strb w3, [x2] + + b mainloop diff --git a/tests/migration/aarch64/a-b-kernel.h b/tests/migration/aarch64/a-b-kernel.h new file mode 100644 index 0000000000..0a9b01137e --- /dev/null +++ b/tests/migration/aarch64/a-b-kernel.h @@ -0,0 +1,18 @@ +/* This file is automatically generated from the assembly file in + * tests/migration/aarch64. Edit that file and then run "make all" + * inside tests/migration to update, and then remember to send both + * the header and the assembler differences in your patch submission. + */ +unsigned char aarch64_kernel[] = { + 0x00, 0x10, 0x38, 0xd5, 0x00, 0xf8, 0x7f, 0x92, 0x00, 0x10, 0x18, 0xd5, + 0xdf, 0x3f, 0x03, 0xd5, 0x00, 0x02, 0xa8, 0xd2, 0x01, 0xc8, 0xa8, 0xd2, + 0x23, 0x08, 0x80, 0x52, 0x02, 0x20, 0xa1, 0xd2, 0x43, 0x00, 0x00, 0x39, + 0x03, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x83, 0x00, 0x00, 0x39, + 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb, 0xad, 0xff, 0xff, 0x54, + 0x05, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x83, 0x00, 0x40, 0x39, + 0x63, 0x04, 0x00, 0x11, 0x63, 0x1c, 0x00, 0x12, 0x83, 0x00, 0x00, 0x39, + 0x24, 0x7e, 0x0b, 0xd5, 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb, + 0x2b, 0xff, 0xff, 0x54, 0xa5, 0x04, 0x00, 0x11, 0xa5, 0x1c, 0x00, 0x12, + 0xbf, 0x00, 0x00, 0x71, 0x81, 0xfe, 0xff, 0x54, 0x43, 0x08, 0x80, 0x52, + 0x43, 0x00, 0x00, 0x39, 0xf1, 0xff, 0xff, 0x17 +}; diff --git a/tests/migration/migration-test.h b/tests/migration/migration-test.h index c4c0c526b6..6939a134c2 100644 --- a/tests/migration/migration-test.h +++ b/tests/migration/migration-test.h @@ -18,4 +18,13 @@ #define PPC_TEST_MEM_START (1 * 1024 * 1024) #define PPC_TEST_MEM_END (100 * 1024 * 1024) +/* ARM */ +#define ARM_TEST_MEM_START (0x40000000 + 1 * 1024 * 1024) +#define ARM_TEST_MEM_END (0x40000000 + 100 * 1024 * 1024) +#define ARM_MACH_VIRT_UART 0x09000000 +/* AArch64 kernel load address is 0x40080000, and the test memory starts at + * 0x40100000. So the maximum allowable kernel size is 512KB. + */ +#define ARM_TEST_MAX_KERNEL_SIZE (512 * 1024) + #endif /* _TEST_MIGRATION_H_ */ From 5571dc824b3a8b9e890caa721a6938970e7596ed Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Tue, 9 Oct 2018 11:51:17 +0200 Subject: [PATCH 2/7] tests/migration: Enable the migration test on s390x, too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can re-use the s390-ccw bios code to implement a small firmware for a s390x guest which prints out the "A" and "B" characters and modifies the memory, as required for the migration test. [quintela: Converted the compile script to Makefile rules] Signed-off-by: Juan Quintela Signed-off-by: Thomas Huth Message-Id: <1539078677-25396-1-git-send-email-thuth@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Dr. David Alan Gilbert Fixed up Makefile since the aarch patch sneaked in first --- tests/Makefile.include | 1 + tests/migration-test.c | 24 +++ tests/migration/Makefile | 2 +- tests/migration/migration-test.h | 4 + tests/migration/s390x/Makefile | 24 +++ tests/migration/s390x/a-b-bios.c | 36 +++++ tests/migration/s390x/a-b-bios.h | 253 +++++++++++++++++++++++++++++++ 7 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 tests/migration/s390x/Makefile create mode 100644 tests/migration/s390x/a-b-bios.c create mode 100644 tests/migration/s390x/a-b-bios.h diff --git a/tests/Makefile.include b/tests/Makefile.include index 9324a8be6b..5eadfd52f9 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -416,6 +416,7 @@ check-qtest-s390x-$(CONFIG_POSIX) += tests/test-filter-redirector$(EXESUF) check-qtest-s390x-y += tests/drive_del-test$(EXESUF) check-qtest-s390x-y += tests/virtio-ccw-test$(EXESUF) check-qtest-s390x-y += tests/cpu-plug-test$(EXESUF) +check-qtest-s390x-y += tests/migration-test$(EXESUF) check-qtest-generic-y += tests/machine-none-test$(EXESUF) check-qtest-generic-y += tests/qom-test$(EXESUF) diff --git a/tests/migration-test.c b/tests/migration-test.c index 5bdc0bd519..b7920255c5 100644 --- a/tests/migration-test.c +++ b/tests/migration-test.c @@ -96,6 +96,17 @@ static void init_bootfile(const char *bootpath, void *content) fclose(bootfile); } +#include "tests/migration/s390x/a-b-bios.h" + +static void init_bootfile_s390x(const char *bootpath) +{ + FILE *bootfile = fopen(bootpath, "wb"); + size_t len = sizeof(s390x_elf); + + g_assert_cmpint(fwrite(s390x_elf, len, 1, bootfile), ==, 1); + fclose(bootfile); +} + /* * Wait for some output in the serial output file, * we get an 'A' followed by an endless string of 'B's @@ -443,6 +454,19 @@ static int test_migrate_start(QTestState **from, QTestState **to, accel, tmpfs, bootpath, uri); start_address = X86_TEST_MEM_START; end_address = X86_TEST_MEM_END; + } else if (g_str_equal(arch, "s390x")) { + init_bootfile_s390x(bootpath); + cmd_src = g_strdup_printf("-machine accel=%s -m 128M" + " -name source,debug-threads=on" + " -serial file:%s/src_serial -bios %s", + accel, tmpfs, bootpath); + cmd_dst = g_strdup_printf("-machine accel=%s -m 128M" + " -name target,debug-threads=on" + " -serial file:%s/dest_serial -bios %s" + " -incoming %s", + accel, tmpfs, bootpath, uri); + start_address = S390_TEST_MEM_START; + end_address = S390_TEST_MEM_END; } else if (strcmp(arch, "ppc64") == 0) { cmd_src = g_strdup_printf("-machine accel=%s -m 256M -nodefaults" " -name source,debug-threads=on" diff --git a/tests/migration/Makefile b/tests/migration/Makefile index 91237a84d9..ff726ed7dd 100644 --- a/tests/migration/Makefile +++ b/tests/migration/Makefile @@ -5,7 +5,7 @@ # See the COPYING file in the top-level directory. # -TARGET_LIST = i386 aarch64 +TARGET_LIST = i386 aarch64 s390x SRC_PATH = ../.. diff --git a/tests/migration/migration-test.h b/tests/migration/migration-test.h index 6939a134c2..03c252368a 100644 --- a/tests/migration/migration-test.h +++ b/tests/migration/migration-test.h @@ -14,6 +14,10 @@ #define X86_TEST_MEM_START (1 * 1024 * 1024) #define X86_TEST_MEM_END (100 * 1024 * 1024) +/* S390 */ +#define S390_TEST_MEM_START (1 * 1024 * 1024) +#define S390_TEST_MEM_END (100 * 1024 * 1024) + /* PPC */ #define PPC_TEST_MEM_START (1 * 1024 * 1024) #define PPC_TEST_MEM_END (100 * 1024 * 1024) diff --git a/tests/migration/s390x/Makefile b/tests/migration/s390x/Makefile new file mode 100644 index 0000000000..6393c3e5b9 --- /dev/null +++ b/tests/migration/s390x/Makefile @@ -0,0 +1,24 @@ +# To specify cross compiler prefix, use CROSS_PREFIX= +# $ make CROSS_PREFIX=s390x-linux-gnu- + +.PHONY: all clean +all: a-b-bios.h +fwdir=../../../pc-bios/s390-ccw + +CFLAGS+=-ffreestanding -fno-delete-null-pointer-checks -fPIE -Os \ + -msoft-float -march=z900 -fno-asynchronous-unwind-tables -Wl,-pie \ + -Wl,--build-id=none -nostdlib + +a-b-bios.h: s390x.elf + echo "$$__note" > header.tmp + xxd -i $< | sed -e 's/.*int.*//' >> header.tmp + mv header.tmp $@ + +# We use common-page-size=16 to avoid big padding in the ELF file +s390x.elf: a-b-bios.c + $(CROSS_PREFIX)gcc $(CFLAGS) -I$(fwdir) $(fwdir)/start.S \ + $(fwdir)/sclp.c -Wl,-zcommon-page-size=16 -o $@ $< + $(CROSS_PREFIX)strip $@ + +clean: + @rm -rf *.elf *.o diff --git a/tests/migration/s390x/a-b-bios.c b/tests/migration/s390x/a-b-bios.c new file mode 100644 index 0000000000..a0327cd153 --- /dev/null +++ b/tests/migration/s390x/a-b-bios.c @@ -0,0 +1,36 @@ +/* + * S390 guest code used in migration tests + * + * Copyright 2018 Thomas Huth, Red Hat Inc. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#define LOADPARM_LEN 8 /* Needed for sclp.h */ + +#include +#include +#include + +char stack[0x8000] __attribute__((aligned(4096))); + +#define START_ADDRESS (1024 * 1024) +#define END_ADDRESS (100 * 1024 * 1024) + +void main(void) +{ + unsigned long addr; + + sclp_setup(); + sclp_print("A"); + + while (1) { + for (addr = START_ADDRESS; addr < END_ADDRESS; addr += 4096) { + *(volatile char *)addr += 1; /* Change pages */ + } + sclp_print("B"); + } +} diff --git a/tests/migration/s390x/a-b-bios.h b/tests/migration/s390x/a-b-bios.h new file mode 100644 index 0000000000..e722dc7c40 --- /dev/null +++ b/tests/migration/s390x/a-b-bios.h @@ -0,0 +1,253 @@ +/* This file is automatically generated from the a-b-bios.c file in + * tests/migration/s390x. Edit that file and then run "make all" + * inside tests/migration to update, and then remember to send both + * the header and the assembler differences in your patch submission. + */ +unsigned char s390x_elf[] = { + 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x16, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x78, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x38, 0x00, 0x07, 0x00, 0x40, + 0x00, 0x0c, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x88, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x0c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x0c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x10, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x17, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x10, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x98, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x07, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x10, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x10, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x64, 0x74, 0xe5, 0x51, + 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x64, 0x74, 0xe5, 0x52, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x10, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x17, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x10, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x2f, 0x6c, 0x69, 0x62, 0x2f, 0x6c, 0x64, 0x36, 0x34, 0x2e, 0x73, 0x6f, + 0x2e, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xeb, 0xef, 0xf0, 0x70, + 0x00, 0x24, 0xa7, 0xfb, 0xff, 0x60, 0xc0, 0xe5, 0x00, 0x00, 0x01, 0x1f, + 0xc0, 0x20, 0x00, 0x00, 0x02, 0x64, 0xc0, 0xe5, 0x00, 0x00, 0x01, 0x35, + 0xa5, 0x1e, 0x00, 0x10, 0xa7, 0x29, 0x63, 0x00, 0xe3, 0x30, 0x10, 0x00, + 0x00, 0x90, 0xa7, 0x3a, 0x00, 0x01, 0x42, 0x30, 0x10, 0x00, 0xa7, 0x1b, + 0x10, 0x00, 0xa7, 0x27, 0xff, 0xf7, 0xc0, 0x20, 0x00, 0x00, 0x02, 0x50, + 0xa7, 0xf4, 0xff, 0xeb, 0x07, 0x07, 0x07, 0x07, 0xc0, 0xf0, 0x00, 0x00, + 0x56, 0xc4, 0xc0, 0x20, 0x00, 0x00, 0x0a, 0xbd, 0xc0, 0x30, 0x00, 0x00, + 0x56, 0xbe, 0xb9, 0x0b, 0x00, 0x32, 0xb9, 0x02, 0x00, 0x33, 0xa7, 0x84, + 0x00, 0x19, 0xa7, 0x3b, 0xff, 0xff, 0xeb, 0x43, 0x00, 0x08, 0x00, 0x0c, + 0xb9, 0x02, 0x00, 0x44, 0xb9, 0x04, 0x00, 0x12, 0xa7, 0x84, 0x00, 0x09, + 0xd7, 0xff, 0x10, 0x00, 0x10, 0x00, 0x41, 0x10, 0x11, 0x00, 0xa7, 0x47, + 0xff, 0xfb, 0xc0, 0x20, 0x00, 0x00, 0x00, 0x07, 0x44, 0x30, 0x20, 0x00, + 0xa7, 0xf4, 0xff, 0xb6, 0xd7, 0x00, 0x10, 0x00, 0x10, 0x00, 0xc0, 0x10, + 0x00, 0x00, 0x00, 0x29, 0xb2, 0xb2, 0x10, 0x00, 0xeb, 0x00, 0xf0, 0x00, + 0x00, 0x25, 0x96, 0x02, 0xf0, 0x06, 0xeb, 0x00, 0xf0, 0x00, 0x00, 0x2f, + 0xc0, 0x10, 0x00, 0x00, 0x00, 0x11, 0xe3, 0x10, 0x01, 0xb8, 0x00, 0x24, + 0xc0, 0x10, 0x00, 0x00, 0x00, 0x26, 0xd2, 0x07, 0x01, 0xb0, 0x10, 0x00, + 0xc0, 0x10, 0x00, 0x00, 0x00, 0x18, 0xb2, 0xb2, 0x10, 0x00, 0xeb, 0x00, + 0xf0, 0x00, 0x00, 0x25, 0x94, 0xfd, 0xf0, 0x06, 0xeb, 0x00, 0xf0, 0x00, + 0x00, 0x2f, 0x07, 0xfe, 0x07, 0x07, 0x07, 0x07, 0x00, 0x02, 0x00, 0x01, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x02, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00, + 0xeb, 0xbf, 0xf0, 0x58, 0x00, 0x24, 0xc0, 0x10, 0x00, 0x00, 0x0e, 0x59, + 0xa7, 0xfb, 0xff, 0x60, 0xb2, 0x20, 0x00, 0x21, 0xb2, 0x22, 0x00, 0xb0, + 0x88, 0xb0, 0x00, 0x1c, 0xc0, 0xe5, 0xff, 0xff, 0xff, 0xba, 0xa7, 0xbe, + 0x00, 0x03, 0xa7, 0x84, 0x00, 0x13, 0xa7, 0xbe, 0x00, 0x02, 0xa7, 0x28, + 0x00, 0x00, 0xa7, 0x74, 0x00, 0x04, 0xa7, 0x28, 0xff, 0xfe, 0xe3, 0x40, + 0xf1, 0x10, 0x00, 0x04, 0xb9, 0x14, 0x00, 0x22, 0xeb, 0xbf, 0xf0, 0xf8, + 0x00, 0x04, 0x07, 0xf4, 0xa7, 0x28, 0xff, 0xff, 0xa7, 0xf4, 0xff, 0xf5, + 0x07, 0x07, 0x07, 0x07, 0xeb, 0xbf, 0xf0, 0x58, 0x00, 0x24, 0xc0, 0xd0, + 0x00, 0x00, 0x01, 0x21, 0xa7, 0xfb, 0xff, 0x60, 0xa7, 0xb9, 0x00, 0x00, + 0xa7, 0x19, 0x00, 0x00, 0xc0, 0x40, 0x00, 0x00, 0x0e, 0x24, 0xa7, 0x3b, + 0x00, 0x01, 0xa7, 0x37, 0x00, 0x23, 0xc0, 0x20, 0x00, 0x00, 0x0e, 0x1d, + 0x18, 0x31, 0xa7, 0x1a, 0x00, 0x06, 0x40, 0x10, 0x20, 0x08, 0xa7, 0x3a, + 0x00, 0x0e, 0xa7, 0x18, 0x1a, 0x00, 0x40, 0x30, 0x20, 0x00, 0x92, 0x00, + 0x20, 0x02, 0x40, 0x10, 0x20, 0x0a, 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, + 0xc0, 0xe5, 0xff, 0xff, 0xff, 0xac, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, + 0xb9, 0x04, 0x00, 0x2b, 0xeb, 0xbf, 0xf0, 0xf8, 0x00, 0x04, 0x07, 0xf4, + 0xb9, 0x04, 0x00, 0x51, 0xa7, 0x5b, 0x00, 0x01, 0xa7, 0x09, 0x0f, 0xf7, + 0xb9, 0x21, 0x00, 0x50, 0xa7, 0x24, 0xff, 0xd7, 0x41, 0xeb, 0x20, 0x00, + 0x95, 0x0a, 0xe0, 0x00, 0xa7, 0x74, 0x00, 0x08, 0x41, 0x11, 0x40, 0x0e, + 0x92, 0x0d, 0x10, 0x00, 0xb9, 0x04, 0x00, 0x15, 0x43, 0x5b, 0x20, 0x00, + 0x42, 0x51, 0x40, 0x0e, 0xa7, 0xbb, 0x00, 0x01, 0x41, 0x10, 0x10, 0x01, + 0xa7, 0xf4, 0xff, 0xbf, 0xc0, 0x50, 0x00, 0x00, 0x00, 0xd4, 0xc0, 0x10, + 0x00, 0x00, 0x0d, 0xd9, 0xa7, 0x48, 0x00, 0x1c, 0x40, 0x40, 0x10, 0x00, + 0x50, 0x20, 0x10, 0x0c, 0xa7, 0x48, 0x00, 0x04, 0xe3, 0x20, 0x50, 0x00, + 0x00, 0x04, 0x40, 0x40, 0x10, 0x0a, 0x50, 0x30, 0x10, 0x10, 0xc0, 0xf4, + 0xff, 0xff, 0xff, 0x6b, 0xa7, 0x39, 0x00, 0x40, 0xa7, 0x29, 0x00, 0x00, + 0xc0, 0xf4, 0xff, 0xff, 0xff, 0xe4, 0x07, 0x07, 0xb9, 0x04, 0x00, 0x13, + 0xa7, 0x2a, 0xff, 0xff, 0xb9, 0x04, 0x00, 0x34, 0xa7, 0x48, 0x00, 0x01, + 0x15, 0x24, 0xa7, 0x24, 0x00, 0x07, 0xb9, 0x04, 0x00, 0x21, 0xc0, 0xf4, + 0xff, 0xff, 0xff, 0x7f, 0xa7, 0x29, 0xff, 0xff, 0x07, 0xfe, 0x07, 0x07, + 0xa7, 0x39, 0x00, 0x00, 0x41, 0x13, 0x20, 0x00, 0x95, 0x00, 0x10, 0x00, + 0xa7, 0x74, 0x00, 0x05, 0xc0, 0xf4, 0xff, 0xff, 0xff, 0x70, 0xa7, 0x3b, + 0x00, 0x01, 0xa7, 0xf4, 0xff, 0xf5, 0x07, 0x07, 0xeb, 0xbf, 0xf0, 0x58, + 0x00, 0x24, 0xc0, 0xd0, 0x00, 0x00, 0x00, 0x91, 0xa7, 0xfb, 0xff, 0x60, + 0xb9, 0x04, 0x00, 0xb2, 0xa7, 0x19, 0x00, 0x20, 0xc0, 0x20, 0x00, 0x00, + 0x0d, 0x8c, 0x92, 0x00, 0x20, 0x00, 0xa7, 0x2b, 0x00, 0x01, 0xa7, 0x17, + 0xff, 0xfc, 0xc0, 0x10, 0x00, 0x00, 0x0d, 0x83, 0xa7, 0x28, 0x00, 0x20, + 0x40, 0x20, 0x10, 0x00, 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, 0xc0, 0xe5, + 0xff, 0xff, 0xff, 0x1d, 0x12, 0x22, 0xa7, 0x74, 0x00, 0x17, 0xa7, 0x19, + 0x00, 0x00, 0xc0, 0x40, 0x00, 0x00, 0x00, 0x75, 0xc0, 0x50, 0x00, 0x00, + 0x0d, 0x7a, 0xa7, 0x29, 0x00, 0x08, 0xe3, 0x31, 0x50, 0x00, 0x00, 0x90, + 0x43, 0x33, 0x40, 0x00, 0x42, 0x31, 0xb0, 0x00, 0xa7, 0x1b, 0x00, 0x01, + 0xa7, 0x27, 0xff, 0xf7, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, 0xeb, 0xbf, + 0xf0, 0xf8, 0x00, 0x04, 0x07, 0xf4, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0xeb, 0xaf, 0xf0, 0x50, 0x00, 0x24, 0xc0, 0xd0, 0x00, 0x00, 0x00, 0x51, + 0xa7, 0xfb, 0xff, 0x60, 0xa7, 0x19, 0x0f, 0xf8, 0xb9, 0x21, 0x00, 0x31, + 0xb9, 0x04, 0x00, 0xa2, 0xa7, 0xc4, 0x00, 0x2d, 0xa7, 0xb9, 0x0f, 0xf8, + 0xc0, 0x10, 0x00, 0x00, 0x0d, 0x42, 0xa7, 0x28, 0x10, 0x00, 0x40, 0x20, + 0x10, 0x00, 0x92, 0x00, 0x10, 0x02, 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, + 0xc0, 0xe5, 0xff, 0xff, 0xfe, 0xda, 0xa7, 0xbb, 0x00, 0x01, 0xa7, 0x19, + 0x00, 0x00, 0xc0, 0x20, 0x00, 0x00, 0x0d, 0x2f, 0xa7, 0xb7, 0x00, 0x17, + 0xc0, 0x10, 0x00, 0x00, 0x0d, 0x2a, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, + 0xe3, 0x20, 0x10, 0x08, 0x00, 0x91, 0xa7, 0x2a, 0xff, 0xf9, 0xb9, 0x14, + 0x00, 0x22, 0xeb, 0xaf, 0xf0, 0xf0, 0x00, 0x04, 0x07, 0xf4, 0xb9, 0x04, + 0x00, 0xb3, 0xa7, 0xf4, 0xff, 0xd5, 0x43, 0x31, 0x20, 0x0f, 0x42, 0x31, + 0xa0, 0x00, 0xa7, 0x1b, 0x00, 0x01, 0xa7, 0xf4, 0xff, 0xe3, 0x07, 0x07, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x78, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x00, 0x05, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x20, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x3c, 0x28, 0x2b, 0x7c, 0x26, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0x2e, 0x2d, 0x2f, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2c, 0x25, 0x5f, 0x3e, 0x3f, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x60, 0x3a, 0x23, + 0x40, 0x27, 0x3d, 0x22, 0x2e, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x6a, 0x6b, 0x6c, + 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5a, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x30, 0x31, 0x32, 0x33, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x41, 0x00, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6f, 0xff, 0xfe, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6f, 0xff, 0xff, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x10, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x47, 0x43, 0x43, 0x3a, 0x20, 0x28, 0x47, 0x4e, 0x55, 0x29, 0x20, 0x38, + 0x2e, 0x32, 0x2e, 0x31, 0x20, 0x32, 0x30, 0x31, 0x38, 0x30, 0x39, 0x30, + 0x35, 0x20, 0x28, 0x52, 0x65, 0x64, 0x20, 0x48, 0x61, 0x74, 0x20, 0x38, + 0x2e, 0x32, 0x2e, 0x31, 0x2d, 0x33, 0x29, 0x00, 0x00, 0x2e, 0x73, 0x68, + 0x73, 0x74, 0x72, 0x74, 0x61, 0x62, 0x00, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x70, 0x00, 0x2e, 0x67, 0x6e, 0x75, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x00, 0x2e, 0x64, 0x79, 0x6e, 0x73, 0x79, 0x6d, 0x00, 0x2e, 0x64, 0x79, + 0x6e, 0x73, 0x74, 0x72, 0x00, 0x2e, 0x74, 0x65, 0x78, 0x74, 0x00, 0x2e, + 0x72, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x00, 0x2e, 0x64, 0x79, 0x6e, 0x61, + 0x6d, 0x69, 0x63, 0x00, 0x2e, 0x67, 0x6f, 0x74, 0x00, 0x2e, 0x62, 0x73, + 0x73, 0x00, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x13, 0x6f, 0xff, 0xff, 0xf6, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xf8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x25, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x28, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x05, 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0xe8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x24, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3b, + 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x10, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x07, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xf8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x07, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x24, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 +}; From 55d0fe8254984321a126efd8db358f754737aa63 Mon Sep 17 00:00:00 2001 From: Ilya Maximets Date: Mon, 8 Oct 2018 19:05:35 +0300 Subject: [PATCH 3/7] migration: Stop postcopy fault thread before notifying POSTCOPY_NOTIFY_INBOUND_END handlers will remove userfault fds from the postcopy_remote_fds array which could be still in use by the fault thread. Let's stop the thread before notification to avoid possible accessing wrong memory. Fixes: 46343570c06e ("vhost+postcopy: Wire up POSTCOPY_END notify") Cc: qemu-stable@nongnu.org Signed-off-by: Ilya Maximets Message-Id: <20181008160536.6332-2-i.maximets@samsung.com> Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Maxime Coquelin Signed-off-by: Dr. David Alan Gilbert --- migration/postcopy-ram.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c index 853d8b32ca..e5c02a32c5 100644 --- a/migration/postcopy-ram.c +++ b/migration/postcopy-ram.c @@ -533,6 +533,12 @@ int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) if (mis->have_fault_thread) { Error *local_err = NULL; + /* Let the fault thread quit */ + atomic_set(&mis->fault_thread_quit, 1); + postcopy_fault_thread_notify(mis); + trace_postcopy_ram_incoming_cleanup_join(); + qemu_thread_join(&mis->fault_thread); + if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_END, &local_err)) { error_report_err(local_err); return -1; @@ -541,11 +547,6 @@ int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) if (qemu_ram_foreach_migratable_block(cleanup_range, mis)) { return -1; } - /* Let the fault thread quit */ - atomic_set(&mis->fault_thread_quit, 1); - postcopy_fault_thread_notify(mis); - trace_postcopy_ram_incoming_cleanup_join(); - qemu_thread_join(&mis->fault_thread); trace_postcopy_ram_incoming_cleanup_closeuf(); close(mis->userfault_fd); From c4f753859ae6da1aeb93cad19c586fea1925e269 Mon Sep 17 00:00:00 2001 From: Ilya Maximets Date: Mon, 8 Oct 2018 19:05:36 +0300 Subject: [PATCH 4/7] vhost-user: Fix userfaultfd leak 'fd' received from the vhost side is never freed. Also, everything (including 'postcopy_listen' state) should be cleaned up on vhost cleanup. Fixes: 46343570c06e ("vhost+postcopy: Wire up POSTCOPY_END notify") Fixes: f82c11165ffa ("vhost+postcopy: Register shared ufd with postcopy") Cc: qemu-stable@nongnu.org Signed-off-by: Ilya Maximets Message-Id: <20181008160536.6332-3-i.maximets@samsung.com> Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Maxime Coquelin Signed-off-by: Dr. David Alan Gilbert --- hw/virtio/vhost-user.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index b041343632..ed47d645d9 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -1291,6 +1291,7 @@ static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp) return ret; } postcopy_unregister_shared_ufd(&u->postcopy_fd); + close(u->postcopy_fd.fd); u->postcopy_fd.handler = NULL; trace_vhost_user_postcopy_end_exit(); @@ -1430,6 +1431,12 @@ static int vhost_user_backend_cleanup(struct vhost_dev *dev) postcopy_remove_notifier(&u->postcopy_notifier); u->postcopy_notifier.notify = NULL; } + u->postcopy_listen = false; + if (u->postcopy_fd.handler) { + postcopy_unregister_shared_ufd(&u->postcopy_fd); + close(u->postcopy_fd.fd); + u->postcopy_fd.handler = NULL; + } if (u->slave_fd >= 0) { qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); close(u->slave_fd); From 5ce43896e1679e706db1562d0e2d86ad428ed1e6 Mon Sep 17 00:00:00 2001 From: Ilya Maximets Date: Tue, 2 Oct 2018 17:09:47 +0300 Subject: [PATCH 5/7] vhost-user: Don't ask for reply on postcopy mem table set According to documentation, NEED_REPLY_MASK should not be set for VHOST_USER_SET_MEM_TABLE request in postcopy mode. This restriction was mistakenly applied to 'reply_supported' variable, which is local and used only for non-postcopy case. CC: Dr. David Alan Gilbert Fixes: 9bb38019942c ("vhost+postcopy: Send address back to qemu") Signed-off-by: Ilya Maximets Message-Id: <20181002140947.4107-1-i.maximets@samsung.com> Reviewed-by: Maxime Coquelin Signed-off-by: Dr. David Alan Gilbert --- hw/virtio/vhost-user.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index ed47d645d9..e09bed0e4a 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -374,8 +374,6 @@ static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev, int fds[VHOST_MEMORY_MAX_NREGIONS]; int i, fd; size_t fd_num = 0; - bool reply_supported = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK); VhostUserMsg msg_reply; int region_i, msg_i; @@ -384,10 +382,6 @@ static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev, .hdr.flags = VHOST_USER_VERSION, }; - if (reply_supported) { - msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; - } - if (u->region_rb_len < dev->mem->nregions) { u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions); u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset, @@ -503,10 +497,6 @@ static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev, return -1; } - if (reply_supported) { - return process_message_reply(dev, &msg); - } - return 0; } @@ -519,8 +509,7 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev, size_t fd_num = 0; bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler; bool reply_supported = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK) && - !do_postcopy; + VHOST_USER_PROTOCOL_F_REPLY_ACK); if (do_postcopy) { /* Postcopy has enough differences that it's best done in it's own From 18613dc6d9f2f33c333440fecab4c584281305ee Mon Sep 17 00:00:00 2001 From: "Denis V. Lunev" Date: Tue, 2 Oct 2018 16:55:38 +0300 Subject: [PATCH 6/7] qmp, hmp: make subsystem/system-vendor identities optional According to PCI specification, subsystem id and subsystem vendor id are present only in type 0 and type 2 headers (at different offsets), but not in type 1 headers. Thus we should make this data optional in struct PciDeviceId and skip reporting them via HMP if the information is not available. Additional (wrong information) about PCI bridges (Type1 devices) has been added in 5383a705 and fortunately not released. This patch fixes that problem. The problem was spotted by Markus. Signed-off-by: Denis V. Lunev CC: "Dr. David Alan Gilbert" CC: Eric Blake CC: Markus Armbruster Message-Id: <20181002135538.12113-1-den@openvz.org> Reported-by: Markus Armbruster Reviewed-by: Eric Blake Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Markus Armbruster Signed-off-by: Dr. David Alan Gilbert --- hmp.c | 6 ++++-- hw/pci/pci.c | 13 ++++++++++--- qapi/misc.json | 4 ++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/hmp.c b/hmp.c index 61ef120423..7828f93a39 100644 --- a/hmp.c +++ b/hmp.c @@ -837,8 +837,10 @@ static void hmp_info_pci_device(Monitor *mon, const PciDeviceInfo *dev) monitor_printf(mon, ": PCI device %04" PRIx64 ":%04" PRIx64 "\n", dev->id->vendor, dev->id->device); - monitor_printf(mon, " PCI subsystem %04" PRIx64 ":%04" PRIx64 "\n", - dev->id->subsystem_vendor, dev->id->subsystem); + if (dev->id->has_subsystem_vendor && dev->id->has_subsystem) { + monitor_printf(mon, " PCI subsystem %04" PRIx64 ":%04" PRIx64 "\n", + dev->id->subsystem_vendor, dev->id->subsystem); + } if (dev->has_irq) { monitor_printf(mon, " IRQ %" PRId64 ".\n", dev->irq); diff --git a/hw/pci/pci.c b/hw/pci/pci.c index 51d0dec466..b937f0dc0a 100644 --- a/hw/pci/pci.c +++ b/hw/pci/pci.c @@ -1737,9 +1737,6 @@ static PciDeviceInfo *qmp_query_pci_device(PCIDevice *dev, PCIBus *bus, info->id = g_new0(PciDeviceId, 1); info->id->vendor = pci_get_word(dev->config + PCI_VENDOR_ID); info->id->device = pci_get_word(dev->config + PCI_DEVICE_ID); - info->id->subsystem = pci_get_word(dev->config + PCI_SUBSYSTEM_ID); - info->id->subsystem_vendor = - pci_get_word(dev->config + PCI_SUBSYSTEM_VENDOR_ID); info->regions = qmp_query_pci_regions(dev); info->qdev_id = g_strdup(dev->qdev.id ? dev->qdev.id : ""); @@ -1752,6 +1749,16 @@ static PciDeviceInfo *qmp_query_pci_device(PCIDevice *dev, PCIBus *bus, if (type == PCI_HEADER_TYPE_BRIDGE) { info->has_pci_bridge = true; info->pci_bridge = qmp_query_pci_bridge(dev, bus, bus_num); + } else if (type == PCI_HEADER_TYPE_NORMAL) { + info->id->has_subsystem = info->id->has_subsystem_vendor = true; + info->id->subsystem = pci_get_word(dev->config + PCI_SUBSYSTEM_ID); + info->id->subsystem_vendor = + pci_get_word(dev->config + PCI_SUBSYSTEM_VENDOR_ID); + } else if (type == PCI_HEADER_TYPE_CARDBUS) { + info->id->has_subsystem = info->id->has_subsystem_vendor = true; + info->id->subsystem = pci_get_word(dev->config + PCI_CB_SUBSYSTEM_ID); + info->id->subsystem_vendor = + pci_get_word(dev->config + PCI_CB_SUBSYSTEM_VENDOR_ID); } return info; diff --git a/qapi/misc.json b/qapi/misc.json index f98de3a58c..3a68af9ca3 100644 --- a/qapi/misc.json +++ b/qapi/misc.json @@ -839,8 +839,8 @@ # Since: 2.4 ## { 'struct': 'PciDeviceId', - 'data': {'device': 'int', 'vendor': 'int', 'subsystem': 'int', - 'subsystem-vendor': 'int'} } + 'data': {'device': 'int', 'vendor': 'int', '*subsystem': 'int', + '*subsystem-vendor': 'int'} } ## # @PciDeviceInfo: From 36bd9e3c8b049cb9124e04ed8bf8f9451d63de03 Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Thu, 13 Sep 2018 15:23:13 +0200 Subject: [PATCH 7/7] migration-test: Only generate a single target architecture Several changes: - We only allow generate header "inside" the tree. Why? Because we need to connit the result, so it makes no sense to generate them on the build dir. - We only generate a single target each time. Getting all the cross-compilers correctly is an impossible task. So know you do: make -C tests/migration $target (native) make CROSS_PREFIX=foo- -C tests/migratiion $target (cross) And you are done. - If we are building out of tree, we have no data about if we are cross-compile or whatever. So instead of guess what is happening, just do what I pointed on previous point. Signed-off-by: Juan Quintela Message-Id: <20180913132313.11370-1-quintela@redhat.com> Signed-off-by: Dr. David Alan Gilbert --- tests/migration/Makefile | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/migration/Makefile b/tests/migration/Makefile index ff726ed7dd..13e99b1692 100644 --- a/tests/migration/Makefile +++ b/tests/migration/Makefile @@ -9,6 +9,19 @@ TARGET_LIST = i386 aarch64 s390x SRC_PATH = ../.. +.PHONY: help $(TARGET_LIST) +help: + @echo "Create migration guest includes. We generate a binary." + @echo "And then convert that binary to an include file that can be" + @echo "run in a guest." + @echo "Possible operations are:" + @echo + @echo " $(MAKE) clean Remove all intermediate files" + @echo " $(MAKE) target Generate for that target" + @echo " $(MAKE) CROSS_PREFIX=... target" + @echo " Cross-compile than target" + @echo " Possible targets are: $(TARGET_LIST)" + override define __note /* This file is automatically generated from the assembly file in * tests/migration/$@. Edit that file and then run "make all" @@ -18,16 +31,8 @@ override define __note endef export __note -find-arch-cross-cc = $(lastword $(shell grep -h "CROSS_CC_GUEST=" $(wildcard $(SRC_PATH)/$(patsubst i386,*86*,$(1))-softmmu/config-target.mak) /dev/null)) -parse-cross-prefix = $(subst gcc,,$(patsubst cc,gcc,$(patsubst CROSS_CC_GUEST="%",%,$(call find-arch-cross-cc,$(1))))) -gen-cross-prefix = $(patsubst %-,CROSS_PREFIX=%-,$(call parse-cross-prefix,$(1))) - -.PHONY: all $(TARGET_LIST) - -all: $(TARGET_LIST) - $(TARGET_LIST): - $(MAKE) -C $@ $(call gen-cross-prefix,$@) + $(MAKE) CROSS_PREFIX=$(CROSS_PREFIX) -C $@ clean: for target in $(TARGET_LIST); do \