Python testing fixes for 6.2

A few more fixes to help eliminate race conditions from
 device-crash-test, along with a fix that allows the SCM_RIGHTS
 functionality to work on hosts that only have Python 3.6.
 
 If this is too much this late in the RC process, I'd advocate for at
 least patch 7/7 by itself.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+ber27ys35W+dsvQfe+BBqr8OQ4FAmGcU90ACgkQfe+BBqr8
 OQ7nIw//UF4tgL2Pmbc6Mh26iVXGBbgg1BkTnefUVHDAQRg5GSf988lERsppHCJl
 HoNVrn4brAVWapor9Za4b5qWAkkwszVraiU3mNzfqQFQfttf3sju+kEs7MvvlPma
 GaKk6iOBGEzX9hWSduzLDPjJn5MwNqVrGNxHU/MkS3WI09KdjnIW7W8HpasIC45V
 XRqHqjTFBklfhdBCH7/oh2pK4TYCfnu3ZNqJ0PGn0a3c+jA7kdTfy33WDTS2GnEN
 pUoHkvcTfjDW0tNIikXSSAT1GgtUk0JJe52zUJrK/sBGVLjGiI6+82Ro9pxA+7kT
 +75xnUAkMq9Fww20duJQxBZ86t1GwEtSkpuyCqa/YmsmDncx2Y+uB9hFf2vZzCZU
 DkaCuyASB7WfpIGRcUknzdfay5ovIjNmp46IjjdN2EbGIsLz8nzMMIXQnDSLnFmU
 tlGDl61vFQiQmbQk/Cka2VAp4o8nvgsJ4TOq+WZsXG4uGXdVOoE7UbcpcnvxnhSJ
 D7Vv87qRPXItBflPJh+3/CsuoUbXcrapIUjQhBPHJNBiZ18cUu9ikVgZynt4d78w
 PkOXF19+dHkyFyUbV+OazFUsR/PHZBOdtOr2upjd7DxQPmJtVa8A3ZC0xz5hJ9a+
 ViBXjpAmyflRE2tGo4lCnNuEfTG02zByjlwiCLwpLCOxtvUcHIY=
 =YtGD
 -----END PGP SIGNATURE-----

Merge tag 'python-pull-request' of https://gitlab.com/jsnow/qemu into staging

Python testing fixes for 6.2

A few more fixes to help eliminate race conditions from
device-crash-test, along with a fix that allows the SCM_RIGHTS
functionality to work on hosts that only have Python 3.6.

If this is too much this late in the RC process, I'd advocate for at
least patch 7/7 by itself.

# gpg: Signature made Tue 23 Nov 2021 03:37:17 AM CET
# gpg:                using RSA key F9B7ABDBBCACDF95BE76CBD07DEF8106AAFC390E
# gpg: Good signature from "John Snow (John Huston) <jsnow@redhat.com>" [full]

* tag 'python-pull-request' of https://gitlab.com/jsnow/qemu:
  python/aqmp: fix send_fd_scm for python 3.6.x
  scripts/device-crash-test: Use a QMP timeout
  python/machine: handle "fast" QEMU terminations
  python/machine: move more variable initializations to _pre_launch
  python/machine: add instance disambiguator to default nickname
  python/machine: remove _remove_monitor_sockfile property
  python/machine: add @sock_dir property

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

python/qemu/aqmp/qmp_client.py

@@ -639,9 +639,12 @@ class QMPClient(AsyncProtocol[Message], Events):
         if sock.family != socket.AF_UNIX:
             raise AQMPError("Sending file descriptors requires a UNIX socket.")
 
-        # Void the warranty sticker.
+        if not hasattr(sock, 'sendmsg'):
-        # Access to sendmsg in asyncio is scheduled for removal in Python 3.11.
+            # We need to void the warranty sticker.
-        sock = sock._sock  # pylint: disable=protected-access
+            # Access to sendmsg is scheduled for removal in Python 3.11.
+            # Find the real backing socket to use it anyway.
+            sock = sock._sock  # pylint: disable=protected-access
+
         sock.sendmsg(
             [b' '],
             [(socket.SOL_SOCKET, socket.SCM_RIGHTS, struct.pack('@i', fd))]

python/qemu/machine/machine.py

@@ -133,19 +133,18 @@ class QEMUMachine:
         self._wrapper = wrapper
         self._qmp_timer = qmp_timer
 
-        self._name = name or "qemu-%d" % os.getpid()
+        self._name = name or f"qemu-{os.getpid()}-{id(self):02x}"
+        self._temp_dir: Optional[str] = None
         self._base_temp_dir = base_temp_dir
-        self._sock_dir = sock_dir or self._base_temp_dir
+        self._sock_dir = sock_dir
         self._log_dir = log_dir
 
         if monitor_address is not None:
             self._monitor_address = monitor_address
-            self._remove_monitor_sockfile = False
         else:
             self._monitor_address = os.path.join(
-                self._sock_dir, f"{self._name}-monitor.sock"
+                self.sock_dir, f"{self._name}-monitor.sock"
             )
-            self._remove_monitor_sockfile = True
 
         self._console_log_path = console_log
         if self._console_log_path:
@@ -163,14 +162,13 @@ class QEMUMachine:
         self._qmp_set = True   # Enable QMP monitor by default.
         self._qmp_connection: Optional[QEMUMonitorProtocol] = None
         self._qemu_full_args: Tuple[str, ...] = ()
-        self._temp_dir: Optional[str] = None
         self._launched = False
         self._machine: Optional[str] = None
         self._console_index = 0
         self._console_set = False
         self._console_device_type: Optional[str] = None
         self._console_address = os.path.join(
-            self._sock_dir, f"{self._name}-console.sock"
+            self.sock_dir, f"{self._name}-console.sock"
         )
         self._console_socket: Optional[socket.socket] = None
         self._remove_files: List[str] = []
@@ -315,8 +313,7 @@ class QEMUMachine:
             self._remove_files.append(self._console_address)
 
         if self._qmp_set:
-            if self._remove_monitor_sockfile:
+            if isinstance(self._monitor_address, str):
-                assert isinstance(self._monitor_address, str)
                 self._remove_files.append(self._monitor_address)
             self._qmp_connection = QEMUMonitorProtocol(
                 self._monitor_address,
@@ -330,6 +327,14 @@ class QEMUMachine:
         self._qemu_log_path = os.path.join(self.log_dir, self._name + ".log")
         self._qemu_log_file = open(self._qemu_log_path, 'wb')
 
+        self._iolog = None
+        self._qemu_full_args = tuple(chain(
+            self._wrapper,
+            [self._binary],
+            self._base_args,
+            self._args
+        ))
+
     def _post_launch(self) -> None:
         if self._qmp_connection:
             self._qmp.accept(self._qmp_timer)
@@ -344,9 +349,6 @@ class QEMUMachine:
         Called to cleanup the VM instance after the process has exited.
         May also be called after a failed launch.
         """
-        # Comprehensive reset for the failed launch case:
-        self._early_cleanup()
-
         try:
             self._close_qmp_connection()
         except Exception as err:  # pylint: disable=broad-except
@@ -393,13 +395,18 @@ class QEMUMachine:
         if self._launched:
             raise QEMUMachineError('VM already launched')
 
-        self._iolog = None
-        self._qemu_full_args = ()
         try:
             self._launch()
-            self._launched = True
         except:
-            self._post_shutdown()
+            # We may have launched the process but it may
+            # have exited before we could connect via QMP.
+            # Assume the VM didn't launch or is exiting.
+            # If we don't wait for the process, exitcode() may still be
+            # 'None' by the time control is ceded back to the caller.
+            if self._launched:
+                self.wait()
+            else:
+                self._post_shutdown()
 
             LOG.debug('Error launching VM')
             if self._qemu_full_args:
@@ -413,12 +420,6 @@ class QEMUMachine:
         Launch the VM and establish a QMP connection
         """
         self._pre_launch()
-        self._qemu_full_args = tuple(
-            chain(self._wrapper,
-                  [self._binary],
-                  self._base_args,
-                  self._args)
-        )
         LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args))
 
         # Cleaning up of this subprocess is guaranteed by _do_shutdown.
@@ -429,6 +430,7 @@ class QEMUMachine:
                                        stderr=subprocess.STDOUT,
                                        shell=False,
                                        close_fds=False)
+        self._launched = True
         self._post_launch()
 
     def _close_qmp_connection(self) -> None:
@@ -460,8 +462,8 @@ class QEMUMachine:
         """
         Perform any cleanup that needs to happen before the VM exits.
 
-        May be invoked by both soft and hard shutdown in failover scenarios.
+        This method may be called twice upon shutdown, once each by soft
-        Called additionally by _post_shutdown for comprehensive cleanup.
+        and hard shutdown in failover scenarios.
         """
         # If we keep the console socket open, we may deadlock waiting
         # for QEMU to exit, while QEMU is waiting for the socket to
@@ -816,6 +818,15 @@ class QEMUMachine:
                                               dir=self._base_temp_dir)
         return self._temp_dir
 
+    @property
+    def sock_dir(self) -> str:
+        """
+        Returns the directory used for sockfiles by this machine.
+        """
+        if self._sock_dir:
+            return self._sock_dir
+        return self.temp_dir
+
     @property
     def log_dir(self) -> str:
         """

scripts/device-crash-test

@@ -353,7 +353,7 @@ def checkOneCase(args, testcase):
             '-device', qemuOptsEscape(device)]
     cmdline = ' '.join([binary] + args)
     dbg("will launch QEMU: %s", cmdline)
-    vm = QEMUMachine(binary=binary, args=args)
+    vm = QEMUMachine(binary=binary, args=args, qmp_timer=15)
 
     exc = None
     exc_traceback = None