There are many existing qcow2 images that specify a backing file but no format. This has been the source of CVEs in the past, but has become more prominent of a problem now that libvirt has switched to -blockdev. With older -drive, at least the probing was always done by qemu (so the only risk of a changed format between successive boots of a guest was if qemu was upgraded and probed differently). But with newer -blockdev, libvirt must specify a format; if libvirt guesses raw where the image was formatted, this results in data corruption visible to the guest; conversely, if libvirt guesses qcow2 where qemu was using raw, this can result in potential security holes, so modern libvirt instead refuses to use images without explicit backing format. The change in libvirt to reject images without explicit backing format has pointed out that a number of tools have been far too reliant on probing in the past. It's time to set a better example in our own iotests of properly setting this parameter. iotest calls to create, rebase, and convert are all impacted to some degree. It's a bit annoying that we are inconsistent on command line - while all of those accept -o backing_file=...,backing_fmt=..., the shortcuts are different: create and rebase have -b and -F, while convert has -B but no -F. (amend has no shortcuts, but the previous patch just deprecated the use of amend to change backing chains). Signed-off-by: Eric Blake <eblake@redhat.com> Message-Id: <20200706203954.341758-9-eblake@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
532 lines
19 KiB
Python
Executable file
532 lines
19 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
#
|
|
# Test bitmap-sync backups (incremental, differential, and partials)
|
|
#
|
|
# Copyright (c) 2019 John Snow for Red Hat, Inc.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
# owner=jsnow@redhat.com
|
|
|
|
import math
|
|
import os
|
|
|
|
import iotests
|
|
from iotests import log, qemu_img
|
|
|
|
SIZE = 64 * 1024 * 1024
|
|
GRANULARITY = 64 * 1024
|
|
|
|
|
|
class Pattern:
|
|
def __init__(self, byte, offset, size=GRANULARITY):
|
|
self.byte = byte
|
|
self.offset = offset
|
|
self.size = size
|
|
|
|
def bits(self, granularity):
|
|
lower = self.offset // granularity
|
|
upper = (self.offset + self.size - 1) // granularity
|
|
return set(range(lower, upper + 1))
|
|
|
|
|
|
class PatternGroup:
|
|
"""Grouping of Pattern objects. Initialize with an iterable of Patterns."""
|
|
def __init__(self, patterns):
|
|
self.patterns = patterns
|
|
|
|
def bits(self, granularity):
|
|
"""Calculate the unique bits dirtied by this pattern grouping"""
|
|
res = set()
|
|
for pattern in self.patterns:
|
|
res |= pattern.bits(granularity)
|
|
return res
|
|
|
|
|
|
GROUPS = [
|
|
PatternGroup([
|
|
# Batch 0: 4 clusters
|
|
Pattern('0x49', 0x0000000),
|
|
Pattern('0x6c', 0x0100000), # 1M
|
|
Pattern('0x6f', 0x2000000), # 32M
|
|
Pattern('0x76', 0x3ff0000)]), # 64M - 64K
|
|
PatternGroup([
|
|
# Batch 1: 6 clusters (3 new)
|
|
Pattern('0x65', 0x0000000), # Full overwrite
|
|
Pattern('0x77', 0x00f8000), # Partial-left (1M-32K)
|
|
Pattern('0x72', 0x2008000), # Partial-right (32M+32K)
|
|
Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K)
|
|
PatternGroup([
|
|
# Batch 2: 7 clusters (3 new)
|
|
Pattern('0x74', 0x0010000), # Adjacent-right
|
|
Pattern('0x69', 0x00e8000), # Partial-left (1M-96K)
|
|
Pattern('0x6e', 0x2018000), # Partial-right (32M+96K)
|
|
Pattern('0x67', 0x3fe0000,
|
|
2*GRANULARITY)]), # Overwrite [(64M-128K)-64M)
|
|
PatternGroup([
|
|
# Batch 3: 8 clusters (5 new)
|
|
# Carefully chosen such that nothing re-dirties the one cluster
|
|
# that copies out successfully before failure in Group #1.
|
|
Pattern('0xaa', 0x0010000,
|
|
3*GRANULARITY), # Overwrite and 2x Adjacent-right
|
|
Pattern('0xbb', 0x00d8000), # Partial-left (1M-160K)
|
|
Pattern('0xcc', 0x2028000), # Partial-right (32M+160K)
|
|
Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
|
|
]
|
|
|
|
|
|
class EmulatedBitmap:
|
|
def __init__(self, granularity=GRANULARITY):
|
|
self._bits = set()
|
|
self.granularity = granularity
|
|
|
|
def dirty_bits(self, bits):
|
|
self._bits |= set(bits)
|
|
|
|
def dirty_group(self, n):
|
|
self.dirty_bits(GROUPS[n].bits(self.granularity))
|
|
|
|
def clear(self):
|
|
self._bits = set()
|
|
|
|
def clear_bits(self, bits):
|
|
self._bits -= set(bits)
|
|
|
|
def clear_bit(self, bit):
|
|
self.clear_bits({bit})
|
|
|
|
def clear_group(self, n):
|
|
self.clear_bits(GROUPS[n].bits(self.granularity))
|
|
|
|
@property
|
|
def first_bit(self):
|
|
return sorted(self.bits)[0]
|
|
|
|
@property
|
|
def bits(self):
|
|
return self._bits
|
|
|
|
@property
|
|
def count(self):
|
|
return len(self.bits)
|
|
|
|
def compare(self, qmp_bitmap):
|
|
"""
|
|
Print a nice human-readable message checking that a bitmap as reported
|
|
by the QMP interface has as many bits set as we expect it to.
|
|
"""
|
|
|
|
name = qmp_bitmap.get('name', '(anonymous)')
|
|
log("= Checking Bitmap {:s} =".format(name))
|
|
|
|
want = self.count
|
|
have = qmp_bitmap['count'] // qmp_bitmap['granularity']
|
|
|
|
log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
|
|
want, have, "OK!" if want == have else "ERROR!"))
|
|
log('')
|
|
|
|
|
|
class Drive:
|
|
"""Represents, vaguely, a drive attached to a VM.
|
|
Includes format, graph, and device information."""
|
|
|
|
def __init__(self, path, vm=None):
|
|
self.path = path
|
|
self.vm = vm
|
|
self.fmt = None
|
|
self.size = None
|
|
self.node = None
|
|
|
|
def img_create(self, fmt, size):
|
|
self.fmt = fmt
|
|
self.size = size
|
|
iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size))
|
|
|
|
def create_target(self, name, fmt, size):
|
|
basename = os.path.basename(self.path)
|
|
file_node_name = "file_{}".format(basename)
|
|
vm = self.vm
|
|
|
|
log(vm.command('blockdev-create', job_id='bdc-file-job',
|
|
options={
|
|
'driver': 'file',
|
|
'filename': self.path,
|
|
'size': 0,
|
|
}))
|
|
vm.run_job('bdc-file-job')
|
|
log(vm.command('blockdev-add', driver='file',
|
|
node_name=file_node_name, filename=self.path))
|
|
|
|
log(vm.command('blockdev-create', job_id='bdc-fmt-job',
|
|
options={
|
|
'driver': fmt,
|
|
'file': file_node_name,
|
|
'size': size,
|
|
}))
|
|
vm.run_job('bdc-fmt-job')
|
|
log(vm.command('blockdev-add', driver=fmt,
|
|
node_name=name,
|
|
file=file_node_name))
|
|
self.fmt = fmt
|
|
self.size = size
|
|
self.node = name
|
|
|
|
def blockdev_backup(vm, device, target, sync, **kwargs):
|
|
# Strip any arguments explicitly nulled by the caller:
|
|
kwargs = {key: val for key, val in kwargs.items() if val is not None}
|
|
result = vm.qmp_log('blockdev-backup',
|
|
device=device,
|
|
target=target,
|
|
sync=sync,
|
|
filter_node_name='backup-top',
|
|
**kwargs)
|
|
return result
|
|
|
|
def blockdev_backup_mktarget(drive, target_id, filepath, sync, **kwargs):
|
|
target_drive = Drive(filepath, vm=drive.vm)
|
|
target_drive.create_target(target_id, drive.fmt, drive.size)
|
|
blockdev_backup(drive.vm, drive.node, target_id, sync, **kwargs)
|
|
|
|
def reference_backup(drive, n, filepath):
|
|
log("--- Reference Backup #{:d} ---\n".format(n))
|
|
target_id = "ref_target_{:d}".format(n)
|
|
job_id = "ref_backup_{:d}".format(n)
|
|
blockdev_backup_mktarget(drive, target_id, filepath, "full",
|
|
job_id=job_id)
|
|
drive.vm.run_job(job_id, auto_dismiss=True)
|
|
log('')
|
|
|
|
def backup(drive, n, filepath, sync, **kwargs):
|
|
log("--- Test Backup #{:d} ---\n".format(n))
|
|
target_id = "backup_target_{:d}".format(n)
|
|
job_id = "backup_{:d}".format(n)
|
|
kwargs.setdefault('auto-finalize', False)
|
|
blockdev_backup_mktarget(drive, target_id, filepath, sync,
|
|
job_id=job_id, **kwargs)
|
|
return job_id
|
|
|
|
def perform_writes(drive, n, filter_node_name=None):
|
|
log("--- Write #{:d} ---\n".format(n))
|
|
for pattern in GROUPS[n].patterns:
|
|
cmd = "write -P{:s} 0x{:07x} 0x{:x}".format(
|
|
pattern.byte,
|
|
pattern.offset,
|
|
pattern.size)
|
|
log(cmd)
|
|
log(drive.vm.hmp_qemu_io(filter_node_name or drive.node, cmd))
|
|
bitmaps = drive.vm.query_bitmaps()
|
|
log({'bitmaps': bitmaps}, indent=2)
|
|
log('')
|
|
return bitmaps
|
|
|
|
|
|
def compare_images(image, reference, baseimg=None, expected_match=True):
|
|
"""
|
|
Print a nice human-readable message comparing these images.
|
|
"""
|
|
expected_ret = 0 if expected_match else 1
|
|
if baseimg:
|
|
assert qemu_img("rebase", "-u", "-b", baseimg, '-F', iotests.imgfmt,
|
|
image) == 0
|
|
ret = qemu_img("compare", image, reference)
|
|
log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format(
|
|
image, reference,
|
|
"Identical" if ret == 0 else "Mismatch",
|
|
"OK!" if ret == expected_ret else "ERROR!"),
|
|
filters=[iotests.filter_testfiles])
|
|
|
|
def test_bitmap_sync(bsync_mode, msync_mode='bitmap', failure=None):
|
|
"""
|
|
Test bitmap backup routines.
|
|
|
|
:param bsync_mode: Is the Bitmap Sync mode, and can be any of:
|
|
- on-success: This is the "incremental" style mode. Bitmaps are
|
|
synchronized to what was copied out only on success.
|
|
(Partial images must be discarded.)
|
|
- never: This is the "differential" style mode.
|
|
Bitmaps are never synchronized.
|
|
- always: This is a "best effort" style mode.
|
|
Bitmaps are always synchronized, regardless of failure.
|
|
(Partial images must be kept.)
|
|
|
|
:param msync_mode: The mirror sync mode to use for the first backup.
|
|
Can be any one of:
|
|
- bitmap: Backups based on bitmap manifest.
|
|
- full: Full backups.
|
|
- top: Full backups of the top layer only.
|
|
|
|
:param failure: Is the (optional) failure mode, and can be any of:
|
|
- None: No failure. Test the normative path. Default.
|
|
- simulated: Cancel the job right before it completes.
|
|
This also tests writes "during" the job.
|
|
- intermediate: This tests a job that fails mid-process and produces
|
|
an incomplete backup. Testing limitations prevent
|
|
testing competing writes.
|
|
"""
|
|
with iotests.FilePaths(['img', 'bsync1', 'bsync2',
|
|
'fbackup0', 'fbackup1', 'fbackup2']) as \
|
|
(img_path, bsync1, bsync2,
|
|
fbackup0, fbackup1, fbackup2), \
|
|
iotests.VM() as vm:
|
|
|
|
mode = "Mode {:s}; Bitmap Sync {:s}".format(msync_mode, bsync_mode)
|
|
preposition = "with" if failure else "without"
|
|
cond = "{:s} {:s}".format(preposition,
|
|
"{:s} failure".format(failure) if failure
|
|
else "failure")
|
|
log("\n=== {:s} {:s} ===\n".format(mode, cond))
|
|
|
|
log('--- Preparing image & VM ---\n')
|
|
drive0 = Drive(img_path, vm=vm)
|
|
drive0.img_create(iotests.imgfmt, SIZE)
|
|
vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
|
|
vm.launch()
|
|
|
|
file_config = {
|
|
'driver': 'file',
|
|
'filename': drive0.path
|
|
}
|
|
|
|
if failure == 'intermediate':
|
|
file_config = {
|
|
'driver': 'blkdebug',
|
|
'image': file_config,
|
|
'set-state': [{
|
|
'event': 'flush_to_disk',
|
|
'state': 1,
|
|
'new_state': 2
|
|
}, {
|
|
'event': 'read_aio',
|
|
'state': 2,
|
|
'new_state': 3
|
|
}],
|
|
'inject-error': [{
|
|
'event': 'read_aio',
|
|
'errno': 5,
|
|
'state': 3,
|
|
'immediately': False,
|
|
'once': True
|
|
}]
|
|
}
|
|
|
|
drive0.node = 'drive0'
|
|
vm.qmp_log('blockdev-add',
|
|
filters=[iotests.filter_qmp_testfiles],
|
|
node_name=drive0.node,
|
|
driver=drive0.fmt,
|
|
file=file_config)
|
|
log('')
|
|
|
|
# 0 - Writes and Reference Backup
|
|
perform_writes(drive0, 0)
|
|
reference_backup(drive0, 0, fbackup0)
|
|
log('--- Add Bitmap ---\n')
|
|
vm.qmp_log("block-dirty-bitmap-add", node=drive0.node,
|
|
name="bitmap0", granularity=GRANULARITY)
|
|
log('')
|
|
ebitmap = EmulatedBitmap()
|
|
|
|
# 1 - Writes and Reference Backup
|
|
bitmaps = perform_writes(drive0, 1)
|
|
ebitmap.dirty_group(1)
|
|
bitmap = vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)
|
|
ebitmap.compare(bitmap)
|
|
reference_backup(drive0, 1, fbackup1)
|
|
|
|
# 1 - Test Backup (w/ Optional induced failure)
|
|
if failure == 'intermediate':
|
|
# Activate blkdebug induced failure for second-to-next read
|
|
log(vm.hmp_qemu_io(drive0.node, 'flush'))
|
|
log('')
|
|
job = backup(drive0, 1, bsync1, msync_mode,
|
|
bitmap="bitmap0", bitmap_mode=bsync_mode)
|
|
|
|
def _callback():
|
|
"""Issue writes while the job is open to test bitmap divergence."""
|
|
# Note: when `failure` is 'intermediate', this isn't called.
|
|
log('')
|
|
bitmaps = perform_writes(drive0, 2, filter_node_name='backup-top')
|
|
# Named bitmap (static, should be unchanged)
|
|
ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0',
|
|
bitmaps=bitmaps))
|
|
# Anonymous bitmap (dynamic, shows new writes)
|
|
anonymous = EmulatedBitmap()
|
|
anonymous.dirty_group(2)
|
|
anonymous.compare(vm.get_bitmap(drive0.node, '', recording=True,
|
|
bitmaps=bitmaps))
|
|
|
|
# Simulate the order in which this will happen:
|
|
# group 1 gets cleared first, then group two gets written.
|
|
if ((bsync_mode == 'on-success' and not failure) or
|
|
(bsync_mode == 'always')):
|
|
ebitmap.clear()
|
|
ebitmap.dirty_group(2)
|
|
|
|
vm.run_job(job, auto_dismiss=True, auto_finalize=False,
|
|
pre_finalize=_callback,
|
|
cancel=(failure == 'simulated'))
|
|
bitmaps = vm.query_bitmaps()
|
|
log({'bitmaps': bitmaps}, indent=2)
|
|
log('')
|
|
|
|
if bsync_mode == 'always' and failure == 'intermediate':
|
|
# TOP treats anything allocated as dirty, expect to see:
|
|
if msync_mode == 'top':
|
|
ebitmap.dirty_group(0)
|
|
|
|
# We manage to copy one sector (one bit) before the error.
|
|
ebitmap.clear_bit(ebitmap.first_bit)
|
|
|
|
# Full returns all bits set except what was copied/skipped
|
|
if msync_mode == 'full':
|
|
fail_bit = ebitmap.first_bit
|
|
ebitmap.clear()
|
|
ebitmap.dirty_bits(range(fail_bit, SIZE // GRANULARITY))
|
|
|
|
ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
|
|
|
|
# 2 - Writes and Reference Backup
|
|
bitmaps = perform_writes(drive0, 3)
|
|
ebitmap.dirty_group(3)
|
|
ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
|
|
reference_backup(drive0, 2, fbackup2)
|
|
|
|
# 2 - Bitmap Backup (In failure modes, this is a recovery.)
|
|
job = backup(drive0, 2, bsync2, "bitmap",
|
|
bitmap="bitmap0", bitmap_mode=bsync_mode)
|
|
vm.run_job(job, auto_dismiss=True, auto_finalize=False)
|
|
bitmaps = vm.query_bitmaps()
|
|
log({'bitmaps': bitmaps}, indent=2)
|
|
log('')
|
|
if bsync_mode != 'never':
|
|
ebitmap.clear()
|
|
ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
|
|
|
|
log('--- Cleanup ---\n')
|
|
vm.qmp_log("block-dirty-bitmap-remove",
|
|
node=drive0.node, name="bitmap0")
|
|
bitmaps = vm.query_bitmaps()
|
|
log({'bitmaps': bitmaps}, indent=2)
|
|
vm.shutdown()
|
|
log('')
|
|
|
|
log('--- Verification ---\n')
|
|
# 'simulated' failures will actually all pass here because we canceled
|
|
# while "pending". This is actually undefined behavior,
|
|
# don't rely on this to be true!
|
|
compare_images(bsync1, fbackup1, baseimg=fbackup0,
|
|
expected_match=failure != 'intermediate')
|
|
if not failure or bsync_mode == 'always':
|
|
# Always keep the last backup on success or when using 'always'
|
|
base = bsync1
|
|
else:
|
|
base = fbackup0
|
|
compare_images(bsync2, fbackup2, baseimg=base)
|
|
compare_images(img_path, fbackup2)
|
|
log('')
|
|
|
|
def test_backup_api():
|
|
"""
|
|
Test malformed and prohibited invocations of the backup API.
|
|
"""
|
|
with iotests.FilePaths(['img', 'bsync1']) as \
|
|
(img_path, backup_path), \
|
|
iotests.VM() as vm:
|
|
|
|
log("\n=== API failure tests ===\n")
|
|
log('--- Preparing image & VM ---\n')
|
|
drive0 = Drive(img_path, vm=vm)
|
|
drive0.img_create(iotests.imgfmt, SIZE)
|
|
vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
|
|
vm.launch()
|
|
|
|
file_config = {
|
|
'driver': 'file',
|
|
'filename': drive0.path
|
|
}
|
|
|
|
drive0.node = 'drive0'
|
|
vm.qmp_log('blockdev-add',
|
|
filters=[iotests.filter_qmp_testfiles],
|
|
node_name=drive0.node,
|
|
driver=drive0.fmt,
|
|
file=file_config)
|
|
log('')
|
|
|
|
target0 = Drive(backup_path, vm=vm)
|
|
target0.create_target("backup_target", drive0.fmt, drive0.size)
|
|
log('')
|
|
|
|
vm.qmp_log("block-dirty-bitmap-add", node=drive0.node,
|
|
name="bitmap0", granularity=GRANULARITY)
|
|
log('')
|
|
|
|
log('-- Testing invalid QMP commands --\n')
|
|
|
|
error_cases = {
|
|
'incremental': {
|
|
None: ['on-success', 'always', 'never', None],
|
|
'bitmap404': ['on-success', 'always', 'never', None],
|
|
'bitmap0': ['always', 'never']
|
|
},
|
|
'bitmap': {
|
|
None: ['on-success', 'always', 'never', None],
|
|
'bitmap404': ['on-success', 'always', 'never', None],
|
|
'bitmap0': [None],
|
|
},
|
|
'full': {
|
|
None: ['on-success', 'always', 'never'],
|
|
'bitmap404': ['on-success', 'always', 'never', None],
|
|
'bitmap0': ['never', None],
|
|
},
|
|
'top': {
|
|
None: ['on-success', 'always', 'never'],
|
|
'bitmap404': ['on-success', 'always', 'never', None],
|
|
'bitmap0': ['never', None],
|
|
},
|
|
'none': {
|
|
None: ['on-success', 'always', 'never'],
|
|
'bitmap404': ['on-success', 'always', 'never', None],
|
|
'bitmap0': ['on-success', 'always', 'never', None],
|
|
}
|
|
}
|
|
|
|
# Dicts, as always, are not stably-ordered prior to 3.7, so use tuples:
|
|
for sync_mode in ('incremental', 'bitmap', 'full', 'top', 'none'):
|
|
log("-- Sync mode {:s} tests --\n".format(sync_mode))
|
|
for bitmap in (None, 'bitmap404', 'bitmap0'):
|
|
for policy in error_cases[sync_mode][bitmap]:
|
|
blockdev_backup(drive0.vm, drive0.node, "backup_target",
|
|
sync_mode, job_id='api_job',
|
|
bitmap=bitmap, bitmap_mode=policy)
|
|
log('')
|
|
|
|
|
|
def main():
|
|
for bsync_mode in ("never", "on-success", "always"):
|
|
for failure in ("simulated", "intermediate", None):
|
|
test_bitmap_sync(bsync_mode, "bitmap", failure)
|
|
|
|
for sync_mode in ('full', 'top'):
|
|
for bsync_mode in ('on-success', 'always'):
|
|
for failure in ('simulated', 'intermediate', None):
|
|
test_bitmap_sync(bsync_mode, sync_mode, failure)
|
|
|
|
test_backup_api()
|
|
|
|
if __name__ == '__main__':
|
|
iotests.script_main(main, supported_fmts=['qcow2'],
|
|
supported_protocols=['file'])
|