Skip to content

Commit a4a8c27

Browse files
committed
test: validate a whole backup chain, including across a mid-chain delete
The existing chain tests each check one property of one backup: the full/incremental cadence, a restore from the tip, a delete that repairs or defers. None of them walks the chain as a whole, and none asserts on backup size. Add a test that builds FULL + 3 INCREMENTALs with a marker file per backup, restores from every backup and creates a VM from every backup, then deletes the 2nd incremental and repeats both sweeps over what survives. The mid-chain delete is what earns the runtime on RBD. An `rbd export-diff` stream carries no pointer to the diff before it, so a restore replays the whole sequence from the full anchor, and a diff whose descendant still needs it cannot be dropped when it is deleted, only hidden. Restoring the tip across that gap is what breaks if the provider ever sends a chain with a hole in it, and nothing else covered it. Sizes are asserted throughout: an incremental must be a small fraction of the full. That is what distinguishes a real export-diff delta from the whole image being re-exported every cycle, and no other test in this file looks at size at all, so a provider that silently fell back to full-every-time still passed. Each restore also asserts that markers written after that backup are gone. Checking only that the expected markers are present would pass even if the restore had quietly done nothing. VMs built from backups are expunged as the test goes rather than through _cleanup: seven of them are created, and holding them all to the end risks exhausting the environment's capacity partway through a run.
1 parent 0cdbf56 commit a4a8c27

1 file changed

Lines changed: 175 additions & 0 deletions

File tree

‎test/integration/smoke/test_backup_recovery_nas.py‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -843,6 +843,181 @@ def test_restore_volume_and_attach_to_vm(self):
843843
pass
844844
self.backup_offering.removeOffering(self.apiclient, self.vm.id)
845845

846+
# ------------------------------------------------------------------
847+
# Whole-chain validation: every backup is restorable and bootable, and
848+
# stays so after a backup is removed from the middle of the chain
849+
# ------------------------------------------------------------------
850+
# The other chain tests each check one property of one backup. This one walks
851+
# the whole chain: FULL + 3 INCREMENTALs, then restores from every backup and
852+
# builds a VM from every backup, deletes the 2nd incremental, and repeats both
853+
# sweeps over what survives.
854+
#
855+
# It is the mid-chain delete that makes this worth its runtime on RBD. An
856+
# `rbd export-diff` stream carries no pointer to the diff before it, so a restore
857+
# replays the whole sequence from the full anchor; a diff whose descendant still
858+
# needs it therefore cannot be dropped when it is deleted, only hidden. Restoring
859+
# the tip across that gap is what breaks if the provider ever sends a chain with a
860+
# hole in it, and no other test exercises it.
861+
#
862+
# Sizes are asserted throughout: an incremental must be a small fraction of the
863+
# full, which is what distinguishes a real `export-diff` delta from the whole
864+
# image being re-exported every cycle. Nothing else in this file asserts on size,
865+
# so a provider that silently fell back to full-every-time would still pass.
866+
867+
def _assert_chain_sizes(self, backups, context):
868+
"""A FULL carries the allocated image; an INCREMENTAL must carry only a delta."""
869+
full = backups[0]
870+
full_size = int(full.size)
871+
self.assertGreater(full_size, 0, "%s: FULL reports a size of 0" % context)
872+
for inc in backups[1:]:
873+
inc_size = int(inc.size)
874+
self.assertGreater(inc_size, 0, "%s: incremental reports a size of 0" % context)
875+
self.assertLess(inc_size, full_size / 10,
876+
"%s: incremental is %d bytes against a FULL of %d -- an export-diff delta "
877+
"should be a small fraction of the full, not the whole image again"
878+
% (context, inc_size, full_size))
879+
880+
def _restore_in_place_and_check(self, backup, expected_markers, absent_markers, context):
881+
"""Restore over the Instance's own volume and verify the guest's contents.
882+
883+
A restore rewinds the volume wholesale, so markers written after the backup
884+
was taken must be gone -- checking only that the expected ones are present
885+
would pass even if the restore had quietly done nothing.
886+
"""
887+
self.vm.stop(self.apiclient)
888+
time.sleep(5)
889+
Backup.restoreVM(self.apiclient, backup.id)
890+
self.vm.start(self.apiclient)
891+
ssh = self.vm.get_ssh_client(reconnect=True, keyPairFileLocation=self.ssh_key_file)
892+
listing = "".join(ssh.execute(self._sudo("ls /root")))
893+
for marker in expected_markers:
894+
self.assertIn(marker, listing,
895+
"%s: marker %s written before this backup is missing after the restore"
896+
% (context, marker))
897+
for marker in absent_markers:
898+
self.assertNotIn(marker, listing,
899+
"%s: marker %s was written AFTER this backup, so the restore should have "
900+
"rolled it back" % (context, marker))
901+
902+
def _create_vm_from_backup_and_check(self, backup, context):
903+
"""Build a VM from a backup and confirm it reaches Running, then clean it up."""
904+
name = "chain-%s-%d" % (context, int(time.time()))
905+
new_vm = Backup.createVMFromBackup(
906+
self.apiclient,
907+
self.services["small"],
908+
mode=self.services["mode"],
909+
backupid=backup.id,
910+
vmname=name,
911+
accountname=self.account.name,
912+
domainid=self.account.domainid,
913+
zoneid=self.zone.id,
914+
networkids=None,
915+
templateid=None
916+
)
917+
self.assertIsNotNone(new_vm, "%s: createVMFromBackup returned nothing" % context)
918+
try:
919+
vms = VirtualMachine.list(self.apiclient, id=new_vm.id, listall=True)
920+
self.assertTrue(vms and str(vms[0].state).lower() == "running",
921+
"%s: VM created from the backup is not Running (state=%s)"
922+
% (context, vms[0].state if vms else "<not listed>"))
923+
finally:
924+
# Destroyed as we go rather than via cls._cleanup: this test builds seven
925+
# of these, and keeping them all alive to the end risks exhausting the
926+
# environment's capacity partway through the run.
927+
try:
928+
new_vm.delete(self.apiclient, expunge=True)
929+
except Exception:
930+
pass
931+
932+
@attr(tags=["advanced", "backup"], required_hardware="true")
933+
def test_full_chain_restore_and_create_vm_survive_mid_chain_delete(self):
934+
"""
935+
FULL + 3 INCREMENTALs: restore from every backup and create a VM from every
936+
backup; then delete the 2nd incremental and repeat over the survivors. Backup
937+
sizes are checked at each stage, and the chain must be gone at the end.
938+
"""
939+
self.backup_offering.assignOffering(self.apiclient, self.vm.id)
940+
original_full_every = self._get_full_every()
941+
# High enough that every backup after the first stays an incremental.
942+
self._set_full_every(10)
943+
backups = []
944+
try:
945+
ssh = self.vm.get_ssh_client(reconnect=True, keyPairFileLocation=self.ssh_key_file)
946+
947+
# --- build FULL -> INC1 -> INC2 -> INC3, a marker per backup ---------
948+
markers = []
949+
for idx in range(4):
950+
marker = "chain_marker_%d.txt" % idx
951+
# sync: without it the file can still be in the guest page cache rather
952+
# than on the RBD image when the snapshot is taken moments later.
953+
ssh.execute(self._sudo("touch /root/%s; sync" % marker))
954+
markers.append(marker)
955+
Backup.create(self.apiclient, self.vm.id, "chain_%d" % idx)
956+
time.sleep(3)
957+
958+
backups = Backup.list(self.apiclient, self.vm.id)
959+
backups.sort(key=lambda b: b.created)
960+
self.assertEqual(len(backups), 4, "Expected FULL + 3 INCREMENTAL backups")
961+
self.assertEqual(self._backup_type(backups[0]).upper(), 'FULL')
962+
for inc in backups[1:]:
963+
self.assertEqual(self._backup_type(inc).upper(), 'INCREMENTAL')
964+
self._assert_chain_sizes(backups, "complete chain")
965+
sizes_before = [int(b.size) for b in backups]
966+
967+
# --- restore from each, then build a VM from each --------------------
968+
for idx, backup in enumerate(backups):
969+
self._restore_in_place_and_check(
970+
backup, markers[:idx + 1], markers[idx + 1:], "complete chain, backup %d" % idx)
971+
for idx, backup in enumerate(backups):
972+
self._create_vm_from_backup_and_check(backup, "full%d" % idx)
973+
974+
# --- delete the 2nd incremental --------------------------------------
975+
# Its diff stays on the repository while INC3 still needs it to replay;
976+
# it is swept once the last backup depending on it is gone.
977+
deleted = backups[2]
978+
Backup.delete(self.apiclient, deleted.id)
979+
survivors = Backup.list(self.apiclient, self.vm.id)
980+
survivors.sort(key=lambda b: b.created)
981+
self.assertEqual(len(survivors), 3,
982+
"Deleting one backup should leave exactly the other three")
983+
self.assertNotIn(deleted.id, [b.id for b in survivors],
984+
"The deleted backup must no longer be listed")
985+
986+
# Removing a sibling must not alter what the others report.
987+
surviving_sizes = [int(b.size) for b in survivors]
988+
self.assertEqual(surviving_sizes,
989+
[sizes_before[0], sizes_before[1], sizes_before[3]],
990+
"Deleting a backup changed the reported size of the backups that remain")
991+
self._assert_chain_sizes(survivors, "after mid-chain delete")
992+
993+
# --- restore from each survivor, then build a VM from each -----------
994+
# survivors[2] is the old INC3: restoring it replays the chain across the
995+
# deleted INC2, so its marker must still come back.
996+
expected = [markers[:1], markers[:2], markers[:4]]
997+
absent = [markers[1:], markers[2:], []]
998+
for idx, backup in enumerate(survivors):
999+
self._restore_in_place_and_check(
1000+
backup, expected[idx], absent[idx], "after delete, backup %d" % idx)
1001+
for idx, backup in enumerate(survivors):
1002+
self._create_vm_from_backup_and_check(backup, "post%d" % idx)
1003+
1004+
# --- the chain must leave nothing behind ------------------------------
1005+
for backup in reversed(survivors):
1006+
Backup.delete(self.apiclient, backup.id)
1007+
backups = []
1008+
remaining = Backup.list(self.apiclient, self.vm.id)
1009+
self.assertFalse(remaining,
1010+
"Every backup was deleted, so none should be left for this Instance "
1011+
"(still listed: %s)" % remaining)
1012+
finally:
1013+
for b in reversed(backups):
1014+
try:
1015+
Backup.delete(self.apiclient, b.id)
1016+
except Exception:
1017+
pass
1018+
self._set_full_every(original_full_every)
1019+
self.backup_offering.removeOffering(self.apiclient, self.vm.id)
1020+
8461021
def _sudo(self, cmd):
8471022
"""Wrap a shell command to run as root via sudo (the cloud-init "ubuntu" user
8481023
this test SSHes in as has passwordless sudo) -- needed for anything touching

0 commit comments

Comments
 (0)