@@ -843,6 +843,181 @@ def test_restore_volume_and_attach_to_vm(self):
843843 pass
844844 self .backup_offering .removeOffering (self .apiclient , self .vm .id )
845845
846+ # ------------------------------------------------------------------
847+ # Whole-chain validation: every backup is restorable and bootable, and
848+ # stays so after a backup is removed from the middle of the chain
849+ # ------------------------------------------------------------------
850+ # The other chain tests each check one property of one backup. This one walks
851+ # the whole chain: FULL + 3 INCREMENTALs, then restores from every backup and
852+ # builds a VM from every backup, deletes the 2nd incremental, and repeats both
853+ # sweeps over what survives.
854+ #
855+ # It is the mid-chain delete that makes this worth its runtime on RBD. An
856+ # `rbd export-diff` stream carries no pointer to the diff before it, so a restore
857+ # replays the whole sequence from the full anchor; a diff whose descendant still
858+ # needs it therefore cannot be dropped when it is deleted, only hidden. Restoring
859+ # the tip across that gap is what breaks if the provider ever sends a chain with a
860+ # hole in it, and no other test exercises it.
861+ #
862+ # Sizes are asserted throughout: an incremental must be a small fraction of the
863+ # full, which is what distinguishes a real `export-diff` delta from the whole
864+ # image being re-exported every cycle. Nothing else in this file asserts on size,
865+ # so a provider that silently fell back to full-every-time would still pass.
866+
867+ def _assert_chain_sizes (self , backups , context ):
868+ """A FULL carries the allocated image; an INCREMENTAL must carry only a delta."""
869+ full = backups [0 ]
870+ full_size = int (full .size )
871+ self .assertGreater (full_size , 0 , "%s: FULL reports a size of 0" % context )
872+ for inc in backups [1 :]:
873+ inc_size = int (inc .size )
874+ self .assertGreater (inc_size , 0 , "%s: incremental reports a size of 0" % context )
875+ self .assertLess (inc_size , full_size / 10 ,
876+ "%s: incremental is %d bytes against a FULL of %d -- an export-diff delta "
877+ "should be a small fraction of the full, not the whole image again"
878+ % (context , inc_size , full_size ))
879+
880+ def _restore_in_place_and_check (self , backup , expected_markers , absent_markers , context ):
881+ """Restore over the Instance's own volume and verify the guest's contents.
882+
883+ A restore rewinds the volume wholesale, so markers written after the backup
884+ was taken must be gone -- checking only that the expected ones are present
885+ would pass even if the restore had quietly done nothing.
886+ """
887+ self .vm .stop (self .apiclient )
888+ time .sleep (5 )
889+ Backup .restoreVM (self .apiclient , backup .id )
890+ self .vm .start (self .apiclient )
891+ ssh = self .vm .get_ssh_client (reconnect = True , keyPairFileLocation = self .ssh_key_file )
892+ listing = "" .join (ssh .execute (self ._sudo ("ls /root" )))
893+ for marker in expected_markers :
894+ self .assertIn (marker , listing ,
895+ "%s: marker %s written before this backup is missing after the restore"
896+ % (context , marker ))
897+ for marker in absent_markers :
898+ self .assertNotIn (marker , listing ,
899+ "%s: marker %s was written AFTER this backup, so the restore should have "
900+ "rolled it back" % (context , marker ))
901+
902+ def _create_vm_from_backup_and_check (self , backup , context ):
903+ """Build a VM from a backup and confirm it reaches Running, then clean it up."""
904+ name = "chain-%s-%d" % (context , int (time .time ()))
905+ new_vm = Backup .createVMFromBackup (
906+ self .apiclient ,
907+ self .services ["small" ],
908+ mode = self .services ["mode" ],
909+ backupid = backup .id ,
910+ vmname = name ,
911+ accountname = self .account .name ,
912+ domainid = self .account .domainid ,
913+ zoneid = self .zone .id ,
914+ networkids = None ,
915+ templateid = None
916+ )
917+ self .assertIsNotNone (new_vm , "%s: createVMFromBackup returned nothing" % context )
918+ try :
919+ vms = VirtualMachine .list (self .apiclient , id = new_vm .id , listall = True )
920+ self .assertTrue (vms and str (vms [0 ].state ).lower () == "running" ,
921+ "%s: VM created from the backup is not Running (state=%s)"
922+ % (context , vms [0 ].state if vms else "<not listed>" ))
923+ finally :
924+ # Destroyed as we go rather than via cls._cleanup: this test builds seven
925+ # of these, and keeping them all alive to the end risks exhausting the
926+ # environment's capacity partway through the run.
927+ try :
928+ new_vm .delete (self .apiclient , expunge = True )
929+ except Exception :
930+ pass
931+
932+ @attr (tags = ["advanced" , "backup" ], required_hardware = "true" )
933+ def test_full_chain_restore_and_create_vm_survive_mid_chain_delete (self ):
934+ """
935+ FULL + 3 INCREMENTALs: restore from every backup and create a VM from every
936+ backup; then delete the 2nd incremental and repeat over the survivors. Backup
937+ sizes are checked at each stage, and the chain must be gone at the end.
938+ """
939+ self .backup_offering .assignOffering (self .apiclient , self .vm .id )
940+ original_full_every = self ._get_full_every ()
941+ # High enough that every backup after the first stays an incremental.
942+ self ._set_full_every (10 )
943+ backups = []
944+ try :
945+ ssh = self .vm .get_ssh_client (reconnect = True , keyPairFileLocation = self .ssh_key_file )
946+
947+ # --- build FULL -> INC1 -> INC2 -> INC3, a marker per backup ---------
948+ markers = []
949+ for idx in range (4 ):
950+ marker = "chain_marker_%d.txt" % idx
951+ # sync: without it the file can still be in the guest page cache rather
952+ # than on the RBD image when the snapshot is taken moments later.
953+ ssh .execute (self ._sudo ("touch /root/%s; sync" % marker ))
954+ markers .append (marker )
955+ Backup .create (self .apiclient , self .vm .id , "chain_%d" % idx )
956+ time .sleep (3 )
957+
958+ backups = Backup .list (self .apiclient , self .vm .id )
959+ backups .sort (key = lambda b : b .created )
960+ self .assertEqual (len (backups ), 4 , "Expected FULL + 3 INCREMENTAL backups" )
961+ self .assertEqual (self ._backup_type (backups [0 ]).upper (), 'FULL' )
962+ for inc in backups [1 :]:
963+ self .assertEqual (self ._backup_type (inc ).upper (), 'INCREMENTAL' )
964+ self ._assert_chain_sizes (backups , "complete chain" )
965+ sizes_before = [int (b .size ) for b in backups ]
966+
967+ # --- restore from each, then build a VM from each --------------------
968+ for idx , backup in enumerate (backups ):
969+ self ._restore_in_place_and_check (
970+ backup , markers [:idx + 1 ], markers [idx + 1 :], "complete chain, backup %d" % idx )
971+ for idx , backup in enumerate (backups ):
972+ self ._create_vm_from_backup_and_check (backup , "full%d" % idx )
973+
974+ # --- delete the 2nd incremental --------------------------------------
975+ # Its diff stays on the repository while INC3 still needs it to replay;
976+ # it is swept once the last backup depending on it is gone.
977+ deleted = backups [2 ]
978+ Backup .delete (self .apiclient , deleted .id )
979+ survivors = Backup .list (self .apiclient , self .vm .id )
980+ survivors .sort (key = lambda b : b .created )
981+ self .assertEqual (len (survivors ), 3 ,
982+ "Deleting one backup should leave exactly the other three" )
983+ self .assertNotIn (deleted .id , [b .id for b in survivors ],
984+ "The deleted backup must no longer be listed" )
985+
986+ # Removing a sibling must not alter what the others report.
987+ surviving_sizes = [int (b .size ) for b in survivors ]
988+ self .assertEqual (surviving_sizes ,
989+ [sizes_before [0 ], sizes_before [1 ], sizes_before [3 ]],
990+ "Deleting a backup changed the reported size of the backups that remain" )
991+ self ._assert_chain_sizes (survivors , "after mid-chain delete" )
992+
993+ # --- restore from each survivor, then build a VM from each -----------
994+ # survivors[2] is the old INC3: restoring it replays the chain across the
995+ # deleted INC2, so its marker must still come back.
996+ expected = [markers [:1 ], markers [:2 ], markers [:4 ]]
997+ absent = [markers [1 :], markers [2 :], []]
998+ for idx , backup in enumerate (survivors ):
999+ self ._restore_in_place_and_check (
1000+ backup , expected [idx ], absent [idx ], "after delete, backup %d" % idx )
1001+ for idx , backup in enumerate (survivors ):
1002+ self ._create_vm_from_backup_and_check (backup , "post%d" % idx )
1003+
1004+ # --- the chain must leave nothing behind ------------------------------
1005+ for backup in reversed (survivors ):
1006+ Backup .delete (self .apiclient , backup .id )
1007+ backups = []
1008+ remaining = Backup .list (self .apiclient , self .vm .id )
1009+ self .assertFalse (remaining ,
1010+ "Every backup was deleted, so none should be left for this Instance "
1011+ "(still listed: %s)" % remaining )
1012+ finally :
1013+ for b in reversed (backups ):
1014+ try :
1015+ Backup .delete (self .apiclient , b .id )
1016+ except Exception :
1017+ pass
1018+ self ._set_full_every (original_full_every )
1019+ self .backup_offering .removeOffering (self .apiclient , self .vm .id )
1020+
8461021 def _sudo (self , cmd ):
8471022 """Wrap a shell command to run as root via sudo (the cloud-init "ubuntu" user
8481023 this test SSHes in as has passwordless sudo) -- needed for anything touching
0 commit comments