|
4 | 4 | import subprocess
|
5 | 5 | from datetime import datetime
|
6 | 6 | import sys
|
7 |
| -import time |
| 7 | +from time import sleep |
8 | 8 |
|
9 | 9 |
|
10 | 10 | module_name = 'restore'
|
@@ -1441,3 +1441,279 @@ def test_zags_block_corrupt_1(self):
|
1441 | 1441 | True,
|
1442 | 1442 | 'Failed to start pg_wal_dump: {0}'.format(
|
1443 | 1443 | pg_receivexlog.communicate()[1]))
|
| 1444 | + |
| 1445 | + # @unittest.skip("skip") |
| 1446 | + def test_restore_chain(self): |
| 1447 | + """ |
| 1448 | + make node, take full backup, take several |
| 1449 | + ERROR delta backups, take valid delta backup, |
| 1450 | + restore must be successfull |
| 1451 | + """ |
| 1452 | + fname = self.id().split('.')[3] |
| 1453 | + node = self.make_simple_node( |
| 1454 | + base_dir=os.path.join(module_name, fname, 'node'), |
| 1455 | + set_replication=True, |
| 1456 | + initdb_params=['--data-checksums'], |
| 1457 | + pg_options={ |
| 1458 | + 'wal_level': 'replica', |
| 1459 | + 'max_wal_senders': '2'}) |
| 1460 | + |
| 1461 | + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') |
| 1462 | + self.init_pb(backup_dir) |
| 1463 | + self.add_instance(backup_dir, 'node', node) |
| 1464 | + self.set_archiving(backup_dir, 'node', node) |
| 1465 | + node.slow_start() |
| 1466 | + |
| 1467 | + self.backup_node( |
| 1468 | + backup_dir, 'node', node) |
| 1469 | + |
| 1470 | + self.backup_node( |
| 1471 | + backup_dir, 'node', node, backup_type='delta') |
| 1472 | + |
| 1473 | + try: |
| 1474 | + self.backup_node( |
| 1475 | + backup_dir, 'node', node, |
| 1476 | + backup_type='delta', options=['--archive-timeout=0s']) |
| 1477 | + except ProbackupException as e: |
| 1478 | + pass |
| 1479 | + |
| 1480 | + try: |
| 1481 | + self.backup_node( |
| 1482 | + backup_dir, 'node', node, |
| 1483 | + backup_type='delta', options=['--archive-timeout=0s']) |
| 1484 | + except ProbackupException as e: |
| 1485 | + pass |
| 1486 | + |
| 1487 | + |
| 1488 | + self.backup_node( |
| 1489 | + backup_dir, 'node', node, backup_type='delta') |
| 1490 | + |
| 1491 | + try: |
| 1492 | + self.backup_node( |
| 1493 | + backup_dir, 'node', node, |
| 1494 | + backup_type='delta', options=['--archive-timeout=0s']) |
| 1495 | + except ProbackupException as e: |
| 1496 | + pass |
| 1497 | + |
| 1498 | + self.assertEqual( |
| 1499 | + 'OK', |
| 1500 | + self.show_pb(backup_dir, 'node')[0]['status'], |
| 1501 | + 'Backup STATUS should be "OK"') |
| 1502 | + |
| 1503 | + self.assertEqual( |
| 1504 | + 'OK', |
| 1505 | + self.show_pb(backup_dir, 'node')[1]['status'], |
| 1506 | + 'Backup STATUS should be "OK"') |
| 1507 | + |
| 1508 | + self.assertEqual( |
| 1509 | + 'ERROR', |
| 1510 | + self.show_pb(backup_dir, 'node')[2]['status'], |
| 1511 | + 'Backup STATUS should be "ERROR"') |
| 1512 | + |
| 1513 | + self.assertEqual( |
| 1514 | + 'ERROR', |
| 1515 | + self.show_pb(backup_dir, 'node')[3]['status'], |
| 1516 | + 'Backup STATUS should be "ERROR"') |
| 1517 | + |
| 1518 | + self.assertEqual( |
| 1519 | + 'OK', |
| 1520 | + self.show_pb(backup_dir, 'node')[4]['status'], |
| 1521 | + 'Backup STATUS should be "OK"') |
| 1522 | + |
| 1523 | + self.assertEqual( |
| 1524 | + 'ERROR', |
| 1525 | + self.show_pb(backup_dir, 'node')[5]['status'], |
| 1526 | + 'Backup STATUS should be "ERROR"') |
| 1527 | + |
| 1528 | + node.cleanup() |
| 1529 | + |
| 1530 | + self.restore_node(backup_dir, 'node', node) |
| 1531 | + |
| 1532 | + # Clean after yourself |
| 1533 | + self.del_test_dir(module_name, fname) |
| 1534 | + |
| 1535 | + # @unittest.skip("skip") |
| 1536 | + def test_restore_chain_with_corrupted_backup(self): |
| 1537 | + """more complex test_restore_chain()""" |
| 1538 | + fname = self.id().split('.')[3] |
| 1539 | + node = self.make_simple_node( |
| 1540 | + base_dir=os.path.join(module_name, fname, 'node'), |
| 1541 | + set_replication=True, |
| 1542 | + initdb_params=['--data-checksums'], |
| 1543 | + pg_options={ |
| 1544 | + 'wal_level': 'replica', |
| 1545 | + 'max_wal_senders': '2'}) |
| 1546 | + |
| 1547 | + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') |
| 1548 | + self.init_pb(backup_dir) |
| 1549 | + self.add_instance(backup_dir, 'node', node) |
| 1550 | + self.set_archiving(backup_dir, 'node', node) |
| 1551 | + node.slow_start() |
| 1552 | + |
| 1553 | + # Take FULL |
| 1554 | + self.backup_node( |
| 1555 | + backup_dir, 'node', node) |
| 1556 | + |
| 1557 | + # Take DELTA |
| 1558 | + self.backup_node( |
| 1559 | + backup_dir, 'node', node, backup_type='page') |
| 1560 | + |
| 1561 | + # Take ERROR DELTA |
| 1562 | + try: |
| 1563 | + self.backup_node( |
| 1564 | + backup_dir, 'node', node, |
| 1565 | + backup_type='page', options=['--archive-timeout=0s']) |
| 1566 | + except ProbackupException as e: |
| 1567 | + pass |
| 1568 | + |
| 1569 | + # Take 1 DELTA |
| 1570 | + self.backup_node( |
| 1571 | + backup_dir, 'node', node, backup_type='delta') |
| 1572 | + |
| 1573 | + # Take ERROR DELTA |
| 1574 | + try: |
| 1575 | + self.backup_node( |
| 1576 | + backup_dir, 'node', node, |
| 1577 | + backup_type='delta', options=['--archive-timeout=0s']) |
| 1578 | + except ProbackupException as e: |
| 1579 | + pass |
| 1580 | + |
| 1581 | + # Take 2 DELTA |
| 1582 | + self.backup_node( |
| 1583 | + backup_dir, 'node', node, backup_type='delta') |
| 1584 | + |
| 1585 | + # Take ERROR DELTA |
| 1586 | + try: |
| 1587 | + self.backup_node( |
| 1588 | + backup_dir, 'node', node, |
| 1589 | + backup_type='delta', options=['--archive-timeout=0s']) |
| 1590 | + except ProbackupException as e: |
| 1591 | + pass |
| 1592 | + |
| 1593 | + # Take 3 DELTA |
| 1594 | + self.backup_node( |
| 1595 | + backup_dir, 'node', node, backup_type='delta') |
| 1596 | + |
| 1597 | + # Corrupted 4 DELTA |
| 1598 | + corrupt_id = self.backup_node( |
| 1599 | + backup_dir, 'node', node, backup_type='delta') |
| 1600 | + |
| 1601 | + # ORPHAN 5 DELTA |
| 1602 | + restore_target_id = self.backup_node( |
| 1603 | + backup_dir, 'node', node, backup_type='delta') |
| 1604 | + |
| 1605 | + # ORPHAN 6 DELTA |
| 1606 | + self.backup_node( |
| 1607 | + backup_dir, 'node', node, backup_type='delta') |
| 1608 | + |
| 1609 | + # NEXT FULL BACKUP |
| 1610 | + self.backup_node( |
| 1611 | + backup_dir, 'node', node, backup_type='full') |
| 1612 | + |
| 1613 | + # Next Delta |
| 1614 | + self.backup_node( |
| 1615 | + backup_dir, 'node', node, backup_type='delta') |
| 1616 | + |
| 1617 | + # do corrupt 6 DELTA backup |
| 1618 | + file = os.path.join( |
| 1619 | + backup_dir, 'backups', 'node', |
| 1620 | + corrupt_id, 'database', 'global', 'pg_control') |
| 1621 | + |
| 1622 | + file_new = os.path.join(backup_dir, 'pg_control') |
| 1623 | + os.rename(file, file_new) |
| 1624 | + |
| 1625 | + # RESTORE BACKUP |
| 1626 | + node.cleanup() |
| 1627 | + |
| 1628 | + print(restore_target_id) |
| 1629 | + exit(1) |
| 1630 | + |
| 1631 | + try: |
| 1632 | + self.restore_node( |
| 1633 | + backup_dir, 'node', node, backup_id=restore_target_id) |
| 1634 | + self.assertEqual( |
| 1635 | + 1, 0, |
| 1636 | + "Expecting Error because restore backup is corrupted.\n " |
| 1637 | + "Output: {0} \n CMD: {1}".format( |
| 1638 | + repr(self.output), self.cmd)) |
| 1639 | + except ProbackupException as e: |
| 1640 | + self.assertIn( |
| 1641 | + 'ERROR: Backup {0} is orphan'.format(restore_target_id), |
| 1642 | + e.message, |
| 1643 | + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( |
| 1644 | + repr(e.message), self.cmd)) |
| 1645 | + |
| 1646 | + self.assertEqual( |
| 1647 | + 'OK', |
| 1648 | + self.show_pb(backup_dir, 'node')[0]['status'], |
| 1649 | + 'Backup STATUS should be "OK"') |
| 1650 | + |
| 1651 | + self.assertEqual( |
| 1652 | + 'OK', |
| 1653 | + self.show_pb(backup_dir, 'node')[1]['status'], |
| 1654 | + 'Backup STATUS should be "OK"') |
| 1655 | + |
| 1656 | + self.assertEqual( |
| 1657 | + 'ERROR', |
| 1658 | + self.show_pb(backup_dir, 'node')[2]['status'], |
| 1659 | + 'Backup STATUS should be "ERROR"') |
| 1660 | + |
| 1661 | + self.assertEqual( |
| 1662 | + 'OK', |
| 1663 | + self.show_pb(backup_dir, 'node')[3]['status'], |
| 1664 | + 'Backup STATUS should be "OK"') |
| 1665 | + |
| 1666 | + self.assertEqual( |
| 1667 | + 'ERROR', |
| 1668 | + self.show_pb(backup_dir, 'node')[4]['status'], |
| 1669 | + 'Backup STATUS should be "ERROR"') |
| 1670 | + |
| 1671 | + self.assertEqual( |
| 1672 | + 'OK', |
| 1673 | + self.show_pb(backup_dir, 'node')[5]['status'], |
| 1674 | + 'Backup STATUS should be "OK"') |
| 1675 | + |
| 1676 | + self.assertEqual( |
| 1677 | + 'ERROR', |
| 1678 | + self.show_pb(backup_dir, 'node')[6]['status'], |
| 1679 | + 'Backup STATUS should be "ERROR"') |
| 1680 | + |
| 1681 | + self.assertEqual( |
| 1682 | + 'OK', |
| 1683 | + self.show_pb(backup_dir, 'node')[7]['status'], |
| 1684 | + 'Backup STATUS should be "OK"') |
| 1685 | + |
| 1686 | + # corruption victim |
| 1687 | + self.assertEqual( |
| 1688 | + 'CORRUPT', |
| 1689 | + self.show_pb(backup_dir, 'node')[8]['status'], |
| 1690 | + 'Backup STATUS should be "CORRUPT"') |
| 1691 | + |
| 1692 | + # orphaned child |
| 1693 | + self.assertEqual( |
| 1694 | + 'ORPHAN', |
| 1695 | + self.show_pb(backup_dir, 'node')[9]['status'], |
| 1696 | + 'Backup STATUS should be "ORPHAN"') |
| 1697 | + |
| 1698 | + # orphaned child |
| 1699 | + self.assertEqual( |
| 1700 | + 'ORPHAN', |
| 1701 | + self.show_pb(backup_dir, 'node')[10]['status'], |
| 1702 | + 'Backup STATUS should be "ORPHAN"') |
| 1703 | + |
| 1704 | + # next FULL |
| 1705 | + self.assertEqual( |
| 1706 | + 'OK', |
| 1707 | + self.show_pb(backup_dir, 'node')[11]['status'], |
| 1708 | + 'Backup STATUS should be "OK"') |
| 1709 | + |
| 1710 | + # next DELTA |
| 1711 | + self.assertEqual( |
| 1712 | + 'OK', |
| 1713 | + self.show_pb(backup_dir, 'node')[12]['status'], |
| 1714 | + 'Backup STATUS should be "OK"') |
| 1715 | + |
| 1716 | + node.cleanup() |
| 1717 | + |
| 1718 | + # Clean after yourself |
| 1719 | + self.del_test_dir(module_name, fname) |
0 commit comments