Skip to content

Commit 5089400

Browse files
kulaginmkulaginm
committed
fix datafile truncation bug
1 parent ce34427 commit 5089400

File tree

7 files changed

+48
-18
lines changed

7 files changed

+48
-18
lines changed

‎src/backup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2392,4 +2392,4 @@ calculate_datasize_of_filelist(parray *filelist)
23922392
bytes += file->size;
23932393
}
23942394
return bytes;
2395-
}
2395+
}

‎src/catchup.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
213213

214214
join_path_components(backup_label_filename, dest_pgdata, "backup_label");
215215
if (fio_access(backup_label_filename, F_OK, FIO_LOCAL_HOST) == 0)
216-
elog(ERROR, "Destination directory contains \"backup_control\" file");
216+
elog(ERROR, "Destination directory contains \"backup_label\" file");
217217
}
218218

219219
if (current.from_replica && exclusive_backup)
@@ -922,7 +922,8 @@ catchup_thread_runner(void *arg)
922922
arguments->nodeInfo->checksum_version,
923923
arguments->nodeInfo->ptrack_version_num,
924924
arguments->nodeInfo->ptrack_schema,
925-
false);
925+
false,
926+
dest_file != NULL ? dest_file->size : 0);
926927
}
927928
else
928929
{

‎src/data.c

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -662,7 +662,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa
662662
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
663663
CompressAlg calg, int clevel, uint32 checksum_version,
664664
int ptrack_version_num, const char *ptrack_schema,
665-
bool is_merge)
665+
bool is_merge, size_t prev_size)
666666
{
667667
int rc;
668668
bool use_pagemap;
@@ -689,7 +689,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa
689689
*/
690690
if (backup_mode == BACKUP_MODE_DIFF_PTRACK &&
691691
file->pagemap.bitmapsize == PageBitmapIsEmpty &&
692-
file->exists_in_prev && !file->pagemap_isabsent)
692+
file->exists_in_prev && file->size == prev_size && !file->pagemap_isabsent)
693693
{
694694
/*
695695
* There are no changed blocks since last backup. We want to make
@@ -793,7 +793,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa
793793
backup_mode == BACKUP_MODE_DIFF_DELTA)
794794
file->n_blocks = file->read_size / BLCKSZ;
795795

796-
/* Determine that file didn`t changed in case of incremental backup */
796+
/* Determine that file didn`t changed in case of incremental catchup */
797797
if (backup_mode != BACKUP_MODE_FULL &&
798798
file->exists_in_prev &&
799799
file->write_size == 0 &&
@@ -2318,6 +2318,12 @@ copy_pages(const char *to_fullpath, const char *from_fullpath,
23182318
elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath,
23192319
strerror(errno));
23202320

2321+
elog(VERBOSE, "ftruncate file \"%s\" to size %lu",
2322+
to_fullpath, file->size);
2323+
if (fio_ftruncate(out, file->size) == -1)
2324+
elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s",
2325+
to_fullpath, file->size, strerror(errno));
2326+
23212327
if (!fio_is_remote_file(out))
23222328
{
23232329
out_buf = pgut_malloc(STDIO_BUFSIZE);

‎src/pg_probackup.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1104,7 +1104,7 @@ extern void catchup_data_file(pgFile *file, const char *from_fullpath, const cha
11041104
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
11051105
CompressAlg calg, int clevel, uint32 checksum_version,
11061106
int ptrack_version_num, const char *ptrack_schema,
1107-
bool missing_ok);
1107+
bool is_merge, size_t prev_size);
11081108
extern void backup_non_data_file(pgFile *file, pgFile *prev_file,
11091109
const char *from_fullpath, const char *to_fullpath,
11101110
BackupMode backup_mode, time_t parent_backup_time,

‎src/utils/file.c

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1939,11 +1939,27 @@ fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file,
19391939
if (use_pagemap)
19401940
IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize);
19411941

1942-
//out = open_local_file_rw_append(to_fullpath, &out_buf, STDIO_BUFSIZE);
19431942
out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST);
19441943
if (out == NULL)
19451944
elog(ERROR, "Cannot open restore target file \"%s\": %s", to_fullpath, strerror(errno));
19461945

1946+
/* update file permission */
1947+
if (fio_chmod(to_fullpath, file->mode, FIO_BACKUP_HOST) == -1)
1948+
elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath,
1949+
strerror(errno));
1950+
1951+
elog(VERBOSE, "ftruncate file \"%s\" to size %lu",
1952+
to_fullpath, file->size);
1953+
if (fio_ftruncate(out, file->size) == -1)
1954+
elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s",
1955+
to_fullpath, file->size, strerror(errno));
1956+
1957+
if (!fio_is_remote_file(out))
1958+
{
1959+
out_buf = pgut_malloc(STDIO_BUFSIZE);
1960+
setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE);
1961+
}
1962+
19471963
while (true)
19481964
{
19491965
fio_header hdr;

‎tests/catchup.py

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import io
12
import os
23
import unittest
34
from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
@@ -24,7 +25,7 @@ def test_multithread_local_transfer(self):
2425
result = source_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
2526

2627
dest_pg = self.make_empty_node(os.path.join(module_name, fname, 'dst'))
27-
dest_pg = self.catchup_node(
28+
self.catchup_node(
2829
backup_mode = 'FULL',
2930
source_pgdata = source_pg.data_dir,
3031
destination_node = dest_pg,
@@ -66,7 +67,7 @@ def test_local_simple_transfer_with_tablespace(self):
6667

6768
dest_pg = self.make_empty_node(os.path.join(module_name, fname, 'dst'))
6869
tblspace1_new_path = self.get_tblspace_path(dest_pg, 'tblspace1_new')
69-
dest_pg = self.catchup_node(
70+
self.catchup_node(
7071
backup_mode = 'FULL',
7172
source_pgdata = source_pg.data_dir,
7273
destination_node = dest_pg,
@@ -113,7 +114,7 @@ def test_multithread_remote_transfer(self):
113114
result = source_pg.safe_psql("postgres", "SELECT * FROM ultimate_question")
114115

115116
dest_pg = self.make_empty_node(os.path.join(module_name, fname, 'dst'))
116-
dest_pg = self.catchup_node(
117+
self.catchup_node(
117118
backup_mode = 'FULL',
118119
source_pgdata = source_pg.data_dir,
119120
destination_node = dest_pg,
@@ -160,7 +161,7 @@ def test_remote_ptrack_catchup(self):
160161

161162
# make clean shutdowned lagging behind replica
162163
dest_pg = self.make_empty_node(os.path.join(module_name, fname, 'dst'))
163-
dest_pg = self.catchup_node(
164+
self.catchup_node(
164165
backup_mode = 'FULL',
165166
source_pgdata = source_pg.data_dir,
166167
destination_node = dest_pg,
@@ -224,7 +225,7 @@ def test_remote_delta_catchup(self):
224225

225226
# make clean shutdowned lagging behind replica
226227
dest_pg = self.make_empty_node(os.path.join(module_name, fname, 'dst'))
227-
dest_pg = self.catchup_node(
228+
self.catchup_node(
228229
backup_mode = 'FULL',
229230
source_pgdata = source_pg.data_dir,
230231
destination_node = dest_pg,
@@ -292,7 +293,7 @@ def test_table_drop(self):
292293
"CREATE TABLE ultimate_question AS SELECT 42 AS answer")
293294

294295
dest_pg = self.make_empty_node(os.path.join(module_name, fname, 'dst'))
295-
dest_pg = self.catchup_node(
296+
self.catchup_node(
296297
backup_mode = 'FULL',
297298
source_pgdata = source_pg.data_dir,
298299
destination_node = dest_pg,
@@ -354,7 +355,7 @@ def test_tablefile_truncation(self):
354355
source_pg.safe_psql("postgres", "VACUUM t_heap")
355356

356357
dest_pg = self.make_empty_node(os.path.join(module_name, fname, 'dst'))
357-
dest_pg = self.catchup_node(
358+
self.catchup_node(
358359
backup_mode = 'FULL',
359360
source_pgdata = source_pg.data_dir,
360361
destination_node = dest_pg,
@@ -380,6 +381,8 @@ def test_tablefile_truncation(self):
380381
source_pgdata = source_pg.data_dir,
381382
destination_node = dest_pg,
382383
options = ['-d', 'postgres', '-p', str(source_pg.port), '--stream'])
384+
with io.open(os.path.join(dest_pg.logs_dir, 'catchup.log'), 'a') as catchup_log:
385+
catchup_log.write(self.output)
383386

384387
source_pgdata = self.pgdata_content(source_pg.data_dir)
385388
dest_pgdata = self.pgdata_content(dest_pg.data_dir)
@@ -391,6 +394,9 @@ def test_tablefile_truncation(self):
391394

392395
# @unittest.skip("skip")
393396
def test_local_tablespace_without_mapping(self):
397+
if self.remote:
398+
return unittest.skip('Skipped because this test tests local catchup error handling')
399+
394400
fname = self.id().split('.')[3]
395401

396402
source_pg = self.make_simple_node(
@@ -409,7 +415,7 @@ def test_local_tablespace_without_mapping(self):
409415

410416
dest_pg = self.make_empty_node(os.path.join(module_name, fname, 'dst'))
411417
try:
412-
dest_pg = self.catchup_node(
418+
self.catchup_node(
413419
backup_mode = 'FULL',
414420
source_pgdata = source_pg.data_dir,
415421
destination_node = dest_pg,

‎tests/helpers/ptrack_helpers.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1052,9 +1052,10 @@ def catchup_node(
10521052
]
10531053
if self.remote:
10541054
cmd_list += ['--remote-proto=ssh', '--remote-host=localhost']
1055+
if self.verbose:
1056+
cmd_list += ['--log-level-console=verbose']
10551057

1056-
self.run_pb(cmd_list + options)
1057-
return destination_node
1058+
return self.run_pb(cmd_list + options)
10581059

10591060
def show_pb(
10601061
self, backup_dir, instance=None, backup_id=None,

0 commit comments

Comments
 (0)