Merge branch 'release_2_5' into release_2_5-issue313

Author: gsmolk

src/utils/file.c

@@ -777,6 +777,34 @@ fio_fwrite(FILE* f, void const* buf, size_t size)
 		return fwrite(buf, 1, size, f);
 }
 
+/*
+ * Write buffer to descriptor by calling write(),
+ * If size of written data is less than buffer size,
+ * then try to write what is left.
+ * We do this to get honest errno if there are some problems
+ * with filesystem, since writing less than buffer size
+ * is not considered an error.
+ */
+static ssize_t
+durable_write(int fd, const char* buf, size_t size)
+{
+	off_t current_pos = 0;
+	size_t bytes_left = size;
+
+	while (bytes_left > 0)
+	{
+		int rc = write(fd, buf + current_pos, bytes_left);
+
+		if (rc <= 0)
+			return rc;
+
+		bytes_left -= rc;
+		current_pos += rc;
+	}
+
+	return size;
+}
+
 /* Write data to the file synchronously */
 ssize_t
 fio_write(int fd, void const* buf, size_t size)
@@ -806,7 +834,7 @@ fio_write(int fd, void const* buf, size_t size)
 	}
 	else
 	{
-		return write(fd, buf, size);
+		return durable_write(fd, buf, size);
 	}
 }
 
@@ -816,7 +844,7 @@ fio_write_impl(int fd, void const* buf, size_t size, int out)
 	int rc;
 	fio_header hdr;
 
-	rc = write(fd, buf, size);
+	rc = durable_write(fd, buf, size);
 
 	hdr.arg = 0;
 	hdr.size = 0;
@@ -838,34 +866,6 @@ fio_fwrite_async(FILE* f, void const* buf, size_t size)
 		: fwrite(buf, 1, size, f);
 }
 
-/*
- * Write buffer to descriptor by calling write(),
- * If size of written data is less than buffer size,
- * then try to write what is left.
- * We do this to get honest errno if there are some problems
- * with filesystem, since writing less than buffer size
- * is not considered an error.
- */
-static ssize_t
-durable_write(int fd, const char* buf, size_t size)
-{
-	off_t current_pos = 0;
-	size_t bytes_left = size;
-
-	while (bytes_left > 0)
-	{
-		int rc = write(fd, buf + current_pos, bytes_left);
-
-		if (rc <= 0)
-			return rc;
-
-		bytes_left -= rc;
-		current_pos += rc;
-	}
-
-	return size;
-}
-
 /* Write data to the file */
 /* TODO: support async report error */
 ssize_t
@@ -950,23 +950,22 @@ fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_
 	}
 	else
 	{
-		char uncompressed_buf[BLCKSZ];
 		char *errormsg = NULL;
-		int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg, &errormsg);
+		char decompressed_buf[BLCKSZ];
+		int32 decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &errormsg);
 
-		if (uncompressed_size < 0)
+		if (decompressed_size < 0)
 			elog(ERROR, "%s", errormsg);
 
-		return fwrite(uncompressed_buf, 1, uncompressed_size, f);
+		return fwrite(decompressed_buf, 1, decompressed_size, f);
 	}
 }
 
 static void
 fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg)
 {
-	int rc;
-	int32 uncompressed_size;
-	char uncompressed_buf[BLCKSZ];
+	int32 decompressed_size;
+	char decompressed_buf[BLCKSZ];
 
 	/* If the previous command already have failed,
 	 * then there is no point in bashing a head against the wall
@@ -975,14 +974,12 @@ fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg
 		return;
 
 	/* decompress chunk */
-	uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg, &async_errormsg);
+	decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &async_errormsg);
 
-	if (uncompressed_size < 0)
+	if (decompressed_size < 0)
 		return;
 
-	rc = write(fd, uncompressed_buf, uncompressed_size);
-
-	if (rc <= 0)
+	if (durable_write(fd, decompressed_buf, decompressed_size) <= 0)
 	{
 		async_errormsg = pgut_malloc(ERRMSG_MAX_LEN);
 		snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno));

tests/helpers/ptrack_helpers.py

@@ -2003,6 +2003,9 @@ def stopped_in_breakpoint(self):
                 return True
         return False
 
+    def quit(self):
+        self.proc.terminate()
+
     # use for breakpoint, run, continue
     def _execute(self, cmd, running=True):
         output = []

tests/restore.py

@@ -9,7 +9,8 @@
 import shutil
 import json
 from shutil import copyfile
-from testgres import QueryException
+from testgres import QueryException, StartNodeException
+from stat import S_ISDIR
 
 
 module_name = 'restore'
@@ -3856,3 +3857,66 @@ def test_concurrent_restore(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_restore_issue_313(self):
+        """
+        Check that partially restored PostgreSQL instance cannot be started
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'])
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        # FULL backup
+        backup_id = self.backup_node(backup_dir, 'node', node)
+        node.cleanup()
+
+        count = 0
+        filelist =  self.get_backup_filelist(backup_dir, 'node', backup_id)
+        for file in filelist:
+            # count only nondata files
+            if int(filelist[file]['is_datafile']) == 0 and int(filelist[file]['size']) > 0:
+                count += 1
+
+        node_restored = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node_restored'))
+        node_restored.cleanup()
+        self.restore_node(backup_dir, 'node', node_restored)
+
+        gdb = self.restore_node(backup_dir, 'node', node, gdb=True, options=['--progress'])
+        gdb.verbose = False
+        gdb.set_breakpoint('restore_non_data_file')
+        gdb.run_until_break()
+        gdb.continue_execution_until_break(count - 2)
+        gdb.quit()
+
+        # emulate the user or HA taking care of PG configuration
+        for fname in os.listdir(node_restored.data_dir):
+            if fname.endswith('.conf'):
+                os.rename(
+                    os.path.join(node_restored.data_dir, fname),
+                    os.path.join(node.data_dir, fname))
+
+        try:
+            node.slow_start()
+            # we should die here because exception is what we expect to happen
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because backup is not fully restored")
+        except StartNodeException as e:
+            self.assertIn(
+                'Cannot start node',
+                e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)