MDEV-5878: Failing assertion: mutex_own(mutex) with innodb_use_fallocate=ON.

Analysis: This was merge error on file fil0fil.cc. fil_system mutex was taken twice because of this.

Fix: Remove unnecessary mutex_enter and fixed the issue with slow posix_fallocate usage.

storage/innobase/fil/fil0fil.cc

@@ -4926,10 +4926,20 @@ retry:
 
 #ifdef HAVE_POSIX_FALLOCATE
 	if (srv_use_posix_fallocate) {
-		ulint n_pages = size_after_extend - start_page_no;
+		os_offset_t	start_offset = start_page_no * page_size;
+		os_offset_t	n_pages = (size_after_extend - start_page_no);
+		os_offset_t	len = n_pages * page_size;
 
-		success = os_file_set_size(node->name, node->handle,
+		if (posix_fallocate(node->handle, start_offset, len) == -1) {
-			n_pages * page_size);
+			ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file "
+				"space for file \'%s\' failed.  Current size "
+				INT64PF ", desired size " INT64PF "\n",
+				node->name, start_offset, len+start_offset);
+			os_file_handle_error_no_exit(node->name, "posix_fallocate", FALSE);
+			success = FALSE;
+		} else {
+			success = TRUE;
+		}
 
 		mutex_enter(&fil_system->mutex);
 		if (success) {
@@ -4937,7 +4947,14 @@ retry:
 			space->size += n_pages;
 			os_has_said_disk_full = FALSE;
 		}
-		goto complete_io;
+
+		/* If posix_fallocate was used to extent the file space
+		we need to complete the io. Because no actual writes were
+		dispatched read operation is enough here. Without this
+		there will be assertion at shutdown indicating that
+		all IO is not completed. */
+		fil_node_complete_io(node, fil_system, OS_FILE_READ);
+		goto file_extended;
 	}
 #endif
 
@@ -4995,12 +5012,10 @@ retry:
 	space->size += pages_added;
 	node->size += pages_added;
 
-#ifdef HAVE_POSIX_FALLOCATE
-complete_io:
-	fil_node_complete_io(node, fil_system, OS_FILE_READ);
-#else
 	fil_node_complete_io(node, fil_system, OS_FILE_WRITE);
-#endif
+
+	/* At this point file has been extended */
+file_extended:
 
 	node->being_extended = FALSE;
 	*actual_size = space->size;

storage/innobase/include/os0file.h

@@ -1277,6 +1277,17 @@ os_aio_linux_handle(
 	ulint*	type);		/*!< out: OS_FILE_WRITE or ..._READ */
 #endif /* LINUX_NATIVE_AIO */
 
+/****************************************************************//**
+Does error handling when a file operation fails.
+@return	TRUE if we should retry the operation */
+ibool
+os_file_handle_error_no_exit(
+/*=========================*/
+	const char*	name,		/*!< in: name of a file or NULL */
+	const char*	operation,	/*!< in: operation */
+	ibool		on_error_silent);/*!< in: if TRUE then don't print
+					any message to the log. */
+
 #ifndef UNIV_NONINL
 #include "os0file.ic"
 #endif

storage/innobase/os/os0file.cc

@@ -671,7 +671,6 @@ os_file_handle_error(
 /****************************************************************//**
 Does error handling when a file operation fails.
 @return	TRUE if we should retry the operation */
-static
 ibool
 os_file_handle_error_no_exit(
 /*=========================*/

storage/xtradb/fil/fil0fil.cc

@@ -4937,20 +4937,35 @@ retry:
 
 #ifdef HAVE_POSIX_FALLOCATE
 	if (srv_use_posix_fallocate) {
+		os_offset_t	start_offset = start_page_no * page_size;
+		os_offset_t	n_pages = (size_after_extend - start_page_no);
+		os_offset_t	len = n_pages * page_size;
+
+		if (posix_fallocate(node->handle, start_offset, len) == -1) {
+			ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file "
+				"space for file \'%s\' failed.  Current size "
+				INT64PF ", desired size " INT64PF "\n",
+				node->name, start_offset, len+start_offset);
+			os_file_handle_error_no_exit(node->name, "posix_fallocate", FALSE);
+			success = FALSE;
+		} else {
+			success = TRUE;
+		}
 
-		mutex_exit(&fil_system->mutex);
-		success = os_file_set_size(node->name, node->handle,
-					   (size_after_extend
-					    - file_start_page_no) * page_size);
 		mutex_enter(&fil_system->mutex);
 		if (success) {
-			node->size += (size_after_extend - start_page_no);
+			node->size += n_pages;
-			space->size += (size_after_extend - start_page_no);
+			space->size += n_pages;
 			os_has_said_disk_full = FALSE;
 		}
-		node->being_extended = FALSE;
+
+		/* If posix_fallocate was used to extent the file space
+		we need to complete the io. Because no actual writes were
+		dispatched read operation is enough here. Without this
+		there will be assertion at shutdown indicating that
+		all IO is not completed. */
 		fil_node_complete_io(node, fil_system, OS_FILE_READ);
-		goto complete_io;
+		goto file_extended;
 	}
 #endif
 
@@ -5007,24 +5022,13 @@ retry:
 
 	space->size += pages_added;
 	node->size += pages_added;
+
+	fil_node_complete_io(node, fil_system, OS_FILE_WRITE);
+
+	/* At this point file has been extended */
+file_extended:
+
 	node->being_extended = FALSE;
-
-#ifdef HAVE_POSIX_FALLOCATE
-complete_io:
-	/* If posix_fallocate was used to extent the file space
-	we need to complete the io. Because no actual writes were
-	dispatched read operation is enough here. Without this
-	there will be assertion at shutdown indicating that
-	all IO is not completed. */
-	if (srv_use_posix_fallocate) {
-		fil_node_complete_io(node, fil_system, OS_FILE_READ);
-	} else {
-		fil_node_complete_io(node, fil_system, OS_FILE_WRITE);
-	}
-#else
-	fil_node_complete_io(node, fil_system, OS_FILE_WRITE);
-#endif
-
 	*actual_size = space->size;
 
 #ifndef UNIV_HOTBACKUP

storage/xtradb/include/os0file.h

@@ -1312,6 +1312,18 @@ os_aio_linux_handle(
 	ulint*	space_id);
 #endif /* LINUX_NATIVE_AIO */
 
+/****************************************************************//**
+Does error handling when a file operation fails.
+@return	TRUE if we should retry the operation */
+ibool
+os_file_handle_error_no_exit(
+/*=========================*/
+	const char*	name,		/*!< in: name of a file or NULL */
+	const char*	operation,	/*!< in: operation */
+	ibool		on_error_silent);/*!< in: if TRUE then don't print
+					any message to the log. */
+
+
 #ifndef UNIV_NONINL
 #include "os0file.ic"
 #endif

storage/xtradb/os/os0file.cc

@@ -769,7 +769,6 @@ os_file_handle_error(
 /****************************************************************//**
 Does error handling when a file operation fails.
 @return	TRUE if we should retry the operation */
-static
 ibool
 os_file_handle_error_no_exit(
 /*=========================*/