From 48821b82fbc55e26f2f2e01f58f51386689e8788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Dec 2018 13:43:30 +0200 Subject: [PATCH] MDEV-18907 Indicate which InnoDB data files are zero-initialized FIXME: The code in fil_space_for_table_exists_in_mem() would cause trouble in DROP TABLE after downgrade. It would actually crash the server unless the fix of commit b74eb5a5feb41164f7e5cab986cf203537c2128a is present. The downgrade problem was introduced in commit ab1e6fefd869242d962cb91a006f37bb9ad534a7 which effectively insists that no 'extra' flags are set. Downgrade to MariaDB 10.0 could be possible. FIXME: Failing tests: mariabackup.partition_datadir mariabackup.apply-log-only-incr main.tc_heuristic_recover encryption.innodb-checksum-algorithm mariabackup.data_directory encryption.innodb-discard-import encryption.innodb_encryption_discard_import innodb.pt encryption.innodb-first-page-read innodb.read_only_recover_committed innodb.read_only_recovery mariabackup.incremental_backup innodb.recovery_shutdown innodb.restart innodb.innodb_bug59641 innodb.xa_recovery --- storage/innobase/fil/fil0fil.cc | 17 ++++++++++++----- storage/innobase/include/fsp0fsp.h | 11 +++++++---- storage/innobase/srv/srv0start.cc | 5 ++++- storage/xtradb/fil/fil0fil.cc | 17 ++++++++++++----- storage/xtradb/include/fsp0fsp.h | 11 +++++++---- storage/xtradb/srv/srv0start.cc | 5 ++++- 6 files changed, 46 insertions(+), 20 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 5891db62758..7bb037be60b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -3573,7 +3573,7 @@ fil_create_new_single_table_tablespace( memset(page, '\0', UNIV_PAGE_SIZE); - flags |= FSP_FLAGS_PAGE_SSIZE(); + flags |= FSP_FLAGS_INIT_UNUSED | FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); @@ -3733,14 +3733,20 @@ fsp_flags_try_adjust(ulint space_id, ulint flags) space_id, fsp_flags_get_zip_size(flags), 0, RW_X_LATCH, &mtr)) { ulint f = fsp_header_get_flags(b->frame); - /* Suppress the message if only the DATA_DIR flag to differs. */ - if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + /* Do not clear the INIT_UNUSED flag + (previously known as DATA_DIR, introduced in MySQL 5.6). */ + if ((f ^ flags) & ~FSP_FLAGS_INIT_UNUSED) { + /* We only get here if the file was created + between MariaDB 10.1.0 and 10.1.20. + Already starting with MariaDB 5.5, we know that + all unused data fields were zero-initialized, so + it is safe to set the INIT_UNUSED flag for + files that were created by 10.1. */ + flags |= FSP_FLAGS_INIT_UNUSED; ib_logf(IB_LOG_LEVEL_WARN, "adjusting FSP_SPACE_FLAGS of tablespace " ULINTPF " from 0x%x to 0x%x", space_id, int(f), int(flags)); - } - if (f != flags) { mlog_write_ulint(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + b->frame, flags, MLOG_4BYTES, &mtr); @@ -5080,6 +5086,7 @@ fil_space_for_table_exists_in_mem( fnamespace = fil_space_get_by_name(name); bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_INIT_UNUSED & ~FSP_FLAGS_MEM_MASK); if (!space) { diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index c11c48d58cf..a9a11e7552a 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -71,7 +71,7 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ /* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 or newer. -MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21+ ==================================================================== Below flags in same offset ==================================================================== @@ -83,7 +83,7 @@ Below flags in same offset Below note the order difference: ===================================================================== 6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) -10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: INIT_UNUSED (5.6 DATA_DIR) ===================================================================== The flags below were in incorrect position in MariaDB 10.1, or have been introduced in MySQL 5.7 or 8.0: @@ -149,6 +149,8 @@ these are only used in MySQL 5.7 and used for compatibility. */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) +/** Unused data fields inside the file are zero-initialized (not garbage) */ +#define FSP_FLAGS_INIT_UNUSED (1U << FSP_FLAGS_POS_RESERVED) /** Bit mask of the RESERVED1 field */ #define FSP_FLAGS_MASK_RESERVED \ ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ @@ -812,7 +814,8 @@ fsp_flags_is_valid(ulint flags, bool is_ibd) return(false); } /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag - of MySQL 5.6 and MariaDB 10.0, which we ignore. + of MySQL 5.6 and MariaDB 10.0, which we ignore + and repurpose as INIT_UNUSED. In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, bits 10..14 would be nonzero 0bsssaa where sss is nonzero PAGE_SSIZE (3, 4, 6, or 7) diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index db52e31881c..383246a993a 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1191,9 +1191,12 @@ open_or_create_data_files( if (i == 0) { flags = FSP_FLAGS_PAGE_SSIZE(); + if (*create_new_db) { + flags |= FSP_FLAGS_INIT_UNUSED; + } fil_space_create(name, 0, flags, FIL_TABLESPACE, - crypt_data, (*create_new_db) == true); + crypt_data, *create_new_db); } ut_a(fil_validate()); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index a8835083165..0b74a808d84 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -3629,7 +3629,7 @@ fil_create_new_single_table_tablespace( memset(page, '\0', UNIV_PAGE_SIZE); - flags |= FSP_FLAGS_PAGE_SSIZE(); + flags |= FSP_FLAGS_INIT_UNUSED | FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); @@ -3922,14 +3922,20 @@ fsp_flags_try_adjust(ulint space_id, ulint flags) space_id, fsp_flags_get_zip_size(flags), 0, RW_X_LATCH, &mtr)) { ulint f = fsp_header_get_flags(b->frame); - /* Suppress the message if only the DATA_DIR flag to differs. */ - if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + /* Do not clear the INIT_UNUSED flag + (previously known as DATA_DIR, introduced in MySQL 5.6). */ + if ((f ^ flags) & ~FSP_FLAGS_INIT_UNUSED) { + /* We only get here if the file was created + between MariaDB 10.1.0 and 10.1.20. + Already starting with MariaDB 5.5, we know that + all unused data fields were zero-initialized, so + it is safe to set the INIT_UNUSED flag for + files that were created by 10.1. */ + flags |= FSP_FLAGS_INIT_UNUSED; ib_logf(IB_LOG_LEVEL_WARN, "adjusting FSP_SPACE_FLAGS of tablespace " ULINTPF " from 0x%x to 0x%x", space_id, int(f), int(flags)); - } - if (f != flags) { mlog_write_ulint(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + b->frame, flags, MLOG_4BYTES, &mtr); @@ -5379,6 +5385,7 @@ fil_space_for_table_exists_in_mem( fnamespace = fil_space_get_by_name(name); bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_INIT_UNUSED & ~FSP_FLAGS_MEM_MASK); if (!space) { diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index bc81a9c126b..35da210f92e 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -70,7 +70,7 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ /* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 or newer. -MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21+ ==================================================================== Below flags in same offset ==================================================================== @@ -82,7 +82,7 @@ Below flags in same offset Below note the order difference: ===================================================================== 6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) -10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: INIT_UNUSED (5.6 DATA_DIR) ===================================================================== The flags below were in incorrect position in MariaDB 10.1, or have been introduced in MySQL 5.7 or 8.0: @@ -148,6 +148,8 @@ these are only used in MySQL 5.7 and used for compatibility. */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) +/** Unused data fields inside the file are zero-initialized (not garbage) */ +#define FSP_FLAGS_INIT_UNUSED (1U << FSP_FLAGS_POS_RESERVED) /** Bit mask of the RESERVED1 field */ #define FSP_FLAGS_MASK_RESERVED \ ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ @@ -811,7 +813,8 @@ fsp_flags_is_valid(ulint flags, bool is_ibd) return(false); } /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag - of MySQL 5.6 and MariaDB 10.0, which we ignore. + of MySQL 5.6 and MariaDB 10.0, which we ignore + and repurpose as INIT_UNUSED. In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, bits 10..14 would be nonzero 0bsssaa where sss is nonzero PAGE_SSIZE (3, 4, 6, or 7) diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 75b664a0558..91300239315 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -1228,9 +1228,12 @@ open_or_create_data_files( if (i == 0) { flags = FSP_FLAGS_PAGE_SSIZE(); + if (*create_new_db) { + flags |= FSP_FLAGS_INIT_UNUSED; + } fil_space_create(name, 0, flags, FIL_TABLESPACE, - crypt_data, (*create_new_db) == true); + crypt_data, *create_new_db); } ut_a(fil_validate()); -- 2.20.1