[Commits] 9f351ce: Add support for backing up MariaDB compressed and encrypted

Jan Lindström jan.lindstrom at mariadb.com
Tue May 3 13:37:28 EEST 2016


revision-id: 9f351ce8061f768e2bb92e0ea47df9cd7c54a33e (mysql-5.6.11-309-g9f351ce)
parent(s): 8443494c50efa867cf119a231e6b1b0cb660a92b
committer: Jan Lindström
timestamp: 2016-05-03 13:36:26 +0300
message:

Add support for backing up MariaDB compressed and encrypted
tablespaces and tables. Furhermore, add support for backing
up MariaDB encrypted redo-logs.

---
 storage/innobase/include/fsp0fsp.h             |  31 ++++
 storage/innobase/include/fsp0fsp.ic            |  11 ++
 storage/innobase/xtrabackup/src/CMakeLists.txt |   1 +
 storage/innobase/xtrabackup/src/fil_cur.cc     |  13 +-
 storage/innobase/xtrabackup/src/fil_cur.h      |   1 +
 storage/innobase/xtrabackup/src/mariadb.cc     | 210 +++++++++++++++++++++++++
 storage/innobase/xtrabackup/src/mariadb.h      |  31 ++++
 storage/innobase/xtrabackup/src/xtrabackup.cc  |  15 +-
 8 files changed, 310 insertions(+), 3 deletions(-)

diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h
index a587ccc..e9d07e6 100644
--- a/storage/innobase/include/fsp0fsp.h
+++ b/storage/innobase/include/fsp0fsp.h
@@ -81,6 +81,37 @@ is found in a remote location, not the default data directory. */
 #define FSP_FLAGS_POS_UNUSED		(FSP_FLAGS_POS_DATA_DIR	\
 					+ FSP_FLAGS_WIDTH_DATA_DIR)
 
+/** MariaDB extra tablespace flag withds,
+PAGE_COMPRESSION = 1,
+PAGE_COMPRESSION_LEVEL = 4
+ATOMIC_WRITES = 2 */
+#define FSP_EXTRA_POS_MARIADB		(1 + 4 + 2)
+
+/** Zero relative shift position of the start of the UNUSED bits for
+MariaDB */
+#define FSP_FLAGS_POS_UNUSED_MARIADB	(FSP_FLAGS_POS_DATA_DIR	   \
+					+ FSP_FLAGS_WIDTH_DATA_DIR \
+					+ FSP_EXTRA_POS_MARIADB)
+
+/** Zero relative shift position of the PAGE_SSIZE field for MariaDB*/
+#define FSP_FLAGS_POS_PAGE_SSIZE_MARIADB		\
+		(FSP_FLAGS_POS_ATOMIC_BLOBS		\
+		 + FSP_EXTRA_POS_MARIADB)
+
+/** Return the contents of the UNUSED bits for MariaDB */
+#define FSP_FLAGS_GET_UNUSED_MARIADB(flags)		\
+		(flags >> FSP_FLAGS_POS_UNUSED_MARIADB)
+
+/** Bit mask of the PAGE_SSIZE field for MariaDB */
+#define FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB		        \
+		((~(~0 << FSP_FLAGS_WIDTH_PAGE_SSIZE))		\
+		<< FSP_FLAGS_POS_PAGE_SSIZE_MARIADB)
+
+/** Return the contents of the SSIZE bits for MariaDB */
+#define FSP_FLAGS_GET_PAGE_SSIZE_MARIADB(flags)			\
+		((flags & FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB)		\
+		>> FSP_FLAGS_POS_PAGE_SSIZE_MARIADB)
+
 /** Bit mask of the POST_ANTELOPE field */
 #define FSP_FLAGS_MASK_POST_ANTELOPE				\
 		((~(~0 << FSP_FLAGS_WIDTH_POST_ANTELOPE))	\
diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic
index 0d81e81..81a7e63 100644
--- a/storage/innobase/include/fsp0fsp.ic
+++ b/storage/innobase/include/fsp0fsp.ic
@@ -63,9 +63,20 @@ fsp_flags_is_valid(
 	ulint	atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(flags);
 	ulint	page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags);
 	ulint	unused = FSP_FLAGS_GET_UNUSED(flags);
+	ulint	mariadb_unused =  FSP_FLAGS_GET_UNUSED_MARIADB(flags);
+	ulint	mariadb_page_ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB(flags);
 
 	DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false););
 
+	/* If there is flags that are MariaDB extended flags,
+	read unused and page size from MariaDB positions. */
+	if ((unused && mariadb_unused == 0) ||
+	    (page_ssize && (page_ssize < UNIV_PAGE_SIZE_MIN ||
+	     page_ssize > UNIV_PAGE_SIZE_MAX))) {
+		unused = mariadb_unused;
+		page_ssize = mariadb_page_ssize;
+	}
+
 	/* fsp_flags is zero unless atomic_blobs is set. */
 	/* Make sure there are no bits that we do not know about. */
 	if (unused != 0 || flags == 1) {
diff --git a/storage/innobase/xtrabackup/src/CMakeLists.txt b/storage/innobase/xtrabackup/src/CMakeLists.txt
index f6f55f5..c2dbb62 100644
--- a/storage/innobase/xtrabackup/src/CMakeLists.txt
+++ b/storage/innobase/xtrabackup/src/CMakeLists.txt
@@ -83,6 +83,7 @@ MYSQL_ADD_EXECUTABLE(xtrabackup
   xbstream_write.c
   backup_mysql.cc
   backup_copy.cc
+  mariadb.cc
   )
 
 SET_TARGET_PROPERTIES(xtrabackup PROPERTIES ENABLE_EXPORTS TRUE)
diff --git a/storage/innobase/xtrabackup/src/fil_cur.cc b/storage/innobase/xtrabackup/src/fil_cur.cc
index 50d6c0c..b825510 100644
--- a/storage/innobase/xtrabackup/src/fil_cur.cc
+++ b/storage/innobase/xtrabackup/src/fil_cur.cc
@@ -33,6 +33,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 #include "common.h"
 #include "read_filt.h"
 #include "xtrabackup.h"
+#include "mariadb.h"
 
 /* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */
 #define XB_FIL_CUR_PAGES 640
@@ -258,6 +259,8 @@ xb_fil_cur_open(
 	cursor->read_filter = read_filter;
 	cursor->read_filter->init(&cursor->read_filter_ctxt, cursor,
 				  node->space->id);
+	cursor->encrypted = mariadb_check_tablespace_encryption(cursor->file,
+		cursor->abs_path, zip_size);
 
 	return(XB_FIL_CUR_SUCCESS);
 }
@@ -339,8 +342,14 @@ read_retry:
 	partially written pages */
 	for (page = cursor->buf, i = 0; i < npages;
 	     page += cursor->page_size, i++) {
-
-		if (buf_page_is_corrupted(TRUE, page, cursor->zip_size)) {
+		bool mariadb_compressed = mariadb_check_compression(page);
+
+		/* MariaDB tablespaces using compression do not have correct
+		checksum. Similarly, encrypted tablespaces. All other
+		tablespaces should have correct checksums. */
+		if (!cursor->encrypted &&
+		    !mariadb_compressed &&
+		    buf_page_is_corrupted(TRUE, page, cursor->zip_size)) {
 
 			ulint page_no = cursor->buf_page_no + i;
 
diff --git a/storage/innobase/xtrabackup/src/fil_cur.h b/storage/innobase/xtrabackup/src/fil_cur.h
index 31f97cd..4f37cb6 100644
--- a/storage/innobase/xtrabackup/src/fil_cur.h
+++ b/storage/innobase/xtrabackup/src/fil_cur.h
@@ -61,6 +61,7 @@ struct xb_fil_cur_t {
 	uint		thread_n;	/*!< thread number for diagnostics */
 	ulint		space_id;	/*!< ID of tablespace */
 	ulint		space_size;	/*!< space size in pages */
+	bool		encrypted;	/*!< true if tablespace encrypted */
 };
 
 typedef enum {
diff --git a/storage/innobase/xtrabackup/src/mariadb.cc b/storage/innobase/xtrabackup/src/mariadb.cc
new file mode 100644
index 0000000..4b36cdc
--- /dev/null
+++ b/storage/innobase/xtrabackup/src/mariadb.cc
@@ -0,0 +1,210 @@
+/*****************************************************************************
+
+Copyright (C) 2016, MariaDB Corporation. All Rights Reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+*****************************************************************************/
+/**************************************************//**
+ at file mariadb.cc
+Helper functions to check MariaDB 10.1 extended features.
+
+Jan Lindström jan.lindstrom at mariadb.com
+*******************************************************/
+
+#include "univ.i"
+#include "log0log.h"
+#include "common.h"
+#include "fil0fil.h"
+#include "fsp0fsp.h"
+
+static const byte redo_log_purpose_byte = 0x02;
+
+#define MY_AES_BLOCK_SIZE 16
+#define LOG_CRYPT_ENTRY_SIZE            (4 + 4 + 2 * MY_AES_BLOCK_SIZE)
+#define LOG_CRYPT_VER		        308
+
+#define FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED 37401 /*!< Page is compressed and
+						 then encrypted */
+#define FIL_PAGE_PAGE_COMPRESSED 34354  /*!< Page compressed page */
+
+#define FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION 26 /*!< for the first page
+					in a system tablespace data file
+					(ibdata*, not *.ibd): the file has
+					been flushed to disk at least up
+					to this lsn
+					for other pages: a 32-bit key version
+					used to encrypt the page + 32-bit checksum
+					or 64 bits of zero if no encryption
+					*/
+/**
+* Magic pattern in start of crypt data on page 0
+*/
+#define MAGIC_SZ 6
+
+static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = {
+	's', 0xE, 0xC, 'R', 'E', 't' };
+
+static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = {
+	0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
+
+#define CRYPT_SCHEME_1 1
+#define CRYPT_SCHEME_1_IV_LEN 16
+#define CRYPT_SCHEME_UNENCRYPTED 0
+
+extern void xtrabackup_io_throttling(void);
+
+/********************************************************//**
+Read from checkpoint is redo log encrypted.
+ at return	true if log is encrypted, false if not. */
+bool
+mariadb_check_encryption(
+/*=====================*/
+	log_group_t*	group)
+{
+	ib_uint64_t	checkpoint_no;
+	ulint		field;
+	bool		encrypted = false;
+
+	for (field = LOG_CHECKPOINT_1; field <= LOG_CHECKPOINT_2;
+	     field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) {
+
+		xtrabackup_io_throttling();
+
+		mutex_enter(&log_sys->mutex);
+		log_group_read_checkpoint_info(group, field);
+
+		byte* buf = log_sys->checkpoint_buf;
+		checkpoint_no = mach_read_from_8(
+				buf + LOG_CHECKPOINT_NO);
+		buf+=LOG_CRYPT_VER;
+		byte scheme = buf[0];
+
+		if (scheme == redo_log_purpose_byte) {
+			buf++;
+			size_t n = buf[0];
+			buf++;
+			for (size_t i = 0; i < n; i++) {
+				ulint key_version = mach_read_from_4(buf + 4);
+				buf += LOG_CRYPT_ENTRY_SIZE;
+
+				if (key_version != 0) {
+					msg("xtrabackup: Redo log encrypted on checkpoint %lu key_version %lu\n",
+						checkpoint_no, key_version);
+					encrypted = true;
+				}
+			}
+		}
+		mutex_exit(&log_sys->mutex);
+	}
+
+	return(encrypted);
+}
+
+/********************************************************//**
+Check from page type is page compressed.
+ at return	true if page is compressed, false if not. */
+bool
+mariadb_check_compression(
+/*======================*/
+	const byte*	page)
+{
+	ulint page_type = mach_read_from_2(page+FIL_PAGE_TYPE);
+
+	return (page_type == FIL_PAGE_PAGE_COMPRESSED ||
+		page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
+}
+
+/**********************************************************************//**
+Compute offset after xdes where crypt data can be stored
+ at return	offset */
+static
+ulint
+fsp_header_get_crypt_offset(
+/*========================*/
+	ulint   zip_size, /*!< in: zip_size */
+	ulint*  max_size) /*!< out: free space available for crypt data */
+{
+	ulint pageno = 0;
+	/* compute first page_no that will have xdes stored on page != 0*/
+	for (ulint i = 0;
+	     (pageno = xdes_calc_descriptor_page(zip_size, i)) == 0; )
+		i++;
+
+	/* use pageno prior to this...i.e last page on page 0 */
+	ut_ad(pageno > 0);
+	pageno--;
+
+	ulint iv_offset = XDES_ARR_OFFSET +
+		XDES_SIZE * (1 + xdes_calc_descriptor_index(zip_size, pageno));
+
+	if (max_size != NULL) {
+		/* return how much free space there is available on page */
+		*max_size = (zip_size ? zip_size : UNIV_PAGE_SIZE) -
+			(FSP_HEADER_OFFSET + iv_offset + FIL_PAGE_DATA_END);
+	}
+
+	return FSP_HEADER_OFFSET + iv_offset;
+}
+
+/********************************************************//**
+Check from tablespace page 0 is tablespace encrypted.
+ at return	true if tablespace is encrypted, false if not. */
+bool
+mariadb_check_tablespace_encryption(
+/*================================*/
+	os_file_t	fd,
+	const char*	name,
+	ulint		zip_size)
+{
+	byte*		buf;
+	byte*		page;
+	ulint		maxsize=0;
+	os_offset_t	offset = fsp_header_get_crypt_offset(zip_size, &maxsize);
+
+	/* Allocate memory for buffer and read page 0 from datafile */
+	buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
+	page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE));
+	os_file_read(fd, page, 0, UNIV_PAGE_SIZE);
+
+	if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0 ||
+	    memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) {
+		return false;
+	}
+
+	ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0);
+
+	if (! (type == CRYPT_SCHEME_UNENCRYPTED ||
+	       type == CRYPT_SCHEME_1)) {
+		return false;
+	}
+
+	ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1);
+	if (iv_length != CRYPT_SCHEME_1_IV_LEN) {
+		return false;
+	}
+	uint min_key_version = mach_read_from_4
+		(page + offset + MAGIC_SZ + 2 + iv_length);
+
+	uint key_id = mach_read_from_4
+		(page + offset + MAGIC_SZ + 2 + iv_length + 4);
+
+	if (type == CRYPT_SCHEME_1) {
+		msg("xtrabackup: Tablespace %s encrypted key_version %u key_id %u\n",
+			name, min_key_version, key_id);
+	}
+
+	ut_free(buf);
+	return (type == CRYPT_SCHEME_1);
+}
+
diff --git a/storage/innobase/xtrabackup/src/mariadb.h b/storage/innobase/xtrabackup/src/mariadb.h
new file mode 100644
index 0000000..4eacacb
--- /dev/null
+++ b/storage/innobase/xtrabackup/src/mariadb.h
@@ -0,0 +1,31 @@
+/*****************************************************************************
+
+Copyright (C) 2016, MariaDB Corporation. All Rights Reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+*****************************************************************************/
+/**************************************************//**
+ at file mariadb.h
+Helper functions to check MariaDB 10.1 extended features.
+
+Jan Lindström jan.lindstrom at mariadb.com
+*******************************************************/
+
+#include "univ.i"
+#include "log0log.h"
+
+bool mariadb_check_encryption(log_group_t* group);
+bool mariadb_check_compression(const byte* page);
+bool mariadb_check_tablespace_encryption(os_file_t fd, const char* name, ulint zip_size);
+
diff --git a/storage/innobase/xtrabackup/src/xtrabackup.cc b/storage/innobase/xtrabackup/src/xtrabackup.cc
index ebb63bf..8cbabaf 100644
--- a/storage/innobase/xtrabackup/src/xtrabackup.cc
+++ b/storage/innobase/xtrabackup/src/xtrabackup.cc
@@ -90,6 +90,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "backup_mysql.h"
 #include "backup_copy.h"
 #include "backup_mysql.h"
+#include "mariadb.h"
 
 /* TODO: replace with appropriate macros used in InnoDB 5.6 */
 #define PAGE_ZIP_MIN_SIZE_SHIFT	10
@@ -2520,6 +2521,7 @@ xtrabackup_scan_log_recs(
 	bool		is_last,	/*!< in: whether it is last segment
 					to copy */
 	lsn_t		start_lsn,	/*!< in: buffer start lsn */
+	bool		encrypted,	/*!< in: true if log encrypted */
 	lsn_t*		contiguous_lsn,	/*!< in/out: it is known that all log
 					groups contain contiguous log data up
 					to this lsn */
@@ -2545,6 +2547,11 @@ xtrabackup_scan_log_recs(
 		ibool	checksum_is_ok =
 			log_block_checksum_is_ok_or_old_format(log_block);
 
+		if (encrypted) {
+			/* MariaDB: Encrypted log blocks have incorrect checksum */
+			checksum_is_ok = true;
+		}
+
 		if (no != scanned_no && checksum_is_ok) {
 			ulint blocks_in_group;
 
@@ -2668,6 +2675,7 @@ xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last)
 	log_group_t*	group;
 	lsn_t		group_scanned_lsn;
 	lsn_t		contiguous_lsn;
+	bool		encrypted = false;
 
 	ut_a(dst_log_file != NULL);
 
@@ -2688,6 +2696,10 @@ xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last)
 
 		start_lsn = contiguous_lsn;
 
+		/* Read checkpoint records to find out is the redo-log
+		encrypted */
+		encrypted = mariadb_check_encryption(group);
+
 		while (!finished) {
 
 			end_lsn = start_lsn + RECV_SCAN_SIZE;
@@ -2700,7 +2712,8 @@ xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last)
 					       group, start_lsn, end_lsn);
 
 			 if (!xtrabackup_scan_log_recs(group, is_last,
-				start_lsn, &contiguous_lsn, &group_scanned_lsn,
+				start_lsn, encrypted,
+				&contiguous_lsn, &group_scanned_lsn,
 				&finished)) {
 				goto error;
 			 }


More information about the commits mailing list