From 1ad23d07c6ba1e317635b2d69aa49987779844cb Mon Sep 17 00:00:00 2001 From: Milind Changire Date: Mon, 27 Aug 2018 08:37:56 -0400 Subject: [PATCH] autobuild v3.12.2-18 Resolves: bz#1524336 bz#1622029 bz#1622452 Signed-off-by: Milind Changire --- ...omatic-gfid-conflict-resolution-opti.patch | 90 +++++ ...tr-xlator-init-if-ctr-is-not-enabled.patch | 332 ++++++++++++++++++ ...d_brick_start-shouldn-t-cleanup-pidf.patch | 69 ++++ glusterfs.spec | 8 +- 4 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 0360-geo-rep-Make-automatic-gfid-conflict-resolution-opti.patch create mode 100644 0361-ctr-skip-ctr-xlator-init-if-ctr-is-not-enabled.patch create mode 100644 0362-glusterd-glusterd_brick_start-shouldn-t-cleanup-pidf.patch diff --git a/0360-geo-rep-Make-automatic-gfid-conflict-resolution-opti.patch b/0360-geo-rep-Make-automatic-gfid-conflict-resolution-opti.patch new file mode 100644 index 0000000..0273c25 --- /dev/null +++ b/0360-geo-rep-Make-automatic-gfid-conflict-resolution-opti.patch @@ -0,0 +1,90 @@ +From 81b5fbe38a022b738aa817444d7564534e0a886e Mon Sep 17 00:00:00 2001 +From: Kotresh HR +Date: Fri, 24 Aug 2018 08:30:54 -0400 +Subject: [PATCH 360/362] geo-rep: Make automatic gfid conflict resolution + optional + +Autmatic gfid conflict resolution needs to be disabled +during failover/failback as it might lead to data loss +in the following scenario. + +1. Master went down without syncing directory "dir1" to slave. +2. When slave is failed over to master, if a new file + is written inside "dir1", creating dir1 again if not + present, "dir1" ends up with different gfid on original + slave. +3. When original master is up and failed back, due to + automatic gfid conflict resolution, "dir1" present in + original master is deleted losing all files and only + new file created on original slave is restored. + +Hence during failover/failback, automatic gfid conflict +resolution should be disabled. So in these cases, appropriate +decision is taken. + +Backport of: + > Patch: https://review.gluster.org/20986/ + > BUG: 1622076 + > Change-Id: I433616f5d3e13d4b6eb675475bd554ca34928573 + +BUG: 1622029 +Signed-off-by: Kotresh HR +Change-Id: I433616f5d3e13d4b6eb675475bd554ca34928573 +Reviewed-on: https://code.engineering.redhat.com/gerrit/148004 +Tested-by: RHGS Build Bot +Reviewed-by: Aravinda Vishwanathapura Krishna Murthy +Reviewed-by: Atin Mukherjee +--- + geo-replication/syncdaemon/gsyncd.py | 2 ++ + geo-replication/syncdaemon/master.py | 22 ++++++++++++---------- + 2 files changed, 14 insertions(+), 10 deletions(-) + +diff --git a/geo-replication/syncdaemon/gsyncd.py b/geo-replication/syncdaemon/gsyncd.py +index fff193b..23d588e 100644 +--- a/geo-replication/syncdaemon/gsyncd.py ++++ b/geo-replication/syncdaemon/gsyncd.py +@@ -323,6 +323,8 @@ def main_i(): + op.add_option('--changelog-archive-format', metavar='N', + type=str, default="%Y%m") + op.add_option('--use-meta-volume', default=False, action='store_true') ++ op.add_option('--gfid-conflict-resolution', default=True, ++ action='store_true') + op.add_option('--meta-volume-mnt', metavar='N', + type=str, default="/var/run/gluster/shared_storage") + op.add_option( +diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py +index 1399378..6de2c77 100644 +--- a/geo-replication/syncdaemon/master.py ++++ b/geo-replication/syncdaemon/master.py +@@ -1163,16 +1163,18 @@ class GMasterChangelogMixin(GMasterCommon): + self.status.inc_value("entry", len(entries)) + + failures = self.slave.server.entry_ops(entries) +- count = 0 +- while failures and count < self.MAX_OE_RETRIES: +- count += 1 +- self.handle_entry_failures(failures, entries) +- logging.info("Retry original entries. count = %s" % count) +- failures = self.slave.server.entry_ops(entries) +- if not failures: +- logging.info("Sucessfully fixed all entry ops with gfid " +- "mismatch") +- break ++ ++ if boolify(gconf.gfid_conflict_resolution): ++ count = 0 ++ while failures and count < self.MAX_OE_RETRIES: ++ count += 1 ++ self.handle_entry_failures(failures, entries) ++ logging.info("Retry original entries. count = %s" % count) ++ failures = self.slave.server.entry_ops(entries) ++ if not failures: ++ logging.info("Sucessfully fixed all entry ops with " ++ "gfid mismatch") ++ break + + self.log_failures(failures, 'gfid', gauxpfx(), 'ENTRY') + +-- +1.8.3.1 + diff --git a/0361-ctr-skip-ctr-xlator-init-if-ctr-is-not-enabled.patch b/0361-ctr-skip-ctr-xlator-init-if-ctr-is-not-enabled.patch new file mode 100644 index 0000000..be26001 --- /dev/null +++ b/0361-ctr-skip-ctr-xlator-init-if-ctr-is-not-enabled.patch @@ -0,0 +1,332 @@ +From 4833168455511785d05f5d20b47644cbda9066cb Mon Sep 17 00:00:00 2001 +From: Mohit Agrawal +Date: Fri, 24 Aug 2018 10:44:59 +0530 +Subject: [PATCH 361/362] ctr: skip ctr xlator init if ctr is not enabled + +Problem: If ctr xlator is not required it consumes + resources unnecessarily + +Solution: Call ctr xlator init only while feature is enabled + +> Fixes: bz#1524323 + +> Change-Id: I378113a390a286be20c4ade1b1bac170a8ef1b14 +> (Cherry pick from commit 30e46a9b3cef868e5c781044c99c3d5b066d4760) +> (Reviewed on upstream link 30e46a9b3cef868e5c781044c99c3d5b066d4760) + +Change-Id: Ie65c89bf27d2f119fa41cfc143f1f828f9d64b62 +BUG: 1524336 +Signed-off-by: Mohit Agrawal +Reviewed-on: https://code.engineering.redhat.com/gerrit/148134 +Tested-by: Mohit Agrawal +Reviewed-by: Atin Mukherjee +--- + tests/basic/tier/tier_lookup_heal.t | 6 - + .../changetimerecorder/src/changetimerecorder.c | 192 +++++++++++++-------- + .../features/changetimerecorder/src/ctr-helper.h | 2 +- + 3 files changed, 120 insertions(+), 80 deletions(-) + +diff --git a/tests/basic/tier/tier_lookup_heal.t b/tests/basic/tier/tier_lookup_heal.t +index 7dac1fd..8f8292c 100755 +--- a/tests/basic/tier/tier_lookup_heal.t ++++ b/tests/basic/tier/tier_lookup_heal.t +@@ -36,12 +36,6 @@ TEST stat . + TEST touch file1 + TEST stat file1 + +-# gf_file_tb and gf_flink_tb should be empty +-ENTRY_COUNT=$(echo "select * from gf_file_tb; select * from gf_flink_tb;" | \ +- sqlite3 $B0/${V0}$LAST_BRICK/.glusterfs/${V0}$LAST_BRICK.db | wc -l ) +-TEST [ $ENTRY_COUNT -eq 0 ] +- +- + #Attach tier and switch ON CTR Xlator. + TEST $CLI volume attach-tier $V0 replica 2 $H0:$B0/${V0}$CACHE_BRICK_FIRST $H0:$B0/${V0}$CACHE_BRICK_LAST + TEST $CLI volume set $V0 features.ctr-enabled on +diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.c b/xlators/features/changetimerecorder/src/changetimerecorder.c +index 5f82d33..9d4f8a3 100644 +--- a/xlators/features/changetimerecorder/src/changetimerecorder.c ++++ b/xlators/features/changetimerecorder/src/changetimerecorder.c +@@ -2086,6 +2086,84 @@ out: + } + + ++/* Call to initialize db for ctr xlator while ctr is enabled */ ++int32_t ++initialize_ctr_resource (xlator_t *this, gf_ctr_private_t *priv) ++{ ++ int ret_db = -1; ++ dict_t *params_dict = NULL; ++ ++ if (!priv) ++ goto error; ++ ++ /* For compaction */ ++ priv->compact_active = _gf_false; ++ priv->compact_mode_switched = _gf_false; ++ ret_db = pthread_mutex_init (&priv->compact_lock, NULL); ++ ++ if (ret_db) { ++ gf_msg (this->name, GF_LOG_ERROR, 0, ++ CTR_MSG_FATAL_ERROR, ++ "FATAL: Failed initializing compaction mutex"); ++ goto error; ++ } ++ ++ params_dict = dict_new (); ++ if (!params_dict) { ++ gf_msg (this->name, GF_LOG_ERROR, 0, ++ CTR_MSG_INIT_DB_PARAMS_FAILED, ++ "DB Params cannot initialized!"); ++ goto error; ++ } ++ ++ /*Extract db params options*/ ++ ret_db = extract_db_params(this, params_dict, priv->gfdb_db_type); ++ if (ret_db) { ++ gf_msg (this->name, GF_LOG_ERROR, 0, ++ CTR_MSG_EXTRACT_DB_PARAM_OPTIONS_FAILED, ++ "Failed extracting db params options"); ++ goto error; ++ } ++ ++ /*Create a memory pool for ctr xlator*/ ++ this->local_pool = mem_pool_new (gf_ctr_local_t, 64); ++ if (!this->local_pool) { ++ gf_msg (this->name, GF_LOG_ERROR, 0, ++ CTR_MSG_CREATE_LOCAL_MEMORY_POOL_FAILED, ++ "failed to create local memory pool"); ++ ret_db = -1; ++ goto error; ++ } ++ ++ /*Initialize Database Connection*/ ++ priv->_db_conn = init_db(params_dict, priv->gfdb_db_type); ++ if (!priv->_db_conn) { ++ gf_msg (this->name, GF_LOG_ERROR, 0, ++ CTR_MSG_FATAL_ERROR, ++ "FATAL: Failed initializing data base"); ++ ret_db = -1; ++ goto error; ++ } ++ ++ ret_db = 0; ++ goto out; ++ ++error: ++ if (this) ++ mem_pool_destroy (this->local_pool); ++ ++ if (priv) { ++ GF_FREE (priv->ctr_db_path); ++ } ++ GF_FREE (priv); ++ ++out: ++ if (params_dict) ++ dict_unref (params_dict); ++ ++ return ret_db; ++} ++ + /******************************************************************************/ + int + reconfigure (xlator_t *this, dict_t *options) +@@ -2095,6 +2173,7 @@ reconfigure (xlator_t *this, dict_t *options) + gf_ctr_private_t *priv = NULL; + + priv = this->private; ++ + if (dict_get_str(options, "changetimerecorder.frequency", + &temp_str)) { + gf_msg(this->name, GF_LOG_TRACE, 0, CTR_MSG_SET, "set"); +@@ -2102,6 +2181,26 @@ reconfigure (xlator_t *this, dict_t *options) + + GF_OPTION_RECONF ("ctr-enabled", priv->enabled, options, + bool, out); ++ if (!priv->enabled) { ++ gf_msg (GFDB_DATA_STORE, GF_LOG_INFO, 0, ++ CTR_MSG_XLATOR_DISABLED, ++ "CTR Xlator is not enabled so skip ctr reconfigure"); ++ goto out; ++ } ++ ++ /* If ctr is enabled after skip init for ctr xlator then call ++ initialize_ctr_resource during reconfigure phase to allocate resources for ++ xlator ++ */ ++ if (priv->enabled && !priv->_db_conn) { ++ ret = initialize_ctr_resource (this, priv); ++ if (ret) { ++ gf_msg (this->name, GF_LOG_ERROR, 0, ++ CTR_MSG_FATAL_ERROR, ++ "FATAL: Failed ctr initialize resource"); ++ goto out; ++ } ++ } + + GF_OPTION_RECONF ("record-counters", priv->ctr_record_counter, options, + bool, out); +@@ -2174,15 +2273,19 @@ init (xlator_t *this) + { + gf_ctr_private_t *priv = NULL; + int ret_db = -1; +- dict_t *params_dict = NULL; + +- GF_VALIDATE_OR_GOTO ("ctr", this, error); ++ if (!this) { ++ gf_msg (this->name, GF_LOG_ERROR, 0, ++ CTR_MSG_FATAL_ERROR, ++ "FATAL: ctr this is not initialized"); ++ return -1; ++ } + + if (!this->children || this->children->next) { + gf_msg (this->name, GF_LOG_ERROR, 0, + CTR_MSG_FATAL_ERROR, + "FATAL: ctr should have exactly one child"); +- goto error; ++ return -1; + } + + if (!this->parents) { +@@ -2196,7 +2299,7 @@ init (xlator_t *this) + gf_msg (this->name, GF_LOG_ERROR, ENOMEM, + CTR_MSG_CALLOC_FAILED, + "Calloc did not work!!!"); +- goto error; ++ return -1; + } + + /*Default values for the translator*/ +@@ -2205,94 +2308,37 @@ init (xlator_t *this) + priv->ctr_hot_brick = _gf_false; + priv->gfdb_db_type = GFDB_SQLITE3; + priv->gfdb_sync_type = GFDB_DB_SYNC; +- priv->enabled = _gf_true; + priv->_db_conn = NULL; + priv->ctr_lookupheal_link_timeout = + CTR_DEFAULT_HARDLINK_EXP_PERIOD; + priv->ctr_lookupheal_inode_timeout = + CTR_DEFAULT_INODE_EXP_PERIOD; + +- /* For compaction */ +- priv->compact_active = _gf_false; +- priv->compact_mode_switched = _gf_false; +- ret_db = pthread_mutex_init (&priv->compact_lock, NULL); +- +- if (ret_db) { +- gf_msg (this->name, GF_LOG_ERROR, 0, +- CTR_MSG_FATAL_ERROR, +- "FATAL: Failed initializing compaction mutex"); +- goto error; +- } +- + /*Extract ctr xlator options*/ + ret_db = extract_ctr_options (this, priv); + if (ret_db) { + gf_msg (this->name, GF_LOG_ERROR, 0, + CTR_MSG_EXTRACT_CTR_XLATOR_OPTIONS_FAILED, + "Failed extracting ctr xlator options"); +- goto error; ++ return -1; + } + +- params_dict = dict_new (); +- if (!params_dict) { +- gf_msg (this->name, GF_LOG_ERROR, 0, +- CTR_MSG_INIT_DB_PARAMS_FAILED, +- "DB Params cannot initialized!"); +- goto error; +- } +- +- /*Extract db params options*/ +- ret_db = extract_db_params(this, params_dict, priv->gfdb_db_type); +- if (ret_db) { +- gf_msg (this->name, GF_LOG_ERROR, 0, +- CTR_MSG_EXTRACT_DB_PARAM_OPTIONS_FAILED, +- "Failed extracting db params options"); +- goto error; ++ if (!priv->enabled) { ++ gf_msg (GFDB_DATA_STORE, GF_LOG_INFO, 0, ++ CTR_MSG_XLATOR_DISABLED, ++ "CTR Xlator is not enabled so skip ctr init"); ++ goto out; + } + +- /*Create a memory pool for ctr xlator*/ +- this->local_pool = mem_pool_new (gf_ctr_local_t, 64); +- if (!this->local_pool) { ++ ret_db = initialize_ctr_resource (this, priv); ++ if (ret_db) { + gf_msg (this->name, GF_LOG_ERROR, 0, +- CTR_MSG_CREATE_LOCAL_MEMORY_POOL_FAILED, +- "failed to create local memory pool"); +- goto error; +- } +- +- /*Initialize Database Connection*/ +- priv->_db_conn = init_db(params_dict, priv->gfdb_db_type); +- if (!priv->_db_conn) { +- gf_msg (this->name, GF_LOG_ERROR, 0, +- CTR_MSG_FATAL_ERROR, +- "FATAL: Failed initializing data base"); +- goto error; +- } +- +- +- ret_db = 0; +- goto out; +- +-/*Error handling */ +-error: +- +- if (this) +- mem_pool_destroy (this->local_pool); +- +- if (priv) { +- GF_FREE (priv->ctr_db_path); ++ CTR_MSG_FATAL_ERROR, ++ "FATAL: Failed ctr initialize resource"); ++ return -1; + } +- GF_FREE (priv); +- +- if (params_dict) +- dict_unref (params_dict); +- +- return -1; + + out: +- +- if (params_dict) +- dict_unref (params_dict); +- + this->private = (void *)priv; + return 0; + } +@@ -2343,7 +2389,7 @@ fini (xlator_t *this) + + priv = this->private; + +- if (priv) { ++ if (priv && priv->enabled) { + if (fini_db (priv->_db_conn)) { + gf_msg (this->name, GF_LOG_WARNING, 0, + CTR_MSG_CLOSE_DB_CONN_FAILED, "Failed closing " +diff --git a/xlators/features/changetimerecorder/src/ctr-helper.h b/xlators/features/changetimerecorder/src/ctr-helper.h +index 4fd4f74..f0b0dce 100644 +--- a/xlators/features/changetimerecorder/src/ctr-helper.h ++++ b/xlators/features/changetimerecorder/src/ctr-helper.h +@@ -408,7 +408,7 @@ do {\ + GF_ASSERT (this);\ + GF_ASSERT (this->private);\ + _priv = this->private;\ +- if (!_priv->enabled)\ ++ if (!_priv->_db_conn)\ + goto label;\ + } while (0) + +-- +1.8.3.1 + diff --git a/0362-glusterd-glusterd_brick_start-shouldn-t-cleanup-pidf.patch b/0362-glusterd-glusterd_brick_start-shouldn-t-cleanup-pidf.patch new file mode 100644 index 0000000..6f14ff9 --- /dev/null +++ b/0362-glusterd-glusterd_brick_start-shouldn-t-cleanup-pidf.patch @@ -0,0 +1,69 @@ +From b55e9f05cb457806d08ac184de4b5651e420f11e Mon Sep 17 00:00:00 2001 +From: Mohit Agrawal +Date: Mon, 27 Aug 2018 12:18:21 +0530 +Subject: [PATCH 362/362] glusterd: glusterd_brick_start shouldn't cleanup + pidfile if only_connect is true + +Problem: Sometime glusterd cleanup pidfile even brick is started + and cli shows volume status "N/A" + +Solution: Update the condition in glusterd_brick_start to avoid + pidfile cleanup in case if only_connect flag is true + +> Fixes: bz#1622422 +> Change-Id: I8decb34597126b848e3a44d957e138833dd97350 +> (cherry picked from commit 1b499e9dfe63ce051a071cacc19e3161b7a32636) +> (Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/21007/) + +Change-Id: Id9145c26de15cf2354a10a482b5f8a45599e0831 +BUG: 1622452 +Signed-off-by: Mohit Agrawal +Reviewed-on: https://code.engineering.redhat.com/gerrit/148184 +Tested-by: Mohit Agrawal +Reviewed-by: Atin Mukherjee +--- + xlators/mgmt/glusterd/src/glusterd-utils.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c +index b9e8d8d..01345cd 100644 +--- a/xlators/mgmt/glusterd/src/glusterd-utils.c ++++ b/xlators/mgmt/glusterd/src/glusterd-utils.c +@@ -6163,6 +6163,8 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo, + brickpath = search_brick_path_from_proc + (pid, brickinfo->path); + if (!brickpath) { ++ if (only_connect) ++ return 0; + gf_log (this->name, GF_LOG_INFO, + "Either pid %d is not running or brick" + " path %s is not consumed so cleanup pidfile", +@@ -6173,14 +6175,14 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo, + if (sys_access (pidfile , R_OK) == 0) { + sys_unlink (pidfile); + } +- if (only_connect) +- return 0; + goto run; + } + GF_FREE (brickpath); + ret = glusterd_get_sock_from_brick_pid (pid, socketpath, + sizeof(socketpath)); + if (ret) { ++ if (only_connect) ++ return 0; + gf_log (this->name, GF_LOG_INFO, + "Either pid %d is not running or does " + "not match with any running brick " +@@ -6189,8 +6191,6 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo, + if (sys_access (pidfile , R_OK) == 0) { + sys_unlink (pidfile); + } +- if (only_connect) +- return 0; + goto run; + } + } +-- +1.8.3.1 + diff --git a/glusterfs.spec b/glusterfs.spec index ea3870d..ec46e09 100644 --- a/glusterfs.spec +++ b/glusterfs.spec @@ -192,7 +192,7 @@ Release: 0.1%{?prereltag:.%{prereltag}}%{?dist} %else Name: glusterfs Version: 3.12.2 -Release: 17%{?dist} +Release: 18%{?dist} %endif License: GPLv2 or LGPLv3+ Group: System Environment/Base @@ -624,6 +624,9 @@ Patch0356: 0356-io-stats-sanitize-the-dump-path-further.patch Patch0357: 0357-snapshot-fix-snapshot-status-failure-due-to-symlink-.patch Patch0358: 0358-glusterd-glusterd_brick_start-shouldn-t-try-to-bring.patch Patch0359: 0359-storage-posix-Increment-trusted.pgfid-in-posix_mknod.patch +Patch0360: 0360-geo-rep-Make-automatic-gfid-conflict-resolution-opti.patch +Patch0361: 0361-ctr-skip-ctr-xlator-init-if-ctr-is-not-enabled.patch +Patch0362: 0362-glusterd-glusterd_brick_start-shouldn-t-cleanup-pidf.patch %description GlusterFS is a distributed file-system capable of scaling to several @@ -2572,6 +2575,9 @@ fi %endif %changelog +* Mon Aug 27 2018 Milind Changire - 3.12.2-18 +- fixes bugs bz#1524336 bz#1622029 bz#1622452 + * Thu Aug 23 2018 Milind Changire - 3.12.2-17 - fixes bugs bz#1615578 bz#1619416 bz#1619538 bz#1620469 bz#1620765