From 3350cc81a349260848f5e373625c47d68ef484b1 Mon Sep 17 00:00:00 2001 From: Rinku Kothiya Date: Wed, 16 Oct 2019 01:32:29 -0400 Subject: [PATCH] autobuild v6.0-18 Resolves: bz#1758432 Signed-off-by: Rinku Kothiya --- ...using-IO-Error-File-descriptor-in-ba.patch | 347 ++++++++++++++++++ glusterfs.spec | 6 +- 2 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 0308-dht-Rebalance-causing-IO-Error-File-descriptor-in-ba.patch diff --git a/0308-dht-Rebalance-causing-IO-Error-File-descriptor-in-ba.patch b/0308-dht-Rebalance-causing-IO-Error-File-descriptor-in-ba.patch new file mode 100644 index 0000000..adbeb43 --- /dev/null +++ b/0308-dht-Rebalance-causing-IO-Error-File-descriptor-in-ba.patch @@ -0,0 +1,347 @@ +From 27f799563c1c2c1986662ed4a3a83d834c04fd98 Mon Sep 17 00:00:00 2001 +From: Mohit Agrawal +Date: Mon, 14 Oct 2019 15:42:31 +0530 +Subject: [PATCH 308/308] dht: Rebalance causing IO Error - File descriptor in + bad state + +Problem : When a file is migrated, dht attempts to re-open all open + fds on the new cached subvol. Earlier, if dht had not opened the fd, + the client xlator would be unable to find the remote fd and would + fall back to using an anon fd for the fop. That behavior changed with + https://review.gluster.org/#/c/glusterfs/+/15804, causing fops to fail + with EBADFD if the fd was not available on the cached subvol. + The client xlator returns EBADFD if the remote fd is not found but + dht only checks for EBADF before re-opening fds on the new cached subvol. + +Solution: Handle EBADFD at dht code path to avoid the issue + +> Change-Id: I43c51995cdd48d05b12e4b2889c8dbe2bb2a72d8 +> Fixes: bz#1758579 +> Signed-off-by: Mohit Agrawal +> (Cherry pick from commit 9314a9fbf487614c736cf6c4c1b93078d37bb9df) +> (Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/23518/) + +Change-Id: I43c51995cdd48d05b12e4b2889c8dbe2bb2a72d8 +BUG: 1758432 +Signed-off-by: Mohit Agrawal +Reviewed-on: https://code.engineering.redhat.com/gerrit/183370 +Tested-by: RHGS Build Bot +Reviewed-by: Sunil Kumar Heggodu Gopala Acharya +--- + xlators/cluster/dht/src/dht-common.c | 27 +++++++++++++++++--- + xlators/cluster/dht/src/dht-common.h | 19 ++++++++++++++ + xlators/cluster/dht/src/dht-helper.c | 29 +++++++++++++++++++++ + xlators/cluster/dht/src/dht-inode-read.c | 42 +++++++++++++++++++++++++++---- + xlators/cluster/dht/src/dht-inode-write.c | 16 ++++++------ + 5 files changed, 116 insertions(+), 17 deletions(-) + +diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c +index 99cccd6..37952ba 100644 +--- a/xlators/cluster/dht/src/dht-common.c ++++ b/xlators/cluster/dht/src/dht-common.c +@@ -53,6 +53,17 @@ dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req); + int + dht_do_fresh_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc); + ++/* Check the xdata to make sure EBADF has been set by client xlator */ ++int32_t ++dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno) ++{ ++ if (op_ret == -1 && (op_errno == EBADF || op_errno == EBADFD) && ++ !(local->fd_checked)) { ++ return 1; ++ } ++ return 0; ++} ++ + /* Sets the blocks and size values to fixed values. This is to be called + * only for dirs. The caller is responsible for checking the type + */ +@@ -4529,6 +4540,7 @@ dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + int this_call_cnt = 0; + dht_local_t *local = NULL; + dht_conf_t *conf = NULL; ++ int ret = 0; + + VALIDATE_OR_GOTO(frame, err); + VALIDATE_OR_GOTO(frame->local, err); +@@ -4537,6 +4549,13 @@ dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + conf = this->private; + local = frame->local; + ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { ++ ret = dht_check_and_open_fd_on_subvol(this, frame); ++ if (ret) ++ goto err; ++ return 0; ++ } ++ + LOCK(&frame->lock); + { + if (!xattr || (op_ret == -1)) { +@@ -5204,8 +5223,8 @@ dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + + local->op_errno = op_errno; + +- if ((local->fop == GF_FOP_FSETXATTR) && op_ret == -1 && +- (op_errno == EBADF) && !(local->fd_checked)) { ++ if ((local->fop == GF_FOP_FSETXATTR) && ++ dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -5929,8 +5948,8 @@ dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + + local->op_errno = op_errno; + +- if ((local->fop == GF_FOP_FREMOVEXATTR) && (op_ret == -1) && +- (op_errno == EBADF) && !(local->fd_checked)) { ++ if ((local->fop == GF_FOP_FREMOVEXATTR) && ++ dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h +index c516271..ce11f02 100644 +--- a/xlators/cluster/dht/src/dht-common.h ++++ b/xlators/cluster/dht/src/dht-common.h +@@ -1230,6 +1230,22 @@ dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata); + + int ++dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, ++ int32_t op_ret, int32_t op_errno, dict_t *xdata); ++ ++int ++dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, ++ int op_errno, dict_t *xattr, dict_t *xdata); ++ ++int ++dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, ++ int32_t op_ret, int32_t op_errno, dict_t *dict, ++ dict_t *xdata); ++int ++dht_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, ++ int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata); ++ ++int + gf_defrag_status_get(dht_conf_t *conf, dict_t *dict); + + void +@@ -1525,4 +1541,7 @@ int + dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, + dict_t *xdata); + ++int32_t ++dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno); ++ + #endif /* _DHT_H */ +diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c +index 1e9fee0..4f7370d 100644 +--- a/xlators/cluster/dht/src/dht-helper.c ++++ b/xlators/cluster/dht/src/dht-helper.c +@@ -366,6 +366,23 @@ dht_check_and_open_fd_on_subvol_complete(int ret, call_frame_t *frame, + + break; + ++ case GF_FOP_FXATTROP: ++ STACK_WIND(frame, dht_common_xattrop_cbk, subvol, ++ subvol->fops->fxattrop, local->fd, ++ local->rebalance.flags, local->rebalance.xattr, ++ local->xattr_req); ++ break; ++ ++ case GF_FOP_FGETXATTR: ++ STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr, ++ local->fd, local->key, NULL); ++ break; ++ ++ case GF_FOP_FINODELK: ++ STACK_WIND(frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk, ++ local->key, local->fd, local->rebalance.lock_cmd, ++ &local->rebalance.flock, local->xattr_req); ++ break; + default: + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, + "Unknown FOP on fd (%p) on file %s @ %s", fd, +@@ -429,6 +446,18 @@ handle_err: + DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL); + break; + ++ case GF_FOP_FXATTROP: ++ DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL); ++ break; ++ ++ case GF_FOP_FGETXATTR: ++ DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL); ++ break; ++ ++ case GF_FOP_FINODELK: ++ DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL); ++ break; ++ + default: + gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, + "Unknown FOP on fd (%p) on file %s @ %s", fd, +diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c +index cacfe35..0c209a5 100644 +--- a/xlators/cluster/dht/src/dht-inode-read.c ++++ b/xlators/cluster/dht/src/dht-inode-read.c +@@ -162,8 +162,8 @@ dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + local = frame->local; + prev = cookie; + +- if ((local->fop == GF_FOP_FSTAT) && (op_ret == -1) && (op_errno == EBADF) && +- !(local->fd_checked)) { ++ if ((local->fop == GF_FOP_FSTAT) && ++ dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -431,7 +431,7 @@ dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + if (local->call_cnt != 1) + goto out; + +- if (op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) { ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -703,7 +703,7 @@ dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + if (local->call_cnt != 1) + goto out; + +- if (op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) { ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -820,7 +820,7 @@ dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + + local->op_errno = op_errno; + +- if (op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) { ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -1223,6 +1223,13 @@ dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + if (local->call_cnt != 1) + goto out; + ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { ++ ret = dht_check_and_open_fd_on_subvol(this, frame); ++ if (ret) ++ goto out; ++ return 0; ++ } ++ + ret = dht_read_iatt_from_xdata(this, xdata, &stbuf); + + if ((!op_ret) && (ret)) { +@@ -1535,8 +1542,26 @@ dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) + + { ++ dht_local_t *local = NULL; ++ int ret = 0; ++ ++ GF_VALIDATE_OR_GOTO("dht", frame, out); ++ GF_VALIDATE_OR_GOTO("dht", this, out); ++ GF_VALIDATE_OR_GOTO("dht", frame->local, out); ++ ++ local = frame->local; ++ ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { ++ ret = dht_check_and_open_fd_on_subvol(this, frame); ++ if (ret) ++ goto out; ++ return 0; ++ } ++ ++out: + dht_lk_inode_unref(frame, op_ret); + DHT_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata); ++ + return 0; + } + +@@ -1574,6 +1599,13 @@ dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd, + if (ret) + goto err; + */ ++ local->rebalance.flock = *lock; ++ local->rebalance.lock_cmd = cmd; ++ local->key = gf_strdup(volume); ++ ++ if (xdata) ++ local->xattr_req = dict_ref(xdata); ++ + STACK_WIND(frame, dht_finodelk_cbk, lock_subvol, + lock_subvol->fops->finodelk, volume, fd, cmd, lock, xdata); + +diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c +index b26b705..b6b349d 100644 +--- a/xlators/cluster/dht/src/dht-inode-write.c ++++ b/xlators/cluster/dht/src/dht-inode-write.c +@@ -49,7 +49,7 @@ dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + * We only check once as this could be a valid bad fd error. + */ + +- if (op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) { ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -262,8 +262,8 @@ dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + * We only check once as this could actually be a valid error. + */ + +- if ((local->fop == GF_FOP_FTRUNCATE) && (op_ret == -1) && +- ((op_errno == EBADF) || (op_errno == EINVAL)) && !(local->fd_checked)) { ++ if ((local->fop == GF_FOP_FTRUNCATE) && ++ dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -489,7 +489,7 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + * We only check once as this could actually be a valid error. + */ + +- if ((op_ret == -1) && (op_errno == EBADF) && !(local->fd_checked)) { ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -666,7 +666,7 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + * and a lookup updated the cached subvol in the inode ctx. + * We only check once as this could actually be a valid error. + */ +- if ((op_ret == -1) && (op_errno == EBADF) && !(local->fd_checked)) { ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -838,7 +838,7 @@ dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, + * and a lookup updated the cached subvol in the inode ctx. + * We only check once as this could actually be a valid error. + */ +- if ((op_ret == -1) && (op_errno == EBADF) && !(local->fd_checked)) { ++ if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +@@ -1005,8 +1005,8 @@ dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + + local->op_errno = op_errno; + +- if ((local->fop == GF_FOP_FSETATTR) && (op_ret == -1) && +- (op_errno == EBADF) && !(local->fd_checked)) { ++ if ((local->fop == GF_FOP_FSETATTR) && ++ dht_check_remote_fd_failed_error(local, op_ret, op_errno)) { + ret = dht_check_and_open_fd_on_subvol(this, frame); + if (ret) + goto out; +-- +1.8.3.1 + diff --git a/glusterfs.spec b/glusterfs.spec index 1244f3a..d15d361 100644 --- a/glusterfs.spec +++ b/glusterfs.spec @@ -231,7 +231,7 @@ Release: 0.1%{?prereltag:.%{prereltag}}%{?dist} %else Name: glusterfs Version: 6.0 -Release: 17%{?dist} +Release: 18%{?dist} ExcludeArch: i686 %endif License: GPLv2 or LGPLv3+ @@ -616,6 +616,7 @@ Patch0304: 0304-cluster-dht-Correct-fd-processing-loop.patch Patch0305: 0305-glusterd-rebalance-start-should-fail-when-quorum-is-.patch Patch0306: 0306-cli-fix-distCount-value.patch Patch0307: 0307-ssl-fix-RHEL8-regression-failure.patch +Patch0308: 0308-dht-Rebalance-causing-IO-Error-File-descriptor-in-ba.patch %description GlusterFS is a distributed file-system capable of scaling to several @@ -2329,6 +2330,9 @@ fi %endif %changelog +* Wed Oct 16 2019 Rinku Kothiya - 6.0-18 +- fixes bugs bz#1758432 + * Fri Oct 11 2019 Rinku Kothiya - 6.0-17 - fixes bugs bz#1704562 bz#1758618 bz#1760261