From 44ca2331774842fc4bf429876571f6c9442698a0 Mon Sep 17 00:00:00 2001 From: Rinku Kothiya Date: Sat, 4 Apr 2020 07:39:29 -0400 Subject: [PATCH] autobuild v6.0-32 Resolves: bz#1781543 bz#1812789 bz#1812824 bz#1817369 bz#1819059 Signed-off-by: Rinku Kothiya --- ...wrong-loop-count-in-__inode_ctx_free.patch | 51 ++ ...ocess_dir-is-called-even-if-gf_defra.patch | 41 ++ 0360-rpc-Make-ssl-log-more-useful.patch | 117 +++++ ...ython3-compatibility-and-new-test-ca.patch | 122 +++++ 0362-write-behind-fix-data-corruption.patch | 454 ++++++++++++++++++ glusterfs.spec | 10 +- 6 files changed, 794 insertions(+), 1 deletion(-) create mode 100644 0358-inode-fix-wrong-loop-count-in-__inode_ctx_free.patch create mode 100644 0359-dht-gf_defrag_process_dir-is-called-even-if-gf_defra.patch create mode 100644 0360-rpc-Make-ssl-log-more-useful.patch create mode 100644 0361-snap_scheduler-python3-compatibility-and-new-test-ca.patch create mode 100644 0362-write-behind-fix-data-corruption.patch diff --git a/0358-inode-fix-wrong-loop-count-in-__inode_ctx_free.patch b/0358-inode-fix-wrong-loop-count-in-__inode_ctx_free.patch new file mode 100644 index 0000000..d7138a6 --- /dev/null +++ b/0358-inode-fix-wrong-loop-count-in-__inode_ctx_free.patch @@ -0,0 +1,51 @@ +From 0d8c6d78130d22c475010bcce8055073b19de82a Mon Sep 17 00:00:00 2001 +From: Xie Changlong +Date: Fri, 17 May 2019 18:33:11 +0800 +Subject: [PATCH 358/362] inode: fix wrong loop count in __inode_ctx_free + +Avoid serious memory leak + +Backport of : +>fixes: bz#1711240 +>Upstream patch link: https://review.gluster.org/#/c/glusterfs/+/22738/ +>Change-Id: Ic61a8fdd0e941e136c98376a87b5a77fa8c22316 +>Signed-off-by: Xie Changlong + +BUG: 1781543 +Change-Id: I601ebb6cd6744a61c64edd3d21d3b9a0edf1e95b +Signed-off-by: Rinku Kothiya +Reviewed-on: https://code.engineering.redhat.com/gerrit/195611 +Tested-by: RHGS Build Bot +Reviewed-by: Sunil Kumar Heggodu Gopala Acharya +--- + libglusterfs/src/inode.c | 11 ++++++----- + 1 file changed, 6 insertions(+), 5 deletions(-) + +diff --git a/libglusterfs/src/inode.c b/libglusterfs/src/inode.c +index 5331e93..9dbb25b 100644 +--- a/libglusterfs/src/inode.c ++++ b/libglusterfs/src/inode.c +@@ -402,14 +402,15 @@ __inode_ctx_free(inode_t *inode) + goto noctx; + } + +- for (index = 0; index < inode->table->xl->graph->xl_count; index++) { ++ for (index = 0; index < inode->table->ctxcount; index++) { + if (inode->_ctx[index].value1 || inode->_ctx[index].value2) { + xl = (xlator_t *)(long)inode->_ctx[index].xl_key; +- old_THIS = THIS; +- THIS = xl; +- if (!xl->call_cleanup && xl->cbks->forget) ++ if (xl && !xl->call_cleanup && xl->cbks->forget) { ++ old_THIS = THIS; ++ THIS = xl; + xl->cbks->forget(xl, inode); +- THIS = old_THIS; ++ THIS = old_THIS; ++ } + } + } + +-- +1.8.3.1 + diff --git a/0359-dht-gf_defrag_process_dir-is-called-even-if-gf_defra.patch b/0359-dht-gf_defrag_process_dir-is-called-even-if-gf_defra.patch new file mode 100644 index 0000000..bd730bc --- /dev/null +++ b/0359-dht-gf_defrag_process_dir-is-called-even-if-gf_defra.patch @@ -0,0 +1,41 @@ +From c0efaa98d777e4520028bf55482846b3ef5fca3a Mon Sep 17 00:00:00 2001 +From: Susant Palai +Date: Wed, 1 Apr 2020 12:14:31 +0530 +Subject: [PATCH 359/362] dht: gf_defrag_process_dir is called even if + gf_defrag_fix_layout has failed + +Currently even though gf_defrag_fix_layout fails with ENOENT or ESTALE, a +subsequent call is made to gf_defrag_process_dir leading to rebalance failure. + +upstream patch: https://review.gluster.org/#/c/glusterfs/+/24225 + +> fixes: #1102 +> Change-Id: Ib0c309fd78e89a000fed3feb4bbe2c5b48e61478 +> Signed-off-by: Susant Palai + +BUG: 1812789 +Change-Id: Ib0c309fd78e89a000fed3feb4bbe2c5b48e61478 +Signed-off-by: Susant Palai +Reviewed-on: https://code.engineering.redhat.com/gerrit/196249 +Reviewed-by: Mohit Agrawal +Tested-by: RHGS Build Bot +Reviewed-by: Sunil Kumar Heggodu Gopala Acharya +--- + xlators/cluster/dht/src/dht-rebalance.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c +index 559f046..f4c62b8 100644 +--- a/xlators/cluster/dht/src/dht-rebalance.c ++++ b/xlators/cluster/dht/src/dht-rebalance.c +@@ -3939,6 +3939,7 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, + defrag->total_failures++; + } + ret = 0; ++ goto out; + } else { + gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED, + "Setxattr failed for %s", loc->path); +-- +1.8.3.1 + diff --git a/0360-rpc-Make-ssl-log-more-useful.patch b/0360-rpc-Make-ssl-log-more-useful.patch new file mode 100644 index 0000000..05e903d --- /dev/null +++ b/0360-rpc-Make-ssl-log-more-useful.patch @@ -0,0 +1,117 @@ +From 2b859d1a5499a215c8c37472d4fc7d7e4d70dac6 Mon Sep 17 00:00:00 2001 +From: Mohit Agrawal +Date: Tue, 31 Mar 2020 16:45:35 +0530 +Subject: [PATCH 360/362] rpc: Make ssl log more useful + +Currently, ssl_setup_connection_params throws 4 messages for every +rpc connection that irritates a user while reading the logs. The same +info we can print in a single log with peerinfo to make it more +useful.ssl_setup_connection_params try to load dh_param even user +has not configured it and if a dh_param file is not available it throws +a failure message.To avoid the message load dh_param only while the user +has configured it. + +> Change-Id: I9ddb57f86a3fa3e519180cb5d88828e59fe0e487 +> Fixes: #1141 +> Signed-off-by: Mohit Agrawal +> Cherry pick from commit 80dd8cceab3b860bf1bc2945c8e2d8d0b3913e48 +> Reviewed on upstream link https://review.gluster.org/#/c/glusterfs/+/24270/ + +BUG: 1812824 +Change-Id: I9ddb57f86a3fa3e519180cb5d88828e59fe0e487 +Signed-off-by: Mohit Agrawal +Reviewed-on: https://code.engineering.redhat.com/gerrit/196371 +Tested-by: RHGS Build Bot +Reviewed-by: Sunil Kumar Heggodu Gopala Acharya +--- + rpc/rpc-transport/socket/src/socket.c | 46 ++++++++++++++++++++--------------- + 1 file changed, 26 insertions(+), 20 deletions(-) + +diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c +index f54ca83..65845ea 100644 +--- a/rpc/rpc-transport/socket/src/socket.c ++++ b/rpc/rpc-transport/socket/src/socket.c +@@ -4240,6 +4240,7 @@ ssl_setup_connection_params(rpc_transport_t *this) + char *cipher_list = DEFAULT_CIPHER_LIST; + char *dh_param = DEFAULT_DH_PARAM; + char *ec_curve = DEFAULT_EC_CURVE; ++ gf_boolean_t dh_flag = _gf_false; + + priv = this->private; + +@@ -4248,6 +4249,10 @@ ssl_setup_connection_params(rpc_transport_t *this) + return 0; + } + ++ if (!priv->ssl_enabled && !priv->mgmt_ssl) { ++ return 0; ++ } ++ + priv->ssl_own_cert = DEFAULT_CERT_PATH; + if (dict_get_str(this->options, SSL_OWN_CERT_OPT, &optstr) == 0) { + if (!priv->ssl_enabled) { +@@ -4294,27 +4299,25 @@ ssl_setup_connection_params(rpc_transport_t *this) + priv->crl_path = gf_strdup(optstr); + } + +- gf_log(this->name, priv->ssl_enabled ? GF_LOG_INFO : GF_LOG_DEBUG, +- "SSL support on the I/O path is %s", +- priv->ssl_enabled ? "ENABLED" : "NOT enabled"); +- gf_log(this->name, priv->mgmt_ssl ? GF_LOG_INFO : GF_LOG_DEBUG, +- "SSL support for glusterd is %s", +- priv->mgmt_ssl ? "ENABLED" : "NOT enabled"); +- + if (!priv->mgmt_ssl) { +- if (!dict_get_int32(this->options, SSL_CERT_DEPTH_OPT, &cert_depth)) { +- gf_log(this->name, GF_LOG_INFO, "using certificate depth %d", +- cert_depth); ++ if (!dict_get_int32_sizen(this->options, SSL_CERT_DEPTH_OPT, ++ &cert_depth)) { + } + } else { + cert_depth = this->ctx->ssl_cert_depth; +- gf_log(this->name, GF_LOG_INFO, "using certificate depth %d", +- cert_depth); + } +- if (!dict_get_str(this->options, SSL_CIPHER_LIST_OPT, &cipher_list)) { ++ gf_log(this->name, priv->ssl_enabled ? GF_LOG_INFO : GF_LOG_DEBUG, ++ "SSL support for MGMT is %s IO path is %s certificate depth is %d " ++ "for peer %s", ++ (priv->mgmt_ssl ? "ENABLED" : "NOT enabled"), ++ (priv->ssl_enabled ? "ENABLED" : "NOT enabled"), cert_depth, ++ this->peerinfo.identifier); ++ ++ if (!dict_get_str_sizen(this->options, SSL_CIPHER_LIST_OPT, &cipher_list)) { + gf_log(this->name, GF_LOG_INFO, "using cipher list %s", cipher_list); + } +- if (!dict_get_str(this->options, SSL_DH_PARAM_OPT, &dh_param)) { ++ if (!dict_get_str_sizen(this->options, SSL_DH_PARAM_OPT, &dh_param)) { ++ dh_flag = _gf_true; + gf_log(this->name, GF_LOG_INFO, "using DH parameters %s", dh_param); + } + if (!dict_get_str(this->options, SSL_EC_CURVE_OPT, &ec_curve)) { +@@ -4349,12 +4352,15 @@ ssl_setup_connection_params(rpc_transport_t *this) + #ifdef SSL_OP_NO_COMPRESSION + SSL_CTX_set_options(priv->ssl_ctx, SSL_OP_NO_COMPRESSION); + #endif +- +- if ((bio = BIO_new_file(dh_param, "r")) == NULL) { +- gf_log(this->name, GF_LOG_INFO, +- "failed to open %s, " +- "DH ciphers are disabled", +- dh_param); ++ /* Upload file to bio wrapper only if dh param is configured ++ */ ++ if (dh_flag) { ++ if ((bio = BIO_new_file(dh_param, "r")) == NULL) { ++ gf_log(this->name, GF_LOG_ERROR, ++ "failed to open %s, " ++ "DH ciphers are disabled", ++ dh_param); ++ } + } + + if (bio != NULL) { +-- +1.8.3.1 + diff --git a/0361-snap_scheduler-python3-compatibility-and-new-test-ca.patch b/0361-snap_scheduler-python3-compatibility-and-new-test-ca.patch new file mode 100644 index 0000000..62b2fe0 --- /dev/null +++ b/0361-snap_scheduler-python3-compatibility-and-new-test-ca.patch @@ -0,0 +1,122 @@ +From 04b824ebfcf80c648d5855f10bc30fde45fd62eb Mon Sep 17 00:00:00 2001 +From: Sunny Kumar +Date: Thu, 26 Mar 2020 10:46:16 +0000 +Subject: [PATCH 361/362] snap_scheduler: python3 compatibility and new test + case +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Problem: +"snap_scheduler.py init" command failing with the below traceback: + +[root@dhcp43-104 ~]# snap_scheduler.py init +Traceback (most recent call last): + File "/usr/sbin/snap_scheduler.py", line 941, in + sys.exit(main(sys.argv[1:])) + File "/usr/sbin/snap_scheduler.py", line 851, in main + initLogger() + File "/usr/sbin/snap_scheduler.py", line 153, in initLogger + logfile = os.path.join(process.stdout.read()[:-1], SCRIPT_NAME + ".log") + File "/usr/lib64/python3.6/posixpath.py", line 94, in join + genericpath._check_arg_types('join', a, *p) + File "/usr/lib64/python3.6/genericpath.py", line 151, in _check_arg_types + raise TypeError("Can't mix strings and bytes in path components") from None +TypeError: Can't mix strings and bytes in path components + +Solution: + +Added the 'universal_newlines' flag to Popen to support backward compatibility. + +Added a basic test for snapshot scheduler. + +Backport Of: +    +        >Upstream Patch: https://review.gluster.org/#/c/glusterfs/+/24257/ +        >Change-Id: I78e8fabd866fd96638747ecd21d292f5ca074a4e +        >Fixes: #1134 +        >Signed-off-by: Sunny Kumar +    +BUG: 1817369 +Change-Id: I78e8fabd866fd96638747ecd21d292f5ca074a4e +Signed-off-by: Sunny Kumar +Reviewed-on: https://code.engineering.redhat.com/gerrit/196482 +Tested-by: RHGS Build Bot +Reviewed-by: Sunil Kumar Heggodu Gopala Acharya +--- + extras/snap_scheduler/snap_scheduler.py | 2 +- + tests/basic/volume-snap-scheduler.t | 49 +++++++++++++++++++++++++++++++++ + 2 files changed, 50 insertions(+), 1 deletion(-) + create mode 100644 tests/basic/volume-snap-scheduler.t + +diff --git a/extras/snap_scheduler/snap_scheduler.py b/extras/snap_scheduler/snap_scheduler.py +index a66c5e3..5a29d41 100755 +--- a/extras/snap_scheduler/snap_scheduler.py ++++ b/extras/snap_scheduler/snap_scheduler.py +@@ -149,7 +149,7 @@ def initLogger(): + sh.setFormatter(formatter) + + process = subprocess.Popen(["gluster", "--print-logdir"], +- stdout=subprocess.PIPE) ++ stdout=subprocess.PIPE, universal_newlines=True) + logfile = os.path.join(process.stdout.read()[:-1], SCRIPT_NAME + ".log") + + fh = logging.FileHandler(logfile) +diff --git a/tests/basic/volume-snap-scheduler.t b/tests/basic/volume-snap-scheduler.t +new file mode 100644 +index 0000000..a638c5c +--- /dev/null ++++ b/tests/basic/volume-snap-scheduler.t +@@ -0,0 +1,49 @@ ++#!/bin/bash ++ ++. $(dirname $0)/../include.rc ++. $(dirname $0)/../volume.rc ++ ++cleanup; ++ ++TEST glusterd; ++TEST pidof glusterd; ++ ++TEST $CLI volume create $V0 replica 2 $H0:$B0/${GMV0}{1,2,3,4}; ++TEST $CLI volume start $V0 ++ ++## Create, start and mount meta_volume as ++## snap_scheduler expects shared storage to be enabled. ++## This test is very basic in nature not creating any snapshot ++## and purpose is to validate snap scheduling commands. ++ ++TEST $CLI volume create $META_VOL replica 3 $H0:$B0/${META_VOL}{1,2,3}; ++TEST $CLI volume start $META_VOL ++TEST mkdir -p $META_MNT ++TEST glusterfs -s $H0 --volfile-id $META_VOL $META_MNT ++ ++##function to check status ++function check_status_scheduler() ++{ ++ local key=$1 ++ snap_scheduler.py status | grep -F "$key" | wc -l ++} ++ ++##Basic snap_scheduler command test init/enable/disable/list ++ ++TEST snap_scheduler.py init ++ ++TEST snap_scheduler.py enable ++ ++EXPECT 1 check_status_scheduler "Enabled" ++ ++TEST snap_scheduler.py disable ++ ++EXPECT 1 check_status_scheduler "Disabled" ++ ++TEST snap_scheduler.py list ++ ++TEST $CLI volume stop $V0; ++ ++TEST $CLI volume delete $V0; ++ ++cleanup; +-- +1.8.3.1 + diff --git a/0362-write-behind-fix-data-corruption.patch b/0362-write-behind-fix-data-corruption.patch new file mode 100644 index 0000000..aeb7242 --- /dev/null +++ b/0362-write-behind-fix-data-corruption.patch @@ -0,0 +1,454 @@ +From 48f6929590157d9a1697e11c02441207afdc1bed Mon Sep 17 00:00:00 2001 +From: Xavi Hernandez +Date: Fri, 27 Mar 2020 23:56:15 +0100 +Subject: [PATCH 362/362] write-behind: fix data corruption + +There was a bug in write-behind that allowed a previous completed write +to overwrite the overlapping region of data from a future write. + +Suppose we want to send three writes (W1, W2 and W3). W1 and W2 are +sequential, and W3 writes at the same offset of W2: + + W2.offset = W3.offset = W1.offset + W1.size + +Both W1 and W2 are sent in parallel. W3 is only sent after W2 completes. +So W3 should *always* overwrite the overlapping part of W2. + +Suppose write-behind processes the requests from 2 concurrent threads: + + Thread 1 Thread 2 + + + + wb_enqueue_tempted(W1) + /* W1 is assigned gen X */ + wb_enqueue_tempted(W2) + /* W2 is assigned gen X */ + + wb_process_queue() + __wb_preprocess_winds() + /* W1 and W2 are sequential and all + * other requisites are met to merge + * both requests. */ + __wb_collapse_small_writes(W1, W2) + __wb_fulfill_request(W2) + + __wb_pick_unwinds() -> W2 + /* In this case, since the request is + * already fulfilled, wb_inode->gen + * is not updated. */ + + wb_do_unwinds() + STACK_UNWIND(W2) + + /* The application has received the + * result of W2, so it can send W3. */ + + + wb_enqueue_tempted(W3) + /* W3 is assigned gen X */ + + wb_process_queue() + /* Here we have W1 (which contains + * the conflicting W2) and W3 with + * same gen, so they are interpreted + * as concurrent writes that do not + * conflict. */ + __wb_pick_winds() -> W3 + + wb_do_winds() + STACK_WIND(W3) + + wb_process_queue() + /* Eventually W1 will be + * ready to be sent */ + __wb_pick_winds() -> W1 + __wb_pick_unwinds() -> W1 + /* Here wb_inode->gen is + * incremented. */ + + wb_do_unwinds() + STACK_UNWIND(W1) + + wb_do_winds() + STACK_WIND(W1) + +So, as we can see, W3 is sent before W1, which shouldn't happen. + +The problem is that wb_inode->gen is only incremented for requests that +have not been fulfilled but, after a merge, the request is marked as +fulfilled even though it has not been sent to the brick. This allows +that future requests are assigned to the same generation, which could +be internally reordered. + +Solution: + +Increment wb_inode->gen before any unwind, even if it's for a fulfilled +request. + +Special thanks to Stefan Ring for writing a reproducer that has been +crucial to identify the issue. + +Upstream patch: +> Upstream patch link: https://review.gluster.org/c/glusterfs/+/24263 +> Change-Id: Id4ab0f294a09aca9a863ecaeef8856474662ab45 +> Signed-off-by: Xavi Hernandez +> Fixes: #884 + +Change-Id: Id4ab0f294a09aca9a863ecaeef8856474662ab45 +Signed-off-by: Xavi Hernandez +BUG: 1819059 +Reviewed-on: https://code.engineering.redhat.com/gerrit/196250 +Tested-by: RHGS Build Bot +Reviewed-by: Sunil Kumar Heggodu Gopala Acharya +--- + tests/bugs/write-behind/issue-884.c | 267 +++++++++++++++++++++ + tests/bugs/write-behind/issue-884.t | 40 +++ + .../performance/write-behind/src/write-behind.c | 4 +- + 3 files changed, 309 insertions(+), 2 deletions(-) + create mode 100644 tests/bugs/write-behind/issue-884.c + create mode 100755 tests/bugs/write-behind/issue-884.t + +diff --git a/tests/bugs/write-behind/issue-884.c b/tests/bugs/write-behind/issue-884.c +new file mode 100644 +index 0000000..e9c33b3 +--- /dev/null ++++ b/tests/bugs/write-behind/issue-884.c +@@ -0,0 +1,267 @@ ++ ++#define _GNU_SOURCE ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++/* Based on a reproducer by Stefan Ring. It seems to be quite sensible to any ++ * timing modification, so the code has been maintained as is, only with minor ++ * changes. */ ++ ++struct glfs *glfs; ++ ++pthread_mutex_t the_mutex = PTHREAD_MUTEX_INITIALIZER; ++pthread_cond_t the_cond = PTHREAD_COND_INITIALIZER; ++ ++typedef struct _my_aiocb { ++ int64_t size; ++ volatile int64_t seq; ++ int which; ++} my_aiocb; ++ ++typedef struct _worker_data { ++ my_aiocb cb; ++ struct iovec iov; ++ int64_t offset; ++} worker_data; ++ ++typedef struct { ++ worker_data wdata[2]; ++ ++ volatile unsigned busy; ++} all_data_t; ++ ++all_data_t all_data; ++ ++static void ++completion_fnc(struct glfs_fd *fd, ssize_t ret, struct glfs_stat *pre, ++ struct glfs_stat *post, void *arg) ++{ ++ void *the_thread; ++ my_aiocb *cb = (my_aiocb *)arg; ++ long seq = cb->seq; ++ ++ assert(ret == cb->size); ++ ++ pthread_mutex_lock(&the_mutex); ++ pthread_cond_broadcast(&the_cond); ++ ++ all_data.busy &= ~(1 << cb->which); ++ cb->seq = -1; ++ ++ the_thread = (void *)pthread_self(); ++ printf("worker %d is done from thread %p, seq %ld!\n", cb->which, ++ the_thread, seq); ++ ++ pthread_mutex_unlock(&the_mutex); ++} ++ ++static void ++init_wdata(worker_data *data, int which) ++{ ++ data->cb.which = which; ++ data->cb.seq = -1; ++ ++ data->iov.iov_base = malloc(1024 * 1024); ++ memset(data->iov.iov_base, 6, ++ 1024 * 1024); /* tail part never overwritten */ ++} ++ ++static void ++init() ++{ ++ all_data.busy = 0; ++ ++ init_wdata(&all_data.wdata[0], 0); ++ init_wdata(&all_data.wdata[1], 1); ++} ++ ++static void ++do_write(struct glfs_fd *fd, int content, int size, int64_t seq, ++ worker_data *wdata, const char *name) ++{ ++ int ret; ++ ++ wdata->cb.size = size; ++ wdata->cb.seq = seq; ++ ++ if (content >= 0) ++ memset(wdata->iov.iov_base, content, size); ++ wdata->iov.iov_len = size; ++ ++ pthread_mutex_lock(&the_mutex); ++ printf("(%d) dispatching write \"%s\", offset %lx, len %x, seq %ld\n", ++ wdata->cb.which, name, (long)wdata->offset, size, (long)seq); ++ pthread_mutex_unlock(&the_mutex); ++ ret = glfs_pwritev_async(fd, &wdata->iov, 1, wdata->offset, 0, ++ completion_fnc, &wdata->cb); ++ assert(ret >= 0); ++} ++ ++#define IDLE 0 // both workers must be idle ++#define ANY 1 // use any worker, other one may be busy ++ ++int ++get_worker(int waitfor, int64_t excl_seq) ++{ ++ int which; ++ ++ pthread_mutex_lock(&the_mutex); ++ ++ while (waitfor == IDLE && (all_data.busy & 3) != 0 || ++ waitfor == ANY && ++ ((all_data.busy & 3) == 3 || ++ excl_seq >= 0 && (all_data.wdata[0].cb.seq == excl_seq || ++ all_data.wdata[1].cb.seq == excl_seq))) ++ pthread_cond_wait(&the_cond, &the_mutex); ++ ++ if (!(all_data.busy & 1)) ++ which = 0; ++ else ++ which = 1; ++ ++ all_data.busy |= (1 << which); ++ ++ pthread_mutex_unlock(&the_mutex); ++ ++ return which; ++} ++ ++static int ++doit(struct glfs_fd *fd) ++{ ++ int ret; ++ int64_t seq = 0; ++ int64_t offset = 0; // position in file, in blocks ++ int64_t base = 0x1000; // where to place the data, in blocks ++ ++ int async_mode = ANY; ++ ++ init(); ++ ++ for (;;) { ++ int which; ++ worker_data *wdata; ++ ++ // for growing to the first offset ++ for (;;) { ++ int gap = base + 0x42 - offset; ++ if (!gap) ++ break; ++ if (gap > 80) ++ gap = 80; ++ ++ which = get_worker(IDLE, -1); ++ wdata = &all_data.wdata[which]; ++ ++ wdata->offset = offset << 9; ++ do_write(fd, 0, gap << 9, seq++, wdata, "gap-filling"); ++ ++ offset += gap; ++ } ++ ++ // 8700 ++ which = get_worker(IDLE, -1); ++ wdata = &all_data.wdata[which]; ++ ++ wdata->offset = (base + 0x42) << 9; ++ do_write(fd, 1, 62 << 9, seq++, wdata, "!8700"); ++ ++ // 8701 ++ which = get_worker(IDLE, -1); ++ wdata = &all_data.wdata[which]; ++ ++ wdata->offset = (base + 0x42) << 9; ++ do_write(fd, 2, 55 << 9, seq++, wdata, "!8701"); ++ ++ // 8702 ++ which = get_worker(async_mode, -1); ++ wdata = &all_data.wdata[which]; ++ ++ wdata->offset = (base + 0x79) << 9; ++ do_write(fd, 3, 54 << 9, seq++, wdata, "!8702"); ++ ++ // 8703 ++ which = get_worker(async_mode, -1); ++ wdata = &all_data.wdata[which]; ++ ++ wdata->offset = (base + 0xaf) << 9; ++ do_write(fd, 4, 81 << 9, seq++, wdata, "!8703"); ++ ++ // 8704 ++ // this writes both 5s and 6s ++ // the range of 5s is the one that overwrites 8703 ++ ++ which = get_worker(async_mode, seq - 1); ++ wdata = &all_data.wdata[which]; ++ ++ memset(wdata->iov.iov_base, 5, 81 << 9); ++ wdata->offset = (base + 0xaf) << 9; ++ do_write(fd, -1, 1623 << 9, seq++, wdata, "!8704"); ++ ++ offset = base + 0x706; ++ base += 0x1000; ++ if (base >= 0x100000) ++ break; ++ } ++ ++ printf("done!\n"); ++ fflush(stdout); ++ ++ pthread_mutex_lock(&the_mutex); ++ ++ while ((all_data.busy & 3) != 0) ++ pthread_cond_wait(&the_cond, &the_mutex); ++ ++ pthread_mutex_unlock(&the_mutex); ++ ++ ret = glfs_close(fd); ++ assert(ret >= 0); ++ /* ++ ret = glfs_fini(glfs); ++ assert(ret >= 0); ++ */ ++ return 0; ++} ++ ++int ++main(int argc, char *argv[]) ++{ ++ int ret; ++ int open_flags = O_RDWR | O_DIRECT | O_TRUNC; ++ struct glfs_fd *fd; ++ ++ glfs = glfs_new(argv[1]); ++ if (!glfs) { ++ printf("glfs_new!\n"); ++ goto out; ++ } ++ ret = glfs_set_volfile_server(glfs, "tcp", "localhost", 24007); ++ if (ret < 0) { ++ printf("set_volfile!\n"); ++ goto out; ++ } ++ ret = glfs_init(glfs); ++ if (ret) { ++ printf("init!\n"); ++ goto out; ++ } ++ fd = glfs_open(glfs, argv[2], open_flags); ++ if (!fd) { ++ printf("open!\n"); ++ goto out; ++ } ++ srand(time(NULL)); ++ return doit(fd); ++out: ++ return 1; ++} +diff --git a/tests/bugs/write-behind/issue-884.t b/tests/bugs/write-behind/issue-884.t +new file mode 100755 +index 0000000..2bcf7d1 +--- /dev/null ++++ b/tests/bugs/write-behind/issue-884.t +@@ -0,0 +1,40 @@ ++#!/bin/bash ++ ++. $(dirname $0)/../../include.rc ++. $(dirname $0)/../../volume.rc ++ ++# This test tries to detect a race condition in write-behind. It's based on a ++# reproducer written by Stefan Ring that is able to hit it sometimes. On my ++# system, it happened around 10% of the runs. This means that if this bug ++# appears again, this test will fail once every 10 runs. Most probably this ++# failure will be hidden by the automatic test retry of the testing framework. ++# ++# Please, if this test fails, it needs to be analyzed in detail. ++ ++function run() { ++ "${@}" >/dev/null ++} ++ ++cleanup ++ ++TEST glusterd ++TEST pidof glusterd ++ ++TEST $CLI volume create $V0 $H0:$B0/$V0 ++# This makes it easier to hit the issue ++TEST $CLI volume set $V0 client-log-level TRACE ++TEST $CLI volume start $V0 ++ ++TEST $GFS --volfile-server=$H0 --volfile-id=$V0 $M0 ++ ++build_tester $(dirname $0)/issue-884.c -lgfapi ++ ++TEST touch $M0/testfile ++ ++# This program generates a file of 535694336 bytes with a fixed pattern ++TEST run $(dirname $0)/issue-884 $V0 testfile ++ ++# This is the md5sum of the expected pattern without corruption ++EXPECT "ad105f9349345a70fc697632cbb5eec8" echo "$(md5sum $B0/$V0/testfile | awk '{ print $1; }')" ++ ++cleanup +diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c +index 70e281a..90a0bcf 100644 +--- a/xlators/performance/write-behind/src/write-behind.c ++++ b/xlators/performance/write-behind/src/write-behind.c +@@ -1284,14 +1284,14 @@ __wb_pick_unwinds(wb_inode_t *wb_inode, list_head_t *lies) + + wb_inode->window_current += req->orig_size; + ++ wb_inode->gen++; ++ + if (!req->ordering.fulfilled) { + /* burden increased */ + list_add_tail(&req->lie, &wb_inode->liability); + + req->ordering.lied = 1; + +- wb_inode->gen++; +- + uuid_utoa_r(req->gfid, gfid); + gf_msg_debug(wb_inode->this->name, 0, + "(unique=%" PRIu64 +-- +1.8.3.1 + diff --git a/glusterfs.spec b/glusterfs.spec index 67de5de..ba4e559 100644 --- a/glusterfs.spec +++ b/glusterfs.spec @@ -231,7 +231,7 @@ Release: 0.1%{?prereltag:.%{prereltag}}%{?dist} %else Name: glusterfs Version: 6.0 -Release: 31%{?dist} +Release: 32%{?dist} ExcludeArch: i686 %endif License: GPLv2 or LGPLv3+ @@ -666,6 +666,11 @@ Patch0354: 0354-core-fix-memory-pool-management-races.patch Patch0355: 0355-core-Prevent-crash-on-process-termination.patch Patch0356: 0356-Update-rfc.sh-to-rhgs-3.5.1-rhel-8.patch Patch0357: 0357-ganesha-ha-updates-for-pcs-0.10.x-i.e.-in-Fedora-29-.patch +Patch0358: 0358-inode-fix-wrong-loop-count-in-__inode_ctx_free.patch +Patch0359: 0359-dht-gf_defrag_process_dir-is-called-even-if-gf_defra.patch +Patch0360: 0360-rpc-Make-ssl-log-more-useful.patch +Patch0361: 0361-snap_scheduler-python3-compatibility-and-new-test-ca.patch +Patch0362: 0362-write-behind-fix-data-corruption.patch %description GlusterFS is a distributed file-system capable of scaling to several @@ -2395,6 +2400,9 @@ fi %endif %changelog +* Sat Apr 04 2020 Rinku Kothiya - 6.0-32 +- fixes bugs bz#1781543 bz#1812789 bz#1812824 bz#1817369 bz#1819059 + * Tue Mar 17 2020 Rinku Kothiya - 6.0-31 - fixes bugs bz#1802727