glusterfs/0506-posix-Attach-a-posix_spawn_disk_thread-with-glusterf.patch

From 17a9ce965ef2fec9ee5c8e4b76981bb7cbcf1352 Mon Sep 17 00:00:00 2001
From: mohit84 <moagrawa@redhat.com>
Date: Mon, 9 Nov 2020 17:15:42 +0530
Subject: [PATCH 506/511] posix: Attach a posix_spawn_disk_thread with
 glusterfs_ctx (#1595)

Currently posix xlator spawns posix_disk_space_threads per brick and in
case of brick_mux environment while glusterd attached bricks at maximum
level(250) with a single brick process in that case 250 threads are
spawned for all bricks and brick process memory size also increased.

Solution: Attach a posix_disk_space thread with glusterfs_ctx to
          spawn a thread per process basis instead of spawning a per brick

> Fixes: #1482
> Change-Id: I8dd88f252a950495b71742e2a7588bd5bb019ec7
> Cherry-picked from commit 3f93be77e1acf5baacafa97a320e91e6879d1c0e
> Reviewed on upstream link https://github.com/gluster/glusterfs/issues/1482
> Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>

Change-Id: I8dd88f252a950495b71742e2a7588bd5bb019ec7
Bug: 1898776
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/220366
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
---
 glusterfsd/src/glusterfsd.c                    |   4 +
 libglusterfs/src/glusterfs/glusterfs.h         |   6 ++
 xlators/storage/posix/src/posix-common.c       |  68 +++++++++++--
 xlators/storage/posix/src/posix-handle.h       |   3 +-
 xlators/storage/posix/src/posix-helpers.c      | 131 ++++++++++++++-----------
 xlators/storage/posix/src/posix-inode-fd-ops.c |   3 +-
 xlators/storage/posix/src/posix-mem-types.h    |   1 +
 xlators/storage/posix/src/posix.h              |  12 ++-
 8 files changed, 160 insertions(+), 68 deletions(-)

diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 955bf1d..ac25255 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -1840,9 +1840,13 @@ glusterfs_ctx_defaults_init(glusterfs_ctx_t *ctx)
     INIT_LIST_HEAD(&cmd_args->xlator_options);
     INIT_LIST_HEAD(&cmd_args->volfile_servers);
     ctx->pxl_count = 0;
+    ctx->diskxl_count = 0;
     pthread_mutex_init(&ctx->fd_lock, NULL);
     pthread_cond_init(&ctx->fd_cond, NULL);
     INIT_LIST_HEAD(&ctx->janitor_fds);
+    pthread_mutex_init(&ctx->xl_lock, NULL);
+    pthread_cond_init(&ctx->xl_cond, NULL);
+    INIT_LIST_HEAD(&ctx->diskth_xl);
 
     lim.rlim_cur = RLIM_INFINITY;
     lim.rlim_max = RLIM_INFINITY;
diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h
index bf6a987..d3400bf 100644
--- a/libglusterfs/src/glusterfs/glusterfs.h
+++ b/libglusterfs/src/glusterfs/glusterfs.h
@@ -740,7 +740,13 @@ struct _glusterfs_ctx {
     pthread_t janitor;
     /* The variable is use to save total posix xlator count */
     uint32_t pxl_count;
+    uint32_t diskxl_count;
 
+    /* List of posix xlator use by disk thread*/
+    struct list_head diskth_xl;
+    pthread_mutex_t xl_lock;
+    pthread_cond_t xl_cond;
+    pthread_t disk_space_check;
     char volume_id[GF_UUID_BUF_SIZE]; /* Used only in protocol/client */
 };
 typedef struct _glusterfs_ctx glusterfs_ctx_t;
diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c
index e5c6e62..2c9030b 100644
--- a/xlators/storage/posix/src/posix-common.c
+++ b/xlators/storage/posix/src/posix-common.c
@@ -138,6 +138,36 @@ posix_inode(xlator_t *this)
     return 0;
 }
 
+static void
+delete_posix_diskxl(xlator_t *this)
+{
+    struct posix_private *priv = this->private;
+    struct posix_diskxl *pxl = priv->pxl;
+    glusterfs_ctx_t *ctx = this->ctx;
+    uint32_t count = 1;
+
+    if (pxl) {
+        pthread_mutex_lock(&ctx->xl_lock);
+        {
+            pxl->detach_notify = _gf_true;
+            while (pxl->is_use)
+                pthread_cond_wait(&pxl->cond, &ctx->xl_lock);
+            list_del_init(&pxl->list);
+            priv->pxl = NULL;
+            count = --ctx->diskxl_count;
+            if (count == 0)
+                pthread_cond_signal(&ctx->xl_cond);
+        }
+        pthread_mutex_unlock(&ctx->xl_lock);
+        pthread_cond_destroy(&pxl->cond);
+        GF_FREE(pxl);
+        if (count == 0) {
+            pthread_join(ctx->disk_space_check, NULL);
+            ctx->disk_space_check = 0;
+        }
+    }
+}
+
 /**
  * notify - when parent sends PARENT_UP, send CHILD_UP event from here
  */
@@ -194,6 +224,8 @@ posix_notify(xlator_t *this, int32_t event, void *data, ...)
             }
             pthread_mutex_unlock(&ctx->fd_lock);
 
+            delete_posix_diskxl(this);
+
             gf_log(this->name, GF_LOG_INFO, "Sending CHILD_DOWN for brick %s",
                    victim->name);
             default_notify(this->parents->xlator, GF_EVENT_CHILD_DOWN, data);
@@ -318,6 +350,7 @@ posix_reconfigure(xlator_t *this, dict_t *options)
     int32_t force_directory_mode = -1;
     int32_t create_mask = -1;
     int32_t create_directory_mask = -1;
+    double old_disk_reserve = 0.0;
 
     priv = this->private;
 
@@ -383,6 +416,7 @@ posix_reconfigure(xlator_t *this, dict_t *options)
                " fallback to <hostname>:<export>");
     }
 
+    old_disk_reserve = priv->disk_reserve;
     GF_OPTION_RECONF("reserve", priv->disk_reserve, options, percent_or_size,
                      out);
     /* option can be any one of percent or bytes */
@@ -390,11 +424,19 @@ posix_reconfigure(xlator_t *this, dict_t *options)
     if (priv->disk_reserve < 100.0)
         priv->disk_unit = 'p';
 
-    if (priv->disk_reserve) {
+    /* Delete a pxl object from a list of disk_reserve while something
+       is changed for reserve option during graph reconfigure
+    */
+    if (old_disk_reserve != priv->disk_reserve) {
+        delete_posix_diskxl(this);
+        old_disk_reserve = 0;
+    }
+
+    if (!old_disk_reserve && priv->disk_reserve) {
         ret = posix_spawn_disk_space_check_thread(this);
         if (ret) {
             gf_msg(this->name, GF_LOG_INFO, 0, P_MSG_DISK_SPACE_CHECK_FAILED,
-                   "Getting disk space check from thread failed");
+                   "Getting disk space check from thread failed ");
             goto out;
         }
     }
@@ -1008,13 +1050,13 @@ posix_init(xlator_t *this)
                " fallback to <hostname>:<export>");
     }
 
-    _private->disk_space_check_active = _gf_false;
     _private->disk_space_full = 0;
 
     GF_OPTION_INIT("reserve", _private->disk_reserve, percent_or_size, out);
 
     /* option can be any one of percent or bytes */
     _private->disk_unit = 0;
+    pthread_cond_init(&_private->fd_cond, NULL);
     if (_private->disk_reserve < 100.0)
         _private->disk_unit = 'p';
 
@@ -1162,12 +1204,6 @@ posix_fini(xlator_t *this)
         priv->health_check = 0;
     }
 
-    if (priv->disk_space_check) {
-        priv->disk_space_check_active = _gf_false;
-        (void)gf_thread_cleanup_xint(priv->disk_space_check);
-        priv->disk_space_check = 0;
-    }
-
     if (priv->janitor) {
         /*TODO: Make sure the synctask is also complete */
         ret = gf_tw_del_timer(this->ctx->tw->timer_wheel, priv->janitor);
@@ -1192,10 +1228,24 @@ posix_fini(xlator_t *this)
         pthread_join(ctx->janitor, NULL);
     }
 
+    pthread_mutex_lock(&ctx->xl_lock);
+    {
+        count = --ctx->diskxl_count;
+        if (count == 0)
+            pthread_cond_signal(&ctx->xl_cond);
+    }
+    pthread_mutex_unlock(&ctx->xl_lock);
+
+    if (count == 0) {
+        pthread_join(ctx->disk_space_check, NULL);
+        ctx->disk_space_check = 0;
+    }
+
     if (priv->fsyncer) {
         (void)gf_thread_cleanup_xint(priv->fsyncer);
         priv->fsyncer = 0;
     }
+
     /*unlock brick dir*/
     if (priv->mount_lock)
         (void)sys_closedir(priv->mount_lock);
diff --git a/xlators/storage/posix/src/posix-handle.h b/xlators/storage/posix/src/posix-handle.h
index c4d7cb1..8e4c719 100644
--- a/xlators/storage/posix/src/posix-handle.h
+++ b/xlators/storage/posix/src/posix-handle.h
@@ -206,5 +206,6 @@ int
 posix_check_internal_writes(xlator_t *this, fd_t *fd, int sysfd, dict_t *xdata);
 
 void
-posix_disk_space_check(xlator_t *this);
+posix_disk_space_check(struct posix_private* priv);
+
 #endif /* !_POSIX_HANDLE_H */
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index ceac52a..110d383 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -2284,9 +2284,8 @@ unlock:
 }
 
 void
-posix_disk_space_check(xlator_t *this)
+posix_disk_space_check(struct posix_private *priv)
 {
-    struct posix_private *priv = NULL;
     char *subvol_path = NULL;
     int op_ret = 0;
     double size = 0;
@@ -2295,16 +2294,14 @@ posix_disk_space_check(xlator_t *this)
     double totsz = 0;
     double freesz = 0;
 
-    GF_VALIDATE_OR_GOTO(this->name, this, out);
-    priv = this->private;
-    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+    GF_VALIDATE_OR_GOTO("posix-helpers", priv, out);
 
     subvol_path = priv->base_path;
 
     op_ret = sys_statvfs(subvol_path, &buf);
 
     if (op_ret == -1) {
-        gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_STATVFS_FAILED,
+        gf_msg("posix-disk", GF_LOG_ERROR, errno, P_MSG_STATVFS_FAILED,
                "statvfs failed on %s", subvol_path);
         goto out;
     }
@@ -2328,78 +2325,102 @@ out:
 }
 
 static void *
-posix_disk_space_check_thread_proc(void *data)
+posix_ctx_disk_thread_proc(void *data)
 {
-    xlator_t *this = NULL;
     struct posix_private *priv = NULL;
+    glusterfs_ctx_t *ctx = NULL;
     uint32_t interval = 0;
-    int ret = -1;
-
-    this = data;
-    priv = this->private;
+    struct posix_diskxl *pthis = NULL;
+    xlator_t *this = NULL;
+    struct timespec sleep_till = {
+        0,
+    };
 
+    ctx = data;
     interval = 5;
-    gf_msg_debug(this->name, 0,
-                 "disk-space thread started, "
+
+    gf_msg_debug("glusterfs_ctx", 0,
+                 "Ctx disk-space thread started, "
                  "interval = %d seconds",
                  interval);
-    while (1) {
-        /* aborting sleep() is a request to exit this thread, sleep()
-         * will normally not return when cancelled */
-        ret = sleep(interval);
-        if (ret > 0)
-            break;
-        /* prevent thread errors while doing the health-check(s) */
-        pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
-
-        /* Do the disk-check.*/
-        posix_disk_space_check(this);
-        if (!priv->disk_space_check_active)
-            goto out;
-        pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
-    }
 
-out:
-    gf_msg_debug(this->name, 0, "disk space check thread exiting");
-    LOCK(&priv->lock);
+    pthread_mutex_lock(&ctx->xl_lock);
     {
-        priv->disk_space_check_active = _gf_false;
+        while (ctx->diskxl_count > 0) {
+            list_for_each_entry(pthis, &ctx->diskth_xl, list)
+            {
+                pthis->is_use = _gf_true;
+                pthread_mutex_unlock(&ctx->xl_lock);
+
+                THIS = this = pthis->xl;
+                priv = this->private;
+
+                posix_disk_space_check(priv);
+
+                pthread_mutex_lock(&ctx->xl_lock);
+                pthis->is_use = _gf_false;
+                /* Send a signal to posix_notify function */
+                if (pthis->detach_notify)
+                    pthread_cond_signal(&pthis->cond);
+            }
+
+            timespec_now_realtime(&sleep_till);
+            sleep_till.tv_sec += 5;
+            (void)pthread_cond_timedwait(&ctx->xl_cond, &ctx->xl_lock,
+                                         &sleep_till);
+        }
     }
-    UNLOCK(&priv->lock);
+    pthread_mutex_unlock(&ctx->xl_lock);
 
     return NULL;
 }
 
 int
-posix_spawn_disk_space_check_thread(xlator_t *xl)
+posix_spawn_disk_space_check_thread(xlator_t *this)
 {
-    struct posix_private *priv = NULL;
-    int ret = -1;
+    int ret = 0;
+    glusterfs_ctx_t *ctx = this->ctx;
+    struct posix_diskxl *pxl = NULL;
+    struct posix_private *priv = this->private;
 
-    priv = xl->private;
+    pxl = GF_CALLOC(1, sizeof(struct posix_diskxl), gf_posix_mt_diskxl_t);
+    if (!pxl) {
+        ret = -ENOMEM;
+        gf_log(this->name, GF_LOG_ERROR,
+               "Calloc is failed to allocate "
+               "memory for diskxl object");
+        goto out;
+    }
+    pthread_cond_init(&pxl->cond, NULL);
 
-    LOCK(&priv->lock);
+    pthread_mutex_lock(&ctx->xl_lock);
     {
-        /* cancel the running thread  */
-        if (priv->disk_space_check_active == _gf_true) {
-            pthread_cancel(priv->disk_space_check);
-            priv->disk_space_check_active = _gf_false;
-        }
+        if (ctx->diskxl_count++ == 0) {
+            ret = gf_thread_create(&ctx->disk_space_check, NULL,
+                                   posix_ctx_disk_thread_proc, ctx,
+                                   "posixctxres");
 
-        ret = gf_thread_create(&priv->disk_space_check, NULL,
-                               posix_disk_space_check_thread_proc, xl,
-                               "posix_reserve");
-        if (ret) {
-            priv->disk_space_check_active = _gf_false;
-            gf_msg(xl->name, GF_LOG_ERROR, errno, P_MSG_DISK_SPACE_CHECK_FAILED,
-                   "unable to setup disk space check thread");
-            goto unlock;
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED,
+                       "spawning disk space check thread failed");
+                ctx->diskxl_count--;
+                pthread_mutex_unlock(&ctx->xl_lock);
+                goto out;
+            }
         }
+        pxl->xl = this;
+        priv->pxl = (void *)pxl;
+        list_add_tail(&pxl->list, &ctx->diskth_xl);
+    }
+    pthread_mutex_unlock(&ctx->xl_lock);
 
-        priv->disk_space_check_active = _gf_true;
+out:
+    if (ret) {
+        if (pxl) {
+            pthread_cond_destroy(&pxl->cond);
+            GF_FREE(pxl);
+        }
     }
-unlock:
-    UNLOCK(&priv->lock);
     return ret;
 }
 
diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c
index 1d37aed..761e018 100644
--- a/xlators/storage/posix/src/posix-inode-fd-ops.c
+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c
@@ -37,6 +37,7 @@
 #include <fcntl.h>
 #endif /* HAVE_LINKAT */
 
+#include "posix-handle.h"
 #include <glusterfs/glusterfs.h>
 #include <glusterfs/checksum.h>
 #include <glusterfs/dict.h>
@@ -713,7 +714,7 @@ posix_do_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
        option behaviour
     */
     if (priv->disk_reserve)
-        posix_disk_space_check(this);
+        posix_disk_space_check(priv);
 
     DISK_SPACE_CHECK_AND_GOTO(frame, priv, xdata, ret, ret, unlock);
 
diff --git a/xlators/storage/posix/src/posix-mem-types.h b/xlators/storage/posix/src/posix-mem-types.h
index 2253f38..bb4c56d 100644
--- a/xlators/storage/posix/src/posix-mem-types.h
+++ b/xlators/storage/posix/src/posix-mem-types.h
@@ -20,6 +20,7 @@ enum gf_posix_mem_types_ {
     gf_posix_mt_paiocb,
     gf_posix_mt_inode_ctx_t,
     gf_posix_mt_mdata_attr,
+    gf_posix_mt_diskxl_t,
     gf_posix_mt_end
 };
 #endif
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index 07f367b..4be979c 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -36,7 +36,6 @@
 #include <glusterfs/compat.h>
 #include <glusterfs/timer.h>
 #include "posix-mem-types.h"
-#include "posix-handle.h"
 #include <glusterfs/call-stub.h>
 
 #ifdef HAVE_LIBAIO
@@ -138,6 +137,14 @@ struct posix_fd {
     char _pad[4]; /* manual padding */
 };
 
+struct posix_diskxl {
+    pthread_cond_t cond;
+    struct list_head list;
+    xlator_t *xl;
+    gf_boolean_t detach_notify;
+    gf_boolean_t is_use;
+};
+
 struct posix_private {
     char *base_path;
     int32_t base_path_length;
@@ -207,6 +214,7 @@ struct posix_private {
     pthread_mutex_t janitor_mutex;
     pthread_cond_t janitor_cond;
     pthread_cond_t fd_cond;
+    pthread_cond_t disk_cond;
     int fsync_queue_count;
 
     enum {
@@ -233,7 +241,6 @@ struct posix_private {
     char disk_unit;
     uint32_t disk_space_full;
     pthread_t disk_space_check;
-    gf_boolean_t disk_space_check_active;
 
 #ifdef GF_DARWIN_HOST_OS
     enum {
@@ -263,6 +270,7 @@ struct posix_private {
     gf_boolean_t ctime;
     gf_boolean_t janitor_task_stop;
     uint32_t rel_fdcount;
+    void *pxl;
 };
 
 typedef struct {
-- 
1.8.3.1