rpms/glusterfs
5
0
Fork 0
Code Issues Pull Requests Releases Activity

autobuild v6.0-27

Browse Source 
Resolves: bz#1789447
Signed-off-by: Rinku Kothiya <rkothiya@redhat.com>
This commit is contained in:
Rinku Kothiya 2020-01-13 05:44:03 -05:00
parent 61de5f4d9d
commit 9d5f2fefff
4 changed files with 412 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

236
0347-tools-glusterfind-handle-offline-bricks.patch Normal file
View File

@ -0,0 +1,236 @@
From 87e6ea2cd63898c5d243b0f0c719f4f6347fb829 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Thu, 5 Jan 2017 19:53:19 +0530
Subject: [PATCH 347/349] tools/glusterfind: handle offline bricks
Problem:
glusterfind is unable to copy remote output file to local node when a
remove-brick is in progress on the remote node. After copying remote
files, in the --full output listing path, a "sort -u" command is run on
the collected files. However, "sort" exits with an error code if it
finds any file missing.
Solution:
Maintain a map of (pid, output file) when the node commands are started
and remove the mapping for the pid for which the command returns an
error. Use the list of files present in the map for the "sort" command.
Backport of:
> Patch: https://review.gluster.org/16332
> Change-Id: Ie6e019037379f4cb163f24b1c65eb382efc2fb3b
> fixes: bz#1410439
> Signed-off-by: Milind Changire <mchangir@redhat.com>
> Signed-off-by: Shwetha K Acharya <sacharya@redhat.com>
BUG: 1789447
Change-Id: Ie6e019037379f4cb163f24b1c65eb382efc2fb3b
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/189214
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Sunny Kumar <sunkumar@redhat.com>
---
tools/glusterfind/src/gfind_py2py3.py | 25 ++++++++++++++
tools/glusterfind/src/main.py | 61 +++++++++++++++++++++--------------
2 files changed, 61 insertions(+), 25 deletions(-)
diff --git a/tools/glusterfind/src/gfind_py2py3.py b/tools/glusterfind/src/gfind_py2py3.py
index 1d41ec5..87324fb 100644
--- a/tools/glusterfind/src/gfind_py2py3.py
+++ b/tools/glusterfind/src/gfind_py2py3.py
@@ -40,6 +40,19 @@ if sys.version_info >= (3,):
def gfind_history_changelog_done(libgfc, clfile):
return libgfc.gf_history_changelog_done(clfile.encode())
+ def gfind_write_row(f, row, field_separator, p_rep, row_2_rep):
+ f.write(u"{0}{1}{2}{3}{4}\n".format(row,
+ field_separator,
+ p_rep,
+ field_separator,
+ row_2_rep))
+
+ def gfind_write(f, row, field_separator, p_rep):
+ f.write(u"{0}{1}{2}\n".format(row,
+ field_separator,
+ p_rep))
+
+
else:
# Raw conversion of bytearray to string
@@ -61,3 +74,15 @@ else:
def gfind_history_changelog_done(libgfc, clfile):
return libgfc.gf_history_changelog_done(clfile)
+
+ def gfind_write_row(f, row, field_separator, p_rep, row_2_rep):
+ f.write(u"{0}{1}{2}{3}{4}\n".format(row,
+ field_separator,
+ p_rep,
+ field_separator,
+ row_2_rep).encode())
+
+ def gfind_write(f, row, field_separator, p_rep):
+ f.write(u"{0}{1}{2}\n".format(row,
+ field_separator,
+ p_rep).encode())
diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py
index cc5a86f..fefe4a3 100644
--- a/tools/glusterfind/src/main.py
+++ b/tools/glusterfind/src/main.py
@@ -16,6 +16,7 @@ from multiprocessing import Process
import os
import xml.etree.cElementTree as etree
from argparse import ArgumentParser, RawDescriptionHelpFormatter, Action
+from gfind_py2py3 import gfind_write_row, gfind_write
import logging
import shutil
import tempfile
@@ -35,9 +36,9 @@ GlusterFS Incremental API
ParseError = etree.ParseError if hasattr(etree, 'ParseError') else SyntaxError
logger = logging.getLogger()
-node_outfiles = []
vol_statusStr = ""
gtmpfilename = None
+g_pid_nodefile_map = {}
class StoreAbsPath(Action):
@@ -111,7 +112,7 @@ def node_cmd(host, host_uuid, task, cmd, args, opts):
def run_cmd_nodes(task, args, **kwargs):
- global node_outfiles
+ global g_pid_nodefile_map
nodes = get_nodes(args.volume)
pool = []
for num, node in enumerate(nodes):
@@ -142,7 +143,6 @@ def run_cmd_nodes(task, args, **kwargs):
if tag == "":
tag = '""' if not is_host_local(host_uuid) else ""
- node_outfiles.append(node_outfile)
# remote file will be copied into this directory
mkdirp(os.path.dirname(node_outfile),
exit_on_err=True, logger=logger)
@@ -180,7 +180,6 @@ def run_cmd_nodes(task, args, **kwargs):
if tag == "":
tag = '""' if not is_host_local(host_uuid) else ""
- node_outfiles.append(node_outfile)
# remote file will be copied into this directory
mkdirp(os.path.dirname(node_outfile),
exit_on_err=True, logger=logger)
@@ -264,6 +263,7 @@ def run_cmd_nodes(task, args, **kwargs):
args=(host, host_uuid, task, cmd, args, opts))
p.start()
pool.append(p)
+ g_pid_nodefile_map[p.pid] = node_outfile
for num, p in enumerate(pool):
p.join()
@@ -271,8 +271,11 @@ def run_cmd_nodes(task, args, **kwargs):
logger.warn("Command %s failed in %s" % (task, nodes[num][1]))
if task in ["create", "delete"]:
fail("Command %s failed in %s" % (task, nodes[num][1]))
- elif task == "pre" and args.disable_partial:
- sys.exit(1)
+ elif task == "pre" or task == "query":
+ if args.disable_partial:
+ sys.exit(1)
+ else:
+ del g_pid_nodefile_map[p.pid]
@cache_output
@@ -512,16 +515,10 @@ def write_output(outfile, outfilemerger, field_separator):
continue
if row_2_rep and row_2_rep != "":
- f.write(u"{0}{1}{2}{3}{4}\n".format(row[0],
- field_separator,
- p_rep,
- field_separator,
- row_2_rep).encode())
- else:
- f.write(u"{0}{1}{2}\n".format(row[0],
- field_separator,
- p_rep).encode())
+ gfind_write_row(f, row[0], field_separator, p_rep, field_separator, row_2_rep)
+ else:
+ gfind_write(f, row[0], field_separator, p_rep)
def mode_create(session_dir, args):
logger.debug("Init is called - Session: %s, Volume: %s"
@@ -571,6 +568,7 @@ def mode_create(session_dir, args):
def mode_query(session_dir, args):
global gtmpfilename
+ global g_pid_nodefile_map
# Verify volume status
cmd = ["gluster", 'volume', 'info', args.volume, "--xml"]
@@ -634,14 +632,20 @@ def mode_query(session_dir, args):
# Merger
if args.full:
- cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
- execute(cmd,
- exit_msg="Failed to merge output files "
- "collected from nodes", logger=logger)
+ if len(g_pid_nodefile_map) > 0:
+ cmd = ["sort", "-u"] + g_pid_nodefile_map.values() + \
+ ["-o", args.outfile]
+ execute(cmd,
+ exit_msg="Failed to merge output files "
+ "collected from nodes", logger=logger)
+ else:
+ fail("Failed to collect any output files from peers. "
+ "Looks like all bricks are offline.", logger=logger)
else:
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
- outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
+ outfilemerger = OutputMerger(args.outfile + ".db",
+ g_pid_nodefile_map.values())
write_output(args.outfile, outfilemerger, args.field_separator)
try:
@@ -656,6 +660,7 @@ def mode_query(session_dir, args):
def mode_pre(session_dir, args):
global gtmpfilename
+ global g_pid_nodefile_map
"""
Read from Session file and write to session.pre file
@@ -696,14 +701,20 @@ def mode_pre(session_dir, args):
# Merger
if args.full:
- cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
- execute(cmd,
- exit_msg="Failed to merge output files "
- "collected from nodes", logger=logger)
+ if len(g_pid_nodefile_map) > 0:
+ cmd = ["sort", "-u"] + g_pid_nodefile_map.values() + \
+ ["-o", args.outfile]
+ execute(cmd,
+ exit_msg="Failed to merge output files "
+ "collected from nodes", logger=logger)
+ else:
+ fail("Failed to collect any output files from peers. "
+ "Looks like all bricks are offline.", logger=logger)
else:
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
- outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
+ outfilemerger = OutputMerger(args.outfile + ".db",
+ g_pid_nodefile_map.values())
write_output(args.outfile, outfilemerger, args.field_separator)
try:
--
1.8.3.1

113
0348-glusterfind-Fix-py2-py3-issues.patch Normal file
View File

@ -0,0 +1,113 @@
From 1ca8a545833e0a6e674984245338b8675ddc58bc Mon Sep 17 00:00:00 2001
From: Kotresh HR <khiremat@redhat.com>
Date: Fri, 10 Jan 2020 16:48:14 +0530
Subject: [PATCH 348/349] glusterfind: Fix py2/py3 issues
1. In dictionary values(), returns list in py2 and not in py3.
So explicitly convert it into list.
2. xattr module returns values in bytes. So explicitly convert
them to str to work both with py2 and py3
Backport of:
> Patch: https://review.gluster.org/23993
> fixes: bz#1789439
> Change-Id: I27a639cda4f7a4ece9744a97c3d16e247906bd94
> Signed-off-by: Kotresh HR <khiremat@redhat.com>
BUG: 1789447
Change-Id: I27a639cda4f7a4ece9744a97c3d16e247906bd94
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/189215
Reviewed-by: Shwetha Acharya <sacharya@redhat.com>
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Hari Gowtham Gopal <hgowtham@redhat.com>
Reviewed-by: Sunny Kumar <sunkumar@redhat.com>
---
tools/glusterfind/src/changelog.py | 14 +++++++++-----
tools/glusterfind/src/main.py | 8 ++++----
2 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/tools/glusterfind/src/changelog.py b/tools/glusterfind/src/changelog.py
index d8f97e0..d972fb5 100644
--- a/tools/glusterfind/src/changelog.py
+++ b/tools/glusterfind/src/changelog.py
@@ -14,6 +14,7 @@ import sys
import time
import xattr
import logging
+from gfind_py2py3 import bytearray_to_str
from argparse import ArgumentParser, RawDescriptionHelpFormatter
import hashlib
try:
@@ -105,9 +106,10 @@ def populate_pgfid_and_inodegfid(brick, changelog_data):
changelog_data.inodegfid_add(os.stat(p).st_ino, gfid)
file_xattrs = xattr.list(p)
for x in file_xattrs:
- if x.startswith("trusted.pgfid."):
+ x_str = bytearray_to_str(x)
+ if x_str.startswith("trusted.pgfid."):
# PGFID in pgfid table
- changelog_data.pgfid_add(x.split(".")[-1])
+ changelog_data.pgfid_add(x_str.split(".")[-1])
except (IOError, OSError):
# All OS Errors ignored, since failures will be logged
# in End. All GFIDs present in gfidpath table
@@ -122,10 +124,12 @@ def enum_hard_links_using_gfid2path(brick, gfid, args):
try:
file_xattrs = xattr.list(p)
for x in file_xattrs:
- if x.startswith("trusted.gfid2path."):
+ x_str = bytearray_to_str(x)
+ if x_str.startswith("trusted.gfid2path."):
# get the value for the xattr i.e. <PGFID>/<BN>
- v = xattr.getxattr(p, x)
- pgfid, bn = v.split(os.sep)
+ v = xattr.getxattr(p, x_str)
+ v_str = bytearray_to_str(v)
+ pgfid, bn = v_str.split(os.sep)
try:
path = symlink_gfid_to_path(brick, pgfid)
fullpath = os.path.join(path, bn)
diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py
index fefe4a3..dfc9d07 100644
--- a/tools/glusterfind/src/main.py
+++ b/tools/glusterfind/src/main.py
@@ -633,7 +633,7 @@ def mode_query(session_dir, args):
# Merger
if args.full:
if len(g_pid_nodefile_map) > 0:
- cmd = ["sort", "-u"] + g_pid_nodefile_map.values() + \
+ cmd = ["sort", "-u"] + list(g_pid_nodefile_map.values()) + \
["-o", args.outfile]
execute(cmd,
exit_msg="Failed to merge output files "
@@ -645,7 +645,7 @@ def mode_query(session_dir, args):
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db",
- g_pid_nodefile_map.values())
+ list(g_pid_nodefile_map.values()))
write_output(args.outfile, outfilemerger, args.field_separator)
try:
@@ -702,7 +702,7 @@ def mode_pre(session_dir, args):
# Merger
if args.full:
if len(g_pid_nodefile_map) > 0:
- cmd = ["sort", "-u"] + g_pid_nodefile_map.values() + \
+ cmd = ["sort", "-u"] + list(g_pid_nodefile_map.values()) + \
["-o", args.outfile]
execute(cmd,
exit_msg="Failed to merge output files "
@@ -714,7 +714,7 @@ def mode_pre(session_dir, args):
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db",
- g_pid_nodefile_map.values())
+ list(g_pid_nodefile_map.values()))
write_output(args.outfile, outfilemerger, args.field_separator)
try:
--
1.8.3.1

56
0349-glusterfind-python3-compatibility.patch Normal file
View File

@ -0,0 +1,56 @@
From 1354a492cbc758f9801568153380ca896fab7765 Mon Sep 17 00:00:00 2001
From: Sunny Kumar <sunkumar@redhat.com>
Date: Fri, 10 Jan 2020 14:28:35 +0000
Subject: [PATCH 349/349] glusterfind: python3 compatibility
Problem:
While we delete gluster volume the hook script 'S57glusterfind-delete-post.py'
is failed to execute and error message can be observed in glusterd log.
Traceback:
File "/var/lib/glusterd/hooks/1/delete/post/S57glusterfind-delete-post", line 69, in <module>
main()
File "/var/lib/glusterd/hooks/1/delete/post/S57glusterfind-delete-post", line 39, in main
glusterfind_dir = os.path.join(get_glusterd_workdir(), "glusterfind")
File "/usr/lib64/python3.7/posixpath.py", line 94, in join
genericpath._check_arg_types('join', a, *p)
File "/usr/lib64/python3.7/genericpath.py", line 155, in _check_arg_types
raise TypeError("Can't mix strings and bytes in path components") from None
TypeError: Can't mix strings and bytes in path components
Solution:
Added the 'universal_newlines' flag to Popen to support backward compatibility.
Backport of:
> Patch: https://review.gluster.org/23994
> Change-Id: Ie5655b11b55535c5ad2338108d0448e6fdaacf4f
> Fixes: bz#1789478
> Signed-off-by: Sunny Kumar <sunkumar@redhat.com>
Change-Id: Ie5655b11b55535c5ad2338108d0448e6fdaacf4f
BUG: 1789447
Signed-off-by: Sunny Kumar <sunkumar@redhat.com>
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/189216
Tested-by: RHGS Build Bot <nigelb@redhat.com>
---
tools/glusterfind/S57glusterfind-delete-post.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/glusterfind/S57glusterfind-delete-post.py b/tools/glusterfind/S57glusterfind-delete-post.py
index 5b5142d..5beece2 100755
--- a/tools/glusterfind/S57glusterfind-delete-post.py
+++ b/tools/glusterfind/S57glusterfind-delete-post.py
@@ -18,7 +18,7 @@ def handle_rm_error(func, path, exc_info):
def get_glusterd_workdir():
p = Popen(["gluster", "system::", "getwd"],
- stdout=PIPE, stderr=PIPE)
+ stdout=PIPE, stderr=PIPE, universal_newlines=True)
out, _ = p.communicate()
--
1.8.3.1

8
glusterfs.spec
View File

@ -231,7 +231,7 @@ Release: 0.1%{?prereltag:.%{prereltag}}%{?dist}
%else
Name: glusterfs
Version: 6.0
Release: 26%{?dist}
Release: 27%{?dist}
ExcludeArch: i686
%endif
License: GPLv2 or LGPLv3+
@ -655,6 +655,9 @@ Patch0343: 0343-extras-hooks-syntactical-errors-in-SELinux-hooks-sci.patch
Patch0344: 0344-Revert-all-fixes-to-include-SELinux-hook-scripts.patch
Patch0345: 0345-read-ahead-io-cache-turn-off-by-default.patch
Patch0346: 0346-fuse-degrade-logging-of-write-failure-to-fuse-device.patch
Patch0347: 0347-tools-glusterfind-handle-offline-bricks.patch
Patch0348: 0348-glusterfind-Fix-py2-py3-issues.patch
Patch0349: 0349-glusterfind-python3-compatibility.patch
%description
GlusterFS is a distributed file-system capable of scaling to several
@ -2382,6 +2385,9 @@ fi
%endif
%changelog
* Mon Jan 13 2020 Rinku Kothiya <rkothiya@redhat.com> - 6.0-27
- fixes bugs bz#1789447
* Fri Jan 10 2020 Rinku Kothiya <rkothiya@redhat.com> - 6.0-26
- fixes bugs bz#1763208 bz#1788656