From 7c4f87d5669ebee9119fd7b127f744bc6e730a32 Mon Sep 17 00:00:00 2001 From: "Kaleb S. KEITHLEY" Date: Wed, 6 Jul 2022 12:39:08 -0400 Subject: [PATCH] enable cephfs-shell Signed-off-by: Kaleb S. KEITHLEY --- 0021-cephfs-shell.patch | 1756 +++++++++++++++++++++++++++++++++++++++ ceph.spec | 8 +- 2 files changed, 1762 insertions(+), 2 deletions(-) create mode 100644 0021-cephfs-shell.patch diff --git a/0021-cephfs-shell.patch b/0021-cephfs-shell.patch new file mode 100644 index 0000000..f85a91e --- /dev/null +++ b/0021-cephfs-shell.patch @@ -0,0 +1,1756 @@ +--- ceph-17.2.1/src/tools/cephfs/CMakeLists.txt.orig 2022-07-05 19:26:04.629170597 -0400 ++++ ceph-17.2.1/src/tools/cephfs/CMakeLists.txt 2022-07-05 19:26:40.710580427 -0400 +@@ -49,12 +49,7 @@ + + option(WITH_CEPHFS_SHELL "install cephfs-shell" OFF) + if(WITH_CEPHFS_SHELL) +- include(Distutils) +- distutils_install_module(cephfs-shell) +- if(WITH_TESTS) +- include(AddCephTest) +- add_tox_test(cephfs-shell) +- endif() ++ add_subdirectory(shell) + endif() + + option(WITH_CEPHFS_TOP "install cephfs-top utility" ON) +--- /dev/null 2022-06-30 09:45:32.996000000 -0400 ++++ ceph-17.2.1/src/tools/cephfs/shell/CMakeLists.txt 2022-07-05 19:27:58.983300150 -0400 +@@ -0,0 +1,7 @@ ++include(Distutils) ++distutils_install_module(cephfs-shell) ++ ++if(WITH_TESTS) ++ include(AddCephTest) ++ add_tox_test(cephfs-shell) ++endif() +--- /dev/null 2022-06-30 09:45:32.996000000 -0400 ++++ ceph-17.2.1/src/tools/cephfs/shell/cephfs-shell 2022-06-23 10:41:35.000000000 -0400 +@@ -0,0 +1,1687 @@ ++#!/usr/bin/python3 ++# coding = utf-8 ++ ++import argparse ++import os ++import os.path ++import sys ++import cephfs as libcephfs ++import shutil ++import traceback ++import colorama ++import fnmatch ++import math ++import re ++import shlex ++import stat ++import errno ++ ++from cmd2 import Cmd ++from cmd2 import __version__ as cmd2_version ++from distutils.version import LooseVersion ++ ++if sys.version_info.major < 3: ++ raise RuntimeError("cephfs-shell is only compatible with python3") ++ ++try: ++ from cmd2 import with_argparser ++except ImportError: ++ def with_argparser(argparser): ++ import functools ++ ++ def argparser_decorator(func): ++ @functools.wraps(func) ++ def wrapper(thiz, cmdline): ++ if isinstance(cmdline, list): ++ arglist = cmdline ++ else: ++ # do not split if it's already a list ++ arglist = shlex.split(cmdline, posix=False) ++ # in case user quotes the command args ++ arglist = [arg.strip('\'""') for arg in arglist] ++ try: ++ args = argparser.parse_args(arglist) ++ except SystemExit: ++ shell.exit_code = 1 ++ # argparse exits at seeing bad arguments ++ return ++ else: ++ return func(thiz, args) ++ argparser.prog = func.__name__[3:] ++ if argparser.description is None and func.__doc__: ++ argparser.description = func.__doc__ ++ ++ return wrapper ++ ++ return argparser_decorator ++ ++ ++cephfs = None # holds CephFS Python bindings ++shell = None # holds instance of class CephFSShell ++exit_codes = {'Misc': 1, ++ 'KeyboardInterrupt': 2, ++ errno.EPERM: 3, ++ errno.EACCES: 4, ++ errno.ENOENT: 5, ++ errno.EIO: 6, ++ errno.ENOSPC: 7, ++ errno.EEXIST: 8, ++ errno.ENODATA: 9, ++ errno.EINVAL: 10, ++ errno.EOPNOTSUPP: 11, ++ errno.ERANGE: 12, ++ errno.EWOULDBLOCK: 13, ++ errno.ENOTEMPTY: 14, ++ errno.ENOTDIR: 15, ++ errno.EDQUOT: 16, ++ errno.EPIPE: 17, ++ errno.ESHUTDOWN: 18, ++ errno.ECONNABORTED: 19, ++ errno.ECONNREFUSED: 20, ++ errno.ECONNRESET: 21, ++ errno.EINTR: 22} ++ ++ ++######################################################################### ++# ++# Following are methods are generically useful through class CephFSShell ++# ++####################################################################### ++ ++ ++def poutput(s, end='\n'): ++ shell.poutput(s, end=end) ++ ++ ++def perror(msg, **kwargs): ++ shell.perror(msg, **kwargs) ++ ++ ++def set_exit_code_msg(errcode='Misc', msg=''): ++ """ ++ Set exit code and print error message ++ """ ++ if isinstance(msg, libcephfs.Error): ++ shell.exit_code = exit_codes[msg.get_error_code()] ++ else: ++ shell.exit_code = exit_codes[errcode] ++ if msg: ++ perror(msg) ++ ++ ++def mode_notation(mode): ++ """ ++ """ ++ permission_bits = {'0': '---', ++ '1': '--x', ++ '2': '-w-', ++ '3': '-wx', ++ '4': 'r--', ++ '5': 'r-x', ++ '6': 'rw-', ++ '7': 'rwx'} ++ mode = str(oct(mode)) ++ notation = '-' ++ if mode[2] == '4': ++ notation = 'd' ++ elif mode[2:4] == '12': ++ notation = 'l' ++ for i in mode[-3:]: ++ notation += permission_bits[i] ++ return notation ++ ++ ++def get_chunks(file_size): ++ chunk_start = 0 ++ chunk_size = 0x20000 # 131072 bytes, default max ssl buffer size ++ while chunk_start + chunk_size < file_size: ++ yield(chunk_start, chunk_size) ++ chunk_start += chunk_size ++ final_chunk_size = file_size - chunk_start ++ yield(chunk_start, final_chunk_size) ++ ++ ++def to_bytes(param): ++ # don't convert as follows as it can lead unusable results like coverting ++ # [1, 2, 3, 4] to '[1, 2, 3, 4]' - ++ # str(param).encode('utf-8') ++ if isinstance(param, bytes): ++ return param ++ elif isinstance(param, str): ++ return bytes(param, encoding='utf-8') ++ elif isinstance(param, list): ++ return [i.encode('utf-8') if isinstance(i, str) else to_bytes(i) for ++ i in param] ++ elif isinstance(param, int) or isinstance(param, float): ++ return str(param).encode('utf-8') ++ elif param is None: ++ return None ++ ++ ++def ls(path, opts=''): ++ # opts tries to be like /bin/ls opts ++ almost_all = 'A' in opts ++ try: ++ with cephfs.opendir(path) as d: ++ while True: ++ dent = cephfs.readdir(d) ++ if dent is None: ++ return ++ elif almost_all and dent.d_name in (b'.', b'..'): ++ continue ++ yield dent ++ except libcephfs.ObjectNotFound as e: ++ set_exit_code_msg(msg=e) ++ ++ ++def glob(path, pattern): ++ paths = [] ++ parent_dir = os.path.dirname(path) ++ if parent_dir == b'': ++ parent_dir = b'/' ++ if path == b'/' or is_dir_exists(os.path.basename(path), parent_dir): ++ for i in ls(path, opts='A'): ++ if fnmatch.fnmatch(i.d_name, pattern): ++ paths.append(os.path.join(path, i.d_name)) ++ return paths ++ ++ ++def locate_file(name, case_sensitive=True): ++ dir_list = sorted(set(dirwalk(cephfs.getcwd()))) ++ if not case_sensitive: ++ return [dname for dname in dir_list if name.lower() in dname.lower()] ++ else: ++ return [dname for dname in dir_list if name in dname] ++ ++ ++def get_all_possible_paths(pattern): ++ complete_pattern = pattern[:] ++ paths = [] ++ is_rel_path = not os.path.isabs(pattern) ++ if is_rel_path: ++ dir_ = cephfs.getcwd() ++ else: ++ dir_ = b'/' ++ pattern = pattern[1:] ++ patterns = pattern.split(b'/') ++ paths.extend(glob(dir_, patterns[0])) ++ patterns.pop(0) ++ for pattern in patterns: ++ for path in paths: ++ paths.extend(glob(path, pattern)) ++ if is_rel_path: ++ complete_pattern = os.path.join(cephfs.getcwd(), complete_pattern) ++ return [path for path in paths if fnmatch.fnmatch(path, complete_pattern)] ++ ++ ++suffixes = ['B', 'K', 'M', 'G', 'T', 'P'] ++ ++ ++def humansize(nbytes): ++ i = 0 ++ while nbytes >= 1024 and i < len(suffixes) - 1: ++ nbytes /= 1024. ++ i += 1 ++ nbytes = math.ceil(nbytes) ++ f = ('%d' % nbytes).rstrip('.') ++ return '%s%s' % (f, suffixes[i]) ++ ++ ++def style_listing(path, is_dir, is_symlink, ls_long=False): ++ if not (is_dir or is_symlink): ++ return path ++ pretty = colorama.Style.BRIGHT ++ if is_symlink: ++ pretty += colorama.Fore.CYAN + path ++ if ls_long: ++ # Add target path ++ pretty += ' -> ' + cephfs.readlink(path, size=255).decode('utf-8') ++ elif is_dir: ++ pretty += colorama.Fore.BLUE + path + '/' ++ pretty += colorama.Style.RESET_ALL ++ return pretty ++ ++ ++def print_long(path, is_dir, is_symlink, human_readable): ++ info = cephfs.stat(path, follow_symlink=(not is_symlink)) ++ pretty = style_listing(os.path.basename(path.decode('utf-8')), is_dir, is_symlink, True) ++ if human_readable: ++ sizefmt = '\t {:10s}'.format(humansize(info.st_size)) ++ else: ++ sizefmt = '{:12d}'.format(info.st_size) ++ poutput(f'{mode_notation(info.st_mode)} {sizefmt} {info.st_uid} {info.st_gid} {info.st_mtime}' ++ f' {pretty}') ++ ++ ++def word_len(word): ++ """ ++ Returns the word length, minus any color codes. ++ """ ++ if word[0] == '\x1b': ++ return len(word) - 9 ++ return len(word) ++ ++ ++def is_dir_exists(path, dir_=b''): ++ path_to_stat = os.path.join(dir_, path) ++ try: ++ return ((cephfs.stat(path_to_stat).st_mode & 0o0040000) != 0) ++ except libcephfs.Error: ++ return False ++ ++ ++def is_file_exists(path, dir_=b''): ++ try: ++ # if its not a directory, then its a file ++ return ((cephfs.stat(os.path.join(dir_, path)).st_mode & 0o0040000) == 0) ++ except libcephfs.Error: ++ return False ++ ++ ++def print_list(words, termwidth=79): ++ if not words: ++ return ++ words = [word.decode('utf-8') if isinstance(word, bytes) else word for word in words] ++ width = max([word_len(word) for word in words]) + 2 ++ nwords = len(words) ++ ncols = max(1, (termwidth + 1) // (width + 1)) ++ nrows = (nwords + ncols - 1) // ncols ++ for row in range(nrows): ++ for i in range(row, nwords, nrows): ++ word = words[i] ++ print_width = width ++ if word[0] == '\x1b': ++ print_width = print_width + 10 ++ ++ poutput('%-*s' % (print_width, words[i]), ++ end='\n' if i + nrows >= nwords else '') ++ ++ ++def copy_from_local(local_path, remote_path): ++ stdin = -1 ++ file_ = None ++ fd = None ++ convert_to_bytes = False ++ if local_path == b'-': ++ file_ = sys.stdin ++ convert_to_bytes = True ++ else: ++ try: ++ file_ = open(local_path, 'rb') ++ except PermissionError as e: ++ set_exit_code_msg(e.errno, 'error: no permission to read local file {}'.format( ++ local_path.decode('utf-8'))) ++ return ++ stdin = 1 ++ try: ++ fd = cephfs.open(remote_path, 'w', 0o666) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ return ++ progress = 0 ++ while True: ++ data = file_.read(65536) ++ if not data or len(data) == 0: ++ break ++ if convert_to_bytes: ++ data = to_bytes(data) ++ wrote = cephfs.write(fd, data, progress) ++ if wrote < 0: ++ break ++ progress += wrote ++ cephfs.close(fd) ++ if stdin > 0: ++ file_.close() ++ poutput('') ++ ++ ++def copy_to_local(remote_path, local_path): ++ fd = None ++ if local_path != b'-': ++ local_dir = os.path.dirname(local_path) ++ dir_list = remote_path.rsplit(b'/', 1) ++ if not os.path.exists(local_dir): ++ os.makedirs(local_dir) ++ if len(dir_list) > 2 and dir_list[1] == b'': ++ return ++ fd = open(local_path, 'wb+') ++ file_ = cephfs.open(remote_path, 'r') ++ file_size = cephfs.stat(remote_path).st_size ++ if file_size <= 0: ++ return ++ progress = 0 ++ for chunk_start, chunk_size in get_chunks(file_size): ++ file_chunk = cephfs.read(file_, chunk_start, chunk_size) ++ progress += len(file_chunk) ++ if fd: ++ fd.write(file_chunk) ++ else: ++ poutput(file_chunk.decode('utf-8')) ++ cephfs.close(file_) ++ if fd: ++ fd.close() ++ ++ ++def dirwalk(path): ++ """ ++ walk a directory tree, using a generator ++ """ ++ path = os.path.normpath(path) ++ for item in ls(path, opts='A'): ++ fullpath = os.path.join(path, item.d_name) ++ src_path = fullpath.rsplit(b'/', 1)[0] ++ ++ yield os.path.normpath(fullpath) ++ if is_dir_exists(item.d_name, src_path): ++ for x in dirwalk(fullpath): ++ yield x ++ ++ ++################################################################## ++# ++# Following methods are implementation for CephFS Shell commands ++# ++################################################################# ++ ++class CephFSShell(Cmd): ++ ++ def __init__(self): ++ super().__init__() ++ self.working_dir = cephfs.getcwd().decode('utf-8') ++ self.set_prompt() ++ self.interactive = False ++ self.umask = '2' ++ ++ def default(self, line): ++ perror('Unrecognized command') ++ ++ def set_prompt(self): ++ self.prompt = ('\033[01;33mCephFS:~' + colorama.Fore.LIGHTCYAN_EX ++ + self.working_dir + colorama.Style.RESET_ALL ++ + '\033[01;33m>>>\033[00m ') ++ ++ def create_argparser(self, command): ++ try: ++ argparse_args = getattr(self, 'argparse_' + command) ++ except AttributeError: ++ set_exit_code_msg() ++ return None ++ doc_lines = getattr( ++ self, 'do_' + command).__doc__.expandtabs().splitlines() ++ if '' in doc_lines: ++ blank_idx = doc_lines.index('') ++ usage = doc_lines[:blank_idx] ++ description = doc_lines[blank_idx + 1:] ++ else: ++ usage = doc_lines ++ description = [] ++ parser = argparse.ArgumentParser( ++ prog=command, ++ usage='\n'.join(usage), ++ description='\n'.join(description), ++ formatter_class=argparse.ArgumentDefaultsHelpFormatter ++ ) ++ for args, kwargs in argparse_args: ++ parser.add_argument(*args, **kwargs) ++ return parser ++ ++ def complete_filenames(self, text, line, begidx, endidx): ++ if not text: ++ completions = [x.d_name.decode('utf-8') + '/' * int(x.is_dir()) ++ for x in ls(b".", opts='A')] ++ else: ++ if text.count('/') > 0: ++ completions = [text.rsplit('/', 1)[0] + '/' ++ + x.d_name.decode('utf-8') + '/' ++ * int(x.is_dir()) for x in ls('/' ++ + text.rsplit('/', 1)[0], opts='A') ++ if x.d_name.decode('utf-8').startswith( ++ text.rsplit('/', 1)[1])] ++ else: ++ completions = [x.d_name.decode('utf-8') + '/' ++ * int(x.is_dir()) for x in ls(b".", opts='A') ++ if x.d_name.decode('utf-8').startswith(text)] ++ if len(completions) == 1 and completions[0][-1] == '/': ++ dir_, file_ = completions[0].rsplit('/', 1) ++ completions.extend([dir_ + '/' + x.d_name.decode('utf-8') ++ + '/' * int(x.is_dir()) for x in ++ ls('/' + dir_, opts='A') ++ if x.d_name.decode('utf-8').startswith(file_)]) ++ return self.delimiter_complete(text, line, begidx, endidx, completions, '/') ++ return completions ++ ++ def onecmd(self, line, **kwargs): ++ """ ++ Global error catcher ++ """ ++ try: ++ res = Cmd.onecmd(self, line, **kwargs) ++ if self.interactive: ++ self.set_prompt() ++ return res ++ except ConnectionError as e: ++ set_exit_code_msg(e.errno, f'***\n{e}') ++ except KeyboardInterrupt: ++ set_exit_code_msg('KeyboardInterrupt', 'Command aborted') ++ except (libcephfs.Error, Exception) as e: ++ if shell.debug: ++ traceback.print_exc(file=sys.stdout) ++ set_exit_code_msg(msg=e) ++ ++ class path_to_bytes(argparse.Action): ++ def __call__(self, parser, namespace, values, option_string=None): ++ values = to_bytes(values) ++ setattr(namespace, self.dest, values) ++ ++ # TODO: move the necessary contents from here to `class path_to_bytes`. ++ class get_list_of_bytes_path(argparse.Action): ++ def __call__(self, parser, namespace, values, option_string=None): ++ values = to_bytes(values) ++ ++ if values == b'.': ++ values = cephfs.getcwd() ++ else: ++ for i in values: ++ if i == b'.': ++ values[values.index(i)] = cephfs.getcwd() ++ ++ setattr(namespace, self.dest, values) ++ ++ def complete_mkdir(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ class ModeAction(argparse.Action): ++ def __init__(self, option_strings, dest, nargs=None, **kwargs): ++ if nargs is not None and nargs != '?': ++ raise ValueError("more than one modes not allowed") ++ super().__init__(option_strings, dest, **kwargs) ++ ++ def __call__(self, parser, namespace, values, option_string=None): ++ o_mode = 0 ++ res = None ++ try: ++ o_mode = int(values, base=8) ++ except ValueError: ++ res = re.match('((u?g?o?)|(a?))(=)(r?w?x?)', values) ++ if res is None: ++ parser.error("invalid mode: %s\n" ++ "mode must be a numeric octal literal\n" ++ "or ((u?g?o?)|(a?))(=)(r?w?x?)" % ++ values) ++ else: ++ # we are supporting only assignment of mode and not + or - ++ # as is generally available with the chmod command ++ # eg. ++ # >>> res = re.match('((u?g?o?)|(a?))(=)(r?w?x?)', 'go=') ++ # >>> res.groups() ++ # ('go', 'go', None, '=', '') ++ val = res.groups() ++ ++ if val[3] != '=': ++ parser.error("need assignment operator between user " ++ "and mode specifiers") ++ if val[4] == '': ++ parser.error("invalid mode: %s\n" ++ "mode must be combination of: r | w | x" % ++ values) ++ users = '' ++ if val[2] is None: ++ users = val[1] ++ else: ++ users = val[2] ++ ++ t_mode = 0 ++ if users == 'a': ++ users = 'ugo' ++ ++ if 'r' in val[4]: ++ t_mode |= 4 ++ if 'w' in val[4]: ++ t_mode |= 2 ++ if 'x' in val[4]: ++ t_mode |= 1 ++ ++ if 'u' in users: ++ o_mode |= (t_mode << 6) ++ if 'g' in users: ++ o_mode |= (t_mode << 3) ++ if 'o' in users: ++ o_mode |= t_mode ++ ++ if o_mode < 0: ++ parser.error("invalid mode: %s\n" ++ "mode cannot be negative" % values) ++ if o_mode > 0o777: ++ parser.error("invalid mode: %s\n" ++ "mode cannot be greater than octal 0777" % values) ++ ++ setattr(namespace, self.dest, str(oct(o_mode))) ++ ++ mkdir_parser = argparse.ArgumentParser( ++ description='Create the directory(ies), if they do not already exist.') ++ mkdir_parser.add_argument('dirs', type=str, ++ action=path_to_bytes, ++ metavar='DIR_NAME', ++ help='Name of new_directory.', ++ nargs='+') ++ mkdir_parser.add_argument('-m', '--mode', type=str, ++ action=ModeAction, ++ help='Sets the access mode for the new directory.') ++ mkdir_parser.add_argument('-p', '--parent', action='store_true', ++ help='Create parent directories as necessary. ' ++ 'When this option is specified, no error is' ++ 'reported if a directory already exists.') ++ ++ @with_argparser(mkdir_parser) ++ def do_mkdir(self, args): ++ """ ++ Create directory. ++ """ ++ for path in args.dirs: ++ if args.mode: ++ permission = int(args.mode, 8) ++ else: ++ permission = 0o777 ++ if args.parent: ++ cephfs.mkdirs(path, permission) ++ else: ++ try: ++ cephfs.mkdir(path, permission) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e) ++ ++ def complete_put(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ index_dict = {1: self.path_complete} ++ return self.index_based_complete(text, line, begidx, endidx, index_dict) ++ ++ put_parser = argparse.ArgumentParser( ++ description='Copy a file/directory to Ceph File System from Local File System.') ++ put_parser.add_argument('local_path', type=str, action=path_to_bytes, ++ help='Path of the file in the local system') ++ put_parser.add_argument('remote_path', type=str, action=path_to_bytes, ++ help='Path of the file in the remote system') ++ put_parser.add_argument('-f', '--force', action='store_true', ++ help='Overwrites the destination if it already exists.') ++ ++ @with_argparser(put_parser) ++ def do_put(self, args): ++ """ ++ Copy a local file/directory to CephFS. ++ """ ++ if args.local_path != b'-' and not os.path.isfile(args.local_path) \ ++ and not os.path.isdir(args.local_path): ++ set_exit_code_msg(errno.ENOENT, ++ msg=f"error: " ++ f"{args.local_path.decode('utf-8')}: " ++ f"No such file or directory") ++ return ++ ++ if (is_file_exists(args.remote_path) or is_dir_exists( ++ args.remote_path)) and not args.force: ++ set_exit_code_msg(msg=f"error: file/directory " ++ f"{args.remote_path.decode('utf-8')} " ++ f"exists, use --force to overwrite") ++ return ++ ++ root_src_dir = args.local_path ++ root_dst_dir = args.remote_path ++ if args.local_path == b'.' or args.local_path == b'./': ++ root_src_dir = os.getcwdb() ++ elif len(args.local_path.rsplit(b'/', 1)) < 2: ++ root_src_dir = os.path.join(os.getcwdb(), args.local_path) ++ else: ++ p = args.local_path.split(b'/') ++ if p[0] == b'.': ++ root_src_dir = os.getcwdb() ++ p.pop(0) ++ while len(p) > 0: ++ root_src_dir += b'/' + p.pop(0) ++ ++ if root_dst_dir == b'.': ++ if args.local_path != b'-': ++ root_dst_dir = root_src_dir.rsplit(b'/', 1)[1] ++ if root_dst_dir == b'': ++ root_dst_dir = root_src_dir.rsplit(b'/', 1)[0] ++ a = root_dst_dir.rsplit(b'/', 1) ++ if len(a) > 1: ++ root_dst_dir = a[1] ++ else: ++ root_dst_dir = a[0] ++ else: ++ set_exit_code_msg(errno.EINVAL, 'error: no filename specified ' ++ 'for destination') ++ return ++ ++ if root_dst_dir[-1] != b'/': ++ root_dst_dir += b'/' ++ ++ if args.local_path == b'-' or os.path.isfile(root_src_dir): ++ if args.local_path == b'-': ++ root_src_dir = b'-' ++ copy_from_local(root_src_dir, root_dst_dir) ++ else: ++ for src_dir, dirs, files in os.walk(root_src_dir): ++ if isinstance(src_dir, str): ++ src_dir = to_bytes(src_dir) ++ dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1) ++ dst_dir = re.sub(rb'\/+', b'/', cephfs.getcwd() ++ + dst_dir) ++ if args.force and dst_dir != b'/' and not is_dir_exists( ++ dst_dir[:-1]) and not locate_file(dst_dir): ++ try: ++ cephfs.mkdirs(dst_dir, 0o777) ++ except libcephfs.Error: ++ pass ++ if (not args.force) and dst_dir != b'/' and not is_dir_exists( ++ dst_dir) and not os.path.isfile(root_src_dir): ++ try: ++ cephfs.mkdirs(dst_dir, 0o777) ++ except libcephfs.Error: ++ # TODO: perhaps, set retval to 1? ++ pass ++ ++ for dir_ in dirs: ++ dir_name = os.path.join(dst_dir, dir_) ++ if not is_dir_exists(dir_name): ++ try: ++ cephfs.mkdirs(dir_name, 0o777) ++ except libcephfs.Error: ++ # TODO: perhaps, set retval to 1? ++ pass ++ ++ for file_ in files: ++ src_file = os.path.join(src_dir, file_) ++ dst_file = re.sub(rb'\/+', b'/', b'/' + dst_dir + b'/' + file_) ++ if (not args.force) and is_file_exists(dst_file): ++ return ++ copy_from_local(src_file, os.path.join(cephfs.getcwd(), ++ dst_file)) ++ ++ def complete_get(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ get_parser = argparse.ArgumentParser( ++ description='Copy a file from Ceph File System to Local Directory.') ++ get_parser.add_argument('remote_path', type=str, action=path_to_bytes, ++ help='Path of the file in the remote system') ++ get_parser.add_argument('local_path', type=str, action=path_to_bytes, ++ help='Path of the file in the local system') ++ get_parser.add_argument('-f', '--force', action='store_true', ++ help='Overwrites the destination if it already exists.') ++ ++ @with_argparser(get_parser) ++ def do_get(self, args): ++ """ ++ Copy a file/directory from CephFS to given path. ++ """ ++ if not is_file_exists(args.remote_path) and not \ ++ is_dir_exists(args.remote_path): ++ set_exit_code_msg(errno.ENOENT, "error: no file/directory" ++ " found at specified remote " ++ "path") ++ return ++ if (os.path.isfile(args.local_path) or os.path.isdir( ++ args.local_path)) and not args.force: ++ set_exit_code_msg(msg=f"error: file/directory " ++ f"{args.local_path.decode('utf-8')}" ++ f" already exists, use --force to " ++ f"overwrite") ++ return ++ root_src_dir = args.remote_path ++ root_dst_dir = args.local_path ++ fname = root_src_dir.rsplit(b'/', 1) ++ if args.local_path == b'.': ++ root_dst_dir = os.getcwdb() ++ if args.remote_path == b'.': ++ root_src_dir = cephfs.getcwd() ++ if args.local_path == b'-': ++ if args.remote_path == b'.' or args.remote_path == b'./': ++ set_exit_code_msg(errno.EINVAL, 'error: no remote file name specified') ++ return ++ copy_to_local(root_src_dir, b'-') ++ elif is_file_exists(args.remote_path): ++ copy_to_local(root_src_dir, root_dst_dir) ++ elif b'/' in root_src_dir and is_file_exists(fname[1], fname[0]): ++ copy_to_local(root_src_dir, root_dst_dir) ++ else: ++ files = list(reversed(sorted(dirwalk(root_src_dir)))) ++ for file_ in files: ++ dst_dirpath, dst_file = file_.rsplit(b'/', 1) ++ if dst_dirpath in files: ++ files.remove(dst_dirpath) ++ dst_path = os.path.join(root_dst_dir, dst_dirpath, dst_file) ++ dst_path = os.path.normpath(dst_path) ++ if is_dir_exists(file_): ++ try: ++ os.makedirs(dst_path) ++ except OSError: ++ pass ++ else: ++ copy_to_local(file_, dst_path) ++ ++ return 0 ++ ++ def complete_ls(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ ls_parser = argparse.ArgumentParser( ++ description='Copy a file from Ceph File System from Local Directory.') ++ ls_parser.add_argument('-l', '--long', action='store_true', ++ help='Detailed list of items in the directory.') ++ ls_parser.add_argument('-r', '--reverse', action='store_true', ++ help='Reverse order of listing items in the directory.') ++ ls_parser.add_argument('-H', action='store_true', help='Human Readable') ++ ls_parser.add_argument('-a', '--all', action='store_true', ++ help='Do not Ignore entries starting with .') ++ ls_parser.add_argument('-S', action='store_true', help='Sort by file_size') ++ ls_parser.add_argument('paths', help='Name of Directories', ++ action=path_to_bytes, nargs='*', default=['.']) ++ ++ @with_argparser(ls_parser) ++ def do_ls(self, args): ++ """ ++ List all the files and directories in the current working directory ++ """ ++ paths = args.paths ++ for path in paths: ++ values = [] ++ items = [] ++ try: ++ if path.count(b'*') > 0: ++ all_items = get_all_possible_paths(path) ++ if len(all_items) == 0: ++ continue ++ path = all_items[0].rsplit(b'/', 1)[0] ++ if path == b'': ++ path = b'/' ++ dirs = [] ++ for i in all_items: ++ for item in ls(path): ++ d_name = item.d_name ++ if os.path.basename(i) == d_name: ++ if item.is_dir(): ++ dirs.append(os.path.join(path, d_name)) ++ else: ++ items.append(item) ++ if dirs: ++ paths.extend(dirs) ++ else: ++ poutput(path.decode('utf-8'), end=':\n') ++ items = sorted(items, key=lambda item: item.d_name) ++ else: ++ if path != b'' and path != cephfs.getcwd() and len(paths) > 1: ++ poutput(path.decode('utf-8'), end=':\n') ++ items = sorted(ls(path), key=lambda item: item.d_name) ++ if not args.all: ++ items = [i for i in items if not i.d_name.startswith(b'.')] ++ if args.S: ++ items = sorted(items, key=lambda item: cephfs.stat( ++ path + b'/' + item.d_name, follow_symlink=( ++ not item.is_symbol_file())).st_size) ++ if args.reverse: ++ items = reversed(items) ++ for item in items: ++ filepath = item.d_name ++ is_dir = item.is_dir() ++ is_sym_lnk = item.is_symbol_file() ++ try: ++ if args.long and args.H: ++ print_long(os.path.join(cephfs.getcwd(), path, filepath), is_dir, ++ is_sym_lnk, True) ++ elif args.long: ++ print_long(os.path.join(cephfs.getcwd(), path, filepath), is_dir, ++ is_sym_lnk, False) ++ elif is_sym_lnk or is_dir: ++ values.append(style_listing(filepath.decode('utf-8'), is_dir, ++ is_sym_lnk)) ++ else: ++ values.append(filepath) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ if not args.long: ++ print_list(values, shutil.get_terminal_size().columns) ++ if path != paths[-1]: ++ poutput('') ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ def complete_rmdir(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ rmdir_parser = argparse.ArgumentParser(description='Remove Directory.') ++ rmdir_parser.add_argument('paths', help='Directory Path.', nargs='+', ++ action=path_to_bytes) ++ rmdir_parser.add_argument('-p', '--parent', action='store_true', ++ help='Remove parent directories as necessary. ' ++ 'When this option is specified, no error ' ++ 'is reported if a directory has any ' ++ 'sub-directories, files') ++ ++ @with_argparser(rmdir_parser) ++ def do_rmdir(self, args): ++ self.do_rmdir_helper(args) ++ ++ def do_rmdir_helper(self, args): ++ """ ++ Remove a specific Directory ++ """ ++ is_pattern = False ++ paths = args.paths ++ for path in paths: ++ if path.count(b'*') > 0: ++ is_pattern = True ++ all_items = get_all_possible_paths(path) ++ if len(all_items) > 0: ++ path = all_items[0].rsplit(b'/', 1)[0] ++ if path == b'': ++ path = b'/' ++ dirs = [] ++ for i in all_items: ++ for item in ls(path): ++ d_name = item.d_name ++ if os.path.basename(i) == d_name: ++ if item.is_dir(): ++ dirs.append(os.path.join(path, d_name)) ++ paths.extend(dirs) ++ continue ++ else: ++ is_pattern = False ++ ++ if args.parent: ++ path = os.path.join(cephfs.getcwd(), path.rsplit(b'/')[0]) ++ files = list(sorted(set(dirwalk(path)), reverse=True)) ++ if not files: ++ path = b'.' ++ for filepath in files: ++ try: ++ cephfs.rmdir(os.path.normpath(filepath)) ++ except libcephfs.Error as e: ++ perror(e) ++ path = b'.' ++ break ++ else: ++ path = os.path.normpath(os.path.join(cephfs.getcwd(), path)) ++ if not is_pattern and path != os.path.normpath(b''): ++ try: ++ cephfs.rmdir(path) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ def complete_rm(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ rm_parser = argparse.ArgumentParser(description='Remove File.') ++ rm_parser.add_argument('paths', help='File Path.', nargs='+', ++ action=path_to_bytes) ++ ++ @with_argparser(rm_parser) ++ def do_rm(self, args): ++ """ ++ Remove a specific file ++ """ ++ file_paths = args.paths ++ for path in file_paths: ++ if path.count(b'*') > 0: ++ file_paths.extend([i for i in get_all_possible_paths( ++ path) if is_file_exists(i)]) ++ else: ++ try: ++ cephfs.unlink(path) ++ except libcephfs.Error as e: ++ # NOTE: perhaps we need a better msg here ++ set_exit_code_msg(msg=e) ++ ++ def complete_mv(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ mv_parser = argparse.ArgumentParser(description='Move File.') ++ mv_parser.add_argument('src_path', type=str, action=path_to_bytes, ++ help='Source File Path.') ++ mv_parser.add_argument('dest_path', type=str, action=path_to_bytes, ++ help='Destination File Path.') ++ ++ @with_argparser(mv_parser) ++ def do_mv(self, args): ++ """ ++ Rename a file or Move a file from source path to the destination ++ """ ++ cephfs.rename(args.src_path, args.dest_path) ++ ++ def complete_cd(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ cd_parser = argparse.ArgumentParser(description='Change working directory') ++ cd_parser.add_argument('path', type=str, help='Name of the directory.', ++ action=path_to_bytes, nargs='?', default='/') ++ ++ @with_argparser(cd_parser) ++ def do_cd(self, args): ++ """ ++ Change working directory ++ """ ++ cephfs.chdir(args.path) ++ self.working_dir = cephfs.getcwd().decode('utf-8') ++ self.set_prompt() ++ ++ def do_cwd(self, arglist): ++ """ ++ Get current working directory. ++ """ ++ poutput(cephfs.getcwd().decode('utf-8')) ++ ++ def complete_chmod(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ chmod_parser = argparse.ArgumentParser(description='Create Directory.') ++ chmod_parser.add_argument('mode', type=str, action=ModeAction, help='Mode') ++ chmod_parser.add_argument('paths', type=str, action=path_to_bytes, ++ help='Name of the file', nargs='+') ++ ++ @with_argparser(chmod_parser) ++ def do_chmod(self, args): ++ """ ++ Change permission of a file ++ """ ++ for path in args.paths: ++ mode = int(args.mode, base=8) ++ try: ++ cephfs.chmod(path, mode) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ def complete_cat(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ cat_parser = argparse.ArgumentParser(description='') ++ cat_parser.add_argument('paths', help='Name of Files', action=path_to_bytes, ++ nargs='+') ++ ++ @with_argparser(cat_parser) ++ def do_cat(self, args): ++ """ ++ Print contents of a file ++ """ ++ for path in args.paths: ++ if is_file_exists(path): ++ copy_to_local(path, b'-') ++ else: ++ set_exit_code_msg(errno.ENOENT, '{}: no such file'.format( ++ path.decode('utf-8'))) ++ ++ umask_parser = argparse.ArgumentParser(description='Set umask value.') ++ umask_parser.add_argument('mode', help='Mode', type=str, action=ModeAction, ++ nargs='?', default='') ++ ++ @with_argparser(umask_parser) ++ def do_umask(self, args): ++ """ ++ Set Umask value. ++ """ ++ if args.mode == '': ++ poutput(self.umask.zfill(4)) ++ else: ++ mode = int(args.mode, 8) ++ self.umask = str(oct(cephfs.umask(mode))[2:]) ++ ++ def complete_write(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ write_parser = argparse.ArgumentParser(description='Writes data into a file') ++ write_parser.add_argument('path', type=str, action=path_to_bytes, ++ help='Name of File') ++ ++ @with_argparser(write_parser) ++ def do_write(self, args): ++ """ ++ Write data into a file. ++ """ ++ ++ copy_from_local(b'-', args.path) ++ ++ def complete_lcd(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ index_dict = {1: self.path_complete} ++ return self.index_based_complete(text, line, begidx, endidx, index_dict) ++ ++ lcd_parser = argparse.ArgumentParser(description='') ++ lcd_parser.add_argument('path', type=str, action=path_to_bytes, help='Path') ++ ++ @with_argparser(lcd_parser) ++ def do_lcd(self, args): ++ """ ++ Moves into the given local directory ++ """ ++ try: ++ os.chdir(os.path.expanduser(args.path)) ++ except OSError as e: ++ set_exit_code_msg(e.errno, "Cannot change to " ++ f"{e.filename.decode('utf-8')}: {e.strerror}") ++ ++ def complete_lls(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ index_dict = {1: self.path_complete} ++ return self.index_based_complete(text, line, begidx, endidx, index_dict) ++ ++ lls_parser = argparse.ArgumentParser( ++ description='List files in local system.') ++ lls_parser.add_argument('paths', help='Paths', action=path_to_bytes, ++ nargs='*') ++ ++ @with_argparser(lls_parser) ++ def do_lls(self, args): ++ """ ++ Lists all files and folders in the current local directory ++ """ ++ if not args.paths: ++ print_list(os.listdir(os.getcwdb())) ++ else: ++ for path in args.paths: ++ try: ++ items = os.listdir(path) ++ poutput("{}:".format(path.decode('utf-8'))) ++ print_list(items) ++ except OSError as e: ++ set_exit_code_msg(e.errno, f"{e.filename.decode('utf-8')}: " ++ f"{e.strerror}") ++ # Arguments to the with_argpaser decorator function are sticky. ++ # The items in args.path do not get overwritten in subsequent calls. ++ # The arguments remain in args.paths after the function exits and we ++ # neeed to clean it up to ensure the next call works as expected. ++ args.paths.clear() ++ ++ def do_lpwd(self, arglist): ++ """ ++ Prints the absolute path of the current local directory ++ """ ++ poutput(os.getcwd()) ++ ++ def complete_df(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ df_parser = argparse.ArgumentParser(description='Show information about\ ++ the amount of available disk space') ++ df_parser.add_argument('file', help='Name of the file', nargs='*', ++ default=['.'], action=path_to_bytes) ++ ++ @with_argparser(df_parser) ++ def do_df(self, arglist): ++ """ ++ Display the amount of available disk space for file systems ++ """ ++ header = True # Set to true for printing header only once ++ if b'.' == arglist.file[0]: ++ arglist.file = ls(b'.') ++ ++ for file in arglist.file: ++ if isinstance(file, libcephfs.DirEntry): ++ file = file.d_name ++ if file == b'.' or file == b'..': ++ continue ++ try: ++ statfs = cephfs.statfs(file) ++ stat = cephfs.stat(file) ++ block_size = (statfs['f_blocks'] * statfs['f_bsize']) // 1024 ++ available = block_size - stat.st_size ++ use = 0 ++ ++ if block_size > 0: ++ use = (stat.st_size * 100) // block_size ++ ++ if header: ++ header = False ++ poutput('{:25s}\t{:5s}\t{:15s}{:10s}{}'.format( ++ "1K-blocks", "Used", "Available", "Use%", ++ "Stored on")) ++ ++ poutput('{:d}\t{:18d}\t{:8d}\t{:10s} {}'.format(block_size, ++ stat.st_size, available, str(int(use)) + '%', ++ file.decode('utf-8'))) ++ except libcephfs.OSError as e: ++ set_exit_code_msg(e.get_error_code(), "could not statfs {}: {}".format( ++ file.decode('utf-8'), e.strerror)) ++ ++ locate_parser = argparse.ArgumentParser( ++ description='Find file within file system') ++ locate_parser.add_argument('name', help='name', type=str, ++ action=path_to_bytes) ++ locate_parser.add_argument('-c', '--count', action='store_true', ++ help='Count list of items located.') ++ locate_parser.add_argument( ++ '-i', '--ignorecase', action='store_true', help='Ignore case') ++ ++ @with_argparser(locate_parser) ++ def do_locate(self, args): ++ """ ++ Find a file within the File System ++ """ ++ if args.name.count(b'*') == 1: ++ if args.name[0] == b'*': ++ args.name += b'/' ++ elif args.name[-1] == '*': ++ args.name = b'/' + args.name ++ args.name = args.name.replace(b'*', b'') ++ if args.ignorecase: ++ locations = locate_file(args.name, False) ++ else: ++ locations = locate_file(args.name) ++ if args.count: ++ poutput(len(locations)) ++ else: ++ poutput((b'\n'.join(locations)).decode('utf-8')) ++ ++ def complete_du(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ du_parser = argparse.ArgumentParser( ++ description='Disk Usage of a Directory') ++ du_parser.add_argument('paths', type=str, action=get_list_of_bytes_path, ++ help='Name of the directory.', nargs='*', ++ default=[b'.']) ++ du_parser.add_argument('-r', action='store_true', ++ help='Recursive Disk usage of all directories.') ++ ++ @with_argparser(du_parser) ++ def do_du(self, args): ++ """ ++ Print disk usage of a given path(s). ++ """ ++ def print_disk_usage(files): ++ if isinstance(files, bytes): ++ files = (files, ) ++ ++ for f in files: ++ try: ++ st = cephfs.lstat(f) ++ ++ if stat.S_ISDIR(st.st_mode): ++ dusage = int(cephfs.getxattr(f, ++ 'ceph.dir.rbytes').decode('utf-8')) ++ else: ++ dusage = st.st_size ++ ++ # print path in local context ++ f = os.path.normpath(f) ++ if f[0] is ord('/'): ++ f = b'.' + f ++ poutput('{:10s} {}'.format(humansize(dusage), ++ f.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ continue ++ ++ for path in args.paths: ++ if args.r: ++ print_disk_usage(sorted(set(dirwalk(path)).union({path}))) ++ else: ++ print_disk_usage(path) ++ ++ quota_parser = argparse.ArgumentParser( ++ description='Quota management for a Directory') ++ quota_parser.add_argument('op', choices=['get', 'set'], ++ help='Quota operation type.') ++ quota_parser.add_argument('path', type=str, action=path_to_bytes, ++ help='Name of the directory.') ++ quota_parser.add_argument('--max_bytes', type=int, default=-1, nargs='?', ++ help='Max cumulative size of the data under ' ++ 'this directory.') ++ quota_parser.add_argument('--max_files', type=int, default=-1, nargs='?', ++ help='Total number of files under this ' ++ 'directory tree.') ++ ++ @with_argparser(quota_parser) ++ def do_quota(self, args): ++ """ ++ Quota management. ++ """ ++ if not is_dir_exists(args.path): ++ set_exit_code_msg(errno.ENOENT, 'error: no such directory {}'.format( ++ args.path.decode('utf-8'))) ++ return ++ ++ if args.op == 'set': ++ if (args.max_bytes == -1) and (args.max_files == -1): ++ set_exit_code_msg(errno.EINVAL, 'please specify either ' ++ '--max_bytes or --max_files or both') ++ return ++ ++ if args.max_bytes >= 0: ++ max_bytes = to_bytes(str(args.max_bytes)) ++ try: ++ cephfs.setxattr(args.path, 'ceph.quota.max_bytes', ++ max_bytes, os.XATTR_CREATE) ++ poutput('max_bytes set to %d' % args.max_bytes) ++ except libcephfs.Error as e: ++ cephfs.setxattr(args.path, 'ceph.quota.max_bytes', ++ max_bytes, os.XATTR_REPLACE) ++ set_exit_code_msg(e.get_error_code(), 'max_bytes reset to ' ++ f'{args.max_bytes}') ++ ++ if args.max_files >= 0: ++ max_files = to_bytes(str(args.max_files)) ++ try: ++ cephfs.setxattr(args.path, 'ceph.quota.max_files', ++ max_files, os.XATTR_CREATE) ++ poutput('max_files set to %d' % args.max_files) ++ except libcephfs.Error as e: ++ cephfs.setxattr(args.path, 'ceph.quota.max_files', ++ max_files, os.XATTR_REPLACE) ++ set_exit_code_msg(e.get_error_code(), 'max_files reset to ' ++ f'{args.max_files}') ++ elif args.op == 'get': ++ max_bytes = '0' ++ max_files = '0' ++ try: ++ max_bytes = cephfs.getxattr(args.path, 'ceph.quota.max_bytes') ++ poutput('max_bytes: {}'.format(max_bytes.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e.get_error_code(), 'max_bytes is not set') ++ ++ try: ++ max_files = cephfs.getxattr(args.path, 'ceph.quota.max_files') ++ poutput('max_files: {}'.format(max_files.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e.get_error_code(), 'max_files is not set') ++ ++ snap_parser = argparse.ArgumentParser(description='Snapshot Management') ++ snap_parser.add_argument('op', type=str, ++ help='Snapshot operation: create or delete') ++ snap_parser.add_argument('name', type=str, action=path_to_bytes, ++ help='Name of snapshot') ++ snap_parser.add_argument('dir', type=str, action=path_to_bytes, ++ help='Directory for which snapshot ' ++ 'needs to be created or deleted') ++ ++ @with_argparser(snap_parser) ++ def do_snap(self, args): ++ """ ++ Snapshot management for the volume ++ """ ++ # setting self.colors to None turns off colorizing and ++ # perror emits plain text ++ self.colors = None ++ ++ snapdir = '.snap' ++ conf_snapdir = cephfs.conf_get('client_snapdir') ++ if conf_snapdir is not None: ++ snapdir = conf_snapdir ++ snapdir = to_bytes(snapdir) ++ if args.op == 'create': ++ try: ++ if is_dir_exists(args.dir): ++ cephfs.mkdir(os.path.join(args.dir, snapdir, args.name), 0o755) ++ else: ++ set_exit_code_msg(errno.ENOENT, "'{}': no such directory".format( ++ args.dir.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e.get_error_code(), ++ "snapshot '{}' already exists".format( ++ args.name.decode('utf-8'))) ++ elif args.op == 'delete': ++ snap_dir = os.path.join(args.dir, snapdir, args.name) ++ try: ++ if is_dir_exists(snap_dir): ++ newargs = argparse.Namespace(paths=[snap_dir], parent=False) ++ self.do_rmdir_helper(newargs) ++ else: ++ set_exit_code_msg(errno.ENOENT, "'{}': no such snapshot".format( ++ args.name.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e.get_error_code(), "error while deleting " ++ "'{}'".format(snap_dir.decode('utf-8'))) ++ else: ++ set_exit_code_msg(errno.EINVAL, "snapshot can only be created or " ++ "deleted; check - help snap") ++ ++ def do_help(self, line): ++ """ ++ Get details about a command. ++ Usage: help - for a specific command ++ help all - for all the commands ++ """ ++ if line == 'all': ++ for k in dir(self): ++ if k.startswith('do_'): ++ poutput('-' * 80) ++ super().do_help(k[3:]) ++ return ++ parser = self.create_argparser(line) ++ if parser: ++ parser.print_help() ++ else: ++ super().do_help(line) ++ ++ def complete_stat(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ stat_parser = argparse.ArgumentParser( ++ description='Display file or file system status') ++ stat_parser.add_argument('paths', type=str, help='file paths', ++ action=path_to_bytes, nargs='+') ++ ++ @with_argparser(stat_parser) ++ def do_stat(self, args): ++ """ ++ Display file or file system status ++ """ ++ for path in args.paths: ++ try: ++ stat = cephfs.stat(path) ++ atime = stat.st_atime.isoformat(' ') ++ mtime = stat.st_mtime.isoformat(' ') ++ ctime = stat.st_mtime.isoformat(' ') ++ ++ poutput("File: {}\nSize: {:d}\nBlocks: {:d}\nIO Block: {:d}\n" ++ "Device: {:d}\tInode: {:d}\tLinks: {:d}\nPermission: " ++ "{:o}/{}\tUid: {:d}\tGid: {:d}\nAccess: {}\nModify: " ++ "{}\nChange: {}".format(path.decode('utf-8'), ++ stat.st_size, stat.st_blocks, ++ stat.st_blksize, stat.st_dev, ++ stat.st_ino, stat.st_nlink, ++ stat.st_mode, ++ mode_notation(stat.st_mode), ++ stat.st_uid, stat.st_gid, atime, ++ mtime, ctime)) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ setxattr_parser = argparse.ArgumentParser( ++ description='Set extended attribute for a file') ++ setxattr_parser.add_argument('path', type=str, action=path_to_bytes, help='Name of the file') ++ setxattr_parser.add_argument('name', type=str, help='Extended attribute name') ++ setxattr_parser.add_argument('value', type=str, help='Extended attribute value') ++ ++ @with_argparser(setxattr_parser) ++ def do_setxattr(self, args): ++ """ ++ Set extended attribute for a file ++ """ ++ val_bytes = to_bytes(args.value) ++ name_bytes = to_bytes(args.name) ++ try: ++ cephfs.setxattr(args.path, name_bytes, val_bytes, os.XATTR_CREATE) ++ poutput('{} is successfully set to {}'.format(args.name, args.value)) ++ except libcephfs.ObjectExists: ++ cephfs.setxattr(args.path, name_bytes, val_bytes, os.XATTR_REPLACE) ++ poutput('{} is successfully reset to {}'.format(args.name, args.value)) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ getxattr_parser = argparse.ArgumentParser( ++ description='Get extended attribute set for a file') ++ getxattr_parser.add_argument('path', type=str, action=path_to_bytes, ++ help='Name of the file') ++ getxattr_parser.add_argument('name', type=str, help='Extended attribute name') ++ ++ @with_argparser(getxattr_parser) ++ def do_getxattr(self, args): ++ """ ++ Get extended attribute for a file ++ """ ++ try: ++ poutput('{}'.format(cephfs.getxattr(args.path, ++ to_bytes(args.name)).decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ listxattr_parser = argparse.ArgumentParser( ++ description='List extended attributes set for a file') ++ listxattr_parser.add_argument('path', type=str, action=path_to_bytes, ++ help='Name of the file') ++ ++ @with_argparser(listxattr_parser) ++ def do_listxattr(self, args): ++ """ ++ List extended attributes for a file ++ """ ++ try: ++ size, xattr_list = cephfs.listxattr(args.path) ++ if size > 0: ++ poutput('{}'.format(xattr_list.replace(b'\x00', b' ').decode('utf-8'))) ++ else: ++ poutput('No extended attribute is set') ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ ++####################################################### ++# ++# Following are methods that get cephfs-shell started. ++# ++##################################################### ++ ++def setup_cephfs(args): ++ """ ++ Mounting a cephfs ++ """ ++ global cephfs ++ try: ++ cephfs = libcephfs.LibCephFS(conffile='') ++ cephfs.mount(filesystem_name=args.fs) ++ except libcephfs.ObjectNotFound as e: ++ print('couldn\'t find ceph configuration not found') ++ sys.exit(e.get_error_code()) ++ except libcephfs.Error as e: ++ print(e) ++ sys.exit(e.get_error_code()) ++ ++ ++def str_to_bool(val): ++ """ ++ Return corresponding bool values for strings like 'true' or 'false'. ++ """ ++ if not isinstance(val, str): ++ return val ++ ++ val = val.replace('\n', '') ++ if val.lower() in ['true', 'yes']: ++ return True ++ elif val.lower() in ['false', 'no']: ++ return False ++ else: ++ return val ++ ++ ++def read_shell_conf(shell, shell_conf_file): ++ import configparser ++ ++ sec = 'cephfs-shell' ++ opts = [] ++ if LooseVersion(cmd2_version) >= LooseVersion("0.10.0"): ++ for attr in shell.settables.keys(): ++ opts.append(attr) ++ else: ++ if LooseVersion(cmd2_version) <= LooseVersion("0.9.13"): ++ # hardcoding options for 0.7.9 because - ++ # 1. we use cmd2 v0.7.9 with teuthology and ++ # 2. there's no way distinguish between a shell setting and shell ++ # object attribute until v0.10.0 ++ opts = ['abbrev', 'autorun_on_edit', 'colors', ++ 'continuation_prompt', 'debug', 'echo', 'editor', ++ 'feedback_to_output', 'locals_in_py', 'prompt', 'quiet', ++ 'timing'] ++ elif LooseVersion(cmd2_version) >= LooseVersion("0.9.23"): ++ opts.append('allow_style') ++ # no equivalent option was defined by cmd2. ++ else: ++ pass ++ ++ # default and only section in our conf file. ++ cp = configparser.ConfigParser(default_section=sec, strict=False) ++ cp.read(shell_conf_file) ++ for opt in opts: ++ if cp.has_option(sec, opt): ++ setattr(shell, opt, str_to_bool(cp.get(sec, opt))) ++ ++ ++def get_shell_conffile_path(arg_conf=''): ++ conf_filename = 'cephfs-shell.conf' ++ env_var = 'CEPHFS_SHELL_CONF' ++ ++ arg_conf = '' if not arg_conf else arg_conf ++ home_dir_conf = os.path.expanduser('~/.' + conf_filename) ++ env_conf = os.environ[env_var] if env_var in os.environ else '' ++ ++ # here's the priority by which conf gets read. ++ for path in (arg_conf, env_conf, home_dir_conf): ++ if os.path.isfile(path): ++ return path ++ else: ++ return '' ++ ++ ++def manage_args(): ++ main_parser = argparse.ArgumentParser(description='') ++ main_parser.add_argument('-b', '--batch', action='store', ++ help='Path to CephFS shell script/batch file' ++ 'containing CephFS shell commands', ++ type=str) ++ main_parser.add_argument('-c', '--config', action='store', ++ help='Path to Ceph configuration file.', ++ type=str) ++ main_parser.add_argument('-f', '--fs', action='store', ++ help='Name of filesystem to mount.', ++ type=str) ++ main_parser.add_argument('-t', '--test', action='store', ++ help='Test against transcript(s) in FILE', ++ nargs='+') ++ main_parser.add_argument('commands', nargs='*', help='Comma delimited ' ++ 'commands. The shell executes the given command ' ++ 'and quits immediately with the return value of ' ++ 'command. In case no commands are provided, the ' ++ 'shell is launched.', default=[]) ++ ++ args = main_parser.parse_args() ++ args.exe_and_quit = False # Execute and quit, don't launch the shell. ++ ++ if args.batch: ++ if LooseVersion(cmd2_version) <= LooseVersion("0.9.13"): ++ args.commands = ['load ' + args.batch, ',quit'] ++ else: ++ args.commands = ['run_script ' + args.batch, ',quit'] ++ if args.test: ++ args.commands.extend(['-t,'] + [arg + ',' for arg in args.test]) ++ if not args.batch and len(args.commands) > 0: ++ args.exe_and_quit = True ++ ++ manage_sys_argv(args) ++ ++ return args ++ ++ ++def manage_sys_argv(args): ++ exe = sys.argv[0] ++ sys.argv.clear() ++ sys.argv.append(exe) ++ sys.argv.extend([i.strip() for i in ' '.join(args.commands).split(',')]) ++ ++ setup_cephfs(args) ++ ++ ++def execute_cmd_args(args): ++ """ ++ Launch a shell session if no arguments were passed, else just execute ++ the given argument as a shell command and exit the shell session ++ immediately at (last) command's termination with the (last) command's ++ return value. ++ """ ++ if not args.exe_and_quit: ++ return shell.cmdloop() ++ return execute_cmds_and_quit(args) ++ ++ ++def execute_cmds_and_quit(args): ++ """ ++ Multiple commands might be passed separated by commas, feed onecmd() ++ one command at a time. ++ """ ++ # do_* methods triggered by cephfs-shell commands return None when they ++ # complete running successfully. Until 0.9.6, shell.onecmd() returned this ++ # value to indicate whether the execution of the commands should stop, but ++ # since 0.9.7 it returns the return value of do_* methods only if it's ++ # not None. When it is None it returns False instead of None. ++ if LooseVersion(cmd2_version) <= LooseVersion("0.9.6"): ++ stop_exec_val = None ++ else: ++ stop_exec_val = False ++ ++ args_to_onecmd = '' ++ if len(args.commands) <= 1: ++ args.commands = args.commands[0].split(' ') ++ for cmdarg in args.commands: ++ if ',' in cmdarg: ++ args_to_onecmd += ' ' + cmdarg[0:-1] ++ onecmd_retval = shell.onecmd(args_to_onecmd) ++ # if the curent command failed, let's abort the execution of ++ # series of commands passed. ++ if onecmd_retval is not stop_exec_val: ++ return onecmd_retval ++ if shell.exit_code != 0: ++ return shell.exit_code ++ ++ args_to_onecmd = '' ++ continue ++ ++ args_to_onecmd += ' ' + cmdarg ++ return shell.onecmd(args_to_onecmd) ++ ++ ++if __name__ == '__main__': ++ args = manage_args() ++ ++ shell = CephFSShell() ++ # TODO: perhaps, we should add an option to pass ceph.conf? ++ read_shell_conf(shell, get_shell_conffile_path(args.config)) ++ # XXX: setting shell.exit_code to zero so that in case there are no errors ++ # and exceptions, it is not set by any method or function of cephfs-shell ++ # and return values from shell.cmdloop() or shell.onecmd() is not an ++ # integer, we can treat it as the return value of cephfs-shell. ++ shell.exit_code = 0 ++ ++ retval = execute_cmd_args(args) ++ sys.exit(retval if retval else shell.exit_code) +--- /dev/null 2022-06-30 09:45:32.996000000 -0400 ++++ ceph-17.2.1/src/tools/cephfs/shell/setup.py 2022-07-05 11:00:12.411260682 -0400 +@@ -0,0 +1,27 @@ ++# -*- coding: utf-8 -*- ++ ++from setuptools import setup ++ ++__version__ = '0.0.1' ++ ++setup( ++ name='cephfs-shell', ++ version=__version__, ++ description='Interactive shell for Ceph file system', ++ keywords='cephfs, shell', ++ scripts=['cephfs-shell'], ++ install_requires=[ ++ 'cephfs', ++ 'cmd2', ++ 'colorama', ++ ], ++ classifiers=[ ++ 'Development Status :: 3 - Alpha', ++ 'Environment :: Console', ++ 'Intended Audience :: System Administrators', ++ 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)', ++ 'Operating System :: POSIX :: Linux', ++ 'Programming Language :: Python :: 3' ++ ], ++ license='LGPLv2+', ++) +--- /dev/null 2022-06-30 09:45:32.996000000 -0400 ++++ ceph-17.2.1/src/tools/cephfs/shell/tox.ini 2022-06-23 10:41:35.000000000 -0400 +@@ -0,0 +1,7 @@ ++[tox] ++envlist = py3 ++skipsdist = true ++ ++[testenv:py3] ++deps = flake8 ++commands = flake8 --ignore=W503 --max-line-length=100 cephfs-shell diff --git a/ceph.spec b/ceph.spec index ce30eab..0d86728 100644 --- a/ceph.spec +++ b/ceph.spec @@ -93,7 +93,7 @@ %endif %bcond_with seastar %bcond_with jaeger -%if (0%{?fedora} && 0%{?fedora} < 37) || 0%{?suse_version} >= 1500 +%if 0%{?fedora} || 0%{?suse_version} >= 1500 # distros that ship cmd2 and/or colorama %bcond_without cephfs_shell %else @@ -159,7 +159,7 @@ ################################################################################# Name: ceph Version: 17.2.1 -Release: 2%{?dist} +Release: 3%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 %endif @@ -188,6 +188,7 @@ Patch0017: 0017-gcc-12-omnibus.patch Patch0018: 0018-src-rgw-store-dbstore-CMakeLists.txt.patch Patch0019: 0019-cmake-modules-CheckCxxAtomic.cmake.patch Patch0020: 0020-src-arrow-cpp-cmake_modules-ThirdpartyToolchain.cmake.patch +Patch0021: 0021-cephfs-shell.patch # ceph 14.0.1 does not support 32-bit architectures, bugs #1727788, #1727787 ExcludeArch: i686 armv7hl %if 0%{?suse_version} @@ -2580,6 +2581,9 @@ exit 0 %config %{_sysconfdir}/prometheus/ceph/ceph_default_alerts.yml %changelog +* Wed Jul 6 2022 Kaleb S. KEITHLEY - 2:17.2.1-3 +- enable cephfs-shell + * Tue Jun 28 2022 Kaleb S. KEITHLEY - 2:17.2.1-2 - ceph-17.2.1, rhbz#2101744, rebuild with gtest, gmock 1.12.0 - fix inconsistent tabs, spaces