rpms/fio
7
0
Fork 0
Code Issues Pull Requests Releases Activity

Finish transition to python3

Browse Source
This commit is contained in:
Eric Sandeen 2018-06-01 14:25:23 -05:00
parent 05c3b5fd1f
commit 579853a79d
2 changed files with 371 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

380
fio-python3.patch
View File

@ -1,27 +1,52 @@
make fio scripts python3-ready From 5eac3b00238b450ac0679121a76f1e566ca8f468 Mon Sep 17 00:00:00 2001
From: Bill O'Donnell <billodo@redhat.com>
Date: Fri, 4 May 2018 14:43:40 -0500
Subject: [PATCH] make fio scripts python3-ready
commit 5eac3b00238b450ac0679121a76f1e566ca8f468 Many distributions are moving to python3 by default. Here's
Author: Bill O'Donnell <billodo@redhat.com> an attempt to make the python scripts in fio python3-ready.
Date: Fri May 4 14:43:40 2018 -0500
make fio scripts python3-ready Conversion was facilitated with automated tools. A few areas
were hand fixed: remove superfluous parentheses introduced by
2to3 converter in print function calls, shebang modifications
to use environment variable for python version, and byte-string
decode correction in steadystate_tests.py following 2to3
conversion.
Many distributions are moving to python3 by default. Here's The modified scripts pass rudimentary testing when run under
an attempt to make the python scripts in fio python3-ready. python2.7 as well as python3.
Conversion was facilitated with automated tools. A few areas Signed-off-by: Bill O'Donnell <billodo@redhat.com>
were hand fixed: remove superfluous parentheses introduced by Signed-off-by: Jens Axboe <axboe@kernel.dk>
2to3 converter in print function calls, shebang modifications
to use environment variable for python version, and byte-string
decode correction in steadystate_tests.py following 2to3
conversion.
The modified scripts pass rudimentary testing when run under doc/conf.py | 3 +++
python2.7 as well as python3. tools/fiologparser.py | 3 +++
unit_tests/steadystate_tests.py | 20 ++++++++++++--------
3 files changed, 18 insertions(+), 8 deletions(-)
Signed-off-by: Bill O'Donnell <billodo@redhat.com> From 50d2a37543908e79c41b1d5a4c2cb2d65c5af21b Mon Sep 17 00:00:00 2001
Signed-off-by: Jens Axboe <axboe@kernel.dk> From: Bill O'Donnell <billodo@redhat.com>
Date: Tue, 22 May 2018 12:49:09 -0500
Subject: [PATCH] make fio scripts python3-ready (part 2)
NOTE: this patch (part 2) is in addition to commit 5eac3b00238b
(this patch was missed on that submit/commit (part 1)).
Many distributions are moving to python3 by default. This is
another patch to make the python scripts in fio python3-ready.
Conversion was facilitated with automated tools.
The modified scripts pass rudimentary testing when run under
python2.7 as well as python3.
Signed-off-by: Bill O'Donnell <billodo@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
--- ---
tools/fio_jsonplus_clat2csv | 13 ++++--
tools/plot/fio2gnuplot | 97 ++++++++++++++++++++++++---------------------
2 files changed, 60 insertions(+), 50 deletions(-)
Index: fio-3.6/doc/conf.py Index: fio-3.6/doc/conf.py
=================================================================== ===================================================================
--- fio-3.6.orig/doc/conf.py --- fio-3.6.orig/doc/conf.py
@ -133,5 +158,326 @@ Index: fio-3.6/unit_tests/steadystate_tests.py
if 'steadystate' in jsonjob: if 'steadystate' in jsonjob:
pp.pprint(jsonjob['steadystate']) pp.pprint(jsonjob['steadystate'])
jobnum += 1 jobnum += 1
Index: fio-3.6/tools/fio_jsonplus_clat2csv
===================================================================
--- fio-3.6.orig/tools/fio_jsonplus_clat2csv
+++ fio-3.6/tools/fio_jsonplus_clat2csv
@@ -1,4 +1,5 @@
-#!/usr/bin/python2.7
+#!/usr/bin/python3
+# Note: this script is python2 and python3 compatible.
#
# fio_jsonplus_clat2csv
#
@@ -60,9 +61,13 @@
# 10304ns is the 100th percentile for read latency
#
+from __future__ import absolute_import
+from __future__ import print_function
import os
import json
import argparse
+import six
+from six.moves import range
def parse_args():
@@ -87,7 +92,7 @@ def percentile(idx, run_total):
def more_lines(indices, bins):
- for key, value in indices.iteritems():
+ for key, value in six.iteritems(indices):
if value < len(bins[key]):
return True
@@ -116,8 +121,8 @@ def main():
"Are you sure you are using json+ output?")
bins[ddir] = [[int(key), value] for key, value in
- jsondata['jobs'][jobnum][ddir][bins_loc]
- ['bins'].iteritems()]
+ six.iteritems(jsondata['jobs'][jobnum][ddir][bins_loc]
+ ['bins'])]
bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0])
run_total[ddir] = [0 for x in range(0, len(bins[ddir]))]
@@ -165,7 +170,7 @@ def main():
output.write(", , , ")
output.write("\n")
- print "{0} generated".format(outfile)
+ print("{0} generated".format(outfile))
if __name__ == '__main__':
Index: fio-3.6/tools/plot/fio2gnuplot
===================================================================
--- fio-3.6.orig/tools/plot/fio2gnuplot
+++ fio-3.6/tools/plot/fio2gnuplot
@@ -1,4 +1,5 @@
-#!/usr/bin/python2.7
+#!/usr/bin/python3
+# Note: this script is python2 and python3 compatible.
#
# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
# Author: Erwan Velu <erwan@enovance.com>
@@ -19,6 +20,8 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+from __future__ import absolute_import
+from __future__ import print_function
import os
import fnmatch
import sys
@@ -26,6 +29,8 @@ import getopt
import re
import math
import shutil
+from six.moves import map
+from six.moves import range
def find_file(path, pattern):
fio_data_file=[]
@@ -39,7 +44,7 @@ def find_file(path, pattern):
return fio_data_file
def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir):
- if verbose: print "Generating rendering scripts"
+ if verbose: print("Generating rendering scripts")
filename=gnuplot_output_dir+'mygraph'
temporary_files.append(filename)
f=open(filename,'w')
@@ -124,7 +129,7 @@ def generate_gnuplot_math_script(title,g
f.close()
def compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir):
- if verbose: print "Processing data file 2/2"
+ if verbose: print("Processing data file 2/2")
temp_files=[]
pos=0
@@ -152,7 +157,7 @@ def compute_temp_file(fio_data_file,disk
end_time=max_time
if end_time == -1:
end_time="infinite"
- if verbose: print "Processing data file 1/2 with %s<time<%s" % (min_time,end_time)
+ if verbose: print("Processing data file 1/2 with %s<time<%s" % (min_time,end_time))
files=[]
temp_outfile=[]
blk_size=0
@@ -198,8 +203,8 @@ def compute_temp_file(fio_data_file,disk
try:
blk_size=int(block_size)
except:
- print "Error while reading the following line :"
- print line
+ print("Error while reading the following line :")
+ print(line)
sys.exit(1);
# We ignore the first 500msec as it doesn't seems to be part of the real benchmark
@@ -225,7 +230,7 @@ def compute_temp_file(fio_data_file,disk
return blk_size
def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir):
- if verbose: print "Computing Maths"
+ if verbose: print("Computing Maths")
global_min=[]
global_max=[]
average_file=open(gnuplot_output_dir+gnuplot_output_filename+'.average', 'w')
@@ -243,14 +248,14 @@ def compute_math(fio_data_file, title,gn
max_file.write('DiskName %s\n'% mode)
average_file.write('DiskName %s\n'% mode)
stddev_file.write('DiskName %s\n'% mode )
- for disk in xrange(len(fio_data_file)):
+ for disk in range(len(fio_data_file)):
# print disk_perf[disk]
min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
avg = average(disk_perf[disk])
- variance = map(lambda x: (x - avg)**2, disk_perf[disk])
+ variance = [(x - avg)**2 for x in disk_perf[disk]]
standard_deviation = math.sqrt(average(variance))
# print "Disk%d [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (disk,min(disk_perf[disk]),max(disk_perf[disk]),avg, standard_deviation)
average_file.write('%d %d\n' % (disk, avg))
@@ -264,7 +269,7 @@ def compute_math(fio_data_file, title,gn
global_disk_perf = sum(disk_perf, [])
avg = average(global_disk_perf)
- variance = map(lambda x: (x - avg)**2, global_disk_perf)
+ variance = [(x - avg)**2 for x in global_disk_perf]
standard_deviation = math.sqrt(average(variance))
global_file.write('min=%.2f\n' % min(global_disk_perf))
@@ -331,52 +336,52 @@ def parse_global_files(fio_data_file, gl
max_file=file
# Let's print the avg output
if global_search == "avg":
- print "Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file)
+ print("Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file))
else:
- print "Global search %s is not yet implemented\n" % global_search
+ print("Global search %s is not yet implemented\n" % global_search)
def render_gnuplot(fio_data_file, gnuplot_output_dir):
- print "Running gnuplot Rendering"
+ print("Running gnuplot Rendering")
try:
# Let's render all the compared files if some
if len(fio_data_file) > 1:
- if verbose: print " |-> Rendering comparing traces"
+ if verbose: print(" |-> Rendering comparing traces")
os.system("cd %s; for i in *.gnuplot; do gnuplot $i; done" % gnuplot_output_dir)
- if verbose: print " |-> Rendering math traces"
+ if verbose: print(" |-> Rendering math traces")
os.system("cd %s; gnuplot mymath" % gnuplot_output_dir)
- if verbose: print " |-> Rendering 2D & 3D traces"
+ if verbose: print(" |-> Rendering 2D & 3D traces")
os.system("cd %s; gnuplot mygraph" % gnuplot_output_dir)
name_of_directory="the current"
if gnuplot_output_dir != "./":
name_of_directory=gnuplot_output_dir
- print "\nRendering traces are available in %s directory" % name_of_directory
+ print("\nRendering traces are available in %s directory" % name_of_directory)
global keep_temp_files
keep_temp_files=False
except:
- print "Could not run gnuplot on mymath or mygraph !\n"
+ print("Could not run gnuplot on mymath or mygraph !\n")
sys.exit(1);
def print_help():
- print 'fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>'
- print
- print '-h --help : Print this help'
- print '-p <pattern> or --pattern <pattern> : A glob pattern to select fio input files'
- print '-b or --bandwidth : A predefined pattern for selecting *_bw.log files'
- print '-i or --iops : A predefined pattern for selecting *_iops.log files'
- print '-g or --gnuplot : Render gnuplot traces before exiting'
- print '-o or --outputfile <file> : The basename for gnuplot traces'
- print ' - Basename is set with the pattern if defined'
- print '-d or --outputdir <dir> : The directory where gnuplot shall render files'
- print '-t or --title <title> : The title of the gnuplot traces'
- print ' - Title is set with the block size detected in fio traces'
- print '-G or --Global <type> : Search for <type> in .global files match by a pattern'
- print ' - Available types are : min, max, avg, stddev'
- print ' - The .global extension is added automatically to the pattern'
- print '-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)'
- print '-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)'
- print '-v or --verbose : Increasing verbosity'
- print '-k or --keep : Keep all temporary files from gnuplot\'s output dir'
+ print('fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>')
+ print()
+ print('-h --help : Print this help')
+ print('-p <pattern> or --pattern <pattern> : A glob pattern to select fio input files')
+ print('-b or --bandwidth : A predefined pattern for selecting *_bw.log files')
+ print('-i or --iops : A predefined pattern for selecting *_iops.log files')
+ print('-g or --gnuplot : Render gnuplot traces before exiting')
+ print('-o or --outputfile <file> : The basename for gnuplot traces')
+ print(' - Basename is set with the pattern if defined')
+ print('-d or --outputdir <dir> : The directory where gnuplot shall render files')
+ print('-t or --title <title> : The title of the gnuplot traces')
+ print(' - Title is set with the block size detected in fio traces')
+ print('-G or --Global <type> : Search for <type> in .global files match by a pattern')
+ print(' - Available types are : min, max, avg, stddev')
+ print(' - The .global extension is added automatically to the pattern')
+ print('-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)')
+ print('-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)')
+ print('-v or --verbose : Increasing verbosity')
+ print('-k or --keep : Keep all temporary files from gnuplot\'s output dir')
def main(argv):
mode='unknown'
@@ -403,14 +408,14 @@ def main(argv):
if not os.path.isfile(gpm_dir+'math.gpm'):
gpm_dir="/usr/local/share/fio/"
if not os.path.isfile(gpm_dir+'math.gpm'):
- print "Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n"
+ print("Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n")
sys.exit(3)
try:
opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep'])
except getopt.GetoptError:
- print "Error: One of the options passed to the cmdline was not supported"
- print "Please fix your command line or read the help (-h option)"
+ print("Error: One of the options passed to the cmdline was not supported")
+ print("Please fix your command line or read the help (-h option)")
sys.exit(2)
for opt, arg in opts:
@@ -457,7 +462,7 @@ def main(argv):
fio_data_file=find_file('.',pattern)
if len(fio_data_file) == 0:
- print "No log file found with pattern %s!" % pattern
+ print("No log file found with pattern %s!" % pattern)
# Try numjob log file format if per_numjob_logs=1
if (pattern == '*_bw.log'):
fio_data_file=find_file('.','*_bw.*.log')
@@ -466,13 +471,13 @@ def main(argv):
if len(fio_data_file) == 0:
sys.exit(1)
else:
- print "Using log file per job format instead"
+ print("Using log file per job format instead")
else:
- print "%d files Selected with pattern '%s'" % (len(fio_data_file), pattern)
+ print("%d files Selected with pattern '%s'" % (len(fio_data_file), pattern))
fio_data_file=sorted(fio_data_file, key=str.lower)
for file in fio_data_file:
- print ' |-> %s' % file
+ print(' |-> %s' % file)
if "_bw.log" in file :
mode="Bandwidth (KB/sec)"
if "_iops.log" in file :
@@ -483,7 +488,7 @@ def main(argv):
if "IO" in mode:
title='IO benchmark with %d fio results' % len(fio_data_file)
- print
+ print()
#We need to adjust the output filename regarding the pattern required by the user
if (pattern_set_by_user == True):
gnuplot_output_filename=pattern
@@ -514,9 +519,9 @@ def main(argv):
# Shall we clean the temporary files ?
if keep_temp_files==False and force_keep_temp_files==False:
# Cleaning temporary files
- if verbose: print "Cleaning temporary files"
+ if verbose: print("Cleaning temporary files")
for f in enumerate(temporary_files):
- if verbose: print " -> %s"%f[1]
+ if verbose: print(" -> %s"%f[1])
try:
os.remove(f[1])
except:
Index: fio-3.6/tools/hist/fiologparser_hist.py
===================================================================
--- fio-3.6.orig/tools/hist/fiologparser_hist.py
+++ fio-3.6/tools/hist/fiologparser_hist.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2.7
+#!/usr/bin/python3
"""
Utility for converting *_clat_hist* files generated by fio into latency statistics.
Index: fio-3.6/tools/hist/half-bins.py
===================================================================
--- fio-3.6.orig/tools/hist/half-bins.py
+++ fio-3.6/tools/hist/half-bins.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2.7
+#!/usr/bin/python3
""" Cut the number bins in half in fio histogram output. Example usage:
$ half-bins.py -c 2 output_clat_hist.1.log > smaller_clat_hist.1.log

5
fio.spec
View File

@ -1,6 +1,6 @@
Name: fio Name: fio
Version: 3.6 Version: 3.6
Release: 2%{?dist} Release: 3%{?dist}
Summary: Multithreaded IO generation tool Summary: Multithreaded IO generation tool
Group: Applications/System Group: Applications/System
@ -52,6 +52,9 @@ make install prefix=%{_prefix} mandir=%{_mandir} DESTDIR=$RPM_BUILD_ROOT INSTALL
%{_datadir}/%{name}/* %{_datadir}/%{name}/*
%changelog %changelog
* Fri Jun 01 2018 Eric Sandeen <sandeen@redhat.com> 3.6-3
- Complete the conversion to python3
* Wed May 16 2018 Eric Sandeen <sandeen@redhat.com> 3.6-2 * Wed May 16 2018 Eric Sandeen <sandeen@redhat.com> 3.6-2
- Make all python scripts python3 compliant and explicit - Make all python scripts python3 compliant and explicit