No longer use our splittree for rawhide.
This commit is contained in:
jkeating@dhcp83-49.boston.redhat.com
Jesse Keating
parent
337a57efdf
commit
f447b2c285
@ -14,7 +14,7 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
#sys.path.append('/usr/lib/anaconda-runtime') # use our patched splittree for now
|
||||
sys.path.append('/usr/lib/anaconda-runtime')
|
||||
import splittree
|
||||
import shutil
|
||||
import re
|
||||
|
||||
@ -1,478 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright (C) 2003-2005 Red Hat, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
#
|
||||
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import string
|
||||
import getopt
|
||||
import time
|
||||
import types
|
||||
import rpm
|
||||
|
||||
global _ts
|
||||
_ts = None
|
||||
|
||||
# returns n-v-r.a from file filename
|
||||
def nvra(pkgfile):
|
||||
global _ts
|
||||
if _ts is None:
|
||||
_ts = rpm.TransactionSet()
|
||||
_ts.setVSFlags(-1)
|
||||
fd = os.open(pkgfile, os.O_RDONLY)
|
||||
h = _ts.hdrFromFdno(fd)
|
||||
os.close(fd)
|
||||
return "%s-%s-%s.%s.rpm" %(h['name'], h['version'], h['release'],
|
||||
h['arch'])
|
||||
|
||||
|
||||
class Timber:
|
||||
"""Split trees like no other"""
|
||||
def __init__(self):
|
||||
|
||||
"""self.release_str : the name and version of the product"
|
||||
|
||||
self.package_order_file : the location of the file which has
|
||||
the package ordering
|
||||
|
||||
self.arch : the arch the tree is intended for
|
||||
|
||||
self.real_arch : the arch found in the unified tree's
|
||||
.discinfo file
|
||||
|
||||
self.dist_dir : the loaction of the unified tree
|
||||
|
||||
self.src_dir : the location of the unified SRPM dir
|
||||
|
||||
self.start_time : the timestamp that's in .discinfo files
|
||||
|
||||
self.dir_info : The info other than start_time that goes
|
||||
into the .discinfo files. The info should already exist
|
||||
after running buildinstall in the unified tree
|
||||
|
||||
self.total_discs : total number of discs
|
||||
|
||||
self.total_bin_discs : total number of discs with RPMs
|
||||
|
||||
self.total_srpm_discs : total number of discs with SRPMs
|
||||
|
||||
self.reverse_sort_srpms : sort the srpms in reverse order to
|
||||
fit. Usually only needed if we share a disc between SRPMs
|
||||
and RPMs. Set to 1 to turn on.
|
||||
|
||||
self.reserve_size : Additional size needed to be reserved on the first disc.
|
||||
"""
|
||||
|
||||
self.target_size = 640.0 * 1024.0 * 1024
|
||||
self.fudge_factor = 1.2 * 1024.0 * 1024
|
||||
self.comps_size = 10.0 * 1024 * 1024
|
||||
self.reserve_size = 0
|
||||
self.release_str = None
|
||||
self.package_order_file = None
|
||||
self.arch = None
|
||||
self.real_arch = None
|
||||
self.dist_dir = None
|
||||
self.src_dir = None
|
||||
self.start_time = None
|
||||
self.dir_info = None
|
||||
self.total_discs = None
|
||||
self.bin_discs = None
|
||||
self.src_discs = None
|
||||
self.product_path = "anaconda"
|
||||
self.bin_list = []
|
||||
self.src_list = []
|
||||
self.shared_list = []
|
||||
self.reverse_sort_srpms=None
|
||||
self.common_files = ['beta_eula.txt', 'EULA', 'README', 'GPL', 'RPM-GPG-KEY', 'RPM-GPG-KEY-beta', 'RPM-GPG-KEY-fedora']
|
||||
self.logfile = []
|
||||
|
||||
|
||||
def getSize(self, path, blocksize=None):
|
||||
"""Gets the size as reported by du -sL"""
|
||||
|
||||
if blocksize:
|
||||
p = os.popen("du -sL --block-size=1 %s" % path, 'r')
|
||||
thesize = p.read()
|
||||
p.close()
|
||||
thesize = long(string.split(thesize)[0])
|
||||
return thesize
|
||||
else:
|
||||
p = os.popen("du -sLh %s" % path, 'r')
|
||||
thesize = p.read()
|
||||
p.close()
|
||||
thesize = string.split(thesize)[0]
|
||||
return thesize
|
||||
|
||||
|
||||
|
||||
def reportSizes(self, disc, firstpkg=None, lastpkg=None):
|
||||
"""appends to self.logfile"""
|
||||
|
||||
if firstpkg:
|
||||
self.logfile.append("First package on disc%d: %s" % (disc, firstpkg))
|
||||
if lastpkg:
|
||||
self.logfile.append("Last package on disc%d : %s" % (disc, lastpkg))
|
||||
|
||||
discsize = self.getSize("%s-disc%d" % (self.dist_dir, disc))
|
||||
self.logfile.append("%s-disc%d size: %s" % (self.arch, disc, discsize))
|
||||
|
||||
|
||||
|
||||
def createDiscInfo(self, discnumber):
|
||||
"""creates the .discinfo files in the split trees"""
|
||||
|
||||
if not os.path.exists("%s/.discinfo" % self.dist_dir):
|
||||
raise RuntimeError, "CRITICAL ERROR : .discinfo doesn't exist in the unified tree, not splitting"
|
||||
|
||||
# if start_time isn't set then we haven't got this info yet
|
||||
if not self.start_time:
|
||||
file = open("%s/.discinfo" % (self.dist_dir), 'r')
|
||||
self.start_time = file.readline()[:-1]
|
||||
self.release_str = file.readline()[:-1]
|
||||
self.real_arch = file.readline()[:-1]
|
||||
|
||||
if self.real_arch != self.arch:
|
||||
raise RuntimeError, "CRITICAL ERROR : self.real_arch is not the same as self.arch"
|
||||
|
||||
# skip the disc number line from the unified tree
|
||||
file.readline()
|
||||
|
||||
# basedir, packagedir, and pixmapdir
|
||||
self.dir_info = [file.readline()[:-1], file.readline()[:-1], file.readline()[:-1]]
|
||||
|
||||
file.close()
|
||||
|
||||
discinfo_file = open("%s-disc%d/.discinfo" % (self.dist_dir, discnumber), 'w')
|
||||
discinfo_file.write("%s\n" % self.start_time)
|
||||
discinfo_file.write(self.release_str + '\n')
|
||||
discinfo_file.write(self.real_arch + '\n')
|
||||
discinfo_file.write("%s\n" % discnumber)
|
||||
for i in range(0, len(self.dir_info)):
|
||||
discinfo_file.write(self.dir_info[i] + '\n')
|
||||
discinfo_file.close()
|
||||
|
||||
|
||||
|
||||
def linkFiles(self, src_dir, dest_dir, filelist):
|
||||
"""Creates hardlinks from files in the unified dir to files in the split dirs. This is not for RPMs or SRPMs"""
|
||||
|
||||
for file in filelist:
|
||||
src = "%s/%s" % (src_dir, file)
|
||||
dest = "%s/%s" % (dest_dir, file)
|
||||
try:
|
||||
os.link(src, dest)
|
||||
except OSError, (errno, msg):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
def createSplitDirs(self):
|
||||
"""Figures out which discs are for RPMs, which are for SRPMs,
|
||||
and which are shared. Also creates links for files on disc1
|
||||
and files which are common across all discs"""
|
||||
|
||||
if self.bin_discs > self.total_discs or self.src_discs > self.total_discs:
|
||||
raise RuntimeError, "CRITICAL ERROR : Number of discs specified exceeds the total number of discs"
|
||||
|
||||
# get a list of discs for each type of disc. shared_list will
|
||||
# be returned for sorting out which discs SRPMS can land on
|
||||
self.bin_list = range(1, self.bin_discs + 1)
|
||||
self.src_list = range(self.total_discs - self.src_discs + 1, self.total_discs + 1)
|
||||
self.shared_list = range(self.total_discs - self.src_discs + 1, self.bin_discs + 1)
|
||||
|
||||
|
||||
for i in range(self.bin_list[0], self.bin_list[-1] + 1):
|
||||
if i == 1:
|
||||
p = os.popen('find %s/ -type f -not -name .discinfo -not -name "*\.rpm"' % self.dist_dir, 'r')
|
||||
filelist = p.read()
|
||||
p.close()
|
||||
filelist = string.split(filelist)
|
||||
|
||||
p = os.popen('find %s/ -type d -not -name RPMS -not -name SRPMS' % self.dist_dir, 'r')
|
||||
dirlist = p.read()
|
||||
p.close()
|
||||
dirlist = string.split(dirlist)
|
||||
|
||||
dont_create = []
|
||||
# we need to clean up the dirlist first. We don't want everything yet
|
||||
for j in range(0, len(dirlist)):
|
||||
dirlist[j] = string.replace(dirlist[j], self.dist_dir, '')
|
||||
|
||||
|
||||
# now create the dirs for disc1
|
||||
for j in range(0, len(dirlist)):
|
||||
os.makedirs("%s-disc%d/%s" % (self.dist_dir, i, dirlist[j]))
|
||||
|
||||
for j in range(0, len(filelist)):
|
||||
filelist[j] = string.replace(filelist[j], self.dist_dir, '')
|
||||
try:
|
||||
os.link(os.path.normpath("%s/%s" % (self.dist_dir, filelist[j])),
|
||||
os.path.normpath("%s-disc%d/%s" % (self.dist_dir, i, filelist[j])))
|
||||
except OSError, (errno, msg):
|
||||
pass
|
||||
|
||||
# now create the product/RPMS dir
|
||||
os.makedirs("%s-disc%d/%s/RPMS" % (self.dist_dir, i, self.product_path))
|
||||
|
||||
else:
|
||||
os.makedirs("%s-disc%d/%s/RPMS" % (self.dist_dir, i, self.product_path))
|
||||
self.linkFiles(self.dist_dir, "%s-disc%d" %(self.dist_dir, i), self.common_files)
|
||||
self.createDiscInfo(i)
|
||||
|
||||
if (self.src_discs != 0):
|
||||
for i in range(self.src_list[0], self.src_list[-1] + 1):
|
||||
os.makedirs("%s-disc%d/SRPMS" % (self.dist_dir, i))
|
||||
self.linkFiles(self.dist_dir,
|
||||
"%s-disc%d" %(self.dist_dir, i),
|
||||
self.common_files)
|
||||
self.createDiscInfo(i)
|
||||
|
||||
|
||||
|
||||
def splitRPMS(self, reportSize = 1):
|
||||
"""Creates links in the split dirs for the RPMs"""
|
||||
|
||||
packages = {}
|
||||
|
||||
if os.path.isdir("%s/%s/RPMS" %(self.dist_dir, self.product_path)):
|
||||
pkgdir = "%s/RPMS" %(self.product_path, )
|
||||
else:
|
||||
pkgdir = "%s" %(self.product_path,)
|
||||
|
||||
rpmlist = os.listdir("%s/%s" %(self.dist_dir, pkgdir))
|
||||
rpmlist.sort()
|
||||
|
||||
# create the packages dictionary in this format: n-v-r.a:['n-v-r.arch.rpm']
|
||||
for filename in rpmlist:
|
||||
filesize = os.path.getsize("%s/%s/%s" % (self.dist_dir, pkgdir, filename))
|
||||
try:
|
||||
pkg_nvr = nvra("%s/%s/%s" %(self.dist_dir, pkgdir, filename))
|
||||
except rpm.error, e:
|
||||
continue
|
||||
|
||||
if packages.has_key(pkg_nvr):
|
||||
# append in case we have multiple packages with the
|
||||
# same n-v-r. Ex: the kernel has multiple n-v-r's for
|
||||
# different arches
|
||||
packages[pkg_nvr].append(filename)
|
||||
else:
|
||||
packages[pkg_nvr] = [filename]
|
||||
|
||||
orderedlist = []
|
||||
|
||||
# read the ordered pacakge list into orderedlist
|
||||
file = open(self.package_order_file, 'r')
|
||||
for pkg_nvr in file.readlines():
|
||||
pkg_nvr = string.rstrip(pkg_nvr)
|
||||
if pkg_nvr[0:8] != "warning:":
|
||||
orderedlist.append(pkg_nvr)
|
||||
file.close()
|
||||
|
||||
# last package is the last package placed on the disc
|
||||
lastpackage = ''
|
||||
|
||||
# packagenum resets when we change discs. It's used to
|
||||
# determine the first package in the split tree and that's
|
||||
# about it
|
||||
packagenum = 0
|
||||
|
||||
disc = self.bin_list[0]
|
||||
|
||||
for rpm_nvr in orderedlist:
|
||||
if not packages.has_key(rpm_nvr):
|
||||
continue
|
||||
for file_name in packages[rpm_nvr]:
|
||||
curused = self.getSize("%s-disc%s" % (self.dist_dir, disc), blocksize=1)
|
||||
filesize = self.getSize("%s/%s/%s" % (self.dist_dir, pkgdir, file_name), blocksize=1)
|
||||
newsize = filesize + curused
|
||||
|
||||
# compensate for the size of the comps package which has yet to be created
|
||||
if disc == 1:
|
||||
maxsize = self.target_size - self.comps_size - self.reserve_size
|
||||
else:
|
||||
maxsize = self.target_size
|
||||
|
||||
packagenum = packagenum + 1
|
||||
|
||||
if packagenum == 1:
|
||||
firstpackage = file_name
|
||||
|
||||
# move to the next disc if true
|
||||
if newsize > maxsize:
|
||||
self.reportSizes(disc, firstpkg=firstpackage, lastpkg=lastpackage)
|
||||
# try it, if we are already on the last disc then complain loudly
|
||||
try:
|
||||
nextdisc=self.bin_list.index(disc+1)
|
||||
disc = self.bin_list[nextdisc]
|
||||
os.link("%s/%s/%s" % (self.dist_dir, pkgdir, file_name),
|
||||
"%s-disc%d/%s/%s" % (self.dist_dir, disc, pkgdir, file_name))
|
||||
packagenum = 1
|
||||
firstpackage = file_name
|
||||
|
||||
except:
|
||||
# back down to the last RPM disc and complain about the overflow
|
||||
disc = disc - 1
|
||||
self.logfile.append("No more discs to put packages, overflowing on disc%d" % disc)
|
||||
continue
|
||||
|
||||
else:
|
||||
os.link("%s/%s/%s" % (self.dist_dir, pkgdir, file_name),
|
||||
"%s-disc%d/%s/%s" % (self.dist_dir, disc, pkgdir, file_name))
|
||||
lastpackage = file_name
|
||||
|
||||
if reportSize == 1:
|
||||
self.reportSizes(disc, firstpkg=firstpackage, lastpkg=lastpackage)
|
||||
|
||||
|
||||
|
||||
def getLeastUsedTree(self):
|
||||
"""Returns the least full tree to use for SRPMS"""
|
||||
|
||||
sizes = []
|
||||
for i in range(0, len(self.src_list)):
|
||||
sizes.append([self.getSize("%s-disc%d" % (self.dist_dir, self.src_list[i]), blocksize=1), self.src_list[i]])
|
||||
sizes.sort()
|
||||
return sizes[0]
|
||||
|
||||
|
||||
|
||||
def splitSRPMS(self):
|
||||
"""Puts the srpms onto the SRPM split discs. The packages are
|
||||
ordered by size, and placed one by one on the disc with the
|
||||
most space available"""
|
||||
|
||||
srpm_list = []
|
||||
|
||||
srpm_disc_list = self.src_list
|
||||
# create a list of [[size, srpm]]
|
||||
for srpm in os.listdir("%s" % self.src_dir):
|
||||
srpm_size = self.getSize("%s/%s" % (self.src_dir, srpm), blocksize=1)
|
||||
srpm_list.append([srpm_size, srpm])
|
||||
|
||||
srpm_list.sort()
|
||||
srpm_list.reverse()
|
||||
|
||||
for i in range(0, len(srpm_list)):
|
||||
# make sure that the src disc is within the size limits,
|
||||
# if it isn't, pull it out of the list. If there's only
|
||||
# one disk make loud noises over the overflow
|
||||
for disc in self.src_list:
|
||||
if self.getSize("%s-disc%s" % (self.dist_dir, disc), blocksize=1) > self.target_size:
|
||||
if len(self.src_list) < 2:
|
||||
self.logfile.append("Overflowing %s on disc%d" % (srpm_list[i][1], disc))
|
||||
break
|
||||
else:
|
||||
discsize = self.getSize("%s-disc%d" % (self.dist_dir, srpm_disc_list[i]))
|
||||
self.logfile.append("%s-disc%d size: %s" % (self.arch, disc, discsize))
|
||||
self.src_list.pop(self.src_list.index(disc))
|
||||
os.link("%s/%s" % (self.src_dir, srpm_list[i][1]),
|
||||
"%s-disc%d/SRPMS/%s" % (self.dist_dir, self.getLeastUsedTree()[1], srpm_list[i][1]))
|
||||
|
||||
for i in range(0, len(srpm_disc_list)):
|
||||
self.reportSizes(srpm_disc_list[i])
|
||||
|
||||
|
||||
def main(self):
|
||||
"""Just runs everything"""
|
||||
self.createSplitDirs()
|
||||
self.splitRPMS()
|
||||
if (self.src_discs != 0):
|
||||
self.splitSRPMS()
|
||||
return self.logfile
|
||||
|
||||
|
||||
|
||||
def usage(theerror):
|
||||
print theerror
|
||||
print """Usage: %s --arch=i386 --total-discs=8 --bin-discs=4 --src-discs=4 --release-string="distro name" --pkgorderfile=/tmp/pkgorder.12345 --distdir=/usr/src/someunifiedtree --srcdir=/usr/src/someunifiedtree/SRPMS --productpath=product""" % sys.argv[0]
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if "__main__" == __name__:
|
||||
import getopt
|
||||
|
||||
timber = Timber()
|
||||
|
||||
theargs = ["arch=", "total-discs=", "bin-discs=",
|
||||
"src-discs=", "release-string=", "pkgorderfile=",
|
||||
"distdir=", "srcdir=", "productpath=", "reserve-size="]
|
||||
|
||||
try:
|
||||
options, args = getopt.getopt(sys.argv[1:], '', theargs)
|
||||
except getopt.error, error:
|
||||
usage(error)
|
||||
|
||||
myopts = {}
|
||||
for i in options:
|
||||
myopts[i[0]] = i[1]
|
||||
|
||||
options = myopts
|
||||
|
||||
if options.has_key("--arch"):
|
||||
timber.arch = options['--arch']
|
||||
else:
|
||||
usage("You forgot to specify --arch")
|
||||
|
||||
if options.has_key("--total-discs"):
|
||||
timber.total_discs = int(options['--total-discs'])
|
||||
else:
|
||||
usage("You forgot to specify --total-discs")
|
||||
|
||||
if options.has_key("--bin-discs"):
|
||||
timber.bin_discs = int(options['--bin-discs'])
|
||||
else:
|
||||
usage("You forgot to specify --bin-discs")
|
||||
|
||||
if options.has_key("--src-discs"):
|
||||
timber.src_discs = int(options['--src-discs'])
|
||||
else:
|
||||
usage("You forgot to specify --src-discs")
|
||||
|
||||
if options.has_key("--release-string"):
|
||||
timber.release_str = options["--release-string"]
|
||||
else:
|
||||
usage("You forgot to specify --release-string")
|
||||
|
||||
if options.has_key("--pkgorderfile"):
|
||||
timber.package_order_file = options["--pkgorderfile"]
|
||||
else:
|
||||
usage("You forgot to specify --pkgorderfile")
|
||||
|
||||
if options.has_key("--distdir"):
|
||||
timber.dist_dir = options["--distdir"]
|
||||
else:
|
||||
usage("You forgot to specify --distdir")
|
||||
|
||||
if options.has_key("--srcdir"):
|
||||
timber.src_dir = options["--srcdir"]
|
||||
else:
|
||||
usage("You forgot to specify --srcdir")
|
||||
|
||||
if options.has_key("--productpath"):
|
||||
timber.product_path = options["--productpath"]
|
||||
|
||||
if options.has_key("--reserve-size"):
|
||||
timber.reserve_size = float(options["--reserve_size"])
|
||||
|
||||
logfile = timber.main()
|
||||
|
||||
for logentry in range(0, len(logfile)):
|
||||
print logfile[logentry]
|
||||
|
||||
sys.exit(0)
|
||||
Loading…
Reference in New Issue
Block a user