From 38fd56c8a896a52bcab4f8921f67c483e36cf728 Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Fri, 28 Sep 2018 07:27:57 +1000 Subject: sb: Monitor the build disk usage. Report the usage, total and various sizes - Track the size of a build of a package in a build set to determine the maximum amout of disk space used. This can be used as a guide to documenting how much space a user needs to set aside to build a specific set of tools. - The `%clean` stage of a build is now split into a separate script. I do not think this is an issue because I could not find any `%clean` sections in any build configs we have. In time support for the `%clean` section will be removed, the package builder cleans up. Closes #3516 --- source-builder/sb/build.py | 125 +++++++++++++++++++++++++++------------- source-builder/sb/config.py | 13 ++++- source-builder/sb/path.py | 68 ++++++++++++++++++++-- source-builder/sb/setbuilder.py | 42 +++++++++++++- 4 files changed, 199 insertions(+), 49 deletions(-) diff --git a/source-builder/sb/build.py b/source-builder/sb/build.py index e28a831..88446bb 100644 --- a/source-builder/sb/build.py +++ b/source-builder/sb/build.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2013 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2018 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -51,6 +51,13 @@ except: print('error: unknown application load error') sys.exit(1) +def humanize_number(num, suffix): + for unit in ['','K','M','G','T','P','E','Z']: + if abs(num) < 1024.0: + return "%5.3f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.3f%s%s" % (size, 'Y', suffix) + class script: """Create and manage a shell script.""" @@ -124,7 +131,8 @@ class build: log.notice('config: ' + name) self.set_macros(macros) self.config = config.file(name, opts, self.macros) - self.script = script() + self.script_build = script() + self.script_clean = script() self.macros['buildname'] = self._name_(self.macros['name']) except error.general as gerr: log.notice(str(gerr)) @@ -288,21 +296,21 @@ class build: raise error.general('setup source tag not found: %d' % (source_tag)) else: name = opt_name - self.script.append(self.config.expand('cd %{_builddir}')) + self.script_build.append(self.config.expand('cd %{_builddir}')) if not deleted_dir and delete_before_unpack: - self.script.append(self.config.expand('%{__rm} -rf ' + name)) + self.script_build.append(self.config.expand('%{__rm} -rf ' + name)) deleted_dir = True if not created_dir and create_dir: - self.script.append(self.config.expand('%{__mkdir_p} ' + name)) + self.script_build.append(self.config.expand('%{__mkdir_p} ' + name)) created_dir = True if not changed_dir and (not unpack_before_chdir or create_dir): - self.script.append(self.config.expand('cd ' + name)) + self.script_build.append(self.config.expand('cd ' + name)) changed_dir = True - self.script.append(self.config.expand(source['script'])) + self.script_build.append(self.config.expand(source['script'])) if not changed_dir and (unpack_before_chdir and not create_dir): - self.script.append(self.config.expand('cd ' + name)) + self.script_build.append(self.config.expand('cd ' + name)) changed_dir = True - self.script.append(self.config.expand('%{__setup_post}')) + self.script_build.append(self.config.expand('%{__setup_post}')) def patch_setup(self, package, args): name = args[1] @@ -360,7 +368,7 @@ class build: else: patch['script'] = '%{__cat} ' + patch['local'] patch['script'] += ' | %%{__patch} %s' % (opts) - self.script.append(self.config.expand(patch['script'])) + self.script_build.append(self.config.expand(patch['script'])) def run(self, command, shell_opts = '', cwd = None): e = execute.capture_execution(log = log.default, dump = self.opts.quiet()) @@ -378,7 +386,7 @@ class build: self.mkdir(builddir) def prep(self, package): - self.script.append('echo "==> %prep:"') + self.script_build.append('echo "==> %prep:"') _prep = package.prep() if _prep: for l in _prep: @@ -400,59 +408,78 @@ class build: sources.hash(args[1:], self.macros, err) self.hash(package, args) else: - self.script.append(' '.join(args)) + self.script_build.append(' '.join(args)) def build(self, package): - self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"') - self.script.append('%s ${SB_BUILD_ROOT}' % - (self.config.expand('%{__rmdir}'))) - self.script.append('%s ${SB_BUILD_ROOT}' % - (self.config.expand('%{__mkdir_p}'))) - self.script.append('echo "==> %build:"') + self.script_build.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"') + self.script_build.append('%s ${SB_BUILD_ROOT}' % + (self.config.expand('%{__rmdir}'))) + self.script_build.append('%s ${SB_BUILD_ROOT}' % + (self.config.expand('%{__mkdir_p}'))) + self.script_build.append('echo "==> %build:"') _build = package.build() if _build: for l in _build: - self.script.append(l) + self.script_build.append(l) def install(self, package): - self.script.append('echo "==> %install:"') + self.script_build.append('echo "==> %install:"') _install = package.install() if _install: for l in _install: args = l.split() - self.script.append(' '.join(args)) + self.script_build.append(' '.join(args)) def files(self, package): if self.create_tar_files \ and not self.macros.get('%{_disable_packaging'): - self.script.append('echo "==> %files:"') + self.script_build.append('echo "==> %files:"') inpath = path.abspath(self.config.expand('%{buildroot}')) tardir = path.abspath(self.config.expand('%{_tardir}')) - self.script.append(self.config.expand('if test -d %s; then' % (inpath))) - self.script.append(self.config.expand(' %%{__mkdir_p} %s' % tardir)) - self.script.append(self.config.expand(' cd ' + inpath)) + self.script_build.append(self.config.expand('if test -d %s; then' % (inpath))) + self.script_build.append(self.config.expand(' %%{__mkdir_p} %s' % tardir)) + self.script_build.append(self.config.expand(' cd ' + inpath)) tar = path.join(tardir, package.long_name() + '.tar.bz2') cmd = self.config.expand(' %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar) - self.script.append(cmd) - self.script.append(self.config.expand(' cd %{_builddir}')) - self.script.append('fi') + self.script_build.append(cmd) + self.script_build.append(self.config.expand(' cd %{_builddir}')) + self.script_build.append('fi') def clean(self, package): - self.script.append('echo "==> %clean:"') + self.script_clean.reset() + self.script_clean.append(self.config.expand('%{___build_template}')) + self.script_clean.append('echo "=> ' + package.name() + ': CLEAN"') + self.script_clean.append('echo "==> %clean:"') _clean = package.clean() if _clean is not None: for l in _clean: args = l.split() - self.script.append(' '.join(args)) + self.script_clean.append(' '.join(args)) + + def sizes(self, package): + def _sizes(package, what, path): + package.set_size(what, path) + s = humanize_number(package.get_size(what), 'B') + log.trace('size: %s (%s): %s (%d)' % (what, path, s, package.get_size(what))) + return s + s = {} + for p in [('build', '%{_builddir}'), + ('build', '%{buildroot}'), + ('installed', '%{buildroot}')]: + hs = _sizes(package, p[0], self.config.expand(p[1])) + s[p[0]] = hs + log.notice('sizes: %s: %s (installed: %s)' % (package.name(), + s['build'], + s['installed'])) def build_package(self, package): if self.canadian_cross(): if not self.config.defined('%{allow_cxc}'): raise error.general('Canadian Cross is not allowed') - self.script.append('echo "==> Candian-cross build/target:"') - self.script.append('SB_CXC="yes"') + self.script_build.append('echo "==> Candian-cross build/target:"') + self.script_build.append('SB_CXC="yes"') else: - self.script.append('SB_CXC="no"') + self.script_build.append('SB_CXC="no"') self.build(package) self.install(package) self.files(package) @@ -498,18 +525,24 @@ class build: log.trace('---- macro maps %s' % ('-' * 55)) log.trace('%s' % (str(self.config.macros))) log.trace('-' * 70) - self.script.reset() - self.script.append(self.config.expand('%{___build_template}')) - self.script.append('echo "=> ' + name + ':"') + self.script_build.reset() + self.script_build.append(self.config.expand('%{___build_template}')) + self.script_build.append('echo "=> ' + name + ': BUILD"') self.prep(package) self.build_package(package) if not self.opts.dry_run(): self.builddir() - sn = path.join(self.config.expand('%{_builddir}'), 'doit') - log.output('write script: ' + sn) - self.script.write(sn) + build_sn = path.join(self.config.expand('%{_builddir}'), 'do-build') + log.output('write script: ' + build_sn) + self.script_build.write(build_sn) + clean_sn = path.join(self.config.expand('%{_builddir}'), 'do-clean') + log.output('write script: ' + clean_sn) + self.script_clean.write(clean_sn) log.notice('building: %s%s' % (cxc_label, name)) - self.run(sn) + self.run(build_sn) + self.sizes(package) + log.notice('cleaning: %s%s' % (cxc_label, name)) + self.run(clean_sn) except error.general as gerr: log.notice(str(gerr)) log.stderr('Build FAILED') @@ -536,6 +569,18 @@ class build: package = packages['main'] return package.disabled() + def get_build_size(self): + package = self.main_package() + if package.disabled(): + return 0 + return package.get_size('build') + + def get_installed_size(self): + package = self.main_package() + if package.disabled(): + return 0 + return package.get_size('installed') + def get_configs(opts): def _scan(_path, ext): diff --git a/source-builder/sb/config.py b/source-builder/sb/config.py index 74c002e..a901038 100644 --- a/source-builder/sb/config.py +++ b/source-builder/sb/config.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2018 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -73,6 +73,7 @@ class package: self.config = config self.directives = {} self.infos = {} + self.sizes = {} def __str__(self): @@ -218,6 +219,16 @@ class package: def disabled(self): return len(self.name()) == 0 + def set_size(self, what, path_): + if what not in self.sizes: + self.sizes[what] = 0 + self.sizes[what] += path.get_size(path_) + + def get_size(self, what): + if what in self.sizes: + return self.sizes[what] + return 0 + class file: """Parse a config file.""" diff --git a/source-builder/sb/path.py b/source-builder/sb/path.py index e2fd3d4..984f3d7 100644 --- a/source-builder/sb/path.py +++ b/source-builder/sb/path.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2018 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -53,11 +53,6 @@ def host(path): path = u'\\'.join([u'\\\\?', path]) return path -def is_abspath(path): - if path is not None and len(path) > 0: - return '/' == path[0] - return False - def shell(path): if path is not None: if windows: @@ -79,6 +74,11 @@ def dirname(path): path = shell(path) return shell(os.path.dirname(path)) +def is_abspath(path): + if path is not None and len(path) > 0: + return '/' == path[0] + return False + def join(path, *args): path = shell(path) for arg in args: @@ -304,6 +304,58 @@ def copy_tree(src, dst): else: raise error.general('copying tree (4): %s -> %s: %s' % (hsrc, hdst, str(why))) +def get_size(path, depth = -1): + # + # Get the size the directory tree manually to the required depth. + # This makes sure on Windows the files are correctly encoded to avoid + # the file name size limit. On Windows the os.walk fails once we + # get to the max path length on Windows. + # + def _isdir(path): + hpath = host(path) + return os.path.isdir(hpath) and not os.path.islink(hpath) + + def _node_size(path): + hpath = host(path) + size = 0 + if not os.path.islink(hpath): + size = os.path.getsize(hpath) + return size + + def _get_size(path, depth, level = 0): + level += 1 + dirs = [] + size = 0 + for name in listdir(path): + path_ = join(path, shell(name)) + hname = host(path_) + if _isdir(path_): + dirs += [shell(name)] + else: + size += _node_size(path_) + if depth < 0 or level < depth: + for name in dirs: + dir = join(path, name) + size += _get_size(dir, depth, level) + return size + + path = shell(path) + hpath = host(path) + size = 0 + + if os.path.exists(hpath): + size = _get_size(path, depth) + + return size + +def get_humanize_size(path, depth = -1): + size = get_size(path, depth) + for unit in ['','K','M','G','T','P','E','Z']: + if abs(size) < 1024.0: + return "%5.3f%sB" % (size, unit) + size /= 1024.0 + return "%.3f%sB" % (size, 'Y') + if __name__ == '__main__': print(host('/a/b/c/d-e-f')) print(host('//a/b//c/d-e-f')) @@ -311,6 +363,10 @@ if __name__ == '__main__': print(basename('/as/sd/df/fg/me.txt')) print(dirname('/as/sd/df/fg/me.txt')) print(join('/d', 'g', '/tyty/fgfg')) + print('size of . depth all: ', get_size('.')) + print('size of . depth 1: ', get_size('.', 1)) + print('size of . depth 2: ', get_size('.', 2)) + print('size of . as human : ', get_humanize_size('.')) windows = True print(host('/a/b/c/d-e-f')) print(host('//a/b//c/d-e-f')) diff --git a/source-builder/sb/setbuilder.py b/source-builder/sb/setbuilder.py index b7cd8f2..2e6d643 100644 --- a/source-builder/sb/setbuilder.py +++ b/source-builder/sb/setbuilder.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chrisj@rtems.org) +# Copyright 2010-2018 Chris Johns (chrisj@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -447,13 +447,48 @@ class buildset: self.install(b.name(), b.config.expand('%{buildroot}'), b.config.expand('%{_prefix}')) - + # + # Sizes ... + # + if len(builds) > 1: + size_build = 0 + size_installed = 0 + size_build_max = 0 + for b in builds: + s = b.get_build_size() + size_build += s + if s > size_build_max: + size_build_max = s + size_installed += b.get_installed_size() + size_sources = 0 + for p in builds[0].config.expand('%{_sourcedir}').split(':'): + size_sources += path.get_size(p) + size_patches = 0 + for p in builds[0].config.expand('%{_patchdir}').split(':'): + size_patches += path.get_size(p) + size_total = size_sources + size_patches + size_installed + build_size = 'usage: %s' % (build.humanize_number(size_build_max + size_installed, 'B')) + build_size += ' total: %s' % (build.humanize_number(size_total, 'B')) + build_size += ' (sources: %s' % (build.humanize_number(size_sources, 'B')) + build_size += ', patches: %s' % (build.humanize_number(size_patches, 'B')) + build_size += ', installed %s)' % (build.humanize_number(size_installed, 'B')) + # + # Cleaning ... + # if deps is None and \ (not self.opts.no_clean() or self.opts.always_clean()): for b in builds: if not b.disabled(): log.notice('cleaning: %s' % (b.name())) b.cleanup() + # + # Log the build size message + # + if len(builds) > 1: + log.notice('Build Sizes: %s' % (build_size)) + # + # Clear out the builds ... + # for b in builds: del b except error.general as gerr: @@ -484,6 +519,9 @@ class buildset: self.write_mail_header('') log.notice('Mailing report: %s' % (mail['to'])) body = self.get_mail_header() + body += 'Sizes' + os.linesep + body += '=====' + os.linesep + os.linesep + body += 'Output' + os.linesep body += '======' + os.linesep + os.linesep body += os.linesep.join(mail['output'].get()) -- cgit v1.2.3