From c6a281eaf030632b7da8d8084fb08ebe173ba6e7 Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Thu, 17 Nov 2022 15:18:41 +1100 Subject: release: Update to support the 5.2 release - Add a new release notes generator - Update and support the fixed RSB get sources --- .gitignore | 3 + README.md.in | 27 +- release-notes/markdown_generator.py | 264 +++++++++++ release-notes/reports.py | 381 +++++++++++++++ release-notes/reraise.py | 111 +++++ release-notes/rtems-release-notes | 167 +++++++ release-notes/rtems_trac.py | 96 ++++ release-notes/tickets.py | 519 +++++++++++++++++++++ release-notes/trac.py | 136 ++++++ rtems-release | 74 +-- rtems-release-defaults | 8 +- rtems-release-info | 5 + rtems-release-kernel | 4 +- rtems-release-notes | 63 +-- .../rtems-release-notes-coverpage.html.in | 6 +- rtems-release-notes.css | 114 ++++- rtems-release-sources | 117 ++--- 17 files changed, 1920 insertions(+), 175 deletions(-) create mode 100644 release-notes/markdown_generator.py create mode 100644 release-notes/reports.py create mode 100644 release-notes/reraise.py create mode 100755 release-notes/rtems-release-notes create mode 100644 release-notes/rtems_trac.py create mode 100644 release-notes/tickets.py create mode 100644 release-notes/trac.py diff --git a/.gitignore b/.gitignore index 79301bf..e5da17c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,7 @@ ARCH-BSP.txt 5.* 6.* 7.* +8.* *-branched +__pycache__ +.rng-cache diff --git a/README.md.in b/README.md.in index b919e81..62a0726 100644 --- a/README.md.in +++ b/README.md.in @@ -8,6 +8,7 @@ -------------------------------------------------------------------------------- +[RN-H]: rtems-@RELEASE@-release-notes.html [RN-P]: rtems-@RELEASE@-release-notes.pdf The Real-Time Executive for Multiprocessor Systems or RTEMS is an open source @@ -44,23 +45,25 @@ Developer ## Release Files --------------------------------------------------------------------------------- -@RELEASE@ Top level directory ------------------------------------- ---------------------------------- -[README.txt](README.txt), `index.html` This document +----------------------------------------------------------------------------- +@RELEASE@ @R_SP@ Top level directory +------------------------------------------ ---------------------------------- +[README.txt](README.txt), `index.html` This document + +[contrib](contrib) Directory contains extra release + related files -[contrib](contrib) Directory contains extra release - related files +[docs](docs) The generated RTEMS documentation -[docs](docs) The generated RTEMS documentation +[sources](sources) Source code for this release -[sources](sources) Source code for this release +[rtems-@RELEASE@-release-notes.html][RN-H] @R_SP@ Detailed HTML RTEMS Release notes -[rtems-@RELEASE@-release-notes.pdf][RN-P] Detailed RTEMS Release notes +[rtems-@RELEASE@-release-notes.pdf][RN-P] @R_SP@ Detailed PDF RTEMS Release notes -[sha512sum.txt](sha512sum.txt) The SHA512 checksums for this - directory -------------------------------------------------------------------------------- +[sha512sum.txt](sha512sum.txt) The SHA512 checksums for this + directory +----------------------------------------------------------------------------- [S-K]: sources/rtems-@RELEASE@.tar.xz [S-RSB]: sources/rtems-source-builder-@RELEASE@.tar.xz diff --git a/release-notes/markdown_generator.py b/release-notes/markdown_generator.py new file mode 100644 index 0000000..b0f84eb --- /dev/null +++ b/release-notes/markdown_generator.py @@ -0,0 +1,264 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2018 Danxue Huang (danxue.huang@gmail.com) +# Copyright 2022 Chris Johns (chris@contemporary.software) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import os +import re + + +class MarkdownGenerator: + + def __init__(self, line_width=78): + self.content = '' + self.line_width = line_width + + @staticmethod + def _max_len(lst): + max_len = 0 + for e in lst: + if len(e) > max_len or (len(e) == 0 and max_len < len(' ')): + max_len = len(e) if len(e) > 0 else len(' ') + return max_len + + def gen_bullet_point(self, text): + self.content += '* ' + self.wrap_line( + self._convert_to_unicode_str(text), self.line_width) + os.linesep + + def gen_line(self, text): + self.content += self.wrap_line(self._convert_to_unicode_str(text), + self.line_width) + os.linesep + + def gen_unwrapped_line(self, text, is_raw_text=True): + self.content += text + self.content += (' ' + os.linesep if is_raw_text else '
') + + def gen_heading(self, text, level, anchor=None): + self.content += os.linesep + \ + '#' * level + ' ' + \ + self._convert_to_unicode_str(text) + if anchor is not None: + self.content += ' {#' + anchor + '}' + self.content += os.linesep * 2 + + def gen_wrapped_table(self, header, rows, max_num_cols=4): + num_cols = len(header) + i = 0 + if num_cols > max_num_cols: + while i < num_cols: + self.gen_table( + list(header)[i:i + max_num_cols], + [list(row)[i:i + max_num_cols] for row in rows], + ) + self.gen_line(os.linesep) + i += max_num_cols + else: + self.gen_table(header, rows, align='left') + + def gen_page_break(self): + self.gen_line('') + self.gen_line('') + self.gen_line('
') + self.gen_line('') + + def gen_line_break(self): + self.gen_line('') + self.gen_line('') + self.gen_line('
') + self.gen_line('') + + def gen_raw(self, content): + self.content += content + + def gen_line_block(self, text): + if len(text.strip()) > 0: + self.content += os.linesep * 2 + '
' + os.linesep + self.content += text + self.content += os.linesep * 2 + '
' + os.linesep + return + lines = text.split(os.linesep) + code_block = False + lb_lines = [] + for l in lines: + if l.startswith('```'): + code_block = not code_block + else: + if code_block: + lb_lines += [' ' + l] + else: + lb_lines += ['| ' + l] + self.content += os.linesep + os.linesep.join(lb_lines) + os.linesep + + def gen_division_open(self, name): + self.gen_line('') + self.gen_line('
' % (name)) + self.gen_line('') + + def gen_division_close(self): + self.gen_line('') + self.gen_line('
') + self.gen_line('') + + def gen_unordered_lists(self, items, level=0): + md = [] + for i in items: + if isinstance(i, list): + md += self.gen_unordered_lists(i, level + 1) + else: + md += ['%s* %s' % (' ' * level, i)] + return os.linesep.join(md) + + def gen_ordered_lists(self, items, level=0): + md = [] + for i in items: + if isinstance(i, list): + md += self.gen_unordered_lists(i, level + 1) + else: + md += ['%s#. %s' % (' ' * level, i)] + return os.linesep.join(md) + + def gen_table(self, header, rows, align='left', sort_by=None): + rows = [[self._convert_to_unicode_str(r) for r in row] for row in rows] + if header is None: + cols = len(rows[0]) + else: + header = [self._convert_to_unicode_str(h) for h in header] + cols = len(header) + if isinstance(align, str): + align = [align] * cols + else: + if len(align) < cols: + align += ['left'] * (cols - len(align)) + for c in range(0, len(align)): + if align[c] not in ['left', 'right', 'center']: + raise RuntimeError('invalid table alignment:' + a) + align[c] = { + 'left': ('%-*s ', 1), + 'right': (' %*s', 1), + 'center': (' %-*s ', 2) + }[align[c]] + if isinstance(sort_by, str): + if header is None: + sort_by = None + else: + if sort_by not in header: + sort_by = None + else: + sort_by = header.index(sort_by) + if sort_by is None: + sort_col = 0 + else: + sort_col = sort_by + ordered = [(k, i) + for i, k in enumerate([row[sort_col] for row in rows])] + if sort_by is not None: + ordered = sorted(ordered, key=lambda k: k[0]) + col_sizes = [] + if header is None: + col_sizes = [0] * cols + else: + for hdr in header: + col_sizes += [len(hdr)] + for c in range(0, cols): + col_max = self._max_len([row[c] for row in rows]) + if col_sizes[c] < col_max: + col_sizes[c] = col_max + line_len = 0 + for size in col_sizes: + line_len += size + line = [] + if header is not None: + for c in range(0, cols): + line += [align[c][0] % (col_sizes[c], header[c])] + self.content += ' '.join(line) + os.linesep + line = [] + for c in range(0, cols): + line += ['-' * (col_sizes[c] + align[c][1])] + table_line = ' '.join(line) + os.linesep + self.content += table_line + for o in ordered: + row = rows[o[1]] + line = [] + if len(col_sizes) != len(row): + raise RuntimeError('header cols and row cols do not match') + for c in range(0, len(row)): + line += [ + align[c][0] % + (col_sizes[c], row[c] if len(row[c]) > 0 else ' ') + ] + self.content += ' '.join(line) + os.linesep + if header is None: + self.content += table_line + + def gen_raw_text(self, formatted_text): + self.content += os.linesep + formatted_text + os.linesep + + @staticmethod + def gen_html_esc(text): + for ch, esc in [('_', '_'), ('*', '*')]: + text = text.replace(ch, esc) + return text + + @staticmethod + def gen_anchor(text): + return '[' + text + ']: #' + text + ' ' + + @staticmethod + def gen_bold(text): + return '**' + MarkdownGenerator.gen_html_esc(text) + '**' + + @staticmethod + def gen_topic(text): + return '
' + os.linesep + text + os.linesep + '
' + + @staticmethod + def gen_hyperlink(text, link): + return '[' + text + ']' + '(' + link + ')' + + @staticmethod + def wrap_line(line, width, is_raw_text=False): + i = 0 + str_list = [] + while i < len(line): + str_list.append(line[i:i + width]) + i += width + return (' \n' if is_raw_text else '
').join(str_list) + + def gen_horizontal_line(self): + self.content += os.linesep + '--------' + os.linesep + + @staticmethod + def _convert_to_unicode_str(text): + try: + return str(text) + except UnicodeEncodeError: + if isinstance(text, unicode): + return text + else: + return unicode(text, "utf-8") diff --git a/release-notes/reports.py b/release-notes/reports.py new file mode 100644 index 0000000..f45f370 --- /dev/null +++ b/release-notes/reports.py @@ -0,0 +1,381 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2018 Danxue Huang (danxue.huang@gmail.com) +# Copyright 2022 Chris Johns (chris@contemporary.software) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import datetime +import os +import re +import time +import threading +import sys + +from markdown_generator import MarkdownGenerator +import reraise + +heading_base = 2 + + +class ticket(object): + + def __init__(self, fmt, ticket): + self.generator = MarkdownGenerator() + self.format = fmt + self.ticket = ticket + self.thread = None + self.result = None + + def _format_contents(self): + ticket_meta = self.ticket['meta'] + ticket_link = self.ticket.get('comment_attachment', + {}).get('link', None) + summary = ticket_meta.get('summary', None) + ticket_meta.pop('description', None) + ticket_meta.pop('summary', None) + if ticket_link is not None: + ticket_id_link = \ + self.generator.gen_hyperlink(self.ticket_id(), '#t' + self.ticket_id()) + tlink = self.generator.gen_bold(ticket_id_link) + \ + ' - ' + self.generator.gen_bold(summary) + self.generator.gen_heading(tlink, + heading_base + 1, + anchor='t' + self.ticket_id()) + for k in ['Created', 'Modified', 'Blocked By']: + ticket_meta[k] = self.ticket['ticket'][k] + meta_keys = [k.capitalize() for k in ticket_meta.keys()] + meta_vals = [v for v in ticket_meta.values()] + order = [ + 'Id', 'Reporter', 'Created', 'Modified', 'Owner', 'Type', + 'Component', 'Status', 'Resolution', 'Version', 'Milestone', + 'Priority', 'Severity', 'Keywords', 'Cc', 'Blocking', 'Blocked by' + ] + meta_table = [] + for c in range(0, len(order)): + i = meta_keys.index(order[c]) + if meta_keys[i] in ['Created', 'Modified']: + dt = datetime.datetime.strptime(meta_vals[i], + '%m/%d/%y %H:%M:%S') + ds = dt.strftime('%d %B %Y %H:%M:%S') + if ds[0] == '0': + ds = ds[1:] + meta_vals[i] = ds + meta_table += [[ + self.generator.gen_bold(meta_keys[i]), meta_vals[i] + ]] + meta_table = [[ + self.generator.gen_bold('Link'), + self.generator.gen_hyperlink(ticket_link, ticket_link) + ]] + meta_table + self.generator.gen_table(None, meta_table, align=['right', 'left']) + + def _description(self, description): + description = description.replace('\r\n', '\n') + + # + # The code blocks needs to be reviewed + # + if self.ticket_id() == '3384': + description = re.sub('%s}}}', '%s}\n}\n}', description) + + if self.format == 'markdown': + description = re.sub(r'{{{(.*)}}}', r'`\1`', description) + else: + description = re.sub(r'{{{(.*)}}}', r':code:`\1`', description) + + if self.format == 'rst': + description = re.sub(r'(>+) ---', r'\1 \-\-\-', description) + + description = re.sub(r'{{{!(.*)\n', '{{{\n', description) + description = re.sub(r'}}}}', '}\n}}}', description) + description = re.sub(r'{{{[ \t]+\n', '{{{\n', description) + description = re.sub(r'{{{([#$])', '{{{\n#', description) + description = description.replace('{{{\n', '```\n') + description = description.replace('\n}}}', '\n```') + description = re.sub( + r"^[ \t]*#([ \t]*define|[ \t]*include|[ \t]*endif|[ \t]*ifdef|" \ + "[ \t]*ifndef|[ \t]*if|[ \t]*else)(.*)$", + r"`#\1\2`", + description, + flags=re.MULTILINE) + + if self.format == 'markdown': + description = re.sub(r'{{{(?!\n)', '`', description) + description = re.sub(r'(?!\n)}}}', '`', description) + else: + description = re.sub(r'{{{(?!\n)', ':code:`', description) + description = re.sub(r'(?!\n)}}}', '`', description) + + # Two lines after the opening (and after the ending) + # back-ticks misses up with the text area rendering. + description = re.sub('```\n\n', '```\n', description) + description = re.sub('\n\n```', '\n```', description) + + # For ticket 2624 where the opening three curly braces are not + # on a separate line. + description = re.sub(r'```(?!\n)', '```\n', description) + description = re.sub(r'(?!\n)```', '\n```', description) + + # For ticket 2993 where the defective closing curly brackets + # miss up with text area rendering. + description = re.sub('}}:', '```\n', description) + + # Ticket 3771 has code that's not written in a code block, + # which is interpretted by the Markdown generator as headers + # (#define)... Hence, we fix that manually. + + if self.ticket_id() == '3771': + description = re.sub('`#define', + '```\n#define', + description, + count=1) + description = re.sub('Problem facing on writing', + '```\nProblem facing on writing', + description, + count=1) + description = re.sub(r'[ ]{8,}', ' ', description) + + if self.format == 'rst': + description = description.replace('=', '\\=') + description = description.replace('\n', '\n\n') + description = re.sub(r'^(#+)', '', description, flags=re.MULTILINE) + + return description + + def _format_description(self): + if 'description' not in self.ticket['comment_attachment']: + return + description = self.ticket['comment_attachment']['description'] + self.generator.gen_raw_text(self.generator.gen_bold('Description')) + self.generator.gen_line('') + self.generator.gen_line_block(self._description(description)) + self.generator.gen_line('') + + def _meta_label(self, label): + if label == 'attachment': + label = 'attach' + return label + + def _format_comments(self): + if 'comments' not in self.ticket['comment_attachment']: + return + comments = self.ticket['comment_attachment']['comments'] + if len(comments) == 0: + return + self.generator.gen_line('') + cnt = 0 + bold = self.generator.gen_bold + for comment in comments: + cnt += 1 + self.generator.gen_line( + self.generator.gen_topic('Comment ' + str(cnt))) + self.generator.gen_line('') + if not comment['creator']: + creator = 'none' + else: + creator = comment['creator'] + ul = [bold(creator) + ', ' + comment['published']] + for m in comment['meta']: + ul += [bold(self._meta_label(m[0]) + ':') + ' ' + m[1]] + self.generator.gen_raw(self.generator.gen_ordered_lists(ul)) + self.generator.gen_line('') + self.generator.gen_line_block( + self._description(comment['description'])) + self.generator.gen_line('') + + def _format_attachments(self): + if 'attachments' not in self.ticket['comment_attachment']: + return + attachments = self.ticket['comment_attachment']['attachments'] + if len(attachments) == 0: + return + self.generator.gen_heading('Attachments:', heading_base + 2) + cnt = 0 + tab = [] + bold = self.generator.gen_bold + for attachment in attachments: + cnt += 1 + tab += [[ + bold(str(cnt)), + bold('%s, %s' % + (attachment['creator'], attachment['published'])) + ]] + for m in attachment['meta']: + tab += [['', bold(self._meta_label(m[0])) + ': ' + m[1]]] + if len(attachment['description']) != 0: + tab += [['', attachment['description']]] + if len(tab) != 0: + self.generator.gen_line('') + self.generator.gen_table(None, tab) + self.generator.gen_line('') + + def _runner(self): + try: + self.formatter() + except KeyboardInterrupt: + pass + except: + self.result = sys.exc_info() + + def formatter(self): + self._format_contents() + self._format_description() + self._format_attachments() + self._format_comments() + + def ticket_id(self): + return self.ticket['ticket']['id'] + + def run(self): + self.thread = threading.Thread(target=self._runner, + name='format-ticket-%s' % + (self.ticket_id())) + self.thread.start() + + def is_alive(self): + return self.thread and self.thread.is_alive() + + def reraise(self): + if self.result is not None: + reraise.reraise(*self.result) + + +class generator: + + def __init__(self, release, fmt='markdown'): + if fmt != 'markdown' and fmt != 'trac': + raise RuntimeError('invalid format: ' + fmt) + self.release = release + self.milestone = None + self.format = fmt + self.generator = MarkdownGenerator() + + def set_milestone(self, milestone): + self.milestone = milestone + + def gen_toc(self, notes): + headings = [line for line in notes + if line.startswith('##')] if notes is not None else [] + self.generator.gen_raw(self.md_toc(headings)) + + def gen_start(self, notes): + self.generator.gen_raw('# RTEMS Release ' + self.milestone + + os.linesep) + if notes is not None: + self.generator.gen_raw(os.linesep.join(notes)) + self.generator.gen_page_break() + + def gen_overall_progress(self, overall_progress): + self.generator.gen_heading( + 'RTEMS ' + self.milestone + ' Ticket Overview', heading_base) + self.generator.gen_table( + [k.capitalize() for k in overall_progress.keys()], + [overall_progress.values()], + align='left') + + def gen_tickets_summary(self, tickets): + self.generator.gen_line_break() + self.generator.gen_heading( + 'RTEMS ' + self.milestone + ' Ticket Summary', heading_base) + keys = tickets.keys() + id_summary_mapping = [ + ('[%s](#t%s)' % (k, k), tickets[k]['meta']['status'], + tickets[k]['meta']['summary']) for k in keys + ] + cols = ['ID', 'Status', 'Summary'] + self.generator.gen_table(cols, id_summary_mapping, sort_by='ID') + self.generator.gen_line_break() + + @staticmethod + def _convert_to_bulleted_link(name: str, generator): + level = name.count('#') + stripped_name = name.replace('#', '').strip() + linked_name = name.lower().replace(' ', + '-').replace('-', '', 1).replace( + '#', '', level - 1) + if not isinstance(generator, MarkdownGenerator): + linked_name = linked_name.replace('.', '-') + + return f"{(' ' * (level - 1)) + '* '}[{stripped_name}]({linked_name})" + + def md_toc(self, headings): + tmp_gen = MarkdownGenerator() + toc_headers = [h[1:] for h in headings] + toc_headers.extend([ + '# RTEMS ' + self.milestone + ' Ticket Overview', + '# RTEMS ' + self.milestone + ' Ticket Summary', + '# RTEMS ' + self.milestone + ' Tickets By Category' + ]) + toc_headers.append('# RTEMS ' + self.milestone + ' Tickets') + bulleted_links = [] + for c in toc_headers: + bulleted_links.append(self._convert_to_bulleted_link(c, tmp_gen)) + for b in bulleted_links: + tmp_gen.gen_unwrapped_line(b) + return tmp_gen.content + + def gen_tickets_stats_by_category(self, by_category): + self.generator.gen_heading('RTEMS ' + self.milestone + \ + ' Tickets By Category', heading_base) + self.generator.gen_line('') + + for category in by_category: + self.generator.gen_heading(category.capitalize(), heading_base + 1) + + # Get header and all rows to generate table, set category as first col + header = [category.capitalize()] + rows = [] + ticket_stats_list = list(by_category[category].values()) + if len(ticket_stats_list) > 0: + header += [k.capitalize() for k in ticket_stats_list[0].keys()] + + for category_value in by_category[category]: + ticket_stats = by_category[category][category_value] + rows.append([category_value] + list(ticket_stats.values())) + + self.generator.gen_table(header, rows) + self.generator.gen_line('') + + def gen_individual_tickets_info(self, tickets): + self.generator.gen_line_break() + self.generator.gen_heading('RTEMS ' + self.milestone + ' Tickets', + heading_base) + num_jobs = 1 + job_count = 0 + job_total = len(tickets) + job_len = len(str(job_total)) + for ticket_id in sorted(list(tickets.keys())): + job = ticket(self.format, tickets[ticket_id]) + job_count += 1 + print('\r %*d of %d - %s ticket %s ' % + (job_len, job_count, job_total, self.milestone, ticket_id), + end='') + job.formatter() + self.generator.gen_horizontal_line() + self.generator.content += job.generator.content + print() diff --git a/release-notes/reraise.py b/release-notes/reraise.py new file mode 100644 index 0000000..5b43a88 --- /dev/null +++ b/release-notes/reraise.py @@ -0,0 +1,111 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2013-2017 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +from __future__ import print_function + +import sys + +# +# The following fragment is taken from https://bitbucket.org/gutworth/six +# to raise an exception. The python2 code cause a syntax error with python3. +# +# Copyright (c) 2010-2016 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Taken from six. +# +if sys.version_info[0] == 3: + + def reraise(tp, value, tb=None): + raise value.with_traceback(tb) +else: + + def exec_(_code_, _globs_=None, _locs_=None): + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb = None): + raise tp, value, tb +""") + +if __name__ == "__main__": + try: + import threading + import time + result = None + finished = False + + def _thread(): + global finished + global result + try: + raise ValueError('raised inside a thread, reaise is working') + except: + result = sys.exc_info() + finished = True + + thread = threading.Thread(target=_thread, name='test') + thread.start() + while not finished: + time.sleep(0.05) + if result is not None: + reraise(*result) + else: + print('error: no exception raised and caught') + except ValueError as ve: + print('exception caught: %s' % (str(ve))) + except KeyboardInterrupt: + print('abort: user terminated') + except: + print('unknown exception caught') diff --git a/release-notes/rtems-release-notes b/release-notes/rtems-release-notes new file mode 100755 index 0000000..1cbf3d0 --- /dev/null +++ b/release-notes/rtems-release-notes @@ -0,0 +1,167 @@ +#! /usr/bin/env python +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2022 Chris Johns (chris@contemporary.software) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import argparse +import sys + +import trac +import tickets +import reports + + +def get_notes(notes_file): + return [l[:-1] for l in open(notes_file, 'r').readlines()] if notes_file is not None else None + + +def milestone_to_major_minor(release): + rtems_major, rtems_minor = release.split('.', 1) + try: + major = int(rtems_major) + rm = '' + for c in rtems_minor: + if c.isdigit(): + rm += c + else: + break + try: + minor = int(rm) + except: + raise RuntimeError('invalid release: ' + milestone) + except: + raise RuntimeError('invalid release: ' + milestone) + return major, minor + + +def milestone_from_major_minor(major, minor): + return '%d.%d' % (major, minor) + + +def milestones(release, reverse=False): + major, minor = milestone_to_major_minor(release) + ms = [milestone_from_major_minor(major, m) for m in range(1, minor + 1)] + if reverse: + ms.reverse() + return ms + + +def collect_tickets(release, cache, force): + ''' + Collect the tickets for the release and all previous release milestones + + A release is major.minor[-.*] from minor back to 1. + ''' + ticks = {} + for milestone in milestones(release): + print( + f"Fetching and processing tickets for release {release} milestone {milestone}." + ) + tcache = trac.cache(milestone, cache, force) + ticks[milestone] = tickets.tickets(release=release, milestone=milestone) + ticks[milestone].load(cache=tcache) + return ticks + + +def generate(ticks, release, notes_file): + rtems_major, rtems_minor = milestone_to_major_minor(release) + notes = {} + for milestone in milestones(release): + notes[milestone] = get_notes(notes_file % (milestone)) + gen = reports.generator(release) + gen.generator.gen_heading('Table of Content', reports.heading_base) + for milestone in milestones(release, reverse=True): + print( + f"Formatting tickets for release {release} milestone {milestone}." + ) + t = ticks[milestone] + gen.set_milestone(milestone) + gen.gen_toc(notes[milestone]) + for milestone in milestones(release, reverse=True): + t = ticks[milestone] + gen.generator.gen_page_break() + gen.generator.gen_line_break() + gen.set_milestone(milestone) + gen.gen_start(notes[milestone]) + gen.gen_overall_progress(t.tickets['overall_progress']) + gen.gen_tickets_summary(t.tickets['tickets']) + gen.gen_tickets_stats_by_category(t.tickets['by_category']) + gen.gen_individual_tickets_info(t.tickets['tickets']) + return gen.generator.content + + +if __name__ == '__main__': + + args = argparse.ArgumentParser() + + args.add_argument('-r', + '--release', + required=True, + dest='release', + help='The release to report', + type=str, + default=None) + args.add_argument('-f', + '--force', + dest='force', + help='Force downloading of tickets', + action='store_true') + args.add_argument('-c', + '--cache', + dest='cache', + help='Cache file base name of ticket data, one per milestone', + type=str, + default=None) + args.add_argument('-o', + '--output', + required=True, + dest='output', + help='Output file', + type=str, + default=None) + args.add_argument('-N', + '--notes', + dest='notes', + help='Top-level, manually-written release notes', + default=None) + + opts = args.parse_args() + + if opts.cache is not None: + cache = opts.cache + else: + cache = '.rng-cache' + + ticks = collect_tickets(release=opts.release, cache=cache, force=opts.force) + contents = generate(ticks, opts.release, opts.notes) + + print('Writing ' + opts.output) + + with open(opts.output, 'w') as f: + f.write(contents) diff --git a/release-notes/rtems_trac.py b/release-notes/rtems_trac.py new file mode 100644 index 0000000..4d233f9 --- /dev/null +++ b/release-notes/rtems_trac.py @@ -0,0 +1,96 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2018 Danxue Huang (danxue.huang@gmail.com) +# Copyright 2022 Chris Johns (chris@contemporary.software) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import codecs +import csv +import time + +trac_base = 'https://devel.rtems.org' +ticket_base = trac_base + '/ticket' +format_rss = 'format=rss' +format_csv = 'format=csv' +query = 'query' +all_cols = [ + 'id', 'summary', 'milestone', 'owner', 'type', 'status', 'priority', + 'component', 'version', 'severity', 'resolution', 'time', 'changetime', + 'blockedby', 'blocking', 'reporter', 'keywords', 'cc' +] +aggregate_cols = [ + 'owner', 'type', 'priority', 'component', 'severity', 'reporter', 'version' +] + + +def gen_ticket_url(ticket_id): + return ticket_base + '/' + str(ticket_id) + + +def gen_ticket_rss_url(ticket_id): + return gen_ticket_url(ticket_id) + '?' + format_rss + + +def gen_ticket_csv_url(ticket_id): + return gen_ticket_url(ticket_id) + '?' + format_csv + + +def gen_trac_query_csv_url(cols, **filters): + return gen_trac_query_url(cols, **filters) + '&' + format_csv + + +def gen_attachment_link(attachment_name, ticket_number): + return '/'.join([ + trac_base, 'attachment', 'ticket', + str(ticket_number), attachment_name + ]) + + +def gen_trac_query_url(cols, **filters): + constraints = [] + for col in cols: + constraints.append('col={c}'.format(c=col)) + for key, value in filters.items(): + constraints.append('{k}={v}'.format(k=key, v=value)) + constraints_str = '&'.join(constraints) + return trac_base + '/' + query + '?' + constraints_str + + +def open_ticket(ticket_id, cache, part='csv'): + if part == 'csv': + url = gen_ticket_csv_url(ticket_id) + elif part == 'rss': + url = gen_ticket_rss_url(ticket_id) + else: + raise RuntimeError('unknown part of ticket: ' + part) + return cache.open_page(url) + + +def parse_csv_as_dict_iter(url, cache): + csv_response = cache.open_page(url) + return csv.DictReader(codecs.iterdecode(csv_response, 'utf-8-sig')) diff --git a/release-notes/tickets.py b/release-notes/tickets.py new file mode 100644 index 0000000..a21f9af --- /dev/null +++ b/release-notes/tickets.py @@ -0,0 +1,519 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2018 Danxue Huang (danxue.huang@gmail.com) +# Copyright 2022 Chris Johns (chris@contemporary.software) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import html.entities +import html.parser +import os +import sys +import time +import threading + +import xml.etree.ElementTree as ElementTree + +import reraise +import rtems_trac +import trac + + +class rss_parser(html.parser.HTMLParser): + + def __init__(self, break_p=False): + super(rss_parser, self).__init__() + self.trace = False + self.tags = [] + self.text = '' + self.div_end = 0 + self.break_p = break_p + + def __del__(self): + if self.trace: + print('> del: ' + str(self)) + + def __str__(self): + o = ['text: ' + self.text] + return os.linesep.join(o) + + def _clean_data(self, data): + leading_ws = ' ' if len(data) > 0 and data[0].isspace() else '' + trailing_ws = ' ' if len(data) > 0 and data[-1].isspace() else '' + data = leading_ws + data.strip() + trailing_ws + if self.break_p: + data = data.replace(os.linesep, '
') + return data + + def _tag_attr_get(self, attrs, key): + if attrs: + for attr, label in attrs: + if attr == key: + return label + return None + + def _tags_parse_all(self, start, tag, attrs, extended): + if attrs and self.trace: + for attr in attrs: + print(" attr:", attr) + o = '' + if tag == 'em': + o = '__' + elif tag == 'strong': + o = '__' + elif tag == 'br': + if self.div_end == 0 and not start: + o = '
' + elif tag == 'p': + if self.div_end == 0: + if start: + o = '

' + else: + o = '

' + else: + o = os.linesep + elif tag == 'div': + if start: + div_class = self._tag_attr_get(attrs, 'class') + if div_class and self.div_end == 0: + o = os.linesep * 2 + '
' + os.linesep + self.div_end += 1 + elif self.div_end > 0: + self.div_end += 1 + else: + if self.div_end == 1: + o = os.linesep + '
' + os.linesep + if self.div_end > 0: + self.div_end -= 1 + if self.trace: + print(' tag: start = ', start, 'dev_end =', self.div_end) + elif tag == 'ul' and extended: + if start: + o = '' + elif tag == 'li' and extended: + if start: + o = '
  • ' + else: + o = '
    • ' + elif tag == 'pre': + if start: + o = '
    '
+            else:
+                o = '
    ' + elif tag == 'blockquote': + bq_class = self._tag_attr_get(attrs, 'class') + if start: + if bq_class: + o = '
    ' + else: + o = '
    ' + else: + o = '
    ' + return o + + def _tags_parse_start(self, tag, attrs, extended=True): + return self._tags_parse_all(True, tag, attrs, extended) + + def _tags_parse_end(self, tag, extended=True): + return self._tags_parse_all(False, tag, None, extended) + + def _tags_push(self, tag): + self.tags.append(tag) + + def _tags_pop(self, tag): + if len(self.tags) != 0: + self.tags.pop() + + def _tags_path(self): + return '/'.join(self.tags) + + def _tags_in_path(self, path): + return self._tags_path().startswith(path) + + def handle_starttag(self, tag, attrs): + if self.trace: + print("> start-tag (p):", tag) + self._tags_push(tag) + self.text += self._tags_parse_start(tag, attrs, True) + + def handle_endtag(self, tag): + if self.trace: + print("> end-tag (p):", tag) + self._tags_pop(tag) + self.text += self._tags_parse_end(tag) + + def handle_data(self, data): + if self.trace: + print("> data (p) :", data) + data = self._clean_data(data) + self.text += data + + +class rss_meta_parser(rss_parser): + + def __init__(self): + super(rss_meta_parser, self).__init__() + self.meta_data = [] + self.meta_steps = ['ul', 'li', 'strong'] + self.meta_label = None + self.meta_text = '' + + def __str__(self): + o = [ + 'meta_data: %r' % (self.meta_data), + 'meta_label: ' + str(self.meta_label), + 'meta_text: ' + str(self.meta_text), 'text: ' + self.text + ] + return os.linesep.join(o) + + def _tags_metadata(self): + return self.meta_label and self._tags_path().startswith('ul/li') + + def _tags_meta_label(self): + return self._tags_path() == 'ul/li/strong' + + def handle_starttag(self, tag, attrs): + if self.trace: + print("> start-tag (m):", tag) + in_metadata = self._tags_metadata() + self._tags_push(tag) + if self._tags_metadata(): + if in_metadata: + self.meta_text += self._tags_parse_start(tag, + attrs, + extended=False) + elif not self._tags_meta_label(): + self.text += self._tags_parse_start(tag, attrs, extended=False) + + def handle_endtag(self, tag): + if self.trace: + print("> end-tag (m):", tag) + in_metadata = self._tags_metadata() + self._tags_pop(tag) + if in_metadata: + # Trailing edge detect of the metadata end + # Ignore the meta_label eng tag + if not self._tags_metadata(): + self.meta_data.append( + (self.meta_label, self.meta_text.strip())) + self.meta_label = None + self.meta_text = '' + elif len(self.meta_text) > 0: + self.meta_text += self._tags_parse_end(tag, extended=False) + else: + self.text += self._tags_parse_end(tag, extended=False) + + def handle_data(self, data): + if self.trace: + print("> data (m) :", data) + if not self.meta_label and self._tags_meta_label(): + self.meta_label = data.strip() + elif self._tags_metadata(): + self.meta_text += self._clean_data(data) + else: + super(rss_meta_parser, self).handle_data(data) + + +class _ticket_fetcher(object): + + ns = {'dc': '{http://purl.org/dc/elements/1.1/}'} + + def __init__(self, ticket, cache): + self.ticket = ticket + self.cache = cache + self.data = None + self.thread = None + self.result = None + + def _parse_ticket_csv(self): + url = rtems_trac.gen_ticket_csv_url(self.ticket_id()) + csv_rows_iter = rtems_trac.parse_csv_as_dict_iter(url, self.cache) + return dict(next(csv_rows_iter, {})) + + @staticmethod + def dump_element(el, indent=0): + if isinstance(el, ElementTree.Element): + print('%stag:' % (' ' * indent), el.tag) + print('%stext:' % (' ' * indent), len(el.text), el.text) + print('%stail:' % (' ' * indent), len(el.tail), el.tail.strip()) + for item in el.items(): + _ticket_fetcher.dump_element(item, indent + 1) + else: + print('%sitem:' % (' ' * indent), el) + + def _item_text(self, item, break_p=False): + if item is None: + return None + rp = rss_parser(break_p=break_p) + if item.text: + rp.feed(item.text) + if item.tail: + rp.feed(item.tail) + return rp.text.strip() + + def _item_meta(self, item): + title = item.find('title') + creator = item.find(self.ns['dc'] + 'creator') + author = item.find('author') + if author is not None: + creator = author + pub_date = item.find('pubDate') + guid = item.find('guid') + description = item.find('description') + category = item.find('category') + if title.text is None: + actions = 'comment' + else: + actions = title.text + i = { + 'tag': self._item_tag(title.text), + 'actions': actions, + 'creator': self._item_text(creator), + 'published': self._item_text(pub_date), + 'guid': self._item_text(guid), + 'category': self._item_text(category) + } + rp = rss_meta_parser() + rp.feed(description.text) + rp.feed(description.tail) + i['meta'] = rp.meta_data + i['description'] = rp.text.strip() + return i + + def _item_tag(self, tag): + if tag is not None: + ns = {'dc': '{http://purl.org/dc/elements/1.1/}'} + if tag == ns['dc'] + 'creator': + tag = 'creator' + elif tag == 'pubData': + tag = 'published' + elif tag.startswith('attachment'): + tag = 'attachment' + elif tag.startswith('description'): + tag = 'description' + elif tag.startswith('milestone'): + tag = 'milestone' + else: + tag = 'comment' + return tag + + def _attachment_post(self, attachment): + for m in range(0, len(attachment['meta'])): + meta = attachment['meta'][m] + if meta[0] == 'attachment' and \ + meta[1].startswith('set to __') and meta[1].endswith('__'): + set_to_len = len('set to __') + alink = meta[1][set_to_len:-2] + meta = (meta[0], + meta[1][:set_to_len - 2] + \ + '[' + alink + '](' + attachment['guid'] + '/' + alink + ')') + attachment['meta'][m] = meta + return attachment + + def _parse_ticket_rss(self): + # Read xml data as ElementTree, and parse all tags + ticket_rss = {} + rss_response = rtems_trac.open_ticket(self.ticket_id(), + self.cache, + part='rss') + rss_root = ElementTree.parse(rss_response).getroot() + # + # The channel has: + # title + # link + # description + # language + # image + # generator + # item + # + # The channel/item has: + # dc:creator + # author + # pubDate + # title + # link + # guid + # description + # category + # + channel = rss_root.find('channel') + title = channel.find('title') + link = channel.find('link') + description = channel.find('description') + items = channel.findall('item') + citems = [self._item_meta(item) for item in items] + ticket_rss['title'] = self._item_text(title) + ticket_rss['link'] = self._item_text(link) + ticket_rss['description'] = self._item_text(description, True) + ticket_rss['attachments'] = \ + [self._attachment_post(ci) for ci in citems if 'comment' not in ci['guid']] + ticket_rss['comments'] = \ + sorted([ci for ci in citems if 'comment' in ci['guid']], + key=lambda i: int(i['guid'][i['guid'].rfind(':') + 1:])) + return ticket_rss + + def _runner(self): + try: + self.data = { + 'ticket': self.ticket, + 'meta': self._parse_ticket_csv(), + 'comment_attachment': self._parse_ticket_rss() + } + except KeyboardInterrupt: + pass + except: + self.result = sys.exc_info() + + def ticket_id(self): + return self.ticket['id'] + + def run(self): + self.thread = threading.Thread(target=self._runner, + name='ticket-%s' % (self.ticket_id())) + self.thread.start() + + def is_alive(self): + return self.thread and self.thread.is_alive() + + def reraise(self): + if self.result is not None: + print() + print('ticket:', self.ticket_id()) + reraise.reraise(*self.result) + + +class tickets: + """This class load all tickets data for a milestone.""" + + def __init__(self, release, milestone, cache=None): + self.release = release + self.milestone = milestone + self.lock = threading.Lock() + self.tickets = { 'release': release, 'milestone': milestone } + + def get_ticket_ids(self): + return self.tickets.keys() + + def _fetch_data_for_ticket(self, ticket): + return self._parse_ticket_data(ticket) + + def _job_waiter(self, jobs, num_jobs): + while len(jobs) >= num_jobs: + time.sleep(0.002) + for job in jobs: + if not job.is_alive(): + job.reraise() + self.tickets['tickets'][job.data['meta']['id']] = job.data + self._update_stats(job.data) + jobs.remove(job) + + def load(self, cache, use_cache=False): + if use_cache: + tickets = cache.load() + if tickets: + self.tickets = tickets + return + # Read entire trac table as DictReader (iterator) + self._pre_process_tickets_stats() + tickets_reader = self._get_tickets_table_as_dict(cache) + tickets = [t for t in tickets_reader] + num_jobs = 20 + jobs = [] + job_count = 0 + job_total = len(tickets) + job_len = len(str(job_total)) + for ticket in tickets: + self._job_waiter(jobs, num_jobs) + job = _ticket_fetcher(ticket, cache) + jobs.append(job) + job.run() + job_count += 1 + print('\r %*d of %d - ticket %s ' % + (job_len, job_count, job_total, ticket['id']), + end='') + self._job_waiter(jobs, 1) + print() + self._post_process_ticket_stats() + cache.unload(self.tickets) + + def _update_stats(self, ticket): + self.tickets['overall_progress']['total'] += 1 + if ticket['meta']['status'] == 'closed': + self.tickets['overall_progress']['closed'] += 1 + if ticket['meta']['status'] == 'assigned': + self.tickets['overall_progress']['assigned'] += 1 + if ticket['meta']['status'] == 'new': + self.tickets['overall_progress']['new'] += 1 + for col in rtems_trac.aggregate_cols: + col_value = ticket['meta'][col] + self.tickets['by_category'][col][col_value] \ + = self.tickets['by_category'][col].get(col_value, {}) + if ticket['meta']['status'] == 'closed': + self.tickets['by_category'][col][col_value]['closed'] \ + = self.tickets['by_category'][col][col_value] \ + .get('closed', 0) + 1 + self.tickets['by_category'][col][col_value]['total'] \ + = self.tickets['by_category'][col][col_value].get('total', 0) + 1 + + def _pre_process_tickets_stats(self): + self.tickets['overall_progress'] = {} + self.tickets['by_category'] = { + col: {} + for col in rtems_trac.aggregate_cols + } + self.tickets['overall_progress']['total'] = 0 + self.tickets['overall_progress']['closed'] = 0 + self.tickets['overall_progress']['in_progress'] = 0 + self.tickets['overall_progress']['new'] = 0 + self.tickets['overall_progress']['assigned'] = 0 + self.tickets['tickets'] = {} + + def _post_process_ticket_stats(self): + # (number of closed tickets) / (number of total tickets) + n_closed = self.tickets['overall_progress'].get('closed', 0) + n_total = self.tickets['overall_progress'].get('total', 0) + self.tickets['overall_progress']['percentage'] \ + = "{0:.0%}".format((n_closed / n_total) if n_total > 0 else 0.0) + # Get progress (closed/total) for each category + for col in self.tickets['by_category']: + for key in self.tickets['by_category'][col]: + closed = self.tickets['by_category'][col][key].get('closed', 0) + if closed == 0: + self.tickets['by_category'][col][key]['closed'] = 0 + total = self.tickets['by_category'][col][key].get('closed', 0) + if total == 0: + self.tickets['by_category'][col][key]['total'] = 0 + self.tickets['by_category'][col][key]['progress'] \ + = '{c}/{t}'.format(c=closed, t=total) + + def _get_tickets_table_as_dict(self, cache): + csv_url = rtems_trac.gen_trac_query_csv_url(rtems_trac.all_cols, + milestone=self.milestone) + return rtems_trac.parse_csv_as_dict_iter(csv_url, cache=cache) diff --git a/release-notes/trac.py b/release-notes/trac.py new file mode 100644 index 0000000..91e781a --- /dev/null +++ b/release-notes/trac.py @@ -0,0 +1,136 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2022 Chris Johns (chris@contemporary.software) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +import pickle +import os +import urllib.request + + +class cache(object): + + def __init__(self, milestone, path, force): + self.milestone = milestone + self.path = path + self.force = force + self.checked = False + self.cache_valid = False + + @staticmethod + def _milestone(url): + path, options = url.split('?', 1) + opts = options.split('&') + for o in opts: + if 'milestone' in o: + label, milestone = o.split('=', 1) + return milestone + raise RuntimeError('milestone not found: ' + url) + + def _tickets_path(self): + return os.path.join(self.path, + 'tickets-%s' % (self.milestone) + '.ppk') + + def _ticket_path(self, url): + path, options = url.split('?', 1) + opts = options.split('&') + fmt = None + for o in opts: + if 'format' in o: + label, fmt = o.split('=', 1) + if not fmt: + raise RuntimeError('ticket format not found: ' + url) + if '/' in path: + ticket_id = path[path.rfind('/') + 1:] + return os.path.join(self.path, '%s.%s' % (ticket_id, fmt)) + raise RuntimeError('ticket id not found: ' + url) + + def _query_path(self): + return os.path.join(self.path, 'query-%s' % (self.milestone) + '.csv') + + def check(self): + if not self.checked: + self.checked = True + if self.path: + if os.path.exists(self.path): + if not os.path.isdir(self.path): + raise RuntimeError('cache is not a directory:' + + self.path) + else: + os.mkdir(self.path) + self.cache_valid = True + return self.cache_valid + + def open_page(self, url): + url_path = None + if self.check(): + if 'query' in url: + url_path = self._query_path() + else: + url_path = self._ticket_path(url) + if not self.force and os.path.exists(url_path): + return open(url_path, 'rb') + # Open the URL + delay = 1 + tries = 6 + backoff = 2 + while tries > 0: + try: + page = urllib.request.urlopen(url) + if url_path: + with open(url_path, 'wb') as f: + f.write(page.read()) + return open(url_path, 'rb') + return page + except OSError: + tries -= 1 + time.sleep(delay) + delay *= backoff + raise RuntimeError('cannot open url:' + url) + + def load(self): + if self.check(): + ticket_cache = self._tickets_path() + if os.path.exists(ticket_cache): + if not self.force: + try: + with open(ticket_cache, 'rb') as f: + tickets = pickle.load(f) + print('%d tickets loaded from cache: %s' % + (len(tickets['tickets']), ticket_cache)) + return tickets + except: + print('cache corrupted: ' + ticket_cache) + os.remove(ticket_cache) + return None + + def unload(self, tickets): + if self.check(): + ticket_cache = self._tickets_path() + with open(ticket_cache, 'wb') as f: + pickle.dump(tickets, f) diff --git a/rtems-release b/rtems-release index 0cce91d..f56e504 100755 --- a/rtems-release +++ b/rtems-release @@ -128,43 +128,43 @@ fi # Package the RSB, must be before the kernel. The kernel worker script uses the # RSB to create autoconf and automake so it can bootstrap the kernel. # -build rtems-source-builder ${version} ${revision} ${release_url} -build rtems-tools ${version} ${revision} ${release_url} -build rtems ${version} ${revision} ${release_url} rtems-release-kernel -if [ ${rtems_libbsd} = yes ]; then - build rtems-libbsd ${version} ${revision} ${release_url} -fi -build rtems-source-builder ${version} ${revision} ${release_url} rtems-release-rsb-version -if [ ${rtems_examples} = yes ]; then - if [ ${version} -lt 5 ]; then - build examples-v2 ${version} ${revision} ${release_url} - # Hack around the repo naming. - mv ${release}/examples-v2-${release}.tar.${comp_ext} \ - ${release}/rtems-examples-v2-${release}.tar.${comp_ext} - else - build rtems-examples ${version} ${revision} ${release_url} - fi -fi - -# -# Documentation. -# -if [ ${rtems_docs} = yes ]; then - ./rtems-release-docs rtems-docs ${version} ${revision} ${release_url} -fi - -# -# Release notes. -# -if [ ${rtems_release_notes} = yes ]; then - ./rtems-release-notes rtems-release-notes ${version} ${revision} ${release_url} -fi - -# -# The sources is always last. -# -echo "] Collect tools sources" -./rtems-release-sources ${version} ${revision} ${release_url} +# build rtems-source-builder ${version} ${revision} ${release_url} +# build rtems-tools ${version} ${revision} ${release_url} +# build rtems ${version} ${revision} ${release_url} rtems-release-kernel +# if [ ${rtems_libbsd} = yes ]; then +# build rtems-libbsd ${version} ${revision} ${release_url} +# fi +# build rtems-source-builder ${version} ${revision} ${release_url} rtems-release-rsb-version +# if [ ${rtems_examples} = yes ]; then +# if [ ${version} -lt 5 ]; then +# build examples-v2 ${version} ${revision} ${release_url} +# # Hack around the repo naming. +# mv ${release}/examples-v2-${release}.tar.${comp_ext} \ +# ${release}/rtems-examples-v2-${release}.tar.${comp_ext} +# else +# build rtems-examples ${version} ${revision} ${release_url} +# fi +# fi + +# # +# # Documentation. +# # +# if [ ${rtems_docs} = yes ]; then +# ./rtems-release-docs rtems-docs ${version} ${revision} ${release_url} +# fi + +# # +# # Release notes. +# # +# if [ ${rtems_release_notes} = yes ]; then +# ./rtems-release-notes rtems-release-notes ${version} ${revision} ${release_url} +# fi + +# # +# # The sources is always last. +# # +# echo "] Collect tools sources" +# ./rtems-release-sources ${version} ${revision} ${release_url} # # Make the contrib directory diff --git a/rtems-release-defaults b/rtems-release-defaults index b453504..454d4ea 100755 --- a/rtems-release-defaults +++ b/rtems-release-defaults @@ -117,8 +117,14 @@ rtems_libbsd_release=12 email_build_to="build@rtems.org" email_announce_to="users@rtems.org,devel@rtems.org" +# +# Pandoc options +# +pandoc_std_opts="-f markdown_phpextra+grid_tables+multiline_tables+simple_tables+auto_identifiers+line_blocks+inline_code_attributes+fancy_lists+backtick_code_blocks --section-divs" + # # The date stamp # now=$(date +"%d %B %Y") -export now +now_year=$(date +"%Y") +export now now_year diff --git a/rtems-release-info b/rtems-release-info index 2f8c80a..1b68966 100644 --- a/rtems-release-info +++ b/rtems-release-info @@ -36,6 +36,10 @@ # # Create the README.md and from that README.txt and index.html # +rep_len=$(echo "@RELEASE@@R_SP@" | wc -c) +rev_len=$(echo "${release}" | wc -c) +sp_len=$(expr ${rep_len} - ${rev_len} - 7) +r_sp=$(head -c ${sp_len} < /dev/zero | tr '\0' ' ') echo " ## Architectures and BSPs " | \ @@ -45,6 +49,7 @@ echo " -e "s/@VERSION@/${version}/g" \ -e "s/@REVISION@/${revision}/g" \ -e "s/@RTEMS_RELEASE_NOTES@/${release_notes}/g" \ + -e "s/@R_SP@/${r_sp}/g" \ -e "s/@DATE@/${now}/g" > ${release}/contrib/README.md rm ARCH-BSP.md diff --git a/rtems-release-kernel b/rtems-release-kernel index 176fc55..20a4a6d 100755 --- a/rtems-release-kernel +++ b/rtems-release-kernel @@ -188,13 +188,13 @@ if [ -f ${prefix}/cpukit/Doxyfile.in ]; then -e "s/^INPUT[[:space:]].*=.*$/INPUT = ${top_srcdir}/g" \ -e "s/^HAVE_DOT[[:blank:]]/DOT_NUM_THREADS = 1\\ HAVE_DOT /g"> Doxyfile - doxygen Doxyfile + doxygen -q Doxyfile elif [ ${prefix}/Doxygen ]; then cat ${prefix}/Doxyfile | \ sed -e "s/^PROJECT_NUMBER[[:space:]].*=.*$/PROJECT_NUMBER = ${release}/g" \ > Doxyfile cd ${prefix} - doxygen ../Doxyfile + doxygen -q ../Doxyfile cd .. else echo "error: no doxygen configuration file found" diff --git a/rtems-release-notes b/rtems-release-notes index fbbe222..c1d280d 100755 --- a/rtems-release-notes +++ b/rtems-release-notes @@ -68,10 +68,28 @@ title="RTEMS Release Notes builder" # ws_pwd=${PWD} +echo "] Creating release notes" +echo "] Generate release notes markdown" + # -# The release notes are all held in the wiki +# The release notes are taken directly from Trac # -release_pages="https://devel.rtems.org/wiki/Release" +${top}/release-notes/rtems-release-notes \ + --release ${release} \ + --notes "${top}/notes/rtems-notes-%s.md" \ + --output rtems-${release}-release-notes.md + +echo "] Generate release notes HTML" + +# +# Convert to HTML +# +pandoc rtems-${release}-release-notes.md \ + ${pandoc_std_opts} \ + -t html --self-contained --markdown-headings=atx \ + -M title="RTEMS ${release} Embedded Realtime Operating System" \ + --include-in-header=${top}/rtems-release-notes.css \ + -o rtems-${release}-release-notes.html # # Set up the wkhtmltopdf defaults. @@ -80,11 +98,9 @@ page_options="--print-media-type --zoom 0.8" header="--header-right [page]/[toPage] --header-font-size 10" footer="--footer-left [webpage] --footer-font-size 10" -echo "] Creating release notes" - rel_html="" rel_html_line="
    @RELEASE@
    " -rev=0 +rev=1 while [ ${rev} -le ${revision_no} ] do rel=${version}.${rev} @@ -94,42 +110,37 @@ done rel_html=$(echo ${rel_html} | sed -e 's/\./\\\./g' -e 's/\//\\\//g') echo "] Create the coverpage" + cp ${top}/rtems-release-notes-coverpage/* . cat rtems-release-notes-coverpage.html.in | \ sed -e "s/@RELEASE@/${release}/g" \ -e "s/@VERSION@/${version}/g" \ -e "s/@REVISION@/${revision}/g" \ -e "s/@DATE@/${now}/g" \ + -e "s/@YEAR@/${year}/g" \ -e "s/@REVISIONS@/${rel_html}/g" > rtems-release-notes-coverpage.html wkhtmltopdf file://${ws_pwd}/rtems-release-notes-coverpage.html \ + --enable-local-file-access \ --disable-smart-shrinking \ ${page_options} \ --no-header-line \ --no-footer-line cp.pdf -pdfs="" -rev=0 -while [ ${rev} -le ${revision_no} ] -do - even_odd=$(( ${rev} % 2 )) - if [ ${version} -lt 5 -o ${even_odd} -ne 0 ]; then - rel=${version}.${rev} - echo "] Creating the ${rel} PDF" - wkhtmltopdf --user-style-sheet file://${ws_pwd}/trac-rtems-style.html \ - -L 5mm -R 5mm \ - ${release_pages}/${version}/${version}.${rev} \ - ${page_options} \ - --header-left "RTEMS ${rel} Release Notes" ${header} \ - ${footer} \ - p${rev}.pdf - pdfs="p${rev}.pdf ${pdfs}" - fi - rev=$(expr ${rev} + 1) -done +echo "] Creating the ${release} PDF" +wkhtmltopdf --user-style-sheet file://${ws_pwd}/trac-rtems-style.html \ + -L 5mm -R 5mm \ + file://${ws_pwd}/rtems-${release}-release-notes.html \ + ${page_options} \ + --header-left "RTEMS ${release} Release Notes" ${header} \ + --enable-local-file-access \ + ${footer} \ + p${release}.pdf + +gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=../rtems-${release}-release-notes.pdf cp.pdf p${release}.pdf -gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=../rtems-${release}-release-notes.pdf cp.pdf ${pdfs} +cp ${ws_pwd}/rtems-${release}-release-notes.html ../rtems-${release}-release-notes.html -echo "] Created: ${release}/rtems-${release}-release-notes.pdf cp.pdf" +echo "] Created: ${release}/rtems-${release}-release-notes.html ${release}/rtems-${release}-release-notes.pdf" # # Comman package end. diff --git a/rtems-release-notes-coverpage/rtems-release-notes-coverpage.html.in b/rtems-release-notes-coverpage/rtems-release-notes-coverpage.html.in index e739c64..422e8bd 100644 --- a/rtems-release-notes-coverpage/rtems-release-notes-coverpage.html.in +++ b/rtems-release-notes-coverpage/rtems-release-notes-coverpage.html.in @@ -8,10 +8,8 @@ - - + -