From ee9695e1b902801dc5a85150403df541d25de1bb Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 29 Sep 2026 22:06:08 -0400 Subject: [PATCH 1/6] releaseWizard: remove solrbot addDepsToChanges step Each solrbot PR now creates its own changelog entry via the renovate-changelog workflows, so the bulk step that built entries from git log only produced duplicates. Remove the step and the addDepsToChanges.py script it called. Co-Authored-By: Claude Opus 5.5 --- dev-tools/scripts/addDepsToChanges.py | 257 -------------------------- dev-tools/scripts/releaseWizard.yaml | 25 +-- 2 files changed, 1 insertion(+), 281 deletions(-) delete mode 100755 dev-tools/scripts/addDepsToChanges.py diff --git a/dev-tools/scripts/addDepsToChanges.py b/dev-tools/scripts/addDepsToChanges.py deleted file mode 100755 index 3286d03cac40..000000000000 --- a/dev-tools/scripts/addDepsToChanges.py +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Script to create changelog YAML entries for solrbot dependency updates -""" -import os -import sys - -sys.path.append(os.path.dirname(__file__)) -from scriptutil import * - -import argparse -import re -import yaml -from pathlib import Path - -line_re = re.compile(r"(.*?) (\(branch_\d+x\) )?\(#(\d+)\)$") - - -class ChangeEntry: - """ - Represents one dependency change entry. - Fields: - - pr_num: string PR number (e.g., '3605') - - message: cleaned commit message text to be shown in CHANGES (noise removed) - - author: Git author (e.g., 'solrbot') - """ - - def __init__(self, pr_num: str, message: str, author: str): - self.pr_num = pr_num - self.message = message - self.author = author - - def dep_key(self) -> str: - """ - Extract a dependency key from the message after 'Update ' and before ' to', '(' or end. - This is used for de-duplication and sorting. Case-insensitive. - """ - m = re.search(r"(?i)update\s+(.+?)(?:\s+to\b|\s*\(|$)", self.message) - if m: - return m.group(1).strip() - return self.message.strip() - - def __str__(self) -> str: - # Keep trailing newline to preserve existing blank-line formatting by update_changes - return f"* PR#{self.pr_num}: {self.message} ({self.author})\n" - - def to_yaml_dict(self) -> dict: - """ - Convert to a dictionary suitable for YAML serialization. - Extracts JIRA IDs from the title and adds them to links. - """ - # Extract JIRA IDs from the message - title, jira_links = extract_jira_issues_from_title(self.message) - - # Build links: JIRA issues first, then PR - links = jira_links.copy() # Start with JIRA links - links.append({ - 'name': f'PR#{self.pr_num}', - 'url': f'https://github.com/apache/solr/pull/{self.pr_num}' - }) - - return { - 'title': title, - 'type': 'dependency_update', - 'authors': [ - { - 'name': self.author - } - ], - 'links': links - } - - def yaml_filename(self) -> str: - """ - Generate a filesystem-safe filename for this entry. - Format: PR#####-slug.yaml - Truncates slug on whitespace boundaries, allowing up to 255 chars total. - """ - # Clean message for slug - slug = self.message.lower() - # Replace whitespace with single space - slug = re.sub(r'\s+', ' ', slug) - # Remove non-alphanumeric except dashes - slug = re.sub(r'[^a-z0-9-._ ]', '', slug) - - # Calculate available space for slug - # Format: "PR" + pr_num + "-" + slug + ".yaml" - # Typical PR#1234 = 8 chars + "-" = 9 chars, ".yaml" = 5 chars, total overhead = 14 chars - # Most filesystems limit filenames to 255 chars - max_filename_length = 255 - overhead = len(f"PR{self.pr_num}-.yaml") - max_slug_length = max_filename_length - overhead - - # Truncate to max length on word boundaries if necessary - if len(slug) > max_slug_length: - # Find the last space within the limit - truncated = slug[:max_slug_length] - last_dash = truncated.rfind(' ') - if last_dash > max_slug_length // 2: # Keep at least half the available space - slug = truncated[:last_dash] - else: - # If no good word boundary, use hard limit and clean up trailing spaces - slug = truncated.rstrip(' ') - else: - # Remove trailing spaces - slug = slug.rstrip(' ') - - return f"PR{self.pr_num}-{slug}.yaml" - - -def get_prev_release_tag(ver): - """ - Based on a given version, compute the git tag for the "previous" version to calculate changes since. - For a major version, we want all solrbot commits since last release, i.e. X-1.Y.Z - For a minor version, we want all solrbot commits since X.Y-1.0 - For a patch version, we want all solrbot commits since X.Y.Z-1 - """ - releases_arr = run('git tag |grep "releases/solr" | cut -c 15-').strip().split("\n") - releases = list(map(lambda x: Version.parse(x), releases_arr)) - if ver.is_major_release(): - last = releases.pop() - return "releases/solr/%s" % last.dot - if ver.is_minor_release(): - return "releases/solr/%s.%s.0" % (ver.major, ver.minor - 1) - if ver.is_bugfix_release(): - return "releases/solr/%s.%s.%s" % (ver.major, ver.minor, ver.bugfix - 1) - return None - - -def read_config(): - parser = argparse.ArgumentParser(description='Adds changelog entries in changelog/ folder') - parser.add_argument('--version', type=Version.parse, help='Solr version to add changes to', required=True) - parser.add_argument('--user', default='solrbot', help='Git user to get changes for. Defaults to solrbot') - newconf = parser.parse_args() - return newconf - - -def get_gitlog_lines(user: str, prev_tag: str): - """ - Run git log and return a list of raw subject lines (%s), newest first. - """ - output = run('git log --author=' + user + ' --oneline --no-merges --pretty=format:"%s" ' + prev_tag + '..') - return list(filter(None, output.split("\n"))) - - -def parse_gitlog_lines(lines, author: str): - """ - Parse raw git log subject lines into ChangeEntry objects. - - Extract pr_num and message - - Strip '(branch_Nx)' noise - - Remove optional leading 'chore(deps): ' - - Normalize 'update dependency X' to 'Update X' (case-insensitive) - """ - entries = [] - for line in lines: - match = line_re.search(line) - if not match: - print("Skipped un-parsable line: %s" % line) - continue - text = match.group(1) - pr_num = match.group(3) - # Clean message noise - msg = text - msg = re.sub(r"^chore\(deps\):\s*", "", msg, flags=re.IGNORECASE) - # Normalize 'update dependency' to 'Update ' - msg = re.sub(r"(?i)^update\s+dependenc(y|ies)\s+", "Update ", msg) - entries.append(ChangeEntry(pr_num=pr_num, message=msg, author=author)) - return entries - - -def write_changelog_yaml(entries): - """ - Write each ChangeEntry to a YAML file in changelog/unreleased/ - """ - changelog_dir = Path('changelog/unreleased') - - # Create directory if it doesn't exist - changelog_dir.mkdir(parents=True, exist_ok=True) - - count = 0 - for entry in entries: - filename = changelog_dir / entry.yaml_filename() - yaml_data = entry.to_yaml_dict() - - # Write YAML file with proper formatting - with open(filename, 'w') as f: - yaml.dump(yaml_data, f, default_flow_style=False, sort_keys=False, allow_unicode=True) - - print(f"Created: {filename}") - count += 1 - - return count - - -def dedupe_entries(entries): - """ - De-duplicate dependency update entries (newest first input) by dep_key. - Keeps the first occurrence for each dependency key. - """ - seen = set() - result = [] - for e in entries: - key = e.dep_key().lower() - if key not in seen: - seen.add(key) - result.append(e) - return result - - -def sort_entries(entries): - """Return a new list sorted alphabetically by dependency key (case-insensitive).""" - return sorted(entries, key=lambda e: e.dep_key().lower()) - - -def main(): - if not os.path.exists('CHANGELOG.md'): - sys.exit("Tool must be run from the root of a source checkout.") - newconf = read_config() - prev_tag = get_prev_release_tag(newconf.version) - print("Creating changelog YAML entries for dependency updates since git tag %s" % prev_tag) - try: - gitlog_lines = get_gitlog_lines(newconf.user, prev_tag) - entries = parse_gitlog_lines(gitlog_lines, author=newconf.user) - if entries: - deduped = dedupe_entries(entries) - sorted_entries = sort_entries(deduped) - count = write_changelog_yaml(sorted_entries) - print(f"Successfully created {count} changelog YAML entries") - else: - print("No changes found for version %s" % newconf.version.dot) - print("Done") - except subprocess.CalledProcessError: - print("Error running git log - check your --version") - sys.exit(1) - - -if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print('\nReceived Ctrl-C, exiting early') diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index 0a3b0201b528..363851af68e3 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -671,29 +671,6 @@ groups: ---- types: - bugfix - - !Todo - id: dependency_updates_changes - title: Add dependency updates to changelog - description: | - Bulk add all 'solrbot' dependency updates since last release. - NOTE: Work in progress to let each Solrbot PR add its own changes to changelog. - This step will be removed once that is done. - Until then, there may be a mix of PRs with and without changes to changelog. - depends: clean_git_checkout - commands: !Commands - root_folder: '{{ git_checkout_folder }}' - commands_text: We call out to a helper script that compiles SolrBot changes into `changelog/unreleased` - confirm_each_command: true - commands: - - !Command - cmd: git checkout {{ release_branch }} - stdout: true - - !Command - cmd: python3 -u dev-tools/scripts/addDepsToChanges.py --user solrbot --version {{ release_version }} - tee: true - - !Command - cmd: git add changelog && git commit -m "Add dependency updates to changelog for {{ release_version }}" && git push - logfile: dependency-changes.log - !Todo id: draft_release_notes title: Get a draft of the release notes in place @@ -765,7 +742,7 @@ groups: step before anything is committed or pushed. Can be run standalone, see `dev-tools/scripts/logchange.py prepare --help`. - depends: dependency_updates_changes + depends: clean_git_checkout commands: !Commands root_folder: '{{ git_checkout_folder }}' commands_text: Prepare changelog folder and regenerate CHANGELOG.md for RC{{ rc_number }} (uncommitted) From 252f29a8af1db100b8e31ef6e6c5ee4e32ae98b0 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 30 Sep 2026 00:33:00 -0400 Subject: [PATCH 2/6] releaseWizard: summarize dependency changes from solr/licenses Replace addDepsToChanges.py (which bulk-converted solrbot commits into changelog entries, duplicating the per-PR entries) with dependencyChanges.py. It diffs the jar checksum files in solr/licenses/ between the previous release tag and HEAD, groups jars that moved between the same versions, and prints a summary or writes one dependency_update changelog entry. The wizard step now runs it. Co-Authored-By: Claude Opus 5.5 --- dev-docs/changelog.adoc | 6 + dev-tools/scripts/dependencyChanges.py | 188 +++++++++++++++++++++++++ dev-tools/scripts/releaseWizard.yaml | 25 +++- 3 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 dev-tools/scripts/dependencyChanges.py diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 0a0e9e5a1bd7..cbe9a25f0133 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -275,6 +275,12 @@ Example report output (Json or Markdown): and the final post-vote forward-porting. It is integrated into the Release Wizard but can also be run standalone. Run with `--help` for usage. +=== 5.5 Dependency changes summary + +`dev-tools/scripts/dependencyChanges.py` summarizes the third-party dependency changes +since the previous release by diffing `solr/licenses/`. By default it prints to stdout; +with `--write`, the Release Wizard uses it to write a single `dependency_update` changelog entry. + == 6. Further Reading * https://github.com/logchange/logchange[Logchange web page] diff --git a/dev-tools/scripts/dependencyChanges.py b/dev-tools/scripts/dependencyChanges.py new file mode 100644 index 000000000000..6d4ccce5b608 --- /dev/null +++ b/dev-tools/scripts/dependencyChanges.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Summarizes changes to the third-party dependencies Solr ships, by diffing the +jar checksum files in solr/licenses/ between two git refs (by default: the +previous release tag and HEAD). Jars that moved between the same two versions +are listed together, since they are typically from the same project. + +Prints the summary to stdout, or with --write, writes a changelog YAML entry. + +This is a stopgap until Solr publishes an SBOM per release, which would be the +better source to diff. +""" + +import argparse +import os +import re +import subprocess +import sys +from collections import defaultdict + +sys.path.append(os.path.dirname(__file__)) +from scriptutil import Version, find_current_version + +LICENSES_DIR = 'solr/licenses' +DEFAULT_OUTPUT = 'changelog/unreleased/dependency-changes.yml' + +# A version starts at the last '-'-delimited segment that looks like a dotted number. +# e.g. log4j-1.2-api-2.25.3, zstd-jni-1.5.6-10, guava-33.4.8-jre +VERSION_SEG_RE = re.compile(r'^\d+\.\d') +# Classifiers that distinguish separate jars of the same artifact and version +CLASSIFIER_RE = re.compile(r'-(tests?|linux-[\w-]+|osx-[\w-]+|windows-[\w-]+)$') +# Test artifacts sharing a LICENSE file with shipped ones (e.g. lucene-test-framework) +TEST_ARTIFACT_RE = re.compile(r'-(tests?|testing|test-framework)$') + + +def git(*args): + return subprocess.run(['git'] + list(args), capture_output=True, text=True, check=True).stdout + + +def parse_jar(jar_name): + """Returns (artifact, version) for a jar name without the '.jar' extension.""" + classifier = '' + m = CLASSIFIER_RE.search(jar_name) + if m: + classifier = m.group(0) + jar_name = jar_name[:m.start()] + segs = jar_name.split('-') + idx = max((i for i, s in enumerate(segs) if i > 0 and VERSION_SEG_RE.match(s)), default=None) + if idx is None: + idx = max((i for i, s in enumerate(segs) if i > 0 and s[:1].isdigit()), default=len(segs)) + return '-'.join(segs[:idx]) + classifier, '-'.join(segs[idx:]) + + +def read_licenses_dir(ref): + """Returns (artifact -> version, set of LICENSE file prefixes) at the given ref.""" + jars = {} + license_prefixes = set() + for name in git('ls-tree', '--name-only', f'{ref}:{LICENSES_DIR}').split(): + if name.endswith('.jar.sha1'): + artifact, version = parse_jar(name[:-len('.jar.sha1')]) + jars[artifact] = version + else: + m = re.match(r'(.+)-LICENSE-.+\.txt$', name) + if m: + license_prefixes.add(m.group(1)) + return jars, license_prefixes + + +def has_license(artifact, license_prefixes): + """Whether some '-'-delimited prefix of the artifact has a LICENSE file (mirrors jar-checks.gradle).""" + name = CLASSIFIER_RE.sub('', artifact) + while True: + if name in license_prefixes: + return True + prefix = re.sub(r'-[^-]+$', '', name) + if prefix == name: + return False + name = prefix + + +def find_previous_release_tag(current): + cur = Version.parse(current) + best = None + for tag in git('tag', '-l', 'releases/solr/*').split(): + m = re.fullmatch(r'releases/solr/(\d+)\.(\d+)\.(\d+)', tag) + if not m: + continue + v = tuple(int(x) for x in m.groups()) + if v < (cur.major, cur.minor, cur.bugfix) and (best is None or v > best[0]): + best = (v, tag) + if best is None: + sys.exit(f'No release tag found before {current}; pass --from') + return best[1] + + +def describe_artifacts(artifacts): + """e.g. 'asm, asm-tree', or 'jetty-* (23 jars)' when all share the first name segment.""" + if len(artifacts) > 3: + first = {a.split('-')[0] for a in artifacts} + if len(first) == 1: + return f'{first.pop()}-* ({len(artifacts)} jars)' + return ', '.join(artifacts) + + +def compute_changes(from_ref, to_ref): + """Returns a sorted list of human-readable change lines.""" + old_jars, _ = read_licenses_dir(from_ref) + new_jars, license_prefixes = read_licenses_dir(to_ref) + + # Removals are omitted, and so is anything lacking a LICENSE file: *.sha1 files also cover jars + # we don't ship (e.g. test dependencies), whereas LICENSE files are only required for shipped ones. + # Older releases have LICENSE files for non-shipped jars too, making removals unreliable. + by_transition = defaultdict(list) # (old_version or None, new_version) -> artifacts + for artifact, new in new_jars.items(): + old = old_jars.get(artifact) + if old != new and has_license(artifact, license_prefixes) and not TEST_ARTIFACT_RE.search(artifact): + by_transition[(old, new)].append(artifact) + + lines = [] + for (old, new), artifacts in by_transition.items(): + names = describe_artifacts(sorted(artifacts, key=str.lower)) + lines.append(f'{names} {new} (new)' if old is None else f'{names} {old} → {new}') + return sorted(lines, key=str.lower) + + +def to_yaml(changes, from_ref, version, author): + def quote(s): + return "'" + s.replace("'", "''") + "'" + title = f'Third-party dependency changes since {from_ref.rsplit("/", 1)[-1]}: ' + '; '.join(changes) + return (f'# Generated by dev-tools/scripts/dependencyChanges.py\n' + f'title: {quote(title)}\n' + f'type: dependency_update\n' + f'authors:\n' + f' - name: {quote(author)}\n' + f'links:\n' + f' - name: solr/licenses\n' + f' url: https://github.com/apache/solr/tree/releases/solr/{version}/solr/licenses\n') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--from', dest='from_ref', + help='Git ref to compare from (default: the release tag preceding the current version)') + parser.add_argument('--to', dest='to_ref', default='HEAD', help='Git ref to compare to (default: HEAD)') + parser.add_argument('--write', nargs='?', const=DEFAULT_OUTPUT, metavar='FILE', + help=f'Write a changelog YAML entry instead of printing (default file: {DEFAULT_OUTPUT})') + parser.add_argument('--author', help='Changelog entry author (default: git user.name)') + args = parser.parse_args() + + version = find_current_version() + from_ref = args.from_ref or find_previous_release_tag(version) + changes = compute_changes(from_ref, args.to_ref) + + if not args.write: + print(f'Dependency changes from {from_ref} to {args.to_ref}:') + for line in changes: + print(f' {line}') + if not changes: + print(' (none)') + return + + if not changes: + print(f'No dependency changes from {from_ref} to {args.to_ref}; nothing written.') + return + author = args.author or git('config', 'user.name').strip() + with open(args.write, 'w', encoding='utf-8') as f: + f.write(to_yaml(changes, from_ref, version, author)) + print(f'Wrote {len(changes)} dependency changes to {args.write}') + + +if __name__ == '__main__': + main() diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index 363851af68e3..a0b35c683f94 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -671,6 +671,29 @@ groups: ---- types: - bugfix + - !Todo + id: dependency_updates_changes + title: Add dependency updates to changelog + description: | + Add a single changelog entry summarizing third-party dependency changes since the previous release, + derived from the jar checksum files in `solr/licenses/`. + Re-running it overwrites the entry, e.g. if dependencies changed before a respin. + Run the script without `--write` to just print the summary. + depends: clean_git_checkout + commands: !Commands + root_folder: '{{ git_checkout_folder }}' + commands_text: We call out to a helper script that writes `changelog/unreleased/dependency-changes.yml` + confirm_each_command: true + commands: + - !Command + cmd: git checkout {{ release_branch }} + stdout: true + - !Command + cmd: python3 -u dev-tools/scripts/dependencyChanges.py --write + tee: true + - !Command + cmd: git add changelog && git commit -m "Add dependency updates to changelog for {{ release_version }}" && git push + logfile: dependency-changes.log - !Todo id: draft_release_notes title: Get a draft of the release notes in place @@ -742,7 +765,7 @@ groups: step before anything is committed or pushed. Can be run standalone, see `dev-tools/scripts/logchange.py prepare --help`. - depends: clean_git_checkout + depends: dependency_updates_changes commands: !Commands root_folder: '{{ git_checkout_folder }}' commands_text: Prepare changelog folder and regenerate CHANGELOG.md for RC{{ rc_number }} (uncommitted) From f5d058ed199e33bf6506d14d60eff3a189f6d442 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 30 Sep 2026 00:41:49 -0400 Subject: [PATCH 3/6] Stop generating changelog entries for solrbot (Renovate) PRs Obsolete now that dependencyChanges.py summarizes dependency changes at release time. Removes the renovate-changelog-prepare/push workflows and generate-renovate-changelog.py. Co-Authored-By: Claude Opus 5.5 --- .../scripts/generate-renovate-changelog.py | 272 ------------------ .../workflows/renovate-changelog-prepare.yml | 123 -------- .github/workflows/renovate-changelog-push.yml | 177 ------------ .github/workflows/validate-changelog.yml | 2 +- 4 files changed, 1 insertion(+), 573 deletions(-) delete mode 100644 .github/scripts/generate-renovate-changelog.py delete mode 100644 .github/workflows/renovate-changelog-prepare.yml delete mode 100644 .github/workflows/renovate-changelog-push.yml diff --git a/.github/scripts/generate-renovate-changelog.py b/.github/scripts/generate-renovate-changelog.py deleted file mode 100644 index ab37fd262880..000000000000 --- a/.github/scripts/generate-renovate-changelog.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate changelog YAML entry for Renovate dependency update PRs. - -This script parses the PR title to extract dependency information, -then generates a changelog YAML file in changelog/unreleased/ with the proper -naming convention and content structure. - -Usage: - python3 generate-renovate-changelog.py --pr-number 1234 --pr-title "Update org.apache.httpcomponents to v1.2.3" -""" - -import argparse -import os -import re -import sys -import yaml -from pathlib import Path -from typing import Optional, Tuple - - -def sanitize_slug(text: str, max_length: int = 50) -> str: - """ - Sanitize text to create a valid filename slug. - - - Convert to lowercase - - Replace dots, colons, slashes with dashes - - Replace other special chars with dashes - - Preserve word boundaries - - Truncate to max_length while preserving word boundaries - """ - # Convert to lowercase - text = text.lower() - - # Replace colons, slashes, dots with dashes - text = re.sub(r'[:/.]+', '-', text) - - # Replace other special characters with dashes - text = re.sub(r'[^a-z0-9\s-]', '-', text) - - # Replace spaces with dashes - text = re.sub(r'\s+', '-', text) - - # Replace multiple dashes with single dash - text = re.sub(r'-+', '-', text) - - # Remove leading/trailing dashes - text = text.strip('-') - - # Truncate to max_length at word boundary - if len(text) > max_length: - text = text[:max_length] - # Find last dash and truncate there - last_dash = text.rfind('-') - if last_dash > 0: - text = text[:last_dash] - text = text.rstrip('-') - - return text - - -def parse_pr_title(title: str) -> Tuple[str, Optional[str]]: - """ - Parse Renovate PR title to extract dependency name and version. - - Handles patterns like: - - "Update dependency org.junit.jupiter:junit-jupiter to v6" - - "Update dependency com.jayway.jsonpath:json-path to v2.10.0" - - "Update netty to v4.2.6.Final" - - "Update apache.kafka to v3.9.1" - - "Update actions/checkout action to v5" - - Returns: - Tuple of (title_for_changelog, dependency_slug_for_filename) - - Note: The slug excludes the version number so the filename remains stable - across version updates. - """ - - # Pattern 1: "Update dependency {group}:{artifact} to {version}" - match = re.match(r'Update dependency (.+?) to (.+?)(?:\s*$|\s*\(|$)', title) - if match: - dep_name = match.group(1) - version = match.group(2).strip() - changelog_title = f"Update {dep_name} to {version}" - # Slug contains only the dependency name, not the version - slug = sanitize_slug(f"Update {dep_name}") - return changelog_title, slug - - # Pattern 2: "Update {owner}/{action} action to {version}" - match = re.match(r'Update ([a-z0-9-]+/[a-z0-9-]+) action to (.+?)(?:\s*$|\s*\(|$)', title) - if match: - action = match.group(1) - version = match.group(2).strip() - changelog_title = f"Update {action} action to {version}" - # Slug contains only the action name, not the version - slug = sanitize_slug(f"Update {action} action") - return changelog_title, slug - - # Pattern 3: "Update {package} to {version}" (short form) - match = re.match(r'Update ([a-z0-9\-_.]+) to (.+?)(?:\s*$|\s*\(|$)', title) - if match: - package = match.group(1) - version = match.group(2).strip() - changelog_title = f"Update {package} to {version}" - # Slug contains only the package name, not the version - slug = sanitize_slug(f"Update {package}") - return changelog_title, slug - - # Fallback: use title as-is if no pattern matches - return title, sanitize_slug(title) - - -def generate_changelog_entry( - pr_number: int, - changelog_title: str, - pr_url: str = None -) -> dict: - """Generate the changelog YAML entry dict.""" - - if pr_url is None: - pr_url = f"https://github.com/apache/solr/pull/{pr_number}" - - return { - 'title': changelog_title, - 'type': 'dependency_update', - 'authors': [ - {'name': 'solrbot'} - ], - 'links': [ - { - 'name': f'PR#{pr_number}', - 'url': pr_url - } - ] - } - - -def find_existing_changelog_file(pr_number: int, changelog_dir: str = 'changelog/unreleased') -> Optional[str]: - """Find existing changelog file for this PR, returns path if exists.""" - pattern = f"PR#{pr_number}-*.yml" - path = Path(changelog_dir) - - if not path.exists(): - return None - - for file in path.glob(f"PR#{pr_number}-*.yml"): - return str(file) - - return None - - -def delete_old_changelog_files(pr_number: int, changelog_dir: str = 'changelog/unreleased') -> int: - """ - Delete all existing changelog files for this PR number. - - This ensures we don't accumulate orphaned files when the PR title/slug changes. - - Returns: - Number of files deleted - """ - pattern = f"PR#{pr_number}-*.yml" - path = Path(changelog_dir) - - if not path.exists(): - return 0 - - deleted_count = 0 - for file in path.glob(pattern): - try: - file.unlink() - print(f"Deleted old changelog file: {file}") - deleted_count += 1 - except Exception as e: - print(f"Warning: Could not delete file {file}: {e}", file=sys.stderr) - - if deleted_count > 0: - print(f"Deleted {deleted_count} old changelog file(s) for PR#{pr_number}") - - return deleted_count - - -def should_update_changelog(existing_file: str, new_title: str) -> bool: - """ - Check if we need to update the changelog file. - - Updates if the title has changed (version was bumped). - """ - if not existing_file or not Path(existing_file).exists(): - return False - - try: - with open(existing_file, 'r') as f: - content = yaml.safe_load(f) - - existing_title = content.get('title', '') - return existing_title != new_title - except Exception as e: - print(f"Warning: Could not read existing file {existing_file}: {e}", file=sys.stderr) - return False - - -def write_changelog_file(filename: str, entry: dict, changelog_dir: str = 'changelog/unreleased') -> None: - """Write the changelog YAML file.""" - path = Path(changelog_dir) - path.mkdir(parents=True, exist_ok=True) - - filepath = path / filename - - # Use YAML dumper that preserves order and formatting - with open(filepath, 'w') as f: - yaml.dump( - entry, - f, - default_flow_style=False, - sort_keys=False, - allow_unicode=True - ) - - print(f"Created/updated changelog file: {filepath}") - - -def main(): - parser = argparse.ArgumentParser( - description='Generate changelog entry for Renovate PR', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - python3 generate-renovate-changelog.py --pr-number 1234 --pr-title "Update org.apache.httpcomponents to v1.2.3" - python3 generate-renovate-changelog.py --pr-number 3751 --pr-title "Update dependency com.microsoft.onnxruntime:onnxruntime to v1.23.1" - """ - ) - - parser.add_argument( - '--pr-number', - type=int, - required=True, - help='GitHub PR number' - ) - parser.add_argument( - '--pr-title', - required=True, - help='GitHub PR title (from the Renovate bot)' - ) - parser.add_argument( - '--changelog-dir', - default='changelog/unreleased', - help='Directory for changelog files (default: changelog/unreleased)' - ) - - args = parser.parse_args() - - # Delete any existing changelog files for this PR to ensure a clean slate - # This prevents orphaned files when the PR title/slug changes - delete_old_changelog_files(args.pr_number, args.changelog_dir) - - # Parse the PR title - changelog_title, slug = parse_pr_title(args.pr_title) - - # Generate filename - filename = f"PR#{args.pr_number}-{slug}.yml" - - # Generate the new entry - entry = generate_changelog_entry(args.pr_number, changelog_title) - - # Write the changelog file - write_changelog_file(filename, entry, args.changelog_dir) - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/.github/workflows/renovate-changelog-prepare.yml b/.github/workflows/renovate-changelog-prepare.yml deleted file mode 100644 index 5dab69c68675..000000000000 --- a/.github/workflows/renovate-changelog-prepare.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Generate Renovate Changelog (Stage 1 - Prepare) - -# Stage 1: runs with pull_request (no secrets, no write access). -# Checks out BASE repo code only, runs the trusted Python script, -# and saves the generated changelog file + PR metadata as an artifact -# for Stage 2 to pick up and push to the fork branch. -# -# Generation is skipped (SKIP=true in the artifact metadata) when the PR -# carries the 'no-changelog' label, or when a user other than solrbot has -# manually edited changelog/unreleased/ in the PR — in that case the bot -# must not overwrite the manual changes. -on: - pull_request: - types: - - opened - - synchronize - branches: - - main - - 'branch_*' - paths-ignore: - - '.github/**' - -concurrency: - group: renovate-changelog-prepare-${{ github.event.pull_request.number }} - cancel-in-progress: true - -permissions: - contents: read - pull-requests: read # Needed to list PR commits for the manual-edit check - -jobs: - generate: - # Only run for Renovate bot PRs from the expected fork - if: | - github.event.pull_request.user.login == 'solrbot' && - github.event.pull_request.head.repo.full_name == 'solrbot/apache-_-solr' - runs-on: ubuntu-latest - - steps: - - name: Checkout BASE repository at base branch (NOT fork code) - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.base.ref }} - repository: ${{ github.repository }} - - - name: Check whether changelog generation should be skipped - id: skip-check - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - REPO: ${{ github.repository }} - HAS_NO_CHANGELOG_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'no-changelog') }} - run: | - set -euo pipefail - skip=false - - if [ "$HAS_NO_CHANGELOG_LABEL" = "true" ]; then - skip=true - echo "::notice::Skipping changelog generation for PR#${PR_NUMBER}: PR has the 'no-changelog' label" - else - # Skip if any PR commit from a user other than solrbot touched - # changelog/unreleased/ — a human has taken over the changelog - # entry and the bot must not overwrite it. Commits with an - # unresolvable login are treated as manual edits (fail safe). - while read -r sha login; do - if [ "$login" != "solrbot" ]; then - if gh api "repos/${REPO}/commits/${sha}" --jq '.files[].filename' | grep -q '^changelog/unreleased/'; then - skip=true - echo "::notice::Skipping changelog generation for PR#${PR_NUMBER}: changelog/unreleased/ was manually edited by '${login}' in commit ${sha}" - break - fi - fi - done < <(gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/commits" \ - --jq '.[] | "\(.sha) \(.author.login // "unknown")"') - fi - - echo "skip=${skip}" >> "$GITHUB_OUTPUT" - - - name: Set up Python - if: steps.skip-check.outputs.skip != 'true' - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.x' - - - name: Install dependencies - if: steps.skip-check.outputs.skip != 'true' - run: python3 -m pip install --quiet pyyaml - - - name: Generate changelog entry - if: steps.skip-check.outputs.skip != 'true' - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_TITLE: ${{ github.event.pull_request.title }} - run: | - python3 .github/scripts/generate-renovate-changelog.py \ - --pr-number "$PR_NUMBER" \ - --pr-title "$PR_TITLE" - - - name: Assemble artifact - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - HEAD_REF: ${{ github.event.pull_request.head.ref }} - HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} - SKIP: ${{ steps.skip-check.outputs.skip }} - run: | - set -euo pipefail - mkdir -p artifact - if [ "$SKIP" != "true" ] && [ -d changelog/unreleased ]; then - cp -r changelog/unreleased artifact/changelog-unreleased - fi - # Use printf + env vars to avoid shell injection from PR metadata values - printf 'PR_NUMBER=%s\nHEAD_REF=%s\nHEAD_REPO=%s\nSKIP=%s\n' \ - "$PR_NUMBER" "$HEAD_REF" "$HEAD_REPO" "$SKIP" \ - > artifact/pr-metadata.env - echo "Artifact contents:"; find artifact/ -type f - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: renovate-changelog-artifact - path: artifact/ - retention-days: 1 - if-no-files-found: error diff --git a/.github/workflows/renovate-changelog-push.yml b/.github/workflows/renovate-changelog-push.yml deleted file mode 100644 index 0e44c9f434e0..000000000000 --- a/.github/workflows/renovate-changelog-push.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Generate Renovate Changelog (Stage 2 - Push) - -# Stage 2: runs after Stage 1 completes, in the base-repo context with access to -# SOLRBOT_GITHUB_TOKEN. Downloads the pre-generated artifact (no fork code executed here), -# validates metadata, and pushes the changelog file to the fork branch. -# -# If Stage 1 flagged SKIP=true in the artifact metadata (PR has the -# 'no-changelog' label, or a user manually edited changelog/unreleased/), -# this workflow leaves the changelog untouched. -on: - workflow_run: - workflows: - - "Generate Renovate Changelog (Stage 1 - Prepare)" - types: - - completed - -# Each Stage 2 run corresponds to a unique Stage 1 run (unique workflow_run.id). -# No cancel-in-progress: Stage 1's concurrency already serializes per-PR. -concurrency: - group: renovate-changelog-push-${{ github.event.workflow_run.id }} - -permissions: - actions: read # Required to download artifacts from another workflow run - contents: read # Minimal; actual write access to fork uses SOLRBOT_GITHUB_TOKEN PAT - -jobs: - push-changelog: - # Only proceed if Stage 1 succeeded for the expected fork. - # Checking head_repository here avoids a spurious artifact-not-found failure - # when Stage 1 ran but skipped its generate job (e.g. non-solrbot PR). - if: | - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.head_repository.full_name == 'solrbot/apache-_-solr' - runs-on: ubuntu-latest - - steps: - - name: Download artifact from Stage 1 - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: renovate-changelog-artifact - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - path: downloaded-artifact/ - - - name: Read and validate PR metadata - id: meta - run: | - set -euo pipefail - META_FILE="downloaded-artifact/pr-metadata.env" - if [ ! -f "$META_FILE" ]; then - echo "::error::Metadata file missing from artifact"; exit 1 - fi - - # Parse with grep/cut rather than source to avoid executing file content as shell. - # Use || true so a missing key doesn't abort under set -euo pipefail before the - # explicit emptiness check below can emit a meaningful error message. - PR_NUMBER=$(grep '^PR_NUMBER=' "$META_FILE" | cut -d= -f2- || true) - HEAD_REF=$(grep '^HEAD_REF=' "$META_FILE" | cut -d= -f2- || true) - HEAD_REPO=$(grep '^HEAD_REPO=' "$META_FILE" | cut -d= -f2- || true) - SKIP=$(grep '^SKIP=' "$META_FILE" | cut -d= -f2- || true) - - if [ -z "$PR_NUMBER" ] || [ -z "$HEAD_REF" ] || [ -z "$HEAD_REPO" ]; then - echo "::error::Missing required metadata fields (PR_NUMBER, HEAD_REF, or HEAD_REPO)"; exit 1 - fi - - # Security: verify this is the expected fork before using SOLRBOT_GITHUB_TOKEN - if [ "$HEAD_REPO" != "solrbot/apache-_-solr" ]; then - echo "::error::Unexpected HEAD_REPO: '$HEAD_REPO'. Expected 'solrbot/apache-_-solr'. Aborting."; exit 1 - fi - - # Validate PR_NUMBER is a plain positive integer (prevents injection) - if ! [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then - echo "::error::PR_NUMBER is not a valid positive integer: '$PR_NUMBER'"; exit 1 - fi - - # Validate HEAD_REF using Git's own branch-name rules. This rejects edge cases - # such as '..', '@{', trailing '.lock', and ':' (dangerous in push refspecs) - # while still accepting valid Renovate branch names like renovate/node@lts. - if ! git check-ref-format --branch "$HEAD_REF" > /dev/null 2>&1; then - echo "::error::HEAD_REF is not a valid Git branch name: '$HEAD_REF'"; exit 1 - fi - - # SKIP is optional (absent in artifacts from older Stage 1 runs); default to false - if [ "$SKIP" = "true" ]; then - echo "::notice::Stage 1 flagged skip for PR#${PR_NUMBER} — leaving changelog untouched" - else - SKIP=false - fi - - echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" - echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT" - echo "head_repo=$HEAD_REPO" >> "$GITHUB_OUTPUT" - echo "skip=$SKIP" >> "$GITHUB_OUTPUT" - echo "Validated: PR#${PR_NUMBER} on ${HEAD_REPO}@${HEAD_REF} (skip=${SKIP})" - - - name: Clone fork branch - if: steps.meta.outputs.skip != 'true' - env: - SOLRBOT_TOKEN: ${{ secrets.SOLRBOT_GITHUB_TOKEN }} - HEAD_REPO: ${{ steps.meta.outputs.head_repo }} - HEAD_REF: ${{ steps.meta.outputs.head_ref }} - run: | - set -euo pipefail - # Store credentials so the token never appears in the command line or process list - git config --global credential.helper store - printf 'https://x-access-token:%s@github.com\n' "$SOLRBOT_TOKEN" > ~/.git-credentials - chmod 600 ~/.git-credentials - - git clone --depth=1 --branch "$HEAD_REF" \ - "https://github.com/${HEAD_REPO}.git" \ - fork-checkout - - - name: Apply changelog to fork checkout - if: steps.meta.outputs.skip != 'true' - id: apply - env: - PR_NUMBER: ${{ steps.meta.outputs.pr_number }} - run: | - set -euo pipefail - CHANGELOG_DIR="fork-checkout/changelog/unreleased" - ARTIFACT_DIR="downloaded-artifact/changelog-unreleased" - - if [ ! -d "$CHANGELOG_DIR" ]; then - echo "::error::changelog/unreleased not found in fork checkout"; exit 1 - fi - - if [ ! -d "$ARTIFACT_DIR" ]; then - echo "::warning::No changelog-unreleased directory in artifact for PR#${PR_NUMBER}" - echo "has_changes=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # Remove stale PR#NNN-*.yml files (handles slug changes between synchronize events) - find "$CHANGELOG_DIR" -maxdepth 1 -name "PR#${PR_NUMBER}-*.yml" -delete -print - - # Copy the new PR#NNN-*.yml file(s) from the artifact - COPIED=0 - for f in "${ARTIFACT_DIR}/PR#${PR_NUMBER}-"*.yml; do - if [ -f "$f" ]; then - cp -v "$f" "$CHANGELOG_DIR/" - COPIED=$((COPIED + 1)) - fi - done - - if [ "$COPIED" -eq 0 ]; then - echo "::warning::No PR#${PR_NUMBER}-*.yml file found in artifact" - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "Copied $COPIED changelog file(s)" - echo "has_changes=true" >> "$GITHUB_OUTPUT" - fi - - - name: Commit and push to fork branch - if: steps.apply.outputs.has_changes == 'true' - env: - HEAD_REF: ${{ steps.meta.outputs.head_ref }} - PR_NUMBER: ${{ steps.meta.outputs.pr_number }} - run: | - set -euo pipefail - cd fork-checkout - - if [ -z "$(git status --porcelain changelog/unreleased/)" ]; then - echo "No changelog changes (already up to date)" - exit 0 - fi - - git config user.name "SolrBot" - git config user.email "solrbot@cominvent.com" - - git add changelog/unreleased/ - git commit -m "Add changelog entry for PR#${PR_NUMBER}" - - # Credential store (configured in Clone step) provides authentication - git push origin "HEAD:refs/heads/${HEAD_REF}" - - # Remove credentials from disk now that the push is complete - rm -f ~/.git-credentials diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index f4adbb83d719..0780c671d40d 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -12,7 +12,7 @@ concurrency: jobs: validate-changelog: name: Check changelog entry - # Skip validation for Renovate PRs (solrbot) - they get changelog entries automatically + # Skip validation for Renovate PRs (solrbot); dependency changes are summarized at release time if: github.event.pull_request.user.login != 'solrbot' runs-on: ubuntu-latest From 55ed2a7c80edf72021ea1bd4f4c1ff80726700d3 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 30 Sep 2026 01:17:42 -0400 Subject: [PATCH 4/6] dependencyChanges.py: separate added, upgraded, and removed Print one line per category, with changes separated by semicolons, and with --write produce one changelog entry per category. Removals are included again, limited to jars that had a LICENSE file. Co-Authored-By: Claude Opus 5.5 --- dev-docs/changelog.adoc | 3 +- dev-tools/scripts/dependencyChanges.py | 76 ++++++++++++++++---------- dev-tools/scripts/releaseWizard.yaml | 8 +-- 3 files changed, 52 insertions(+), 35 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index cbe9a25f0133..5055922ca969 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -279,7 +279,8 @@ Wizard but can also be run standalone. Run with `--help` for usage. `dev-tools/scripts/dependencyChanges.py` summarizes the third-party dependency changes since the previous release by diffing `solr/licenses/`. By default it prints to stdout; -with `--write`, the Release Wizard uses it to write a single `dependency_update` changelog entry. +with `--write`, the Release Wizard uses it to write `dependency_update` changelog entries +(one each for added, upgraded, and removed). == 6. Further Reading diff --git a/dev-tools/scripts/dependencyChanges.py b/dev-tools/scripts/dependencyChanges.py index 6d4ccce5b608..c50fce4f1a02 100644 --- a/dev-tools/scripts/dependencyChanges.py +++ b/dev-tools/scripts/dependencyChanges.py @@ -21,7 +21,7 @@ previous release tag and HEAD). Jars that moved between the same two versions are listed together, since they are typically from the same project. -Prints the summary to stdout, or with --write, writes a changelog YAML entry. +Prints the summary to stdout, or with --write, writes changelog YAML entries. This is a stopgap until Solr publishes an SBOM per release, which would be the better source to diff. @@ -38,7 +38,7 @@ from scriptutil import Version, find_current_version LICENSES_DIR = 'solr/licenses' -DEFAULT_OUTPUT = 'changelog/unreleased/dependency-changes.yml' +DEFAULT_OUTPUT_DIR = 'changelog/unreleased' # A version starts at the last '-'-delimited segment that looks like a dotted number. # e.g. log4j-1.2-api-2.25.3, zstd-jni-1.5.6-10, guava-33.4.8-jre @@ -118,31 +118,43 @@ def describe_artifacts(artifacts): return ', '.join(artifacts) +CATEGORIES = ('added', 'upgraded', 'removed') + + def compute_changes(from_ref, to_ref): - """Returns a sorted list of human-readable change lines.""" - old_jars, _ = read_licenses_dir(from_ref) - new_jars, license_prefixes = read_licenses_dir(to_ref) - - # Removals are omitted, and so is anything lacking a LICENSE file: *.sha1 files also cover jars - # we don't ship (e.g. test dependencies), whereas LICENSE files are only required for shipped ones. - # Older releases have LICENSE files for non-shipped jars too, making removals unreliable. - by_transition = defaultdict(list) # (old_version or None, new_version) -> artifacts - for artifact, new in new_jars.items(): - old = old_jars.get(artifact) - if old != new and has_license(artifact, license_prefixes) and not TEST_ARTIFACT_RE.search(artifact): + """Returns category ('added', 'upgraded', 'removed') -> sorted list of human-readable changes.""" + old_jars, old_license_prefixes = read_licenses_dir(from_ref) + new_jars, new_license_prefixes = read_licenses_dir(to_ref) + + # Only jars with a LICENSE file count: *.sha1 files also cover jars we don't ship (e.g. test + # dependencies), whereas LICENSE files are only required for shipped ones (since SOLR-15465; older + # releases have them for non-shipped jars too, making removals from such a release unreliable). + def shipped(artifact, license_prefixes): + return has_license(artifact, license_prefixes) and not TEST_ARTIFACT_RE.search(artifact) + + by_transition = defaultdict(list) # (old_version or None, new_version or None) -> artifacts + for artifact in old_jars.keys() | new_jars.keys(): + old, new = old_jars.get(artifact), new_jars.get(artifact) + if old == new: + continue + if shipped(artifact, new_license_prefixes if new else old_license_prefixes): by_transition[(old, new)].append(artifact) - lines = [] + changes = {c: [] for c in CATEGORIES} for (old, new), artifacts in by_transition.items(): names = describe_artifacts(sorted(artifacts, key=str.lower)) - lines.append(f'{names} {new} (new)' if old is None else f'{names} {old} → {new}') - return sorted(lines, key=str.lower) + if old is None: + changes['added'].append(f'{names} {new}') + elif new is None: + changes['removed'].append(f'{names} {old}') + else: + changes['upgraded'].append(f'{names} {old} → {new}') + return {c: sorted(lines, key=str.lower) for c, lines in changes.items()} -def to_yaml(changes, from_ref, version, author): +def to_yaml(title, version, author): def quote(s): return "'" + s.replace("'", "''") + "'" - title = f'Third-party dependency changes since {from_ref.rsplit("/", 1)[-1]}: ' + '; '.join(changes) return (f'# Generated by dev-tools/scripts/dependencyChanges.py\n' f'title: {quote(title)}\n' f'type: dependency_update\n' @@ -158,30 +170,34 @@ def main(): parser.add_argument('--from', dest='from_ref', help='Git ref to compare from (default: the release tag preceding the current version)') parser.add_argument('--to', dest='to_ref', default='HEAD', help='Git ref to compare to (default: HEAD)') - parser.add_argument('--write', nargs='?', const=DEFAULT_OUTPUT, metavar='FILE', - help=f'Write a changelog YAML entry instead of printing (default file: {DEFAULT_OUTPUT})') + parser.add_argument('--write', nargs='?', const=DEFAULT_OUTPUT_DIR, metavar='DIR', + help='Write changelog YAML entries (one per category) instead of printing' + f' (default dir: {DEFAULT_OUTPUT_DIR})') parser.add_argument('--author', help='Changelog entry author (default: git user.name)') args = parser.parse_args() version = find_current_version() from_ref = args.from_ref or find_previous_release_tag(version) changes = compute_changes(from_ref, args.to_ref) + since = from_ref.rsplit('/', 1)[-1] if not args.write: print(f'Dependency changes from {from_ref} to {args.to_ref}:') - for line in changes: - print(f' {line}') - if not changes: - print(' (none)') + for category in CATEGORIES: + print(f'{category.capitalize()}: ' + ('; '.join(changes[category]) or '(none)')) return - if not changes: - print(f'No dependency changes from {from_ref} to {args.to_ref}; nothing written.') - return author = args.author or git('config', 'user.name').strip() - with open(args.write, 'w', encoding='utf-8') as f: - f.write(to_yaml(changes, from_ref, version, author)) - print(f'Wrote {len(changes)} dependency changes to {args.write}') + for category in CATEGORIES: + path = os.path.join(args.write, f'dependency-changes-{category}.yml') + if changes[category]: + title = f'Third-party dependencies {category} since {since}: ' + '; '.join(changes[category]) + with open(path, 'w', encoding='utf-8') as f: + f.write(to_yaml(title, version, author)) + print(f'Wrote {len(changes[category])} {category} dependency changes to {path}') + elif os.path.exists(path): + os.remove(path) + print(f'Removed {path}; no {category} dependency changes') if __name__ == '__main__': diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index a0b35c683f94..ea416b95c737 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -675,14 +675,14 @@ groups: id: dependency_updates_changes title: Add dependency updates to changelog description: | - Add a single changelog entry summarizing third-party dependency changes since the previous release, - derived from the jar checksum files in `solr/licenses/`. - Re-running it overwrites the entry, e.g. if dependencies changed before a respin. + Add changelog entries summarizing third-party dependencies added, upgraded, and removed since the + previous release, derived from the jar checksum files in `solr/licenses/`. + Re-running it overwrites the entries, e.g. if dependencies changed before a respin. Run the script without `--write` to just print the summary. depends: clean_git_checkout commands: !Commands root_folder: '{{ git_checkout_folder }}' - commands_text: We call out to a helper script that writes `changelog/unreleased/dependency-changes.yml` + commands_text: We call out to a helper script that writes `changelog/unreleased/dependency-changes-*.yml` confirm_each_command: true commands: - !Command From d3e810f04efe37e3c08ed4894c9bea714b6327b4 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 30 Sep 2026 01:56:32 -0400 Subject: [PATCH 5/6] dependencyChanges.py: attribute entries to "various contributors" Drop the --author option (defaulting to git user.name); the entries summarize many people's changes. Co-Authored-By: Claude Opus 5.5 --- dev-tools/scripts/dependencyChanges.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dev-tools/scripts/dependencyChanges.py b/dev-tools/scripts/dependencyChanges.py index c50fce4f1a02..3b038bbb60a4 100644 --- a/dev-tools/scripts/dependencyChanges.py +++ b/dev-tools/scripts/dependencyChanges.py @@ -152,14 +152,14 @@ def shipped(artifact, license_prefixes): return {c: sorted(lines, key=str.lower) for c, lines in changes.items()} -def to_yaml(title, version, author): +def to_yaml(title, version): def quote(s): return "'" + s.replace("'", "''") + "'" return (f'# Generated by dev-tools/scripts/dependencyChanges.py\n' f'title: {quote(title)}\n' f'type: dependency_update\n' f'authors:\n' - f' - name: {quote(author)}\n' + f' - name: various contributors\n' f'links:\n' f' - name: solr/licenses\n' f' url: https://github.com/apache/solr/tree/releases/solr/{version}/solr/licenses\n') @@ -173,7 +173,6 @@ def main(): parser.add_argument('--write', nargs='?', const=DEFAULT_OUTPUT_DIR, metavar='DIR', help='Write changelog YAML entries (one per category) instead of printing' f' (default dir: {DEFAULT_OUTPUT_DIR})') - parser.add_argument('--author', help='Changelog entry author (default: git user.name)') args = parser.parse_args() version = find_current_version() @@ -187,13 +186,12 @@ def main(): print(f'{category.capitalize()}: ' + ('; '.join(changes[category]) or '(none)')) return - author = args.author or git('config', 'user.name').strip() for category in CATEGORIES: path = os.path.join(args.write, f'dependency-changes-{category}.yml') if changes[category]: title = f'Third-party dependencies {category} since {since}: ' + '; '.join(changes[category]) with open(path, 'w', encoding='utf-8') as f: - f.write(to_yaml(title, version, author)) + f.write(to_yaml(title, version)) print(f'Wrote {len(changes[category])} {category} dependency changes to {path}') elif os.path.exists(path): os.remove(path) From aba77af219128584bb58097f851dff590c4689c9 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 30 Sep 2026 01:58:03 -0400 Subject: [PATCH 6/6] dependencyChanges.py: number the entry files to order added, upgraded, removed Co-Authored-By: Claude Opus 5.5 --- dev-tools/scripts/dependencyChanges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/scripts/dependencyChanges.py b/dev-tools/scripts/dependencyChanges.py index 3b038bbb60a4..c5d71a381f4e 100644 --- a/dev-tools/scripts/dependencyChanges.py +++ b/dev-tools/scripts/dependencyChanges.py @@ -186,8 +186,8 @@ def main(): print(f'{category.capitalize()}: ' + ('; '.join(changes[category]) or '(none)')) return - for category in CATEGORIES: - path = os.path.join(args.write, f'dependency-changes-{category}.yml') + for i, category in enumerate(CATEGORIES, 1): # numbered so the changelog lists them in this order + path = os.path.join(args.write, f'dependency-changes-{i}-{category}.yml') if changes[category]: title = f'Third-party dependencies {category} since {since}: ' + '; '.join(changes[category]) with open(path, 'w', encoding='utf-8') as f: