[ML] Add per-PR changelog YAML entries with schema validation

.buildkite/pipelines/format_and_validation.yml.sh

@@ -18,4 +18,13 @@ steps:
     notify:
       - github_commit_status:
           context: "Validate formatting with clang-format"
+  - label: "Validate changelog entries"
+    key: "validate_changelogs"
+    command: ".buildkite/scripts/steps/validate-changelogs.sh"
+    agents:
+      image: "python:3.11-slim"
+    soft_fail: true
+    notify:
+      - github_commit_status:
+          context: "Validate changelog entries"
 EOL

.buildkite/scripts/steps/validate-changelogs.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+set -euo pipefail
+
+SKIP_LABELS=">test >refactoring >docs >build >non-issue"
+
+# On PR builds, check if the PR has a label that skips changelog validation.
+# BUILDKITE_PULL_REQUEST_LABELS is a comma-separated list set by Buildkite.
+if [[ -n "${BUILDKITE_PULL_REQUEST_LABELS:-}" ]]; then
+  IFS=',' read -ra LABELS <<< "${BUILDKITE_PULL_REQUEST_LABELS}"
+  for label in "${LABELS[@]}"; do
+    label="$(echo "${label}" | xargs)"  # trim whitespace
+    for skip in ${SKIP_LABELS}; do
+      if [[ "${label}" == "${skip}" ]]; then
+        echo "Skipping changelog validation: PR has label '${label}'"
+        exit 0
+      fi
+    done
+  done
+fi
+
+# Install system and Python dependencies
+if ! command -v git &>/dev/null; then
+  apt-get update -qq && apt-get install -y -qq git >/dev/null 2>&1
+fi
+python3 -m pip install --quiet --break-system-packages pyyaml jsonschema 2>/dev/null \
+  || python3 -m pip install --quiet pyyaml jsonschema
+
+# Find changelog files changed in this PR (compared to main/target branch)
+TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
+
+# Fetch the target branch so we can diff against it
+git fetch origin "${TARGET_BRANCH}" --depth=1 2>/dev/null || true
+
+CHANGED_CHANGELOGS=$(git diff --name-only --diff-filter=ACM "origin/${TARGET_BRANCH}"...HEAD -- 'docs/changelog/*.yaml' || true)
+
+if [[ -z "${CHANGED_CHANGELOGS}" ]]; then
+  echo "No changelog files found in this PR."
+  echo "If this PR changes user-visible behaviour, please add a changelog entry."
+  echo "See docs/changelog/README.md for details."
+  echo "To skip this check, add one of these labels: ${SKIP_LABELS}"
+
+  # Soft warning rather than hard failure during rollout
+  if [[ "${CHANGELOG_REQUIRED:-false}" == "true" ]]; then
+    exit 1
+  fi
+  exit 0
+fi
+
+echo "Validating changelog files:"
+echo "${CHANGED_CHANGELOGS}"
+echo ""
+
+python3 dev-tools/validate_changelogs.py ${CHANGED_CHANGELOGS}

build.gradle

@@ -169,6 +169,20 @@ task format(type: Exec) {
   workingDir "${projectDir}"
 }
 
+task validateChangelogs(type: Exec) {
+  commandLine 'python3', 'dev-tools/validate_changelogs.py'
+  workingDir "${projectDir}"
+  description = 'Validate changelog YAML entries against the schema'
+  group = 'verification'
+}
+
+task bundleChangelogs(type: Exec) {
+  commandLine 'python3', 'dev-tools/bundle_changelogs.py', '--version', project.version
+  workingDir "${projectDir}"
+  description = 'Generate consolidated changelog from per-PR YAML entries'
+  group = 'documentation'
+}
+
 task precommit(type: Exec) {
   commandLine shell
   workingDir "${projectDir}"

dev-tools/bundle_changelogs.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""
+Bundle per-PR changelog YAML files into a consolidated changelog for release.
+
+Usage:
+    python3 bundle_changelogs.py [--dir DIR] [--version VERSION] [--format FORMAT]
+
+Outputs a formatted changelog grouped by type and area, suitable for inclusion
+in release notes.
+
+Formats:
+    markdown (default) - Markdown suitable for GitHub releases
+    asciidoc          - AsciiDoc suitable for Elastic docs
+"""
+
+import argparse
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    print("Missing pyyaml. Install with: pip3 install pyyaml", file=sys.stderr)
+    sys.exit(2)
+
+
+TYPE_ORDER = [
+    ("known-issue", "Known issues"),
+    ("security", "Security fixes"),
+    ("breaking", "Breaking changes"),
+    ("breaking-java", "Breaking Java changes"),
+    ("deprecation", "Deprecations"),
+    ("feature", "New features"),
+    ("new-aggregation", "New aggregations"),
+    ("enhancement", "Enhancements"),
+    ("bug", "Bug fixes"),
+    ("regression", "Regression fixes"),
+    ("upgrade", "Upgrades"),
+]
+
+ML_CPP_PULL_URL = "https://github.com/elastic/ml-cpp/pull"
+ML_CPP_ISSUE_URL = "https://github.com/elastic/ml-cpp/issues"
+
+
+def load_entries(changelog_dir):
+    entries = []
+    for path in sorted(changelog_dir.glob("*.yaml")):
+        with open(path) as f:
+            data = yaml.safe_load(f)
+            if data and isinstance(data, dict):
+                data["_file"] = path.name
+                entries.append(data)
+    return entries
+
+
+def format_markdown(entries, version=None):
+    lines = []
+    if version:
+        lines.append(f"## {version}\n")
+
+    grouped = defaultdict(lambda: defaultdict(list))
+    for entry in entries:
+        area = entry.get("area", "General")
+        grouped[entry["type"]][area].append(entry)
+
+    for type_key, type_label in TYPE_ORDER:
+        if type_key not in grouped:
+            continue
+        lines.append(f"### {type_label}\n")
+        for area in sorted(grouped[type_key].keys()):
+            lines.append(f"**{area}**")
+            for entry in sorted(grouped[type_key][area], key=lambda e: e.get("pr", 0)):
+                pr = entry.get("pr")
+                summary = entry["summary"]
+                issues = entry.get("issues", [])
+                issue_refs = ", ".join(f"#{i}" for i in issues)
+                if pr:
+                    line = f"- {summary} [#{pr}]({ML_CPP_PULL_URL}/{pr})"
+                else:
+                    line = f"- {summary}"
+                if issue_refs:
+                    line += f" ({issue_refs})"
+                lines.append(line)
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def format_asciidoc(entries, version=None):
+    lines = []
+    if version:
+        lines.append(f"== {version}\n")
+
+    grouped = defaultdict(lambda: defaultdict(list))
+    for entry in entries:
+        area = entry.get("area", "General")
+        grouped[entry["type"]][area].append(entry)
+
+    for type_key, type_label in TYPE_ORDER:
+        if type_key not in grouped:
+            continue
+        lines.append(f"=== {type_label}\n")
+        for area in sorted(grouped[type_key].keys()):
+            lines.append(f"*{area}*")
+            for entry in sorted(grouped[type_key][area], key=lambda e: e.get("pr", 0)):
+                pr = entry.get("pr")
+                summary = entry["summary"]
+                issues = entry.get("issues", [])
+                issue_refs = ", ".join(
+                    f"{ML_CPP_ISSUE_URL}/{i}[#{i}]" for i in issues
+                )
+                if pr:
+                    line = f"* {summary} {{ml-pull}}{pr}[#{pr}]"
+                else:
+                    line = f"* {summary}"
+                if issue_refs:
+                    line += f" ({issue_refs})"
+                lines.append(line)
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Bundle changelog YAML files")
+    parser.add_argument("--dir", default=None, help="Changelog directory")
+    parser.add_argument("--version", default=None, help="Version string for heading")
+    parser.add_argument("--format", default="markdown", choices=["markdown", "asciidoc"])
+    args = parser.parse_args()
+
+    repo_root = Path(__file__).resolve().parent.parent
+    changelog_dir = Path(args.dir) if args.dir else repo_root / "docs" / "changelog"
+
+    entries = load_entries(changelog_dir)
+    if not entries:
+        print("No changelog entries found.", file=sys.stderr)
+        sys.exit(0)
+
+    if args.format == "asciidoc":
+        print(format_asciidoc(entries, args.version))
+    else:
+        print(format_markdown(entries, args.version))
+
+
+if __name__ == "__main__":
+    main()