diff --git a/AGENTS.md b/AGENTS.md index 70806b1ef8..73d7abd1dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -240,3 +240,7 @@ To inspect the events the SDK emits from the sample app, use the `android-sdk-ev - Keep PRs focused. Don't include unrelated refactors, formatting churn, debug leftovers, stray files or "may be needed later" code (unused properties, schemas, files). - Don't edit `CHANGELOG.md` in feature PRs. It is updated only in release preparation PRs. - When preparing a release changelog: tags reflect customer impact, not change size. Internal-only changes are `[MAINTENANCE]`, not `[IMPROVEMENT]`. Follow the existing entry format: `* [TAG] Description. See [#N](link)`. + +# Dogfooding Branch + +`dogfooding` is managed only through the scripts in `ci/scripts/dogfooding/`; read `ci/scripts/dogfooding/README.md` before helping with it. Never push to `dogfooding`, open a PR from a feature branch into it, or merge into it with squash or rebase: use `feature.sh`, `sync.sh` and `reset.sh`, which open the PRs. The only change made by hand is reverting a feature's dogfood merges, as described in the README's "Remove a single feature". Show the command and ask before running the scripts, since they push branches and open PRs. diff --git a/build-logic/src/main/kotlin/com/datadog/gradle/config/AndroidConfig.kt b/build-logic/src/main/kotlin/com/datadog/gradle/config/AndroidConfig.kt index 2c839cad12..d0970c6c02 100644 --- a/build-logic/src/main/kotlin/com/datadog/gradle/config/AndroidConfig.kt +++ b/build-logic/src/main/kotlin/com/datadog/gradle/config/AndroidConfig.kt @@ -9,6 +9,7 @@ package com.datadog.gradle.config import com.android.build.api.dsl.CompileOptions import com.android.build.api.dsl.LibraryExtension import com.datadog.gradle.utils.Version +import com.datadog.gradle.utils.forCi import org.gradle.api.JavaVersion import org.gradle.api.Project @@ -19,7 +20,7 @@ object AndroidConfig { const val MIN_SDK_FOR_AUTO = 29 const val BUILD_TOOLS_VERSION = "36.0.0" - val VERSION = Version(3, 16, 0, Version.Type.Snapshot) + val VERSION = Version(3, 16, 0, Version.Type.Snapshot).forCi() } // TODO RUM-628 Switch to Java 17 bytecode diff --git a/build-logic/src/main/kotlin/com/datadog/gradle/config/MavenConfig.kt b/build-logic/src/main/kotlin/com/datadog/gradle/config/MavenConfig.kt index a412c31d8d..d70d301a7a 100644 --- a/build-logic/src/main/kotlin/com/datadog/gradle/config/MavenConfig.kt +++ b/build-logic/src/main/kotlin/com/datadog/gradle/config/MavenConfig.kt @@ -7,9 +7,11 @@ package com.datadog.gradle.config import com.android.build.api.dsl.LibraryExtension +import com.datadog.gradle.utils.Version import org.gradle.api.Project import org.gradle.api.publish.PublishingExtension import org.gradle.api.publish.maven.MavenPublication +import org.gradle.authentication.aws.AwsImAuthentication import org.gradle.kotlin.dsl.create import org.gradle.kotlin.dsl.findByType import org.gradle.plugins.signing.SigningExtension @@ -17,6 +19,8 @@ import org.gradle.plugins.signing.SigningExtension object MavenConfig { const val GROUP_ID = "com.datadoghq" const val PUBLICATION = "release" + const val DOGFOOD_REPOSITORY = "dogfood" + const val DOGFOOD_REPOSITORY_URL = "s3://binaries-ddbuild-io-prod/dd-sdk-android/dogfood" } internal fun Project.publishingConfig( @@ -88,6 +92,20 @@ internal fun Project.publishingConfig( } } } + + // Dogfood builds go to binaries.ddbuild.io (internal only), using the CI runner's IAM role. + // Only registered for dogfood versions, so other builds can't be uploaded there. + if (AndroidConfig.VERSION.type is Version.Type.Dogfood) { + repositories { + maven { + name = MavenConfig.DOGFOOD_REPOSITORY + url = uri(MavenConfig.DOGFOOD_REPOSITORY_URL) + authentication { + create("awsIm") + } + } + } + } } signingExtension.apply { diff --git a/build-logic/src/main/kotlin/com/datadog/gradle/utils/Version.kt b/build-logic/src/main/kotlin/com/datadog/gradle/utils/Version.kt index f5d1f4770c..7cf9e32bdc 100644 --- a/build-logic/src/main/kotlin/com/datadog/gradle/utils/Version.kt +++ b/build-logic/src/main/kotlin/com/datadog/gradle/utils/Version.kt @@ -42,6 +42,10 @@ data class Version( object Snapshot : Type() { override val suffix: String = "-SNAPSHOT" } + + data class Dogfood(val shortSha: String) : Type() { + override val suffix: String = "-dogfood-$shortSha" + } } // endregion @@ -79,3 +83,18 @@ data class Version( internal const val MAX_MAJOR = 100 } } + +/** + * @return this version with a [Version.Type.Dogfood] type when building the `dogfooding` branch on CI, + * or this version unchanged otherwise. + */ +internal fun Version.forCi( + ciBranch: String? = System.getenv("CI_COMMIT_BRANCH"), + ciShortSha: String? = System.getenv("CI_COMMIT_SHORT_SHA") +): Version { + return if (ciBranch == "dogfooding" && !ciShortSha.isNullOrBlank()) { + copy(type = Version.Type.Dogfood(ciShortSha)) + } else { + this + } +} diff --git a/build-logic/src/test/kotlin/com/datadog/gradle/utils/VersionTest.kt b/build-logic/src/test/kotlin/com/datadog/gradle/utils/VersionTest.kt index 55f81566cc..bb0c053b8c 100644 --- a/build-logic/src/test/kotlin/com/datadog/gradle/utils/VersionTest.kt +++ b/build-logic/src/test/kotlin/com/datadog/gradle/utils/VersionTest.kt @@ -130,4 +130,63 @@ class VersionTest { val expected = "4.11.5-SNAPSHOT" assertThat(name).isEqualTo(expected) } + + @Test + fun addSuffixForDogfood() { + // When + val name = Version(4, 11, 5, Version.Type.Dogfood("a1b2c3d")).name + + // Then + val expected = "4.11.5-dogfood-a1b2c3d" + assertThat(name).isEqualTo(expected) + } + + @Test + fun forCiReturnsDogfoodVersionOnDogfoodingBranch() { + // When + val version = baseVersion.forCi(ciBranch = "dogfooding", ciShortSha = "a1b2c3d") + + // Then + assertThat(version).isEqualTo(Version(4, 11, 5, Version.Type.Dogfood("a1b2c3d"))) + } + + @Test + fun forCiKeepsVersionOnOtherBranch() { + // When + val version = baseVersion.forCi(ciBranch = "develop", ciShortSha = "a1b2c3d") + + // Then + assertThat(version).isEqualTo(baseVersion) + } + + @Test + fun forCiKeepsVersionWithoutBranch() { + // When + val version = baseVersion.forCi(ciBranch = null, ciShortSha = "a1b2c3d") + + // Then + assertThat(version).isEqualTo(baseVersion) + } + + @Test + fun forCiKeepsVersionOnDogfoodingBranchWithoutSha() { + // When + val version = baseVersion.forCi(ciBranch = "dogfooding", ciShortSha = null) + + // Then + assertThat(version).isEqualTo(baseVersion) + } + + @Test + fun forCiKeepsVersionOnDogfoodingBranchWithBlankSha() { + // When + val version = baseVersion.forCi(ciBranch = "dogfooding", ciShortSha = " ") + + // Then + assertThat(version).isEqualTo(baseVersion) + } + + private companion object { + val baseVersion = Version(4, 11, 5, Version.Type.Snapshot) + } } diff --git a/ci/pipelines/default-pipeline.yml b/ci/pipelines/default-pipeline.yml index 93bd4fbeb4..04c7d00f87 100644 --- a/ci/pipelines/default-pipeline.yml +++ b/ci/pipelines/default-pipeline.yml @@ -672,6 +672,22 @@ publish:release-all: paths: - "**/verification-metadata.xml" +# Publishes every dogfooding build once its pipeline has passed +publish:dogfooding: + extends: + - .cpu-memory-intensive-job + tags: [ "arch:amd64" ] + only: + - dogfooding + image: $CI_IMAGE_DOCKER + stage: publish + timeout: 30m + interruptible: false + script: + - !reference [.snippets, set-publishing-credentials] + - !reference [.snippets, setup-gradle] + - ./gradlew publishReleasePublicationToDogfoodRepository --stacktrace --no-daemon + # SLACK NOTIFICATIONS diff --git a/ci/scripts/dogfooding/README.md b/ci/scripts/dogfooding/README.md new file mode 100644 index 0000000000..5b39f07dee --- /dev/null +++ b/ci/scripts/dogfooding/README.md @@ -0,0 +1,126 @@ +# Dogfooding scripts + +`dogfooding` is a long-lived branch where features that aren't ready for `develop` yet are combined and published as internal builds for Shopist and the Datadog mobile app. These scripts are the only supported way to bring changes into it, apart from reverting a feature (see [Remove a single feature](#remove-a-single-feature)). + +| Script | What it does | +|---|---| +| `feature.sh ` | Brings a feature branch into `dogfooding` as a single squash commit. | +| `sync.sh` | Brings `dogfooding` up to date with `develop`, keeping the dogfooded features. | +| `reset.sh` | Makes `dogfooding` exactly match `develop` again, dropping all dogfooded features. | + +None of them change `dogfooding` directly: each one pushes a side branch and opens a PR against `dogfooding`, which someone has to review and merge. They work in a temporary worktree (or don't check anything out at all), so your current checkout and uncommitted changes are never touched. + +Requirements: `git`, and the [GitHub CLI](https://cli.github.com/) (`gh`) logged in with push access to `DataDog/dd-sdk-android`. Run the scripts from the root of your `dd-sdk-android` checkout. + +## Rules + +- **Features only reach `dogfooding` through `feature.sh`.** Never open a PR from a feature branch straight into `dogfooding`: see [Dogfood a feature](#dogfood-a-feature). +- **Merge every PR into `dogfooding` with a merge commit**, never squash or rebase: that drops the `develop` parent of sync and reset merges (and can rewrite the dogfood commits' trailers), which the scripts rely on. The repository settings already only allow merge commits. + +## Common tasks + +### Dogfood a feature + +```bash +./ci/scripts/dogfooding/feature.sh feature/my-feature +``` + +This opens a PR from `dogfood-feature/my-feature-` with one squash commit containing the whole feature. Review it and merge it with a merge commit. + +Always use this script instead of merging the feature branch into `dogfooding`. A reset doesn't force-push, so a feature merged directly keeps its commits in `dogfooding`'s history even after the reset removes its changes. Git then treats it as already merged: merging it again brings nothing. When it later reaches `develop`, the next sync silently leaves it out. The script avoids this by bringing each dogfood in as a new squash commit, never the feature's own commits. + +The script refuses to run if your branch is based on `develop` commits that `dogfooding` doesn't have yet. Run `sync.sh` first, merge the sync PR, then run it again. + +If the feature conflicts with what's already in `dogfooding` (usually another dogfooded feature touching the same code), nothing is pushed. The temporary worktree is kept, and the script prints its path and the steps to finish: resolve the conflicts, commit with the prepared message, push, and open the PR. **Keep that commit message as is: its trailers are how the scripts track the feature.** + +### Update a dogfooded feature + +Push your changes to the feature branch and run the same command again. The new PR only contains what changed since the last dogfood (including deleted files), even if you rebased or force-pushed the branch in between. If it conflicts, it stops the same way as above. + +### Bring `dogfooding` up to date with `develop` + +```bash +./ci/scripts/dogfooding/sync.sh +``` + +This opens a PR from `sync-dogfooding-` with `develop`'s latest commits. It does nothing if `dogfooding` already contains `develop`. + +If `develop` changed the same code as a dogfooded feature, the PR is opened anyway and GitHub marks it as having conflicts. Resolve them on the sync branch, as described in the PR body: check it out, merge `dogfooding` into it, fix the conflicts, and push to the same branch. + +### When a dogfooded feature is merged into `develop` + +If the version merged into `develop` is exactly the version you dogfooded, there's nothing to do: the next sync brings `develop` in and the feature's code is identical on both sides. + +If you changed the feature after dogfooding it (review fixes, a rebase, removed files), `dogfooding` still has the old version. The sync brings in the new code, but anything that only existed in the old version stays in `dogfooding`: a file you deleted, a line you removed. Git has no reason to remove it, because `develop` never had it. + +To clean it up: + +1. Run `./ci/scripts/dogfooding/sync.sh`. Its PR lists the feature under "Dogfooded features that graduated to develop at a different commit", with the exact command to run afterwards. +2. Merge that sync PR. If it has conflicts in the feature's code, keep `develop`'s version. The build published after this merge may still contain code from the old version: to test the feature, use the one published after step 3. +3. Run the command from the PR, for example `./ci/scripts/dogfooding/feature.sh feature/my-feature --at `, and merge the PR it opens. It removes what's left of the old version, so `dogfooding` matches `develop` for that feature. `--at` is needed because the feature branch is usually deleted after merging into `develop`. + +The warning only appears in that sync PR. If nobody follows it, the leftovers stay until the next reset, which also removes them (along with every other dogfooded feature, which then has to be dogfooded again). + +### Start over from `develop` + +```bash +./ci/scripts/dogfooding/reset.sh +``` + +This opens a PR from `reset-dogfooding--` whose content is exactly `develop`'s. If the reset would discard anything or there are open dogfood PRs, the script shows the following and asks for confirmation (`--force` skips this). The same lists are always in the PR body: + +- What the reset discards. +- The dogfooded features that aren't in `develop` yet, mentioning their authors, with the `feature.sh` command to bring each one back. Run those commands **after** the reset PR is merged. +- Features whose status it can't determine. Check those by hand. +- Open dogfood PRs, which must be closed (deleting their branch) and re-created after the reset. + +While a reset PR is open, `feature.sh` refuses to run. If `dogfooding` already matches `develop`, the script does nothing. A reset never conflicts, since it takes `develop`'s files as they are. + +### Remove a single feature + +Revert, in one PR against `dogfooding`, every merge of a `dogfood--` PR for that feature since the last reset, newest first (`git revert -m 1 `). Or wait for the next reset. + +`feature.sh` doesn't detect reverts: it still considers the feature dogfooded. To dogfood it again afterwards, use `--full`: + +```bash +./ci/scripts/dogfooding/feature.sh feature/my-feature --full +``` + +## Publishing + +Every build of `dogfooding` is published automatically by the `publish:dogfooding` job in GitLab, once the pipeline on `dogfooding` has passed. The version is `-dogfood-`, for example `3.15.0-dogfood-1a2b3c4d`, published to `https://binaries.ddbuild.io/dd-sdk-android/dogfood/`, which is only reachable from Datadog's network (VPN on laptops). Add that repository to the consuming app: + +```kotlin +maven("https://binaries.ddbuild.io/dd-sdk-android/dogfood/") { + content { includeGroup("com.datadoghq") } +} +``` + +Then point Shopist or the Datadog app at that exact version. + +## When a script stops + +| Message | What to do | +|---|---| +| `... is already open: ` | Nothing: a PR for exactly this state already exists. | +| `origin/ already exists but has no open PR` | A previous PR was closed, or a run failed halfway. Check the branch, delete it with the printed command, and re-run. | +| `Couldn't list the PRs merged from into develop` | GitHub couldn't be queried (rate limit, network, authentication). Nothing was pushed: re-run once `gh` works. | +| `is based on develop commits that dogfooding doesn't have yet` | Run `sync.sh` (or merge the open sync PR), then re-run. | +| `A reset of dogfooding is pending` | Wait until the reset PR is merged, then re-run. | +| `Another dogfood PR for is still open` | Merge or close it (deleting its branch), then re-run. | +| `Conflicts while applying onto dogfooding` | See [Dogfood a feature](#dogfood-a-feature). | +| `Can't find , the previously dogfooded tip` | Your clone doesn't have the old tip, usually because someone else force-pushed the branch. Whoever still has it either re-runs the script from their clone, or pushes it (`git push origin :refs/heads/tmp-recover`). Then re-run, and delete `tmp-recover`. | +| `Branch not found on origin` | Pass the commit to dogfood with `--at `. | +| `Can't find commit locally or on origin` | Pass the full 40-character SHA to `--at`. | +| `Nothing to dogfood` | Nothing to do, `dogfooding` already has these changes. | +| `already dogfooded` | Nothing to do. If its dogfood commit was reverted, re-run with `--full`. | + +## How it works + +- `feature.sh` never merges a feature's own commits into `dogfooding`. Each dogfood is a single squash commit with a `Dogfood-Source: @` trailer, which records exactly what was dogfooded. Later dogfoods of the same branch only apply the difference from that commit. +- `sync.sh` doesn't merge anything locally. It pushes `develop`'s current tip as a `sync-dogfooding-` branch and opens a PR, so GitHub computes the merge and a conflicting sync still shows up as a PR to resolve. To find dogfooded features whose leftovers need cleaning up, it reads the `Dogfood-Source` trailers since the last reset and asks GitHub, for each of those branches, which PRs were merged into `develop` in this sync and at which commit. If the branch was at a different commit when its PR merged than the one dogfooded, it's flagged in the PR with the command to run. +- `reset.sh` doesn't force-push. It creates a merge commit with `dogfooding` and `develop` as parents and exactly `develop`'s files, so history is kept and the next syncs see `develop` as already merged. All three scripts find the last reset by that commit's subject, so don't change it. + +## Known limits + +- The scripts run locally. Running sync and reset from CI is planned for later. diff --git a/ci/scripts/dogfooding/feature.sh b/ci/scripts/dogfooding/feature.sh new file mode 100755 index 0000000000..0353d1745b --- /dev/null +++ b/ci/scripts/dogfooding/feature.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2016-Present Datadog, Inc. +# +# Brings a feature branch into `dogfooding` as a single squash commit, through a PR. This is +# the only way features should reach `dogfooding`: the feature's own commits never enter its +# history, so resets and syncs can't silently drop the feature later on. +# +# Usage: ./ci/scripts/dogfooding/feature.sh [--at ] [--full] +# The feature branch to dogfood (its tip on origin, unless --at is given). +# --at Dogfood this commit instead of the branch tip, e.g. once the branch was deleted +# after merging into develop. Use the full SHA if the commit isn't local. +# --full Ignore earlier dogfoods of the branch and squash the whole feature again, +# e.g. after its dogfood commit was reverted in `dogfooding`. +# +# The first dogfood of a branch since the last reset squashes the whole feature. Later ones +# only bring the changes since the previous dogfood (tracked with a `Dogfood-Source` trailer +# on each squash commit), so updates, removed files and force-pushed rebases all apply +# correctly. The PR is opened from a `dogfood--` branch and must be merged with +# a merge commit. +# +# Safe to run locally: the squash is done in a temporary worktree, so the checkout it's run +# from (current branch, uncommitted changes) is never touched. On a conflict the worktree is +# kept and the script prints how to finish by hand. +# +# NOTE: This script does not modify `dogfooding` directly — it opens a PR against it. +# A human must review and merge that PR (branch protection requires this). + +set -euo pipefail + +REPO="DataDog/dd-sdk-android" +DOGFOODING_BRANCH="dogfooding" +RESET_SUBJECT="Reset dogfooding to develop" + +usage() { + echo "Usage: $0 [--at ] [--full]" >&2 + exit 1 +} + +BRANCH="" +AT="" +FULL=false +while [ $# -gt 0 ]; do + case "$1" in + --at) [ -n "${2:-}" ] || usage; AT="$2"; shift 2 ;; + --full) FULL=true; shift ;; + -*) echo "Unknown option: $1" >&2; usage ;; + *) [ -z "$BRANCH" ] || usage; BRANCH="$1"; shift ;; + esac +done +[ -n "$BRANCH" ] || usage + +# Makes a commit available locally: uses it if it's already there (e.g. whoever force-pushed +# the branch still has the old tip), otherwise fetches it by SHA from origin. +ensure_commit() { + git cat-file -e "$1^{commit}" 2>/dev/null && return 0 + git fetch --quiet origin "$1" 2>/dev/null && git cat-file -e "$1^{commit}" 2>/dev/null +} + +echo "Fetching latest from origin..." +if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then + # Unshallow so the checks below see the real history, not a truncated one. + git fetch --unshallow origin +fi +git fetch origin "+refs/heads/develop:refs/remotes/origin/develop" \ + "+refs/heads/$DOGFOODING_BRANCH:refs/remotes/origin/$DOGFOODING_BRANCH" + +if [ -n "$AT" ]; then + if ! ensure_commit "$AT"; then + echo "Can't find commit $AT locally or on origin. Pass the full SHA." >&2 + exit 1 + fi + TARGET=$(git rev-parse "$AT^{commit}") +else + if ! git fetch --quiet origin "+refs/heads/$BRANCH:refs/remotes/origin/$BRANCH" 2>/dev/null; then + echo "Branch $BRANCH not found on origin. If it was deleted, pass the commit with --at ." >&2 + exit 1 + fi + TARGET=$(git rev-parse "refs/remotes/origin/$BRANCH") +fi +SHA12=${TARGET:0:12} +DOGFOOD_BRANCH="dogfood-$BRANCH-$SHA12" + +# Everything merged into dogfooding since the last reset. Earlier dogfoods were discarded by +# that reset, so they don't count. The reset commit is found by the subject the reset script +# writes itself; --topo-order makes "newest" independent of commit dates. +LAST_RESET=$(git log -1 --topo-order --format=%H --grep="^$RESET_SUBJECT" \ + origin/develop.."origin/$DOGFOODING_BRANCH") +if [ -n "$LAST_RESET" ]; then + SINCE_RESET=("$LAST_RESET..origin/$DOGFOODING_BRANCH" --not origin/develop) +else + SINCE_RESET=("origin/develop..origin/$DOGFOODING_BRANCH") +fi + +# Newest Dogfood-Source trailer for this branch since the last reset ("@", split +# on the last @ so branch names are compared exactly). +LAST_DOGFOOD="" +if [ "$FULL" = false ]; then + trailers=$(git log --no-merges --topo-order --format='%(trailers:key=Dogfood-Source,valueonly)' "${SINCE_RESET[@]}") + while IFS= read -r source; do + [ -n "$source" ] || continue + if [ "${source%@*}" = "$BRANCH" ]; then + LAST_DOGFOOD="${source##*@}" + break + fi + done <<< "$trailers" +fi + +open_prs=$(gh pr list --repo "$REPO" --base "$DOGFOODING_BRANCH" --state open \ + --json number,headRefName,url --limit 200 -q '.[] | "\(.number)\t\(.headRefName)\t\(.url)"') + +# 1. A PR for this exact commit is already open: nothing to do. +existing_pr=$(awk -F'\t' -v head="$DOGFOOD_BRANCH" '$2 == head { print $3; exit }' <<< "$open_prs") +if [ -n "$existing_pr" ]; then + echo "A dogfood PR for $BRANCH at $SHA12 is already open: $existing_pr" + exit 0 +fi + +# 2. This exact commit is already dogfooded. +if [ "$LAST_DOGFOOD" = "$TARGET" ]; then + echo "$BRANCH at $SHA12 is already dogfooded. If its dogfood commit was reverted, re-run with --full." + exit 0 +fi + +# 3. The branch exists without an open PR (its PR was closed, or a previous run failed before +# opening one). Never overwrite it: someone may have pushed to it. +if git ls-remote --exit-code --heads origin "refs/heads/$DOGFOOD_BRANCH" >/dev/null; then + echo "origin/$DOGFOOD_BRANCH already exists but has no open PR (its PR was closed, or a previous run failed before opening one)." >&2 + echo "Inspect it, delete it with 'git push origin --delete $DOGFOOD_BRANCH', then re-run." >&2 + exit 1 +fi + +# 4. Another dogfood of this branch is still open: its delta would overlap with this one. +# 5. A reset is pending: a delta computed now would apply on content the reset removes. +while IFS=$'\t' read -r number head url; do + [ -n "$number" ] || continue + suffix="${head#"dogfood-$BRANCH-"}" + if [ "$suffix" != "$head" ] && [[ "$suffix" =~ ^[0-9a-f]{12}$ ]]; then + echo "Another dogfood PR for $BRANCH is still open: $url" >&2 + echo "Merge or close it (deleting its branch) first, then re-run." >&2 + exit 1 + fi + if [[ "$head" == reset-dogfooding-* ]]; then + echo "A reset of $DOGFOODING_BRANCH is pending (#$number): $url" >&2 + echo "Wait until it's merged, then re-run." >&2 + exit 1 + fi +done <<< "$open_prs" + +# 6. The feature must not be based on develop commits that dogfooding doesn't have yet, +# otherwise the squash would bring those commits in too. +base=$(git merge-base "$TARGET" origin/develop || true) +if [ -z "$base" ] || ! git merge-base --is-ancestor "$base" "origin/$DOGFOODING_BRANCH"; then + echo "$BRANCH at $SHA12 is based on develop commits that $DOGFOODING_BRANCH doesn't have yet." >&2 + echo "Run ./ci/scripts/dogfooding/sync.sh (or merge the open sync PR) first, then re-run." >&2 + exit 1 +fi + +if [ -n "$LAST_DOGFOOD" ] && ! ensure_commit "$LAST_DOGFOOD"; then + echo "Can't find $LAST_DOGFOOD, the previously dogfooded tip of $BRANCH (it may have been garbage-collected after a force-push)." >&2 + echo "Your clone doesn't have it, usually because someone else force-pushed $BRANCH. Whoever still has it can re-run from their clone, or push it (git push origin $LAST_DOGFOOD:refs/heads/tmp-recover) so it can be fetched; then re-run." >&2 + exit 1 +fi + +# Source PR into develop, for the PR link and to notify its author. +source_pr=$(gh pr list --repo "$REPO" --head "$BRANCH" --base develop --state all \ + --json url,author,state -q '(map(select(.state == "OPEN")) + .)[0] // empty | "\(.url)\t\(.author.login)"' 2>/dev/null || true) +source_pr_url=$(cut -f1 <<< "$source_pr") +author=$(cut -s -f2 <<< "$source_pr") +[ -n "$author" ] || author=$(gh api user -q .login 2>/dev/null || true) +if [ -n "$author" ]; then author="@$author"; else author=$(git config user.name || echo unknown); fi + +if [ -n "$LAST_DOGFOOD" ]; then + scope="Changes in $BRANCH since its previous dogfood at ${LAST_DOGFOOD:0:12}, up to $SHA12." +else + scope="Full squash of $BRANCH at $SHA12." +fi +TITLE="Dogfood $BRANCH @ $SHA12" + +repo_dir=$(pwd) + +# Do the squash in a separate worktree so this never checks out a branch (or touches +# uncommitted work) in whatever repo the script happens to be run from. +worktree_dir=$(mktemp -d) +trap 'cd "$repo_dir"; git worktree remove --force "$worktree_dir" >/dev/null 2>&1 || true' EXIT +git worktree add --quiet -b "$DOGFOOD_BRANCH" "$worktree_dir" "origin/$DOGFOODING_BRANCH" +# Only now do we know this run created $DOGFOOD_BRANCH - safe to delete it on exit. +trap 'cd "$repo_dir"; git worktree remove --force "$worktree_dir" >/dev/null 2>&1 || true; git branch -D "$DOGFOOD_BRANCH" >/dev/null 2>&1 || true' EXIT +cd "$worktree_dir" + +# The commit message and PR body live in the git dir, not the worktree, so resolving a +# conflict with `git add -A` can't commit them. +git_dir=$(git rev-parse --absolute-git-dir) +msg_file="$git_dir/DOGFOOD_MSG" +body_file="$git_dir/DOGFOOD_PR_BODY" +{ + echo "$TITLE" + echo "" + echo "$scope" + [ -z "$source_pr_url" ] || echo "Source PR: $source_pr_url" + echo "" + echo "Dogfood-Source: $BRANCH@$TARGET" + echo "Dogfood-Author: $author" +} > "$msg_file" +{ + echo "Dogfoods \`$BRANCH\` at $SHA12 as a single squash commit, created by \`ci/scripts/dogfooding/feature.sh\`." + echo "" + echo "$scope" + [ -z "$source_pr_url" ] || echo "Source PR: $source_pr_url" + echo "Author: $author" + echo "" + echo "Merge this PR with a merge commit, never squash: the \`Dogfood-Source\` trailer on the squash commit is how the reset and sync scripts track this feature." +} > "$body_file" + +if [ -n "$LAST_DOGFOOD" ]; then + echo "Applying the changes in $BRANCH since ${LAST_DOGFOOD:0:12}..." + # A throwaway commit whose diff is exactly "previous dogfood -> target", even if the + # branch was rebased in between. + delta=$(git commit-tree "$TARGET^{tree}" -p "$LAST_DOGFOOD" -m "dogfood delta") + git cherry-pick --no-commit "$delta" || true +else + echo "Squashing $BRANCH at $SHA12..." + git merge --squash "$TARGET" || true +fi + +conflicts=$(git diff --name-only --diff-filter=U) +if [ -n "$conflicts" ]; then + echo "" >&2 + echo "Conflicts while applying $BRANCH onto $DOGFOODING_BRANCH:" >&2 + echo "$conflicts" | sed 's/^/ /' >&2 + trap - EXIT + cat >&2 < + git commit --no-verify -F "$msg_file" + git push origin "HEAD:refs/heads/$DOGFOOD_BRANCH" + gh pr create --repo $REPO --base $DOGFOODING_BRANCH --head "$DOGFOOD_BRANCH" --title "$TITLE" --body-file "$body_file" +Then clean up: + cd "$repo_dir" && git worktree remove --force "$worktree_dir" && git branch -D "$DOGFOOD_BRANCH" +EOF + exit 1 +fi + +if git diff --cached --quiet; then + echo "Nothing to dogfood: $DOGFOODING_BRANCH already has the changes of $BRANCH at $SHA12." + exit 0 +fi + +# --no-verify: this commit only squashes code that is reviewed on the feature branch, and the +# repo's pre-commit hook (lint, tests) would take minutes on every dogfood. +git commit --quiet --no-verify -F "$msg_file" + +echo "Pushing $DOGFOOD_BRANCH..." +git push origin "HEAD:refs/heads/$DOGFOOD_BRANCH" + +echo "Opening PR against $DOGFOODING_BRANCH..." +gh pr create --repo "$REPO" --base "$DOGFOODING_BRANCH" --head "$DOGFOOD_BRANCH" \ + --title "$TITLE" --body-file "$body_file" + +echo "" +echo "Done. Review and merge the PR above (with a merge commit) to dogfood $BRANCH." diff --git a/ci/scripts/dogfooding/reset.sh b/ci/scripts/dogfooding/reset.sh new file mode 100755 index 0000000000..2f40743bd8 --- /dev/null +++ b/ci/scripts/dogfooding/reset.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash +# +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2016-Present Datadog, Inc. +# +# Resets `dogfooding`'s content to exactly match `develop`, without force-pushing and +# without losing history: creates a merge commit whose tree matches `develop` exactly +# but whose parents are both the current `dogfooding` tip and `develop`'s tip (so future +# syncs won't try to reintroduce discarded work), then opens a PR for review. +# +# Usage: ./ci/scripts/dogfooding/reset.sh [--force] +# --force Skip the confirmation prompt. +# +# Safe to run locally: the reset is done in a temporary worktree, so the checkout it's run +# from (current branch, uncommitted changes) is never touched. +# +# The reset branch is named after the current `dogfooding` and `develop` tips. Re-running +# against the same tips prints the already-open PR instead of opening a new one; if that +# branch exists without an open PR, the script fails and asks you to delete it first. Exits +# without opening a PR if `dogfooding` already has `develop`'s exact content. +# +# Features dogfooded since the last reset (with dogfooding/feature.sh) that haven't graduated to +# `develop` yet are listed in the prompt and the PR body, mentioning their authors, with the +# command to dogfood them again once this reset is merged. Open dogfood PRs are listed too: +# they were computed against the content this reset removes, so they must be closed and +# re-created after it. +# +# NOTE: This script does not modify `dogfooding` directly — it opens a PR against it. +# A human must review and merge that PR (branch protection requires this). + +set -euo pipefail + +REPO="DataDog/dd-sdk-android" +DOGFOODING_BRANCH="dogfooding" +# Subject of the reset commit. The dogfood, reset and sync scripts find the last reset by it, +# so don't change it. +RESET_SUBJECT="Reset dogfooding to develop" +FORCE=false + +for arg in "$@"; do + case "$arg" in + --force) FORCE=true ;; + *) echo "Unknown argument: $arg" >&2; exit 1 ;; + esac +done + +# Makes a commit available locally: uses it if it's already there, otherwise fetches it by SHA +# from origin (it may only be reachable by SHA, e.g. a feature tip that was force-pushed). +ensure_commit() { + git cat-file -e "$1^{commit}" 2>/dev/null && return 0 + git fetch --quiet origin "$1" 2>/dev/null && git cat-file -e "$1^{commit}" 2>/dev/null +} + +echo "Fetching latest from origin..." +if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then + # Unshallow so the discard list and the merge below see the real history of both branches. + git fetch --unshallow origin +fi +git fetch origin "+refs/heads/develop:refs/remotes/origin/develop" \ + "+refs/heads/$DOGFOODING_BRANCH:refs/remotes/origin/$DOGFOODING_BRANCH" + +# Already has develop's exact content (e.g. right after a reset was merged): nothing to reset. +if git diff --quiet origin/develop "origin/$DOGFOODING_BRANCH"; then + echo "$DOGFOODING_BRANCH already matches develop, nothing to reset." + exit 0 +fi + +# Named after both tips, so re-running against the same tips maps to the same branch/PR. +DOGFOODING_SHA=$(git rev-parse "origin/$DOGFOODING_BRANCH") +DEVELOP_SHA=$(git rev-parse origin/develop) +RESET_BRANCH="reset-dogfooding-${DOGFOODING_SHA:0:12}-${DEVELOP_SHA:0:12}" + +open_prs=$(gh pr list --repo "$REPO" --base "$DOGFOODING_BRANCH" --state open \ + --json number,headRefName,url,author --limit 200 \ + -q '.[] | "\(.number)\t\(.headRefName)\t\(.url)\t\(.author.login)"') + +# Already have a PR for these exact tips: nothing to do. +existing_pr=$(awk -F'\t' -v head="$RESET_BRANCH" '$2 == head { print $3; exit }' <<< "$open_prs") +if [ -n "$existing_pr" ]; then + echo "A reset PR for the current $DOGFOODING_BRANCH and develop tips is already open: $existing_pr" + exit 0 +fi + +# The branch exists without an open PR (its PR was closed, or a previous run failed before +# opening one). Never overwrite it: someone may have pushed to it. +if git ls-remote --exit-code --heads origin "refs/heads/$RESET_BRANCH" >/dev/null; then + echo "origin/$RESET_BRANCH already exists but has no open PR (its PR was closed, or a previous run failed before opening one)." >&2 + echo "Inspect it, delete it with 'git push origin --delete $RESET_BRANCH', then re-run." >&2 + exit 1 +fi + +# Everything merged into dogfooding since the last reset; what came before was already +# discarded by that reset. The reset commit is found by the subject this script writes +# below; --topo-order makes "newest" independent of commit dates. +LAST_RESET=$(git log -1 --topo-order --format=%H --grep="^$RESET_SUBJECT" \ + origin/develop.."origin/$DOGFOODING_BRANCH") +if [ -n "$LAST_RESET" ]; then + SINCE_RESET=("$LAST_RESET..origin/$DOGFOODING_BRANCH" --not origin/develop) +else + SINCE_RESET=("origin/develop..origin/$DOGFOODING_BRANCH") +fi + +# What this reset discards, one line per merged PR (--first-parent skips the commits inside +# each merged branch), leaving out the merge that brought the previous reset in and plain +# sync merges (their second parent is part of develop, so they bring nothing of their own). +AHEAD=$(git log --first-parent --format='%H %s' "${SINCE_RESET[@]}" | while read -r sha subject; do + if [ -n "$LAST_RESET" ] && git merge-base --is-ancestor "$LAST_RESET" "$sha" \ + && ! git merge-base --is-ancestor "$LAST_RESET" "$sha^1"; then + continue + fi + if git merge-base --is-ancestor "$sha^2" origin/develop 2>/dev/null; then + continue + fi + echo "${sha:0:12} $subject" +done) + +# Features dogfooded since the last reset (newest Dogfood-Source trailer per branch) that +# haven't graduated to develop yet. This reset removes them from dogfooding, so they must be +# dogfooded again once it's merged. +IN_FLIGHT="" +# Features whose dogfooded commit can't be found locally or on origin, when it's needed to +# tell whether they graduated or to dogfood them again. +UNKNOWN="" +seen=" " +while IFS=$'\t' read -r source author; do + [ -n "$source" ] || continue + branch="${source%@*}" + sha="${source##*@}" + case "$seen" in *" $branch "*) continue ;; esac + seen="$seen$branch " + unknown_entry="- $branch (dogfooded at ${sha:0:12}) ${author:-} +" + if git ls-remote --exit-code --heads origin "refs/heads/$branch" >/dev/null 2>&1; then + command="./ci/scripts/dogfooding/feature.sh $branch" + needs_sha=false + else + command="./ci/scripts/dogfooding/feature.sh $branch --at $sha" + needs_sha=true + fi + # Graduated: a PR from this branch into develop merged everything that was dogfooded, i.e. + # its head (the branch tip when it merged) is the dogfooded commit, contains it, or is + # unrelated to it (the branch was rebased before merging). A merged head that is a strict + # ancestor of the dogfooded commit only brought in part of it: an older PR that reused the + # branch name, or the branch kept going after an earlier PR merged, so it doesn't count. + # If gh fails, stop: without the merged PRs there's no reliable way to tell whether the feature + # graduated (its branch may have been rebased before merging), and nothing has been pushed yet. + # Comparing the merged heads needs the dogfooded commit; without merged PRs it's only needed + # for the --at command. + if ! merged_heads=$(gh pr list --repo "$REPO" --head "$branch" --base develop --state merged \ + --json headRefOid --limit 100 -q '.[].headRefOid'); then + echo "Couldn't list the PRs merged from $branch into develop (gh failed). Nothing was pushed: re-run once gh works." >&2 + exit 1 + fi + if [ -z "$merged_heads" ]; then + graduated=false + elif ! ensure_commit "$sha"; then + UNKNOWN="$UNKNOWN$unknown_entry" + continue + else + graduated=false + while read -r head; do + [ -n "$head" ] || continue + if [ "$head" = "$sha" ] || ! git merge-base --is-ancestor "$head" "$sha" 2>/dev/null; then + graduated=true + fi + done <<< "$merged_heads" + fi + [ "$graduated" = false ] || continue + if [ "$needs_sha" = true ] && ! ensure_commit "$sha"; then + UNKNOWN="$UNKNOWN$unknown_entry" + continue + fi + IN_FLIGHT="$IN_FLIGHT- $branch (dogfooded at ${sha:0:12}) ${author:-}: $command +" +done <<< "$(git log --no-merges --topo-order \ + --format='%(trailers:key=Dogfood-Source,valueonly,separator=%x2C)%x09%(trailers:key=Dogfood-Author,valueonly,separator=%x2C)' \ + "${SINCE_RESET[@]}")" + +# Open dogfood PRs were computed against content this reset removes: merging one after it +# would apply a partial feature. They must be closed and re-created once this reset is merged. +OPEN_DOGFOOD="" +older_prs="" +while IFS=$'\t' read -r number head url author; do + [ -n "$number" ] || continue + if [[ "$head" =~ ^dogfood-(.+)-[0-9a-f]{12}$ ]]; then + OPEN_DOGFOOD="$OPEN_DOGFOOD- #$number $head @$author: gh pr close $number --delete-branch, then once this reset is merged ./ci/scripts/dogfooding/feature.sh ${BASH_REMATCH[1]} +" + elif [[ "$head" == reset-dogfooding-* ]]; then + older_prs="${older_prs:+$older_prs, }#$number" + fi +done <<< "$open_prs" + +if { [ -n "$AHEAD" ] || [ -n "$OPEN_DOGFOOD" ]; } && [ "$FORCE" = false ]; then + if [ -n "$AHEAD" ]; then + echo "⚠️ The following commits on $DOGFOODING_BRANCH will be discarded:" + echo "$AHEAD" + echo "" + fi + if [ -n "$IN_FLIGHT" ]; then + echo "⚠️ Dogfooded features not in develop yet. Dogfood each one again once the reset PR is merged:" + echo "$IN_FLIGHT" + fi + if [ -n "$UNKNOWN" ]; then + echo "⚠️ Dogfooded features with unknown status (the dogfooded commit can't be found locally or on origin). Check manually whether each one is in develop:" + echo "$UNKNOWN" + fi + if [ -n "$OPEN_DOGFOOD" ]; then + echo "⚠️ Open dogfood PRs, computed against content this reset removes. Close each one without merging, then re-create it once the reset PR is merged:" + echo "$OPEN_DOGFOOD" + fi + read -p "Are you sure you want to reset? [y/N] " confirm + if [[ "$confirm" != "y" && "$confirm" != "Y" ]]; then + echo "Aborted." + exit 0 + fi +fi + +repo_dir=$(pwd) + +# Do the reset in a separate worktree so this never checks out a branch (or +# touches uncommitted work) in whatever repo the script happens to be run from. +worktree_dir=$(mktemp -d) +trap 'cd "$repo_dir"; git worktree remove --force "$worktree_dir" >/dev/null 2>&1 || true' EXIT +echo "Creating $RESET_BRANCH from origin/$DOGFOODING_BRANCH in a temporary worktree..." +git worktree add --quiet -b "$RESET_BRANCH" "$worktree_dir" "origin/$DOGFOODING_BRANCH" +# Only now do we know this run created $RESET_BRANCH - safe to delete it on exit. +# Removing the worktree doesn't delete the branch it checked out, so without this +# a retry after a failed run would fail at `git worktree add -b` with +# "branch already exists". +trap 'cd "$repo_dir"; git worktree remove --force "$worktree_dir" >/dev/null 2>&1 || true; git branch -D "$RESET_BRANCH" >/dev/null 2>&1 || true' EXIT +cd "$worktree_dir" + +echo "Recording origin/develop as merged while keeping our tree for now..." +git merge -s ours --no-commit origin/develop + +echo "Overwriting tree/index to match origin/develop exactly..." +git read-tree -u --reset origin/develop + +echo "Committing reset..." +git commit --no-verify -m "$RESET_SUBJECT (discard in-flight dogfood-only work)" + +echo "Pushing $RESET_BRANCH..." +git push origin "HEAD:refs/heads/$RESET_BRANCH" + +pr_body="Discards: +$AHEAD" +if [ -n "$IN_FLIGHT" ]; then + pr_body="$pr_body + +Dogfooded features not in develop yet. This reset removes them from dogfooding; once it's merged, dogfood each one again: +${IN_FLIGHT%$'\n'}" +fi +if [ -n "$UNKNOWN" ]; then + pr_body="$pr_body + +Dogfooded features with unknown status (the dogfooded commit can't be found locally or on origin). Check manually whether each one is in develop, and dogfood it again once this PR is merged if it isn't: +${UNKNOWN%$'\n'}" +fi +if [ -n "$OPEN_DOGFOOD" ]; then + pr_body="$pr_body + +Open dogfood PRs, computed against content this reset removes (merging one after it would apply a partial feature). Close each one without merging and delete its branch, then re-create it once this PR is merged: +${OPEN_DOGFOOD%$'\n'}" +fi +# Older reset PRs were computed against older tips; merging one of them would leave stray +# content behind, so point reviewers away from them. +if [ -n "$older_prs" ]; then + pr_body="$pr_body + +Supersedes $older_prs (computed against older $DOGFOODING_BRANCH/develop tips). Close them without merging." +fi + +echo "Opening PR against $DOGFOODING_BRANCH..." +gh pr create --repo "$REPO" --base "$DOGFOODING_BRANCH" --head "$RESET_BRANCH" \ + --title "Reset dogfooding branch to develop" \ + --body "$pr_body" + +echo "" +echo "Done. Review and merge the PR above to complete the reset." diff --git a/ci/scripts/dogfooding/sync.sh b/ci/scripts/dogfooding/sync.sh new file mode 100755 index 0000000000..8b80a6f719 --- /dev/null +++ b/ci/scripts/dogfooding/sync.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2016-Present Datadog, Inc. +# +# Brings `dogfooding` up to date with `develop` without discarding in-flight dogfood-only +# work: pushes a snapshot of develop's current tip as a side branch and opens a PR against +# `dogfooding`. No local merge is attempted — GitHub computes the merge itself when the PR +# is viewed/merged, so a conflicting sync still produces a visible, trackable PR (marked +# "must be resolved") instead of silently failing with nothing to show for it. +# +# Usage: ./ci/scripts/dogfooding/sync.sh +# +# Never checks anything out locally, so it's safe to run from any checkout. Exits without +# pushing or opening a PR if `dogfooding` already contains `develop`. +# +# The sync branch is named after `develop`'s tip. Re-running against the same tip prints the +# already-open PR instead of opening a new one; if that branch exists without an open PR, the +# script fails and asks you to delete it first. +# +# Features dogfooded with dogfooding/feature.sh that graduated to `develop` at a different commit +# than the one dogfooded are flagged in the output and the PR body, mentioning their authors: +# once the sync is merged they must be dogfooded again at their graduated commit, otherwise +# code that only existed in the dogfooded version stays in `dogfooding`. +# +# NOTE: This script does not modify `dogfooding` directly — it opens a PR against it. +# A human must review and merge that PR (branch protection requires this). + +set -euo pipefail + +REPO="DataDog/dd-sdk-android" +DOGFOODING_BRANCH="dogfooding" +# Subject of the reset commit written by dogfooding/reset.sh; the last reset is found by it. +RESET_SUBJECT="Reset dogfooding to develop" + +echo "Fetching latest from origin..." +if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then + # Unshallow so the "nothing to sync" check below can see whether develop's tip is + # already part of dogfooding's history. + git fetch --unshallow origin +fi +git fetch origin "+refs/heads/develop:refs/remotes/origin/develop" \ + "+refs/heads/$DOGFOODING_BRANCH:refs/remotes/origin/$DOGFOODING_BRANCH" + +if git merge-base --is-ancestor origin/develop "origin/$DOGFOODING_BRANCH"; then + echo "$DOGFOODING_BRANCH already contains origin/develop, nothing to sync." + exit 0 +fi + +# Named after develop's tip, so re-running against the same tip maps to the same branch/PR. +DEVELOP_SHA=$(git rev-parse origin/develop) +SYNC_BRANCH="sync-dogfooding-${DEVELOP_SHA:0:12}" + +open_prs=$(gh pr list --repo "$REPO" --base "$DOGFOODING_BRANCH" --state open \ + --json number,headRefName,url --limit 200 -q '.[] | "\(.number)\t\(.headRefName)\t\(.url)"') + +# Already have a PR for this exact tip: nothing to do. +existing_pr=$(awk -F'\t' -v head="$SYNC_BRANCH" '$2 == head { print $3; exit }' <<< "$open_prs") +if [ -n "$existing_pr" ]; then + echo "A sync PR for the current develop tip is already open: $existing_pr" + exit 0 +fi + +# The branch exists without an open PR (its PR was closed, or a previous run failed before +# opening one). Never overwrite it: someone may have pushed conflict fixes to it. +if git ls-remote --exit-code --heads origin "refs/heads/$SYNC_BRANCH" >/dev/null; then + echo "origin/$SYNC_BRANCH already exists but has no open PR (its PR was closed, or a previous run failed before opening one)." >&2 + echo "Inspect it, delete it with 'git push origin --delete $SYNC_BRANCH', then re-run." >&2 + exit 1 +fi + +# Older sync PRs bring in an older develop tip; merging this one includes everything in them. +older_prs=$(awk -F'\t' '$2 ~ /^sync-dogfooding-/ { printf "%s#%s", sep, $1; sep = ", " }' <<< "$open_prs") + +# Graduation check: features dogfooded since the last reset whose PR into develop is part of +# this sync, but graduated at a different commit than the one dogfooded (e.g. updated or +# rebased before merging). The sync brings develop's version in, but code that only existed +# in the dogfooded version stays in dogfooding until the feature is dogfooded again. +LAST_RESET=$(git log -1 --topo-order --format=%H --grep="^$RESET_SUBJECT" \ + origin/develop.."origin/$DOGFOODING_BRANCH") +if [ -n "$LAST_RESET" ]; then + SINCE_RESET=("$LAST_RESET..origin/$DOGFOODING_BRANCH" --not origin/develop) +else + SINCE_RESET=("origin/develop..origin/$DOGFOODING_BRANCH") +fi +GRADUATED="" +seen=" " +while IFS=$'\t' read -r source author; do + [ -n "$source" ] || continue + branch="${source%@*}" + sha="${source##*@}" + case "$seen" in *" $branch "*) continue ;; esac + seen="$seen$branch " + if ! merged=$(gh pr list --repo "$REPO" --head "$branch" --base develop --state merged \ + --json mergeCommit,headRefOid --limit 100 -q '.[] | "\(.mergeCommit.oid)\t\(.headRefOid)"'); then + echo "Couldn't list the PRs merged from $branch into develop (gh failed). Nothing was pushed: re-run once gh works." >&2 + exit 1 + fi + latest_merge="" + latest_head="" + while IFS=$'\t' read -r merge head; do + [ -n "$merge" ] || continue + # Only PRs merged as part of this sync count (not an older PR with the same name). + git merge-base --is-ancestor "$merge" origin/develop 2>/dev/null || continue + ! git merge-base --is-ancestor "$merge" "origin/$DOGFOODING_BRANCH" 2>/dev/null || continue + # Several PRs from this branch merged in this sync: develop now has the last one's version. + if [ -z "$latest_merge" ] || git merge-base --is-ancestor "$latest_merge" "$merge"; then + latest_merge=$merge + latest_head=$head + fi + done <<< "$merged" + [ -n "$latest_head" ] || continue + [ "$latest_head" != "$sha" ] || continue + # The merged head only brought in part of what was dogfooded (the branch kept going after + # it): dogfooding already has everything develop now brings, so there's nothing left over. + ! git merge-base --is-ancestor "$latest_head" "$sha" 2>/dev/null || continue + GRADUATED="$GRADUATED- $branch ${author:-}: dogfooded at ${sha:0:12}, graduated at ${latest_head:0:12}. After merging this sync, run ./ci/scripts/dogfooding/feature.sh $branch --at $latest_head +" +done <<< "$(git log --no-merges --topo-order \ + --format='%(trailers:key=Dogfood-Source,valueonly,separator=%x2C)%x09%(trailers:key=Dogfood-Author,valueonly,separator=%x2C)' \ + "${SINCE_RESET[@]}")" + +if [ -n "$GRADUATED" ]; then + echo "" + echo "⚠️ Dogfooded features that graduated to develop at a different commit than the one dogfooded:" + echo "$GRADUATED" +fi + +echo "Pushing origin/develop as $SYNC_BRANCH..." +git push origin "refs/remotes/origin/develop:refs/heads/$SYNC_BRANCH" + +pr_body="Brings dogfooding up to date with the latest develop. If this PR shows conflicts, resolve them by checking out $SYNC_BRANCH locally, merging $DOGFOODING_BRANCH into it, fixing the conflicts, and pushing back to the same branch." +if [ -n "$GRADUATED" ]; then + pr_body="$pr_body + +Dogfooded features that graduated to develop at a different commit than the one dogfooded. Code that only existed in the dogfooded version stays in dogfooding after this sync: +${GRADUATED%$'\n'} + +- On conflicts in these features' code, take develop's side for their hunks. +- The build published after merging this PR may still contain code from the old dogfooded version: to test these features, use a build published after they're dogfooded again. +- This warning won't be shown again after merging: leftovers stay until the feature is dogfooded again or the next reset." +fi +if [ -n "$older_prs" ]; then + pr_body="$pr_body + +Newer than $older_prs (older develop tip). Merging this PR includes everything in them, so they can be closed without merging." +fi + +echo "Opening PR against $DOGFOODING_BRANCH..." +gh pr create --repo "$REPO" --base "$DOGFOODING_BRANCH" --head "$SYNC_BRANCH" \ + --title "Sync dogfooding with develop" \ + --body "$pr_body" + +echo "" +echo "Done. Review and merge the PR above to complete the sync."