fetch.yml

name: Fetch collections

on:
  workflow_dispatch:
  schedule:
    - cron: "0 6 * * *"

env:
  # collections discovery is public for dedl
  EODAG__DEDL__SEARCH__NEED_AUTH: false
  EODAG__WEKEO_MAIN__AUTH__CREDENTIALS__USERNAME: ${{ secrets.EODAG__WEKEO_MAIN__AUTH__CREDENTIALS__USERNAME }}
  EODAG__WEKEO_MAIN__AUTH__CREDENTIALS__PASSWORD: ${{ secrets.EODAG__WEKEO_MAIN__AUTH__CREDENTIALS__PASSWORD }}

jobs:
  fetch-collections:
    name: Fetch providers for new collections
    runs-on: ubuntu-latest
    steps:
    - name: Checkout the repo
      uses: actions/checkout@v4
      with:
        # Get history and tags for SCM versioning to work
        fetch-depth: 0
    - name: Install the latest version of uv with cache enabled
      uses: astral-sh/setup-uv@v9.0.0
      with:
        version: "latest"
        enable-cache: true
    - name: Install eodag in venv
      run: |
        uv venv
        source .venv/bin/activate
        uv pip install ".[all-providers]"
    - name: Fetch external collections data
      env:
        JSON_OUTPUT_FILE: "eodag/resources/ext_collections.json"
      run: |
        source .venv/bin/activate
        eodag -vvv discover --storage ${JSON_OUTPUT_FILE}
        # add empty line at end of file
        sed -i -e '$a\' ${JSON_OUTPUT_FILE}
    - name: List collections by allowing the fetch and hightlight warning logs
      env:
        EODAG_VALIDATE_COLLECTIONS: True
        EODAG_EXT_COLLECTIONS_CFG_FILE: "eodag/resources/ext_collections.json"
      run: |
        source .venv/bin/activate
        eodag -vvv list 2> >(tee /tmp/logs.txt)
        cat /tmp/logs.txt | grep -i "\[WARNING" > /tmp/warnings.txt
    - name: Upload full logs
      uses: actions/upload-artifact@v5
      id: artifact-upload-step
      with:
        name: collection-logs-full
        path: /tmp/logs.txt
        retention-days: 30
    - name: logs artifact URL
      run:  |
        echo "${{ steps.artifact-upload-step.outputs.artifact-url }}" > /tmp/logs_url.txt
        cat /tmp/logs_url.txt
    - name: Commit changes if any
      id: commit
      run: |
        git config user.name "github-actions[bot]"
        git config user.email "'github-actions[bot]@users.noreply.github.com"
        git add "eodag/resources/ext_collections.json"

        # Check if there are changes to commit
        if git diff --cached --quiet; then
          echo "No changes detected in ext_collections.json"
          echo "has_changes=false" >> $GITHUB_OUTPUT
          echo "commit_sha=${GITHUB_SHA}" >> $GITHUB_OUTPUT
        else
          git commit -m "chore: update external collections reference"
          echo "has_changes=true" >> $GITHUB_OUTPUT
          echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
        fi

    - name: Download reference file for comparison
      run: |
        source .venv/bin/activate
        JSON_REF_FILE=$(python -c "import eodag; print(eodag.config.EXT_COLLECTIONS_CONF_URI)")
        curl -s "${JSON_REF_FILE}" -o /tmp/ref_collections.json

    - name: Generate comparison and update job summary
      run: |
        source .venv/bin/activate
        COMMIT_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${{ steps.commit.outputs.commit_sha }}"

        echo "Update external collections reference from daily fetch. See [Python API User Guide / Collections discovery](https://eodag.readthedocs.io/en/latest/notebooks/api_user_guide/1_providers_collections_available.html#Collections-discovery)" >> $GITHUB_STEP_SUMMARY

        echo '### Collections validation' >> $GITHUB_STEP_SUMMARY
        if [ -s "/tmp/warnings.txt" ]; then
          WARN_COUNT=$(wc -l < "/tmp/warnings.txt")
          echo "⚠️ **${WARN_COUNT} warning(s) detected** - please review the [validation logs (logs.txt)](`cat /tmp/logs_url.txt`)" >> $GITHUB_STEP_SUMMARY
        else
          echo "✅ **No warnings detected**" >> $GITHUB_STEP_SUMMARY
        fi

        echo '### Changed file' >> $GITHUB_STEP_SUMMARY
        if [ "${{ steps.commit.outputs.has_changes }}" = "true" ]; then
          git show --name-only --format=tformat: >> $GITHUB_STEP_SUMMARY
        else
          echo "No changes detected" >> $GITHUB_STEP_SUMMARY
        fi
        echo '' >> $GITHUB_STEP_SUMMARY
        echo "commit [${{ steps.commit.outputs.commit_sha }}](${COMMIT_URL})" >> $GITHUB_STEP_SUMMARY
        echo '' >> $GITHUB_STEP_SUMMARY
        # Use our Python comparison script for better diff analysis
        python utils/ext_collections_cmp.py /tmp/ref_collections.json "eodag/resources/ext_collections.json" > /tmp/comparison_output.md
        # Add comparison output to step summary
        cat /tmp/comparison_output.md >> $GITHUB_STEP_SUMMARY
    - name: Create PR body
      run: |
        source .venv/bin/activate
        COMMIT_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${{ steps.commit.outputs.commit_sha }}"

        {
          # Include header info
          echo "Update external collections reference from daily fetch. See [Python API User Guide / Collections discovery](https://eodag.readthedocs.io/en/latest/notebooks/api_user_guide/1_providers_collections_available.html#Collections-discovery)"
          echo ""
          echo '### Collections validation'
          if [ -s "/tmp/warnings.txt" ]; then
            WARN_COUNT=$(wc -l < "/tmp/warnings.txt")
            echo "⚠️ **${WARN_COUNT} warning(s) detected** - please review the [validation logs (logs.txt)](`cat /tmp/logs_url.txt`)"
          else
            echo "✅ **No warnings detected**"
          fi
          echo ""
          echo "### Changed file"
          if [ "${{ steps.commit.outputs.has_changes }}" = "true" ]; then
            git show --name-only --format=tformat:
          else
            echo "No changes detected"
          fi
          echo ""
          echo "commit [${{ steps.commit.outputs.commit_sha }}](${COMMIT_URL})"
          echo ""
          echo "---"
          echo "**Note:** Detailed diffs are available in the [job summary](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
          echo ""
          # Generate comparison without detailed diffs for PR body
          if ! python utils/ext_collections_cmp.py /tmp/ref_collections.json "eodag/resources/ext_collections.json" --no-details --job-url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"; then
            echo "Error running comparison script, but proceeding with PR creation."
            echo ""
            echo "### Summary:"
            echo "- Comparison script failed - please check job logs"
          fi
        } > /tmp/pr_col_body.md
    - name: Create Pull Request
      uses: peter-evans/create-pull-request@v6
      with:
        branch: external-collections-ref-update
        delete-branch: true
        title: 'chore: update external collections reference'
        body-path: /tmp/pr_col_body.md
        labels: |
          automated pr

  fetch-product-types:
    name: Fetch providers for new product-types
    runs-on: ubuntu-latest
    steps:
    - name: Checkout the repo
      uses: actions/checkout@v4
      with:
        # Get history and tags for SCM versioning to work
        fetch-depth: 0
    - name: Install the latest version of uv with cache enabled
      uses: astral-sh/setup-uv@v9.0.0
      with:
        version: "latest"
        enable-cache: true
    - name: Install eodag v3 in venv
      run: |
        uv venv
        source .venv/bin/activate
        # Run from /tmp to avoid installing eodag module
        cd /tmp && uv pip install "eodag[all-providers]~=3.0"
    - name: Fetch external product-types data
      env:
        JSON_OUTPUT_FILE: "eodag/resources/ext_product_types.json"
      run: |
        source .venv/bin/activate
        cd "$GITHUB_WORKSPACE"
        eodag -vvv discover --storage ${JSON_OUTPUT_FILE}
        # add empty line at end of file
        sed -i -e '$a\' ${JSON_OUTPUT_FILE}

    - name: Commit changes if any
      id: commit
      run: |
        git config user.name "github-actions[bot]"
        git config user.email "'github-actions[bot]@users.noreply.github.com"
        git add "eodag/resources/ext_product_types.json"

        # Check if there are changes to commit
        if git diff --cached --quiet; then
          echo "No changes detected in ext_product_types.json"
          echo "has_changes=false" >> $GITHUB_OUTPUT
          echo "commit_sha=${GITHUB_SHA}" >> $GITHUB_OUTPUT
        else
          git commit -m "chore: update external product-types reference"
          echo "has_changes=true" >> $GITHUB_OUTPUT
          echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
        fi

    - name: Download reference file for comparison
      run: |
        source .venv/bin/activate
        # Run from /tmp to avoid importing local eodag module
        JSON_REF_FILE=$(cd /tmp && python -c "import eodag; print(eodag.config.EXT_PRODUCT_TYPES_CONF_URI)")
        curl -s "${JSON_REF_FILE}" -o /tmp/ref_product_types.json

    - name: Generate comparison and update job summary
      run: |
        source .venv/bin/activate
        COMMIT_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${{ steps.commit.outputs.commit_sha }}"

        echo "Update external product-types reference from daily fetch. See [Python API User Guide / Collections discovery](https://eodag.readthedocs.io/en/latest/notebooks/api_user_guide/1_providers_collections_available.html#Collections-discovery)" >> $GITHUB_STEP_SUMMARY
        echo '### Changed file' >> $GITHUB_STEP_SUMMARY
        if [ "${{ steps.commit.outputs.has_changes }}" = "true" ]; then
          git show --name-only --format=tformat: >> $GITHUB_STEP_SUMMARY
        else
          echo "No changes detected" >> $GITHUB_STEP_SUMMARY
        fi
        echo '' >> $GITHUB_STEP_SUMMARY
        echo "commit [${{ steps.commit.outputs.commit_sha }}](${COMMIT_URL})" >> $GITHUB_STEP_SUMMARY
        echo '' >> $GITHUB_STEP_SUMMARY

        # Use our Python comparison script for better diff analysis
        python utils/ext_product_types_cmp.py /tmp/ref_product_types.json "eodag/resources/ext_product_types.json" > /tmp/pt_cmp_output.md
        # Add comparison output to step summary
        cat /tmp/pt_cmp_output.md >> $GITHUB_STEP_SUMMARY

    - name: Create PR body
      run: |
        source .venv/bin/activate
        COMMIT_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${{ steps.commit.outputs.commit_sha }}"

        {
          # Include header info
          echo "Update external product-types reference from daily fetch. See [Python API User Guide / Collections discovery](https://eodag.readthedocs.io/en/latest/notebooks/api_user_guide/1_providers_collections_available.html#Collections-discovery)"
          echo ""
          echo "### Changed file"
          if [ "${{ steps.commit.outputs.has_changes }}" = "true" ]; then
            git show --name-only --format=tformat:
          else
            echo "No changes detected"
          fi
          echo ""
          echo "commit [${{ steps.commit.outputs.commit_sha }}](${COMMIT_URL})"
          echo ""
          echo "---"
          echo "**Note:** Detailed diffs are available in the [job summary](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
          echo ""
          # Generate comparison without detailed diffs for PR body
          if ! python utils/ext_product_types_cmp.py /tmp/ref_product_types.json "eodag/resources/ext_product_types.json" --no-details --job-url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"; then
            echo "Error running comparison script, but proceeding with PR creation."
            echo ""
            echo "### Summary:"
            echo "- Comparison script failed - please check job logs"
          fi
        } > /tmp/pr_pt_body.md
    - name: Create Pull Request
      uses: peter-evans/create-pull-request@v6
      with:
        branch: external-product-types-ref-update
        delete-branch: true
        title: 'chore: update external product types reference'
        body-path: /tmp/pr_pt_body.md
        labels: |
          automated pr