Added org github tokens to check untracked private repositories

.github/workflows/check-untracked-repos.yml

@@ -36,8 +36,22 @@ jobs:
       - name: Install Rust stable
         uses: ./.github/actions/setup-rust
 
+      # Used to detect private repos in allowed-github-orgs
+      - name: Generate GitHub App tokens
+        uses: ./.github/actions/generate-tokens
+        id: generate-tokens
+        with:
+          app-id: ${{ secrets.SYNC_TEAM_GH_APP_ID }}
+          private-key: ${{ secrets.SYNC_TEAM_GH_APP_PRIVATE_KEY }}
+
       - name: Check untracked repositories
         id: check
+        env:
+          GITHUB_TOKEN_RUST_LANG: ${{ steps.generate-tokens.outputs.rust-lang-token }}
+          GITHUB_TOKEN_RUST_LANG_DEPRECATED: ${{ steps.generate-tokens.outputs.rust-lang-deprecated-token }}
+          GITHUB_TOKEN_RUST_LANG_NURSERY: ${{ steps.generate-tokens.outputs.rust-lang-nursery-token }}
+          GITHUB_TOKEN_RUST_ANALYZER: ${{ steps.generate-tokens.outputs.rust-analyzer-token }}
+          GITHUB_TOKEN_RUST_DEV_TOOLS: ${{ steps.generate-tokens.outputs.rust-dev-tools-token }}
         run: |
           cargo build --release
 

src/api/github.rs

@@ -1,3 +1,4 @@
+use crate::sync::GitHubTokens;
 use crate::sync::utils::ResponseExt;
 use anyhow::{Context, Error, bail};
 use base64::Engine;
@@ -6,6 +7,7 @@ use chrono::{DateTime, Duration, Utc};
 use reqwest::header::{self, HeaderValue};
 use reqwest::{Client, ClientBuilder, RequestBuilder};
 use reqwest::{Method, StatusCode};
+use secrecy::ExposeSecret;
 use std::borrow::Cow;
 use std::collections::HashMap;
 
@@ -40,6 +42,7 @@ struct GraphNodes<T> {
 pub(crate) struct GitHubApi {
     http: Client,
     token: Option<String>,
+    org_tokens: Option<GitHubTokens>,
 }
 
 impl GitHubApi {
@@ -50,12 +53,25 @@ impl GitHubApi {
                 .build()
                 .unwrap(),
             token: std::env::var(TOKEN_VAR).ok(),
+            org_tokens: None,
+        }
+    }
+
+    pub(crate) fn new_with_org_tokens() -> Self {
+        GitHubApi {
+            http: ClientBuilder::new()
+                .user_agent(crate::USER_AGENT)
+                .build()
+                .unwrap(),
+            token: std::env::var(TOKEN_VAR).ok(),
+            org_tokens: Some(GitHubTokens::from_env_org_tokens_only()),
         }
     }
 
     fn prepare(
         &self,
         require_auth: bool,
+        org: Option<&str>,
         method: Method,
         url: &str,
     ) -> Result<RequestBuilder, Error> {
@@ -69,7 +85,16 @@ impl GitHubApi {
         }
 
         let mut req = self.http.request(method, url.as_ref());
-        if let Some(token) = &self.token {
+        let token = match org {
+            Some(org) => self
+                .org_tokens
+                .as_ref()
+                .and_then(|tokens| tokens.get_organization_token(org).ok())
+                .map(|token| token.expose_secret()),
+            None => self.token.as_deref(),
+        };
+
+        if let Some(token) = token {
             req = req.header(
                 header::AUTHORIZATION,
                 HeaderValue::from_str(&format!("token {token}"))?,
@@ -89,7 +114,7 @@ impl GitHubApi {
             variables: V,
         }
         let res: GraphResult<R> = self
-            .prepare(true, Method::POST, "graphql")?
+            .prepare(true, None, Method::POST, "graphql")?
             .json(&Request { query, variables })
             .send()
             .await?
@@ -113,20 +138,20 @@ impl GitHubApi {
     }
 
     pub(crate) async fn user(&self, login: &str) -> Result<User, Error> {
-        self.prepare(false, Method::GET, &format!("users/{login}"))?
+        self.prepare(false, None, Method::GET, &format!("users/{login}"))?
             .send()
             .await?
             .error_for_status()?
             .json_annotated()
             .await
     }
 
-    pub(crate) async fn get<T>(&self, url: &str) -> Result<T, Error>
+    pub(crate) async fn get<T>(&self, org: Option<&str>, url: &str) -> Result<T, Error>
     where
         T: serde::de::DeserializeOwned,
     {
         loop {
-            let response = self.prepare(false, Method::GET, url)?.send().await?;
+            let response = self.prepare(false, org, Method::GET, url)?.send().await?;
 
             let status = response.status();
             if status != StatusCode::OK {
@@ -378,7 +403,7 @@ query($query: String!, $issueLimit: Int!, $commentLimit: Int!) {
             items: Vec<octocrab::models::search::CommitSearchResultItem>,
         }
 
-        let response: Response = self.get(&format!("search/commits?q=author:{username}+org:{org}&sort=author-date&order=desc&per_page={limit}")).await?;
+        let response: Response = self.get(None, &format!("search/commits?q=author:{username}+org:{org}&sort=author-date&order=desc&per_page={limit}")).await?;
         Ok(response
             .items
             .into_iter()
@@ -419,3 +444,78 @@ pub struct CommitInfo {
     pub repo_name: String,
     pub created_at: DateTime<Utc>,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use secrecy::SecretString;
+
+    #[test]
+    fn prepare_uses_organization_token() {
+        let github = github_with_org_tokens(HashMap::from([(
+            "rust-lang".to_string(),
+            SecretString::from("organization-token"),
+        )]));
+
+        let request = github
+            .prepare(
+                false,
+                Some("rust-lang"),
+                Method::GET,
+                "orgs/rust-lang/repos",
+            )
+            .unwrap()
+            .build()
+            .unwrap();
+
+        assert_eq!(
+            request.headers().get(header::AUTHORIZATION).unwrap(),
+            "token organization-token"
+        );
+    }
+
+    #[test]
+    fn prepare_without_organization_token_is_unauthenticated() {
+        let github = github_with_org_tokens(HashMap::new());
+
+        let request = github
+            .prepare(
+                false,
+                Some("rust-lang"),
+                Method::GET,
+                "orgs/rust-lang/repos",
+            )
+            .unwrap()
+            .build()
+            .unwrap();
+
+        assert!(!request.headers().contains_key(header::AUTHORIZATION));
+    }
+
+    #[test]
+    fn prepare_without_organization_uses_default_token() {
+        let github = github_with_org_tokens(HashMap::new());
+
+        let request = github
+            .prepare(false, None, Method::GET, "users/rust-lang-owner")
+            .unwrap()
+            .build()
+            .unwrap();
+
+        assert_eq!(
+            request.headers().get(header::AUTHORIZATION).unwrap(),
+            "token default-token"
+        );
+    }
+
+    fn github_with_org_tokens(org_tokens: HashMap<String, SecretString>) -> GitHubApi {
+        GitHubApi {
+            http: ClientBuilder::new().build().unwrap(),
+            token: Some("default-token".to_string()),
+            org_tokens: Some(GitHubTokens::App {
+                org_tokens,
+                enterprise_client_ctx: None,
+            }),
+        }
+    }
+}

src/ci.rs

@@ -202,7 +202,7 @@ pub async fn check_untracked_repos(
     data_dir: &Path,
     create_missing: bool,
 ) -> anyhow::Result<CheckUntrackedReposResult> {
-    let github = crate::api::github::GitHubApi::new();
+    let github = crate::api::github::GitHubApi::new_with_org_tokens();
 
     // Get allowed GitHub organizations from config instead of hardcoding
     let orgs_to_monitor: Vec<&str> = data
@@ -276,7 +276,7 @@ async fn fetch_all_github_repos(
             let url = format!("orgs/{}/repos?per_page=100&page={}", org, page);
 
             let repos: Vec<GitHubRepo> = github
-                .get(&url)
+                .get(Some(org), &url)
                 .await
                 .with_context(|| format!("Failed to fetch repos for org: {}", org))?;
 

src/sync/github/api/mod.rs

@@ -19,11 +19,11 @@ use serde::{Deserialize, de::DeserializeOwned};
 use std::collections::BTreeSet;
 use std::fmt;
 use thiserror::Error;
-use tokens::GitHubTokens;
 use url::GitHubUrl;
 
 use crate::sync::Config;
 pub(crate) use read::{GitHubApiRead, GithubRead};
+pub(crate) use tokens::GitHubTokens;
 pub(crate) use write::GitHubWrite;
 
 #[derive(Debug, Error)]

src/sync/github/api/tokens.rs

@@ -43,7 +43,7 @@ pub enum GitHubTokens {
         /// The token has to be available for the whole duration of the process.
         org_tokens: HashMap<String, SecretString>,
         /// Context for using enterprise GitHub App.
-        enterprise_client_ctx: EnterpriseAppCtx,
+        enterprise_client_ctx: Option<EnterpriseAppCtx>,
     },
     /// One token for all API calls (used with Personal Access Token).
     Pat(SecretString),
@@ -55,13 +55,7 @@ impl GitHubTokens {
     /// Parses environment variables in the format GITHUB_TOKEN_{ORG_NAME}
     /// to retrieve GitHub tokens.
     pub async fn from_env(config: &Config) -> anyhow::Result<Self> {
-        let mut tokens = HashMap::new();
-
-        for (key, value) in std::env::vars() {
-            if let Some(org_name) = org_name_from_env_var(&key) {
-                tokens.insert(org_name, SecretString::from(value));
-            }
-        }
+        let tokens = collect_org_envs();
 
         if tokens.is_empty() {
             let pat_token = std::env::var("GITHUB_TOKEN")
@@ -136,15 +130,26 @@ impl GitHubTokens {
 
             Ok(GitHubTokens::App {
                 org_tokens: tokens,
-                enterprise_client_ctx: EnterpriseAppCtx {
+                enterprise_client_ctx: Some(EnterpriseAppCtx {
                     enterprise_token,
                     org_tokens: enterprise_org_tokens,
                     enterprise_name,
-                },
+                }),
             })
         }
     }
 
+    pub fn from_env_org_tokens_only() -> Self {
+        GitHubTokens::App {
+            org_tokens: collect_org_envs(),
+            enterprise_client_ctx: None,
+        }
+    }
+
+    pub fn get_organization_token(&self, org: &str) -> anyhow::Result<&SecretString> {
+        Self::get_token_for_org(self, org, &TokenType::Organization)
+    }
+
     /// Get a token for a GitHub organization.
     /// Return an error if not present.
     pub fn get_token_for_org(
@@ -163,13 +168,21 @@ impl GitHubTokens {
                     )
                 }),
                 TokenType::EnterpriseOrganization => {
+                    let enterprise_client_ctx = enterprise_client_ctx
+                        .as_ref()
+                        .context("No enterprise GitHub App is configured")?;
+
                     enterprise_client_ctx.org_tokens.get(org).with_context(|| {
                         format!(
                             "failed to get the GitHub token environment variable for organization `{org}` for the enterprise GH app"
                         )
                     })
                 }
                 TokenType::Enterprise => {
+                    let enterprise_client_ctx = enterprise_client_ctx
+                        .as_ref()
+                        .context("No enterprise GitHub App is configured")?;
+
                     Ok(&enterprise_client_ctx.enterprise_token)
                 }
             },
@@ -183,14 +196,30 @@ impl GitHubTokens {
             GitHubTokens::App {
                 enterprise_client_ctx,
                 ..
-            } => Ok(enterprise_client_ctx.enterprise_name.as_str()),
+            } => Ok(enterprise_client_ctx
+                .as_ref()
+                .context("No enterprise GitHub App is configured")?
+                .enterprise_name
+                .as_str()),
             GitHubTokens::Pat(_) => Err(anyhow::anyhow!(
                 "No enterprise is configured when using a PAT"
             )),
         }
     }
 }
 
+fn collect_org_envs() -> HashMap<String, SecretString> {
+    let mut tokens = HashMap::new();
+
+    for (key, value) in std::env::vars() {
+        if let Some(org_name) = org_name_from_env_var(&key) {
+            tokens.insert(org_name, SecretString::from(value));
+        }
+    }
+
+    tokens
+}
+
 fn org_name_from_env_var(env_var: &str) -> Option<String> {
     env_var.strip_prefix("GITHUB_TOKEN_").map(|org| {
         // GitHub environment variables can't contain `-`, while GitHub organizations

src/sync/github/mod.rs

@@ -2,6 +2,7 @@ mod api;
 #[cfg(test)]
 mod tests;
 
+pub(crate) use self::api::GitHubTokens;
 pub(crate) use self::api::{GitHubApiRead, GitHubWrite, HttpClient};
 use self::api::{TeamPrivacy, TeamRole};
 use crate::schema;

src/sync/mod.rs

@@ -6,6 +6,8 @@ pub mod team_api;
 pub mod utils;
 mod zulip;
 
+pub(crate) use github::GitHubTokens;
+
 use std::collections::BTreeSet;
 
 use anyhow::Context;