From 0b7311cae58329fccb16404d615053a009f2e600 Mon Sep 17 00:00:00 2001 From: James B Date: Wed, 28 May 2025 15:14:34 +0100 Subject: [PATCH] Github access - switch to Oauth app credentials. And reduce API calls. https://github.com/IATI/validator-services/issues/496 Previously, this app used a BASIC_GITHUB_TOKEN with a personal access token. Also fixes a bug in getFileCommitSha where an error message could contain the wrong code. Was using branchRes var instead of fileRes --- .env.example | 3 +- .github/workflows/develop-func-deploy.yml | 12 +++- .github/workflows/prod-func-deploy.yml | 12 +++- README.md | 5 +- config/config.js | 3 +- utils/utils.js | 70 +++++++++++++---------- 6 files changed, 65 insertions(+), 40 deletions(-) diff --git a/.env.example b/.env.example index 0d68c71..28eb4e3 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,8 @@ NODE_ENV=development APPLICATIONINSIGHTS_CONNECTION_STRING= -BASIC_GITHUB_TOKEN= +GITHUB_OAUTH_APP_CLIENT_ID= +GITHUB_OAUTH_APP_CLIENT_SECRET= REDIS_PORT=6379 REDIS_HOSTNAME=redis # replace with localhost if you run outside docker diff --git a/.github/workflows/develop-func-deploy.yml b/.github/workflows/develop-func-deploy.yml index 48d1fe0..6ae99a0 100644 --- a/.github/workflows/develop-func-deploy.yml +++ b/.github/workflows/develop-func-deploy.yml @@ -32,7 +32,8 @@ env: ACR_USERNAME: ${{ secrets.ACR_USERNAME }} ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} KV_URI: ${{ secrets.DEV_KV_URI }} - BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }} + GITHUB_OAUTH_APP_CLIENT_ID: ${{ secrets.IATI_GITHUB_OAUTH_APP_CLIENT_ID }} + GITHUB_OAUTH_APP_CLIENT_SECRET: ${{ secrets.IATI_GITHUB_OAUTH_APP_CLIENT_SECRET }} DATASTORE_SERVICES_URL: ${{ secrets.DEV_DATASTORE_SERVICES_URL }} DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }} DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }} @@ -134,8 +135,13 @@ jobs: "slotSetting": false }, { - "name": "BASIC_GITHUB_TOKEN", - "value": "${{ env.BASIC_GITHUB_TOKEN }}", + "name": "GITHUB_OAUTH_APP_CLIENT_ID", + "value": "${{ env.GITHUB_OAUTH_APP_CLIENT_ID }}", + "slotSetting": false + }, + { + "name": "GITHUB_OAUTH_APP_CLIENT_SECRET", + "value": "${{ env.GITHUB_OAUTH_APP_CLIENT_SECRET }}", "slotSetting": false }, { diff --git a/.github/workflows/prod-func-deploy.yml b/.github/workflows/prod-func-deploy.yml index 59b109d..9bf6665 100644 --- a/.github/workflows/prod-func-deploy.yml +++ b/.github/workflows/prod-func-deploy.yml @@ -22,7 +22,8 @@ env: ACR_USERNAME: ${{ secrets.ACR_USERNAME }} ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} KV_URI: ${{ secrets.PROD_KV_URI }} - BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }} + GITHUB_OAUTH_APP_CLIENT_ID: ${{ secrets.IATI_GITHUB_OAUTH_APP_CLIENT_ID }} + GITHUB_OAUTH_APP_CLIENT_SECRET: ${{ secrets.IATI_GITHUB_OAUTH_APP_CLIENT_SECRET }} DATASTORE_SERVICES_URL: ${{ secrets.PROD_DATASTORE_SERVICES_URL }} DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }} DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }} @@ -119,8 +120,13 @@ jobs: "slotSetting": false }, { - "name": "BASIC_GITHUB_TOKEN", - "value": "${{ env.BASIC_GITHUB_TOKEN }}", + "name": "GITHUB_OAUTH_APP_CLIENT_ID", + "value": "${{ env.GITHUB_OAUTH_APP_CLIENT_ID }}", + "slotSetting": false + }, + { + "name": "GITHUB_OAUTH_APP_CLIENT_SECRET", + "value": "${{ env.GITHUB_OAUTH_APP_CLIENT_SECRET }}", "slotSetting": false }, { diff --git a/README.md b/README.md index 4f98b22..108837b 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,10 @@ APPLICATIONINSIGHTS_CONNECTION_STRING - Needs to be set for running locally, but will not actually report telemetry to the AppInsights instance in my experience -BASIC_GITHUB_TOKEN +GITHUB_OAUTH_APP_CLIENT_ID +GITHUB_OAUTH_APP_CLIENT_SECRET -- GitHub personal access token. This is needed to pull in the Codelists from the `IATI/IATI-Validator-Codelists` repository. Note that you cannot use a "Personal Access Token (Classic)"; you must generate a fine-grained access token. +- GitHub OAuth app. No special permissions or access needed. Optional, but you may get rate limited very easily if you don't. REDIS_PORT=6379 REDIS_HOSTNAME=redis diff --git a/config/config.js b/config/config.js index 9e7439f..33fb4c4 100644 --- a/config/config.js +++ b/config/config.js @@ -11,7 +11,8 @@ const config = { NS_PER_SEC: 1e9, VERSIONS: process.env.VERSIONS || ['2.01', '2.02', '2.03'], MAX_FILESIZE: process.env.MAX_FILESIZE || 60, - BASIC_GITHUB_TOKEN: process.env.BASIC_GITHUB_TOKEN, + GITHUB_OAUTH_APP_CLIENT_ID: process.env.GITHUB_OAUTH_APP_CLIENT_ID, + GITHUB_OAUTH_APP_CLIENT_SECRET: process.env.GITHUB_OAUTH_APP_CLIENT_SECRET, REDIS_PORT: process.env.REDIS_PORT || 6379, REDIS_CACHE_SEC: process.env.REDIS_CACHE_SEC || 86400, REDIS_KEY: process.env.REDIS_KEY, diff --git a/utils/utils.js b/utils/utils.js index 492ff7f..9e604b3 100644 --- a/utils/utils.js +++ b/utils/utils.js @@ -6,18 +6,31 @@ import { spawn } from 'child_process'; import redisclient from '../config/redis.js'; import config from '../config/config.js'; -const GITHUB_RAW = 'https://raw.githubusercontent.com'; const GITHUB_API = 'https://api.github.com'; const getFileBySha = async (owner, repo, sha, filePath) => { - // https://raw.githubusercontent.com/IATI/IATI-Validator-Codelists/{sha}/codelist_rules.json - const res = await fetch(`${GITHUB_RAW}/${owner}/${repo}/${sha}/${filePath}`, { - method: 'GET', - headers: { - Accept: 'text/plain', - Authorization: `token ${config.BASIC_GITHUB_TOKEN}`, + const headers = { Accept: "application/vnd.github.raw+json" }; + if ( + config.GITHUB_OAUTH_APP_CLIENT_ID && + config.GITHUB_OAUTH_APP_CLIENT_SECRET + ) { + headers["Authorization"] = + "Basic " + + Buffer.from( + config.GITHUB_OAUTH_APP_CLIENT_ID + + ":" + + config.GITHUB_OAUTH_APP_CLIENT_SECRET, + ).toString("base64"); + } + const res = await fetch( + `${GITHUB_API}/repos/${owner}/${repo}/contents/${filePath}?ref=${sha}`, + { + method: "GET", + headers: headers, }, - }); + ); + // This can be useful to check auth. You should see headers like x-ratelimit-limit, x-ratelimit-remaining + //console.log(res.headers); const body = res.json(); if (res.status !== 200) throw new Error( @@ -27,35 +40,32 @@ const getFileBySha = async (owner, repo, sha, filePath) => { }; const getFileCommitSha = async (owner, repo, branch, filePath) => { - // https://api.github.com/repos/IATI/IATI-Validator-Codelists/branches/version-2.03 - const branchRes = await fetch(`${GITHUB_API}/repos/${owner}/${repo}/branches/${branch}`, { - method: 'GET', - headers: { - Accept: 'application/vnd.github.v3+json', - Authorization: `token ${config.BASIC_GITHUB_TOKEN}`, - }, - }); - const branchBody = await branchRes.json(); - if (branchRes.status !== 200) - throw new Error( - `Error fetching sha from github api. Status: ${branchRes.status} Message: ${branchBody.message} `, - ); - const { sha } = branchBody.commit; - // https://api.github.com/repos/IATI/IATI-Validator-Codelists/commits?sha={sha}&path=codelist_rules.json + const headers = { Accept: "application/vnd.github.v3+json" }; + if ( + config.GITHUB_OAUTH_APP_CLIENT_ID && + config.GITHUB_OAUTH_APP_CLIENT_SECRET + ) { + headers["Authorization"] = + "Basic " + + Buffer.from( + config.GITHUB_OAUTH_APP_CLIENT_ID + + ":" + + config.GITHUB_OAUTH_APP_CLIENT_SECRET, + ).toString("base64"); + } const fileRes = await fetch( - `${GITHUB_API}/repos/${owner}/${repo}/commits?sha=${sha}&path=${filePath}`, + `${GITHUB_API}/repos/${owner}/${repo}/commits?sha=${branch}&path=${filePath}`, { - method: 'GET', - headers: { - Accept: 'application/vnd.github.v3+json', - Authorization: `token ${config.BASIC_GITHUB_TOKEN}`, - }, + method: "GET", + headers: headers, }, ); + // This can be useful to check auth. You should see headers like x-ratelimit-limit, x-ratelimit-remaining + //console.log(fileRes.headers); const fileBody = await fileRes.json(); if (fileRes.status !== 200) throw new Error( - `Error fetching sha from github api. Status: ${branchRes.status} Message: ${fileBody.message} `, + `Error fetching sha from github api. Status: ${fileRes.status} Message: ${fileBody.message} `, ); // sort to get newest commit fileBody.sort(