Github access - switch to Oauth app credentials. And reduce API calls.

.env.example

@@ -2,7 +2,8 @@ NODE_ENV=development
 
 APPLICATIONINSIGHTS_CONNECTION_STRING=
 
-BASIC_GITHUB_TOKEN=
+GITHUB_OAUTH_APP_CLIENT_ID=
+GITHUB_OAUTH_APP_CLIENT_SECRET=
 
 REDIS_PORT=6379
 REDIS_HOSTNAME=redis  # replace with localhost if you run outside docker

.github/workflows/develop-func-deploy.yml

@@ -32,7 +32,8 @@ env:
   ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
   ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
   KV_URI: ${{ secrets.DEV_KV_URI }}
-  BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }}
+  GITHUB_OAUTH_APP_CLIENT_ID: ${{ secrets.IATI_GITHUB_OAUTH_APP_CLIENT_ID }}
+  GITHUB_OAUTH_APP_CLIENT_SECRET: ${{ secrets.IATI_GITHUB_OAUTH_APP_CLIENT_SECRET }}
   DATASTORE_SERVICES_URL: ${{ secrets.DEV_DATASTORE_SERVICES_URL }}
   DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}
   DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}
@@ -134,8 +135,13 @@ jobs:
                     "slotSetting": false
                 },
                 {
-                    "name": "BASIC_GITHUB_TOKEN",
-                    "value": "${{ env.BASIC_GITHUB_TOKEN }}",
+                    "name": "GITHUB_OAUTH_APP_CLIENT_ID",
+                    "value": "${{ env.GITHUB_OAUTH_APP_CLIENT_ID }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "GITHUB_OAUTH_APP_CLIENT_SECRET",
+                    "value": "${{ env.GITHUB_OAUTH_APP_CLIENT_SECRET }}",
                     "slotSetting": false
                 },
                 {

.github/workflows/prod-func-deploy.yml

@@ -22,7 +22,8 @@ env:
   ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
   ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
   KV_URI: ${{ secrets.PROD_KV_URI }}
-  BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }}
+  GITHUB_OAUTH_APP_CLIENT_ID: ${{ secrets.IATI_GITHUB_OAUTH_APP_CLIENT_ID }}
+  GITHUB_OAUTH_APP_CLIENT_SECRET: ${{ secrets.IATI_GITHUB_OAUTH_APP_CLIENT_SECRET }}
   DATASTORE_SERVICES_URL: ${{ secrets.PROD_DATASTORE_SERVICES_URL }}
   DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}
   DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}
@@ -119,8 +120,13 @@ jobs:
                     "slotSetting": false
                 },
                 {
-                    "name": "BASIC_GITHUB_TOKEN",
-                    "value": "${{ env.BASIC_GITHUB_TOKEN }}",
+                    "name": "GITHUB_OAUTH_APP_CLIENT_ID",
+                    "value": "${{ env.GITHUB_OAUTH_APP_CLIENT_ID }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "GITHUB_OAUTH_APP_CLIENT_SECRET",
+                    "value": "${{ env.GITHUB_OAUTH_APP_CLIENT_SECRET }}",
                     "slotSetting": false
                 },
                 {

README.md

@@ -92,9 +92,10 @@ APPLICATIONINSIGHTS_CONNECTION_STRING
 
 -   Needs to be set for running locally, but will not actually report telemetry to the AppInsights instance in my experience
 
-BASIC_GITHUB_TOKEN
+GITHUB_OAUTH_APP_CLIENT_ID
+GITHUB_OAUTH_APP_CLIENT_SECRET
 
--   GitHub personal access token. This is needed to pull in the Codelists from the `IATI/IATI-Validator-Codelists` repository. Note that you cannot use a "Personal Access Token (Classic)"; you must generate a fine-grained access token.
+-   GitHub OAuth app. No special permissions or access needed. Optional, but you may get rate limited very easily if you don't.
 
 REDIS_PORT=6379  
 REDIS_HOSTNAME=redis

config/config.js

@@ -11,7 +11,8 @@ const config = {
     NS_PER_SEC: 1e9,
     VERSIONS: process.env.VERSIONS || ['2.01', '2.02', '2.03'],
     MAX_FILESIZE: process.env.MAX_FILESIZE || 60,
-    BASIC_GITHUB_TOKEN: process.env.BASIC_GITHUB_TOKEN,
+    GITHUB_OAUTH_APP_CLIENT_ID: process.env.GITHUB_OAUTH_APP_CLIENT_ID,
+    GITHUB_OAUTH_APP_CLIENT_SECRET: process.env.GITHUB_OAUTH_APP_CLIENT_SECRET,
     REDIS_PORT: process.env.REDIS_PORT || 6379,
     REDIS_CACHE_SEC: process.env.REDIS_CACHE_SEC || 86400,
     REDIS_KEY: process.env.REDIS_KEY,

utils/utils.js

@@ -6,18 +6,31 @@ import { spawn } from 'child_process';
 import redisclient from '../config/redis.js';
 import config from '../config/config.js';
 
-const GITHUB_RAW = 'https://raw.githubusercontent.com';
 const GITHUB_API = 'https://api.github.com';
 
 const getFileBySha = async (owner, repo, sha, filePath) => {
-    // https://raw.githubusercontent.com/IATI/IATI-Validator-Codelists/{sha}/codelist_rules.json
-    const res = await fetch(`${GITHUB_RAW}/${owner}/${repo}/${sha}/${filePath}`, {
-        method: 'GET',
-        headers: {
-            Accept: 'text/plain',
-            Authorization: `token ${config.BASIC_GITHUB_TOKEN}`,
+    const headers = { Accept: "application/vnd.github.raw+json" };
+    if (
+        config.GITHUB_OAUTH_APP_CLIENT_ID &&
+        config.GITHUB_OAUTH_APP_CLIENT_SECRET
+    ) {
+        headers["Authorization"] =
+            "Basic " +
+            Buffer.from(
+                config.GITHUB_OAUTH_APP_CLIENT_ID +
+                ":" +
+                config.GITHUB_OAUTH_APP_CLIENT_SECRET,
+            ).toString("base64");
+    }
+    const res = await fetch(
+        `${GITHUB_API}/repos/${owner}/${repo}/contents/${filePath}?ref=${sha}`,
+        {
+            method: "GET",
+            headers: headers,
         },
-    });
+    );
+    // This can be useful to check auth. You should see headers like x-ratelimit-limit, x-ratelimit-remaining
+    //console.log(res.headers);
     const body = res.json();
     if (res.status !== 200)
         throw new Error(
@@ -27,35 +40,32 @@ const getFileBySha = async (owner, repo, sha, filePath) => {
 };
 
 const getFileCommitSha = async (owner, repo, branch, filePath) => {
-    // https://api.github.com/repos/IATI/IATI-Validator-Codelists/branches/version-2.03
-    const branchRes = await fetch(`${GITHUB_API}/repos/${owner}/${repo}/branches/${branch}`, {
-        method: 'GET',
-        headers: {
-            Accept: 'application/vnd.github.v3+json',
-            Authorization: `token ${config.BASIC_GITHUB_TOKEN}`,
-        },
-    });
-    const branchBody = await branchRes.json();
-    if (branchRes.status !== 200)
-        throw new Error(
-            `Error fetching sha from github api. Status: ${branchRes.status} Message: ${branchBody.message} `,
-        );
-    const { sha } = branchBody.commit;
-    // https://api.github.com/repos/IATI/IATI-Validator-Codelists/commits?sha={sha}&path=codelist_rules.json
+    const headers = { Accept: "application/vnd.github.v3+json" };
+    if (
+        config.GITHUB_OAUTH_APP_CLIENT_ID &&
+        config.GITHUB_OAUTH_APP_CLIENT_SECRET
+    ) {
+        headers["Authorization"] =
+            "Basic " +
+            Buffer.from(
+                config.GITHUB_OAUTH_APP_CLIENT_ID +
+                ":" +
+                config.GITHUB_OAUTH_APP_CLIENT_SECRET,
+            ).toString("base64");
+    }
     const fileRes = await fetch(
-        `${GITHUB_API}/repos/${owner}/${repo}/commits?sha=${sha}&path=${filePath}`,
+        `${GITHUB_API}/repos/${owner}/${repo}/commits?sha=${branch}&path=${filePath}`,
         {
-            method: 'GET',
-            headers: {
-                Accept: 'application/vnd.github.v3+json',
-                Authorization: `token ${config.BASIC_GITHUB_TOKEN}`,
-            },
+            method: "GET",
+            headers: headers,
         },
     );
+    // This can be useful to check auth. You should see headers like x-ratelimit-limit, x-ratelimit-remaining
+    //console.log(fileRes.headers);
     const fileBody = await fileRes.json();
     if (fileRes.status !== 200)
         throw new Error(
-            `Error fetching sha from github api. Status: ${branchRes.status} Message: ${fileBody.message} `,
+            `Error fetching sha from github api. Status: ${fileRes.status} Message: ${fileBody.message} `,
         );
     // sort to get newest commit
     fileBody.sort(