From f152d3966b24f329e320e950ea16d070fd73b67f Mon Sep 17 00:00:00 2001 From: gitcommitshow <56937085+gitcommitshow@users.noreply.github.com> Date: Fri, 18 Oct 2024 06:41:29 +0530 Subject: [PATCH] feat: add a pr list page (#57) --- .env.sample | 13 ++- app.js | 9 ++ src/helpers.js | 226 ++++++++++++++++++++++++++++++++++++++++++-- src/routes.js | 188 ++++++++++++++++++++++++++++++++++++ src/storage.js | 11 +++ views/download.html | 1 + 6 files changed, 433 insertions(+), 15 deletions(-) diff --git a/.env.sample b/.env.sample index 8885605..402a84d 100644 --- a/.env.sample +++ b/.env.sample @@ -1,7 +1,10 @@ -APP_ID="11" -GITHUB_APP_PRIVATE_KEY_BASE64="base64 encoded private key" -PRIVATE_KEY_PATH="very/secure/location/gh_app_key.pem" -WEBHOOK_SECRET="secret" WEBSITE_ADDRESS="https://github.app.home" LOGIN_USER=username -LOGIN_PASSWORD=strongpassword \ No newline at end of file +LOGIN_PASSWORD=strongpassword +DEFAULT_GITHUB_ORG=Git-Commit-Show +GITHUB_BOT_USERS=dependabot[bot],devops-github-rudderstack +GITHUB_ORG_MEMBERS= +APP_ID="11" +WEBHOOK_SECRET="secret" +PRIVATE_KEY_PATH="very/secure/location/gh_app_key.pem" +GITHUB_APP_PRIVATE_KEY_BASE64="base64 encoded private key" \ No newline at end of file diff --git a/app.js b/app.js index ccc01ef..a5b3cdf 100644 --- a/app.js +++ b/app.js @@ -233,6 +233,15 @@ http case "POST /cla": routes.submitCla(req, res, app); break; + case "GET /contributions/sync": + routes.syncPullRequests(req, res, app); + break; + case "GET /contributions": + routes.listPullRequests(req, res, app); + break; + case "GET /contributions/pr": + routes.getPullRequestDetail(req, res, app); + break; case "POST /api/webhook": middleware(req, res); break; diff --git a/src/helpers.js b/src/helpers.js index e864065..abc44a3 100644 --- a/src/helpers.js +++ b/src/helpers.js @@ -1,12 +1,14 @@ import { storage } from "./storage.js"; import { resolve } from "path"; import { PROJECT_ROOT_PATH } from "./config.js"; +import url from "node:url"; export function parseUrlQueryParams(urlString) { if(!urlString) return urlString; try{ - const url = new URL(urlString); - const params = new URLSearchParams(url.search); + const parsedUrl = url.parse(urlString) + const query = parsedUrl.query; + const params = new URLSearchParams(query); return Object.fromEntries(params.entries()); } catch(err){ console.error(err); @@ -32,7 +34,7 @@ export function isCLARequired(pullRequest) { console.log("This PR is from a bot. So no CLA required."); return false; } - if (!isExternalContribution(pullRequest)) { + if (!isExternalContributionMaybe(pullRequest)) { console.log("This PR is an internal contribution. So no CLA required."); return false; } @@ -48,7 +50,7 @@ export function isMessageAfterMergeRequired(pullRequest) { console.log("This PR is from a bot. So no message after merge required."); return false; } - if (!isExternalContribution(pullRequest)) { + if (!isExternalContributionMaybe(pullRequest)) { console.log( "This PR is an internal contribution. So no message after merge required.", ); @@ -57,13 +59,67 @@ export function isMessageAfterMergeRequired(pullRequest) { return true; } -export function isExternalContribution(pullRequest) { - if ( - pullRequest?.head?.repo?.full_name !== pullRequest?.base?.repo?.full_name - ) { +/** + * Whether a pull request is a contribution by external user who has bot been associated with the repo + * @param {Object} pullRequest + * @returns {boolean | undefined} - boolean when confirmed, undefined when not confirmed + */ +export function isExternalContributionMaybe(pullRequest) { + const { owner, repo } = parseRepoUrl(pullRequest?.repository_url || pullRequest?.base?.repo?.html_url) || {}; + const username = pullRequest?.user?.login; + if (typeof pullRequest?.author_association === "string") { + // OWNER: Author is the owner of the repository. + // MEMBER: Author is a member of the organization that owns the repository. + // COLLABORATOR: Author has been invited to collaborate on the repository. + // CONTRIBUTOR: Author has previously committed to the repository. + // FIRST_TIMER: Author has not previously committed to GitHub. + // FIRST_TIME_CONTRIBUTOR: Author has not previously committed to the repository. + // MANNEQUIN: Author is a placeholder for an unclaimed user. + // NONE: Author has no association with the repository (or doesn't want to make his association public). + switch (pullRequest.author_association.toUpperCase()) { + case "OWNER": + storage.cache.set(false, username, "contribution", "external", owner, repo); + return false; + case "MEMBER": + storage.cache.set(false, username, "contribution", "external", owner, repo); + return false; + case "COLLABORATOR": + pullRequest.isExternalContribution = false; + storage.cache.set(false, username, "contribution", "external", owner, repo); + return false; + default: + //Will need more checks to verify author relation with the repo + break; + } + } + if (pullRequest?.head?.repo?.full_name !== pullRequest?.base?.repo?.full_name) { + storage.cache.set(true, username, "contribution", "external", owner, repo); return true; } - return false; + // Utilize cache if possible + const isConfirmedToBeExternalContributionInPast = storage.cache.get(username, "contribution", "external", owner, repo); + if (typeof isConfirmedToBeExternalContributionInPast === "boolean") { + return isConfirmedToBeExternalContributionInPast + } + // Ambigous results after this point. + // Cannot confirm whether an external contribution or not. + // Need more reliable check. + return undefined; +} + +async function isExternalContribution(octokit, pullRequest) { + const probablisticResult = isExternalContributionMaybe(pullRequest); + if (typeof probablisticResult === "boolean") { + // Boolean is returned when the probabilistic check is sufficient + return probablisticResult; + } + const username = pullRequest?.user?.login; + const { owner, repo } = parseRepoUrl(pullRequest?.repository_url || pullRequest?.base?.repo?.html_url) || {}; + //TODO: Handle failure in checking permissions for the user + const deterministicPermissionCheck = await isAllowedToWriteToTheRepo(octokit, username, owner, repo); + pullRequest.isExternalContribution = deterministicPermissionCheck; + storage.cache.set(pullRequest, username, "contribution", "external", owner, repo); + return deterministicPermissionCheck; } export function isABot(user) { @@ -229,6 +285,7 @@ export function getMessage(name, context) { } export function isCLASigned(username) { + if (!username) return const userData = storage.get({ username: username, terms: "on" }); if (userData?.length > 0) { return true; @@ -261,6 +318,17 @@ export function jsonToCSV(arr) { return csvRows.join('\n'); } +/** + * Authenticate as app installation for the org + * Authenticating as an app installation lets your app access resources that are owned by the user or organization + * that installed the app. Authenticating as an app installation is ideal for automation workflows + * that don't involve user input. + * Check out { @link https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/about-authentication-with-a-github-app GitHub Docs for Authentication } + * and { @tutorial https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation GitHub Docs for Authenticating App Installation} + * @param {Object} app + * @param {string} org + * @returns + */ export async function getOctokitForOrg(app, org) { // Find the installation for the organization for await (const { installation } of app.eachInstallation.iterator()) { @@ -270,6 +338,12 @@ export async function getOctokitForOrg(app, org) { return octokit } } + console.error("No GitHub App installation found for " + org); + // Fall back authentication method + const DEFAULT_GITHUB_ORG = process.env.DEFAULT_GITHUB_ORG; + if (DEFAULT_GITHUB_ORG && org !== DEFAULT_GITHUB_ORG) { + return await getOctokitForOrg(app, DEFAULT_GITHUB_ORG); + } } export async function verifyGitHubAppAuthenticationAndAccess(app) { @@ -330,7 +404,139 @@ function parseRepoUrl(repoUrl) { return { owner: segments[segments.length - 2], repo: segments[segments.length - 1] }; } catch (error) { - // Handle cases where URL constructor fails (e.g., SSH URLs) + //TODO: Handle cases where URL constructor fails (e.g., SSH URLs) return null; } +} + +export async function getOpenPullRequests(octokit, owner, repo, options) { + let query = `is:pr is:open` + (repo ? ` repo:${owner + "/" + repo}` : ` org:${owner}`); + const BOT_USERS = process.env.GITHUB_BOT_USERS ? process.env.GITHUB_BOT_USERS.split(",")?.map((item) => item?.trim()) : null; + const GITHUB_ORG_MEMBERS = process.env.GITHUB_ORG_MEMBERS ? process.env.GITHUB_ORG_MEMBERS.split(",")?.map((item) => item?.trim()) : null; + // Remove results from bots or internal team members + BOT_USERS?.forEach((botUser) => query += (" -author:" + botUser)); + GITHUB_ORG_MEMBERS?.forEach((orgMember) => query += (" -author:" + orgMember)); + const response = await octokit.rest.search.issuesAndPullRequests({ + q: query, + per_page: 100, + page: options?.page || 1, + sort: 'created', + order: 'desc' + }); + console.log(response?.data?.total_count + " results found for search: " + query); + const humanPRs = response?.data?.items?.filter(pr => pr.user && pr.user.type === 'User'); + return humanPRs; +} + +export async function getOpenExternalPullRequests(app, owner, repo, options) { + try { + const octokit = await getOctokitForOrg(app, owner); + if (!octokit) { + throw new Error("Failed to search PR because of undefined octokit intance") + } + const openPRs = await getOpenPullRequests(octokit, owner, repo, options); + if (!Array.isArray(openPRs)) { + return; + } + // Send only the external PRs + const openExternalPRs = [] + for (const pr of openPRs) { + try { + pr.isExternalContribution = await isExternalContribution(octokit, pr); + if (pr.isExternalContribution) { + openExternalPRs.push(pr); + } + } catch (err) { + // Some error occurred, so we cannot deterministically say whether it is an external contribution or not + pr.isExternalContribution = undefined; + // We are anyways going to send this in the external open PR list + openExternalPRs.push(pr); + } + } + return openExternalPRs + } catch (err) { + return + } +} + +export function timeAgo(date) { + if (!date) return ''; + if (typeof date === 'string') { + date = new Date(date); + } + const now = new Date(); + const seconds = Math.floor((now - date) / 1000); + let interval = Math.floor(seconds / 31536000); + + if (interval > 1) { + return `${interval} years ago`; + } + interval = Math.floor(seconds / 2592000); + if (interval > 1) { + return `${interval} months ago`; + } + interval = Math.floor(seconds / 604800); + if (interval > 1) { + return `${interval} weeks ago`; + } + interval = Math.floor(seconds / 86400); + if (interval > 1) { + return `${interval} days ago`; + } + interval = Math.floor(seconds / 3600); + if (interval > 1) { + return `${interval} hours ago`; + } + interval = Math.floor(seconds / 60); + if (interval > 1) { + return `${interval} minutes ago`; + } + return `${seconds} seconds ago`; +} + +/** + * Check user permissions for a repository + * The authenticating octokit instance must have "Metadata" repository permissions (read) + * @param {string} username + * @param {string} owner + * @param {string} repo + * @returns {boolean} + */ +async function isAllowedToWriteToTheRepo(octokit, username, owner, repo,) { + try { + const result = await octokit.rest.repos.getCollaboratorPermissionLevel({ + owner, + repo, + username, + }); + if (["admin", "write"].includes(result?.permission)) { + return true + } + if (["admin", "maintain", "write"].includes(result?.role_name)) { + return true + } + return false; + } catch (err) { + // If 403 error "HttpError: Resource not accessible by integration" + // The app is not installed in that repo + // Only "metadata:repository" permission is needed for this api, which all gh apps have wherever they are installed + console.log("Failed to check if a " + username + " is allowed to write to " + owner + "/" + repo); + console.error(err); + throw new Error("Failed to check user permission for the repo") + } +} + +export async function getPullRequestDetail(app, owner, repo, number) { + const octokit = await getOctokitForOrg(app, owner); + if (!octokit) { + throw new Error("Failed to search PR because of undefined octokit intance") + } + const { data } = await octokit.rest.pulls.get({ + owner: owner, + repo: repo, + pull_number: number + }); + if (!data) return data; + const pr = Object.assign({}, data, { isExternalContribution: isExternalContributionMaybe(data) }); + return pr; } \ No newline at end of file diff --git a/src/routes.js b/src/routes.js index 64af41b..97833ba 100644 --- a/src/routes.js +++ b/src/routes.js @@ -5,10 +5,14 @@ import { PROJECT_ROOT_PATH } from "./config.js"; import { storage } from "./storage.js"; import { sanitizeInput } from "./sanitize.js"; import { + isCLASigned, afterCLA, queryStringToJson, parseUrlQueryParams, jsonToCSV, + getOpenExternalPullRequests, + getPullRequestDetail, + timeAgo } from "./helpers.js"; import { isPasswordValid } from "./auth.js"; @@ -166,9 +170,193 @@ export const routes = { }) }, + syncPullRequests(req, res, app) { + if (err) { + res.writeHead(404); + res.write("Not implemented yet"); + return res.end(); + } + res.writeHead(302, { + Location: "/pr", + }); + return res.end(); + }, + + async listPullRequests(req, res, app) { + const { org, repo, page } = parseUrlQueryParams(req.url) || {}; + if (!org) { + res.writeHead(400); + return res.end("Please add org parameter in the url e.g. ?org=my-github-org-name"); + } + const prs = await getOpenExternalPullRequests(app, org, repo, { page: page }); + if (req.headers['content-type']?.toLowerCase() === 'application/json') { + res.setHeader('Content-Type', 'application/json'); + const jsonString = prs ? JSON.stringify(prs, null, 2) : ("No Open Pull Requests found (or you don't have access to search PRs for " + org); + return res.end(jsonString); + } + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.write(` + + + Recent Contributions (Open) + + + +

Recent Contributions (Open)

+
+ ${Array.isArray(prs) && prs.length > 0 ? `` : ""} +
+
+ ${groupPullRequestsByUser(prs)} +
+
+ ${groupPullRequestsByRepo(prs)} +
+

+ + + + `); + res.end(); + }, + async getPullRequestDetail(req, res, app) { + const { org, repo, number } = parseUrlQueryParams(req.url) || {}; + if (!org) { + res.writeHead(400); + return res.end("Please add org parameter in the url e.g. ?org=my-github-org-name"); + } + const pr = await getPullRequestDetail(app, org, repo, number); + if (req.headers['content-type']?.toLowerCase() === 'application/json') { + res.setHeader('Content-Type', 'application/json'); + const jsonString = pr ? JSON.stringify(pr, null, 2) : ("No Pull Requests found (or you don't have access to get this PR " + org + "/" + repo + "/" + number); + return res.end(jsonString); + } + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end(` + + + Pull Request Detail + + + +

Pull Request Details

+ + 
+                          ${JSON.stringify(pr, null, 2)}
+
+ +

+ + + `); + }, + // ${!Array.isArray(prs) || prs?.length < 1 ? "No contributions found! (Might be an access issue)" : prs?.map(pr => `
  • ${pr?.user?.login} contributed a PR - ${pr?.title} [${pr?.labels?.map(label => label?.name).join('] [')}] updated ${timeAgo(pr?.updated_at)}
    • `).join('')} default(req, res) { res.writeHead(404); res.write("Path not found!"); return res.end(); }, }; + + + +function groupPullRequestsByUser(prs) { + if (!Array.isArray(prs) || prs?.length < 1) { + return "No recent contributions found" + } + const grouped = prs?.reduce((acc, pr) => { + if (!acc[pr?.user?.login]) { + acc[pr?.user?.login] = []; + } + acc[pr?.user?.login].push(pr); + return acc; + }, {}); + let html = ''; + for (const user in grouped) { + html += `

    ${user} ${isCLASigned(user) ? "✅" : ""}

    '; + } + return html; +} + +function groupPullRequestsByRepo(prs) { + if (!Array.isArray(prs) || prs?.length < 1) { + return "No recent contributions found" + } + const grouped = prs?.reduce((acc, pr) => { + if (!acc[pr?.repository_url]) { + acc[pr?.repository_url] = []; + } + acc[pr?.repository_url].push(pr); + return acc; + }, {}); + let html = ''; + for (const repo in grouped) { + const repoName = repo.split('/').slice(-1)[0]; + const org = repo.split('/').slice(-2)[0]; + html += `

    ${repoName}

    '; + } + return html; +} \ No newline at end of file diff --git a/src/storage.js b/src/storage.js index 493452d..a6a1b45 100644 --- a/src/storage.js +++ b/src/storage.js @@ -4,6 +4,7 @@ import { PROJECT_ROOT_PATH } from "./config.js"; const dbPath = process.env.DB_PATH || resolve(PROJECT_ROOT_PATH, "db.json"); createFileIfMissing(dbPath); +const CACHE = new Map(); function createFileIfMissing(path) { try { @@ -43,4 +44,14 @@ export const storage = { return true; }); }, + cache: { + get: function (...args) { + const key = args.join("/"); + return CACHE.get(key); + }, + set: function (value, ...args) { + const key = args.join("/"); + return CACHE.set(key, value); + } + } }; diff --git a/views/download.html b/views/download.html index 825a553..d9bbdcd 100644 --- a/views/download.html +++ b/views/download.html @@ -41,6 +41,7 @@

    ⬇️ Download Center