-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable Dependency Graph Integrator to raise PRs to add workflow (#1164)
* feat: add dep graph integrator topic and flag * feat: send dep graph integrator msgs to sns * feat: query actions usage table * feat: add tests to sns functions * feat: add actions usage table to DEV db * refactor: remove dep graph PR flag * fix: correct logic after local testing * fix: remove flag * refactor: only send messages if on PROD * refactor: rename sns function to make clearer * refactor: move db call up with others
- Loading branch information
Showing
7 changed files
with
297 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
154 changes: 154 additions & 0 deletions
154
packages/repocop/src/remediation/dependency_graph-integrator/send-to-sns.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| import type { | ||
| github_languages, | ||
| guardian_github_actions_usage, | ||
| } from '@prisma/client'; | ||
| import type { Repository } from 'common/src/types'; | ||
| import { removeRepoOwner } from '../shared-utilities'; | ||
| import { | ||
| checkRepoForLanguage, | ||
| createSnsEventsForDependencyGraphIntegration, | ||
| doesRepoHaveWorkflow, | ||
| } from './send-to-sns'; | ||
|
|
||
| const fullName = 'guardian/repo-name'; | ||
| const fullName2 = 'guardian/repo2'; | ||
| const scalaLang = 'Scala'; | ||
|
|
||
| function createActionsUsage( | ||
| fullName: string, | ||
| workflowUses: string[], | ||
| ): guardian_github_actions_usage { | ||
| return { | ||
| evaluated_on: new Date('2024-01-01'), | ||
| full_name: fullName, | ||
| workflow_path: '.github/workflows/some-workflow.yaml', | ||
| workflow_uses: workflowUses, | ||
| }; | ||
| } | ||
|
|
||
| function repoWithLanguages( | ||
| fullName: string, | ||
| languages: string[], | ||
| ): github_languages { | ||
| return { | ||
| cq_sync_time: null, | ||
| cq_parent_id: null, | ||
| cq_id: '', | ||
| cq_source_name: null, | ||
| full_name: fullName, | ||
| name: fullName, | ||
| languages, | ||
| }; | ||
| } | ||
|
|
||
| function repository(fullName: string): Repository { | ||
| return { | ||
| archived: false, | ||
| name: removeRepoOwner(fullName), | ||
| full_name: fullName, | ||
| id: BigInt(1), | ||
| default_branch: null, | ||
| created_at: null, | ||
| pushed_at: null, | ||
| updated_at: null, | ||
| topics: [], | ||
| }; | ||
| } | ||
|
|
||
| function repoWithTargetLanguage(fullName: string): github_languages { | ||
| return repoWithLanguages(fullName, ['Scala', 'TypeScript']); | ||
| } | ||
|
|
||
| function repoWithoutTargetLanguage(fullName: string): github_languages { | ||
| return repoWithLanguages(fullName, ['Rust', 'Typescript']); | ||
| } | ||
|
|
||
| function repoWithDepSubmissionWorkflow( | ||
| fullName: string, | ||
| ): guardian_github_actions_usage { | ||
| return createActionsUsage(fullName, [ | ||
| 'actions/checkout@v2', | ||
| 'scalacenter/sbt-dependency-submission@v2', | ||
| 'aws-actions/configure-aws-credentials@v1', | ||
| ]); | ||
| } | ||
|
|
||
| function repoWithoutWorkflow(fullName: string): guardian_github_actions_usage { | ||
| return createActionsUsage(fullName, [ | ||
| 'actions/checkout@v2', | ||
| 'aws-actions/configure-aws-credentials@v1', | ||
| ]); | ||
| } | ||
|
|
||
| describe('When trying to find repos using Scala', () => { | ||
| test('return true if Scala is found in the repo', () => { | ||
| const result = checkRepoForLanguage( | ||
| repository(fullName), | ||
| [repoWithTargetLanguage(fullName)], | ||
| scalaLang, | ||
| ); | ||
|
|
||
| expect(result).toBe(true); | ||
| }); | ||
| test('return false if Scala is not found in the repo', () => { | ||
| const result = checkRepoForLanguage( | ||
| repository(fullName), | ||
| [repoWithoutTargetLanguage(fullName)], | ||
| scalaLang, | ||
| ); | ||
| expect(result).toBe(false); | ||
| }); | ||
| }); | ||
|
|
||
| describe('When checking a repo for an existing dependency submission workflow', () => { | ||
| test('return true if repo workflow is present', () => { | ||
| const result = doesRepoHaveWorkflow(repository(fullName), [ | ||
| repoWithDepSubmissionWorkflow(fullName), | ||
| ]); | ||
| expect(result).toBe(true); | ||
| }); | ||
| test('return false if workflow is not present', () => { | ||
| const result = doesRepoHaveWorkflow(repository(fullName), [ | ||
| repoWithoutWorkflow(fullName), | ||
| ]); | ||
| expect(result).toBe(false); | ||
| }); | ||
| }); | ||
|
|
||
| describe('When getting suitable events to send to SNS', () => { | ||
| test('return an event when a Scala repo is found without an existing workflow', () => { | ||
| const result = createSnsEventsForDependencyGraphIntegration( | ||
| [repoWithTargetLanguage(fullName)], | ||
| [repository(fullName)], | ||
| [repoWithoutWorkflow(fullName)], | ||
| ); | ||
| expect(result).toEqual([{ name: removeRepoOwner(fullName) }]); | ||
| }); | ||
| test('return empty event array when a Scala repo is found with an existing workflow', () => { | ||
| const result = createSnsEventsForDependencyGraphIntegration( | ||
| [repoWithTargetLanguage(fullName)], | ||
| [repository(fullName)], | ||
| [repoWithDepSubmissionWorkflow(fullName)], | ||
| ); | ||
| expect(result).toEqual([]); | ||
| }); | ||
| test('return empty array when non-Scala repo is found with without an existing workflow', () => { | ||
| const result = createSnsEventsForDependencyGraphIntegration( | ||
| [repoWithoutTargetLanguage(fullName)], | ||
| [repository(fullName)], | ||
| [repoWithoutWorkflow(fullName)], | ||
| ); | ||
| expect(result).toEqual([]); | ||
| }); | ||
| test('return 2 events when 2 Scala repos are found without an existing workflow', () => { | ||
| const result = createSnsEventsForDependencyGraphIntegration( | ||
| [repoWithTargetLanguage(fullName), repoWithTargetLanguage(fullName2)], | ||
| [repository(fullName), repository(fullName2)], | ||
| [repoWithoutWorkflow(fullName), repoWithoutWorkflow(fullName2)], | ||
| ); | ||
| expect(result).toEqual([ | ||
| { name: removeRepoOwner(fullName) }, | ||
| { name: removeRepoOwner(fullName2) }, | ||
| ]); | ||
| }); | ||
| }); |
103 changes: 103 additions & 0 deletions
103
packages/repocop/src/remediation/dependency_graph-integrator/send-to-sns.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| import { PublishCommand, SNSClient } from '@aws-sdk/client-sns'; | ||
| import type { | ||
| github_languages, | ||
| guardian_github_actions_usage, | ||
| } from '@prisma/client'; | ||
| import { awsClientConfig } from 'common/src/aws'; | ||
| import { shuffle } from 'common/src/functions'; | ||
| import type { | ||
| DependencyGraphIntegratorEvent, | ||
| Repository, | ||
| } from 'common/src/types'; | ||
| import type { Config } from '../../config'; | ||
| import { removeRepoOwner } from '../shared-utilities'; | ||
|
|
||
| export function checkRepoForLanguage( | ||
| repo: Repository, | ||
| languages: github_languages[], | ||
| targetLanguage: string, | ||
| ): boolean { | ||
| const languagesInRepo: string[] = | ||
| languages.find((language) => language.full_name === repo.full_name) | ||
| ?.languages ?? []; | ||
| return languagesInRepo.includes(targetLanguage); | ||
| } | ||
|
|
||
| export function doesRepoHaveWorkflow( | ||
| repo: Repository, | ||
| workflow_usages: guardian_github_actions_usage[], | ||
| ): boolean { | ||
| const actionsForRepo = workflow_usages | ||
| .filter((usages) => repo.full_name === usages.full_name) | ||
| .flatMap((workflow) => workflow.workflow_uses); | ||
|
|
||
| const dependencySubmissionWorkflow = actionsForRepo.find((action) => | ||
| action.includes('scalacenter/sbt-dependency-submission'), | ||
| ); | ||
| if (dependencySubmissionWorkflow) { | ||
| return true; | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| export function createSnsEventsForDependencyGraphIntegration( | ||
| languages: github_languages[], | ||
| productionRepos: Repository[], | ||
| workflow_usages: guardian_github_actions_usage[], | ||
| ): DependencyGraphIntegratorEvent[] { | ||
| const scalaRepos = productionRepos.filter((repo) => | ||
| checkRepoForLanguage(repo, languages, 'Scala'), | ||
| ); | ||
|
|
||
| console.log(`Found ${scalaRepos.length} Scala repos in production`); | ||
|
|
||
| const scalaReposWithoutWorkflows = scalaRepos.filter( | ||
| (repo) => !doesRepoHaveWorkflow(repo, workflow_usages), | ||
| ); | ||
|
|
||
| console.log( | ||
| `Found ${scalaRepos.length} production Scala repos without dependency submission workflows`, | ||
| ); | ||
|
|
||
| const events = scalaReposWithoutWorkflows.map((repo) => ({ | ||
| name: removeRepoOwner(repo.full_name), | ||
| })); | ||
| console.log(`Found ${events.length} events to send to SNS`); | ||
| return events; | ||
| } | ||
|
|
||
| export async function sendOneRepoToDepGraphIntegrator( | ||
| config: Config, | ||
| repoLanguages: github_languages[], | ||
| productionRepos: Repository[], | ||
| workflowUsages: guardian_github_actions_usage[], | ||
| ) { | ||
| const eventToSend = shuffle( | ||
| createSnsEventsForDependencyGraphIntegration( | ||
| repoLanguages, | ||
| productionRepos, | ||
| workflowUsages, | ||
| ), | ||
| )[0]; | ||
|
|
||
| if (eventToSend) { | ||
| if (config.stage === 'PROD') { | ||
| const publishRequestEntry = new PublishCommand({ | ||
| Message: JSON.stringify(eventToSend), | ||
| TopicArn: config.dependencyGraphIntegratorTopic, | ||
| }); | ||
| console.log(`Sending ${eventToSend.name} to Dependency Graph Integrator`); | ||
| await new SNSClient(awsClientConfig(config.stage)).send( | ||
| publishRequestEntry, | ||
| ); | ||
| } else { | ||
| console.log( | ||
| `Would have sent ${eventToSend.name} to Dependency Graph Integrator`, | ||
| ); | ||
| } | ||
| } else { | ||
| console.log( | ||
| 'No Scala repos found without SBT dependency submission workflow', | ||
| ); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters