CLI Deploy: support collections (#860)

* CLI: support collections

* fix failing tests

* support deletion

* add release changeset

* version: [email protected]

---------

Co-authored-by: Joe Clark <[email protected]>

Author: midigofrank

packages/cli/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @openfn/cli
 
+## 1.10.3
+
+### Patch Changes
+
+- Updated dependencies [6363e4a]
+  - @openfn/deploy@0.9.0
+
 ## 1.10.2
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.10.2",
+  "version": "1.10.3",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

packages/deploy/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/deploy
 
+## 0.9.0
+
+### Minor Changes
+
+- 6363e4a: Support collections in deploy package
+
 ## 0.8.2
 
 ### Patch Changes

packages/deploy/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/deploy",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "description": "Deploy projects to Lightning instances",
   "type": "module",
   "exports": {

packages/deploy/src/stateTransform.ts

@@ -287,6 +287,43 @@ export function mergeSpecIntoState(
     )
   );
 
+  const nextCollections = Object.fromEntries(
+    splitZip(oldState.collections || {}, spec.collections || {}).map(
+      ([collectionKey, stateCollection, specCollection]) => {
+        if (specCollection && !stateCollection) {
+          return [
+            collectionKey,
+            {
+              id: crypto.randomUUID(),
+              name: specCollection.name,
+            },
+          ];
+        }
+
+        if (specCollection && stateCollection) {
+          return [
+            collectionKey,
+            {
+              id: stateCollection.id,
+              name: specCollection.name,
+            },
+          ];
+        }
+
+        if (!specCollection && stateCollection) {
+          return [collectionKey, { id: stateCollection.id, delete: true }];
+        }
+
+        throw new DeployError(
+          `Invalid collection spec or corrupted state for collection: ${
+            stateCollection?.name || specCollection?.name
+          }`,
+          'VALIDATION_ERROR'
+        );
+      }
+    )
+  );
+
   const nextWorkflows = Object.fromEntries(
     splitZip(oldState.workflows, spec.workflows).map(
       ([workflowKey, stateWorkflow, specWorkflow]) => {
@@ -352,6 +389,7 @@ export function mergeSpecIntoState(
     name: spec.name,
     workflows: nextWorkflows,
     project_credentials: nextCredentials,
+    collections: nextCollections,
   };
 
   if (spec.description) projectState.description = spec.description;
@@ -398,8 +436,11 @@ export function getStateFromProjectPayload(
     {} as Record<string, CredentialState>
   );
 
+  const collections = reduceByKey('name', project.collections || []);
+
   return {
     ...project,
+    collections,
     project_credentials,
     workflows,
   };
@@ -460,8 +501,17 @@ export function mergeProjectPayloadIntoState(
     })
   );
 
+  const nextCollections = Object.fromEntries(
+    idKeyPairs(project.collections || {}, state.collections || {}).map(
+      ([key, nextCollection, _state]) => {
+        return [key, nextCollection];
+      }
+    )
+  );
+
   return {
     ...project,
+    collections: nextCollections,
     project_credentials: nextCredentials,
     workflows: nextWorkflows,
   };
@@ -503,8 +553,13 @@ export function toProjectPayload(state: ProjectState): ProjectPayload {
   const project_credentials: ProjectPayload['project_credentials'] =
     Object.values(state.project_credentials);
 
+  const collections: ProjectPayload['collections'] = Object.values(
+    state.collections
+  );
+
   return {
     ...state,
+    collections,
     project_credentials,
     workflows,
   };

packages/deploy/src/types.ts

@@ -94,11 +94,22 @@ export type CredentialState = {
   owner: string;
 };
 
+export type CollectionSpec = {
+  name: string;
+};
+
+export type CollectionState = {
+  id: string;
+  name: string;
+  delete?: boolean;
+};
+
 export interface ProjectSpec {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowSpec>;
   credentials: Record<string | symbol, CredentialSpec>;
+  collections: Record<string | symbol, CollectionSpec>;
 }
 
 export interface WorkflowState {
@@ -121,12 +132,14 @@ export interface ProjectState {
   description: string;
   workflows: Record<string | symbol, WorkflowState>;
   project_credentials: Record<string | symbol, CredentialState>;
+  collections: Record<string | symbol, CollectionState>;
 }
 
 export interface ProjectPayload {
   id: string;
   name: string;
   description: string;
+  collections: Concrete<CollectionState>[];
   project_credentials: Concrete<CredentialState>[];
   workflows: {
     id: string;

packages/deploy/src/validator.ts

@@ -123,6 +123,12 @@ export async function parseAndValidate(
         }
       }
 
+      if (pair.key && pair.key.value === 'collections') {
+        if (pair.value.value === null) {
+          return doc.createPair('collections', {});
+        }
+      }
+
       if (pair.key && pair.key.value === 'jobs') {
         if (pair.value.value === null) {
           errors.push({

packages/deploy/test/fixtures.ts

@@ -4,6 +4,7 @@ export function fullExampleSpec() {
   return {
     name: 'my project',
     description: 'some helpful description',
+    collections: {},
     credentials: {},
     workflows: {
       'workflow-one': {
@@ -56,6 +57,7 @@ export function fullExampleState() {
     id: 'be156ab1-8426-4151-9a18-4045142f9ec0',
     name: 'my project',
     description: 'some helpful description',
+    collections: {},
     project_credentials: {},
     workflows: {
       'workflow-one': {
@@ -116,6 +118,12 @@ export const lightningProjectPayload = {
   updated_at: '2023-08-25T08:57:31',
   scheduled_deletion: null,
   requires_mfa: false,
+  collections: [
+    {
+      id: 'f8e1c1e1-5c5a-4d9b-8e9f-4b2f6b1c2f4e',
+      name: 'collection-one',
+    },
+  ],
   project_credentials: [
     {
       id: '25f48989-d349-4eb8-99c3-923ebba5b116',
@@ -240,6 +248,12 @@ export const lightningProjectState = {
   updated_at: '2023-08-25T08:57:31',
   scheduled_deletion: null,
   requires_mfa: false,
+  collections: {
+    'collection-one': {
+      id: 'f8e1c1e1-5c5a-4d9b-8e9f-4b2f6b1c2f4e',
+      name: 'collection-one',
+    },
+  },
   project_credentials: {
     '[email protected]': {
       id: '25f48989-d349-4eb8-99c3-923ebba5b116',

packages/deploy/test/stateTransform.test.ts

@@ -90,6 +90,46 @@ test('toNextState adding a job', (t) => {
     name: 'project-name',
     description: 'my test project',
     project_credentials: {},
+    collections: {},
+  });
+});
+
+test('toNextState deleting a credential', (t) => {
+  const spec = {
+    name: 'project-name',
+    description: 'my test project',
+    collections: {},
+    workflows: {},
+  };
+
+  const state = {
+    workflows: {},
+    project_credentials: {},
+    collections: {
+      'test-collection': {
+        id: 'f8e1c1e1-5c5a-4d9b-8e9f-4b2f6b1c2f4e',
+        name: 'test-collection',
+      },
+    },
+    id: 'ecb683d1-5e5a-4c4f-9165-e143e2eeeb48',
+    name: 'project-name',
+    description: 'my test project',
+  };
+
+  let result = mergeSpecIntoState(state, spec);
+
+  t.deepEqual(result, {
+    workflows: {},
+    id: 'ecb683d1-5e5a-4c4f-9165-e143e2eeeb48',
+    name: 'project-name',
+    description: 'my test project',
+    project_credentials: {},
+    collections: {
+      'test-collection': {
+        id: 'f8e1c1e1-5c5a-4d9b-8e9f-4b2f6b1c2f4e',
+        delete: true,
+      },
+    },
   });
 });
 
@@ -117,6 +157,7 @@ test('toNextState with empty state', (t) => {
     name: 'my project',
     description: 'some helpful description',
     project_credentials: {},
+    collections: {},
     workflows: {
       'workflow-one': {
         id: jp.query(result, '$..workflows["workflow-one"].id')[0],
@@ -176,6 +217,7 @@ test('toNextState with no changes', (t) => {
     name: 'my project',
     description: 'for the humans',
     project_credentials: {},
+    collections: {},
     workflows: {
       'workflow-one': {
         id: '8124e88c-566f-472f-be38-363e588af55a',
@@ -291,6 +333,7 @@ test('toNextState with a new job', (t) => {
     name: 'my project',
     description: 'some other description',
     project_credentials: {},
+    collections: {},
     workflows: {
       'workflow-one': {
         id: '8124e88c-566f-472f-be38-363e588af55a',
@@ -538,6 +581,7 @@ test('getStateFromProjectPayload with minimal project', (t) => {
     id: 'xyz',
     name: 'project',
     project_credentials: {},
+    collections: {},
     workflows: {
       a: {
         id: 'wf-a',