ws-worker: support job log level in lightning plan (#867)

* support job log level in lightning plan * add integration test * add changeset * bump lexicon too * version: [email protected] * package lock * bump cli * package lock, always package lock * bundle lexicon --------- Co-authored-by: Joe Clark <[email protected]>
OpenFn · Feb 19, 2025 · 7cbc8cc · 7cbc8cc
1 parent 10e5721
commit 7cbc8cc
Show file tree

Hide file tree

Showing 22 changed files with 522 additions and 41 deletions.
diff --git a/integration-tests/execute/CHANGELOG.md b/integration-tests/execute/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/integration-tests-execute
 
+## 1.0.16
+
+### Patch Changes
+
+- @openfn/compiler@1.0.1
+- @openfn/runtime@1.6.2
+
 ## 1.0.15
 
 ### Patch Changes

diff --git a/integration-tests/execute/package.json b/integration-tests/execute/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-execute",
   "private": true,
-  "version": "1.0.15",
+  "version": "1.0.16",
   "description": "Job execution tests",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md
@@ -1,5 +1,14 @@
 # @openfn/integration-tests-worker
 
+## 1.0.76
+
+### Patch Changes
+
+- Updated dependencies [1857b46]
+  - @openfn/engine-multi@1.5.0
+  - @openfn/ws-worker@1.10.0
+  - @openfn/lightning-mock@2.0.31
+
 ## 1.0.75
 
 ### Patch Changes

diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-worker",
   "private": true,
-  "version": "1.0.75",
+  "version": "1.0.76",
   "description": "Lightning WOrker integration tests",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts
@@ -974,5 +974,48 @@ test.serial('Redact logs which exceed the payload limit', (t) => {
   });
 });
 
+test.serial(
+  "Don't send job logs to stdout when job_log_level is set to none",
+  (t) => {
+    return new Promise(async (done) => {
+      await worker.destroy();
+      ({ worker, engineLogger } = await createDummyWorker());
+
+      const message = 'log that will never exist';
+
+      const run = {
+        id: crypto.randomUUID(),
+        jobs: [
+          {
+            adaptor: '@openfn/[email protected]',
+            body: `fn((s) => { console.log("${message}"); return s;})`,
+          },
+        ],
+        options: {
+          job_log_level: 'none',
+        },
+      };
+
+      lightning.once('run:complete', () => {
+        const jsonLogs = engineLogger._history;
+        // The engine logger shouldn't print out any job logs
+        const jobLog = jsonLogs.find((l) => l.name === 'JOB');
+        t.falsy(jobLog);
+        const jobLog2 = jsonLogs.find((l) => l.message[0] === message);
+        t.falsy(jobLog2);
+
+        // But it SHOULD log engine stuff
+        const runtimeLog = jsonLogs.find(
+          (l) => l.name === 'engine' && l.message[0].match(/complete workflow/i)
+        );
+        t.truthy(runtimeLog);
+        done();
+      });
+
+      lightning.enqueueRun(run);
+    });
+  }
+);
+
 // REMEMBER the default worker was destroyed at this point!
 // If you want to use a worker, you'll have to create your own
diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
@@ -1,5 +1,11 @@
 # @openfn/cli
 
+## 1.11.2
+
+### Patch Changes
+
+- Updated dependencies
+
 ## 1.11.1
 
 ### Patch Changes

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.11.1",
+  "version": "1.11.2",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

diff --git a/packages/compiler/CHANGELOG.md b/packages/compiler/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/compiler
 
+## 1.0.1
+
+### Patch Changes
+
+- Updated dependencies [1857b46]
+  - @openfn/lexicon@1.2.0
+
 ## 1.0.0
 
 ### Major Changes

diff --git a/packages/compiler/package.json b/packages/compiler/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/compiler",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Compiler and language tooling for openfn jobs.",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md
@@ -1,5 +1,18 @@
 # engine-multi
 
+## 1.5.0
+
+### Minor Changes
+
+- 1857b46: Allow configuration of job log level
+
+### Patch Changes
+
+- Updated dependencies [1857b46]
+  - @openfn/lexicon@1.2.0
+  - @openfn/compiler@1.0.1
+  - @openfn/runtime@1.6.2
+
 ## 1.4.9
 
 ### Patch Changes

diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/engine-multi",
-  "version": "1.4.9",
+  "version": "1.5.0",
   "description": "Multi-process runtime engine",
   "main": "dist/index.js",
   "type": "module",

diff --git a/packages/engine-multi/src/worker/thread/helpers.ts b/packages/engine-multi/src/worker/thread/helpers.ts
@@ -3,7 +3,7 @@
 import process from 'node:process';
 import stringify from 'fast-safe-stringify';
 import createLogger, { SanitizePolicies } from '@openfn/logger';
-import type { JSONLog } from '@openfn/logger';
+import type { JSONLog, LogLevel } from '@openfn/logger';
 
 import * as workerEvents from '../events';
 import { HANDLED_EXIT_CODE } from '../../events';
@@ -14,7 +14,8 @@ import serializeError from '../../util/serialize-error';
 export const createLoggers = (
   workflowId: string,
   sanitize: SanitizePolicies = 'none',
-  publish?: any
+  publish?: any,
+  jobLogLevel: LogLevel = 'debug'
 ) => {
   const log = (message: JSONLog) => {
     publish(workerEvents.LOG, {
@@ -47,7 +48,7 @@ export const createLoggers = (
 
   const jobLogger = createLogger('JOB', {
     logger: emitter,
-    level: 'debug',
+    level: jobLogLevel,
     json: true,
     sanitize,
   });

diff --git a/packages/engine-multi/src/worker/thread/run.ts b/packages/engine-multi/src/worker/thread/run.ts
@@ -3,7 +3,7 @@
 import run from '@openfn/runtime';
 import type { NotifyEvents } from '@openfn/runtime';
 import type { ExecutionPlan, State } from '@openfn/lexicon';
-import type { SanitizePolicies } from '@openfn/logger';
+import type { LogLevel, SanitizePolicies } from '@openfn/logger';
 
 import { register, publish } from './runtime';
 import { execute, createLoggers } from './helpers';
@@ -15,6 +15,7 @@ export type RunOptions = {
   whitelist?: RegExp[];
   sanitize: SanitizePolicies;
   statePropsToRemove?: string[];
+  jobLogLevel?: LogLevel;
 };
 
 const eventMap = {
@@ -26,11 +27,13 @@ const eventMap = {
 
 register({
   run: (plan: ExecutionPlan, input: State, runOptions: RunOptions) => {
-    const { repoDir, whitelist, sanitize, statePropsToRemove } = runOptions;
+    const { repoDir, whitelist, sanitize, statePropsToRemove, jobLogLevel } =
+      runOptions;
     const { logger, jobLogger, adaptorLogger } = createLoggers(
       plan.id!,
       sanitize,
-      publish
+      publish,
+      jobLogLevel
     );
 
     // Save the debug function so that we can use it

diff --git a/packages/lexicon/CHANGELOG.md b/packages/lexicon/CHANGELOG.md
@@ -1,5 +1,11 @@
 # lexicon
 
+## 1.2.0
+
+### Minor Changes
+
+- 1857b46: Allow configuration of job log level
+
 ## 1.1.0
 
 ### Minor Changes

diff --git a/packages/lexicon/lightning.d.ts b/packages/lexicon/lightning.d.ts
@@ -1,4 +1,4 @@
-import type { SanitizePolicies } from '@openfn/logger';
+import type { LogLevel, SanitizePolicies } from '@openfn/logger';
 import { LegacyJob, State } from './core';
 
 export const API_VERSION: number;
@@ -50,6 +50,7 @@ export type LightningPlanOptions = {
 
   run_memory_limit_mb?: number;
   payload_limit_mb?: number;
+  job_log_level?: LogLevel;
 };
 
 /**

diff --git a/packages/lexicon/package.json b/packages/lexicon/package.json
@@ -1,11 +1,14 @@
 {
   "name": "@openfn/lexicon",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Central repo of names and type definitions",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",
   "type": "module",
   "main": "index.js",
+  "scripts": {
+    "pack": "pnpm pack --pack-destination ../../dist"
+  },
   "exports": {
     ".": {
       "import": {

diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md
@@ -1,5 +1,14 @@
 # @openfn/lightning-mock
 
+## 2.0.31
+
+### Patch Changes
+
+- Updated dependencies [1857b46]
+  - @openfn/engine-multi@1.5.0
+  - @openfn/lexicon@1.2.0
+  - @openfn/runtime@1.6.2
+
 ## 2.0.30
 
 ### Patch Changes

diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/lightning-mock",
-  "version": "2.0.30",
+  "version": "2.0.31",
   "private": true,
   "description": "A mock Lightning server",
   "main": "dist/index.js",

diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md
@@ -1,5 +1,18 @@
 # ws-worker
 
+## 1.10.0
+
+### Minor Changes
+
+- 1857b46: Allow configuration of job log level
+
+### Patch Changes
+
+- Updated dependencies [1857b46]
+  - @openfn/engine-multi@1.5.0
+  - @openfn/lexicon@1.2.0
+  - @openfn/runtime@1.6.2
+
 ## 1.9.2
 
 ### Patch Changes

diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.9.2",
+  "version": "1.10.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",

diff --git a/packages/ws-worker/src/util/convert-lightning-plan.ts b/packages/ws-worker/src/util/convert-lightning-plan.ts
@@ -11,6 +11,7 @@ import type {
   WorkflowOptions,
   Lazy,
 } from '@openfn/lexicon';
+import type { LogLevel } from '@openfn/logger';
 import { LightningPlan, LightningEdge } from '@openfn/lexicon/lightning';
 import { ExecuteOptions } from '@openfn/engine-multi';
 import { getNameAndVersion } from '@openfn/runtime';
@@ -46,6 +47,7 @@ export type WorkerRunOptions = ExecuteOptions & {
   // Defaults to true - must be explicity false to stop dataclips being sent
   outputDataclips?: boolean;
   payloadLimitMb?: number;
+  jobLogLevel?: LogLevel;
 };
 
 type ConversionOptions = {
@@ -124,6 +126,9 @@ export default (
     if ('output_dataclips' in run.options) {
       engineOpts.outputDataclips = run.options.output_dataclips;
     }
+    if ('job_log_level' in run.options) {
+      engineOpts.jobLogLevel = run.options.job_log_level;
+    }
   }
 
   const plan: Partial<ExecutionPlan> = {