frontend/latex/jobExec: refactoring, finally

Author: haraldschilly

src/packages/frontend/frame-editors/latex-editor/knitr.ts

@@ -13,11 +13,9 @@ import {
   ExecOutput,
 } from "@cocalc/frontend/frame-editors/generic/client";
 import { ExecuteCodeOutputAsync } from "@cocalc/util/types/execute-code";
-import { TIMEOUT_LATEX_JOB_S } from "./constants";
 import { Error as ErrorLog, ProcessedLatexLog } from "./latex-log-parser";
 import { BuildLog } from "./types";
-import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project";
-import { gatherJobInfo } from "./util";
+import { runJob } from "./util";
 
 // this still respects the environment variables and init files
 const R_CMD = "R";
@@ -39,50 +37,15 @@ export async function knitr(
   const { directory, filename } = parse_path(path);
   const expr = `require(knitr); opts_knit$set(concordance = TRUE, progress = FALSE); knit("${filename}")`;
   status(`${expr}`);
-  const job_info = await exec({
-    timeout: TIMEOUT_LATEX_JOB_S,
+
+  return runJob({
+    project_id,
     command: R_CMD,
     args: [...R_ARGS, expr],
-    bash: true, // so timeout is enforced by ulimit
-    project_id: project_id,
-    path: directory,
-    err_on_exit: false,
-    aggregate: time ? { value: time } : undefined, // one might think to aggregate on hash, but the output could be random!
-    async_call: true,
+    rundir: directory,
+    aggregate: time ? { value: time } : undefined,
+    set_job_info,
   });
-
-  if (job_info.type !== "async") {
-    // this is not an async job. This could happen for old projects.
-    return job_info;
-  }
-
-  set_job_info(job_info);
-  gatherJobInfo(project_id, job_info, set_job_info);
-
-  while (true) {
-    try {
-      const output = await exec({
-        project_id,
-        async_get: job_info.job_id,
-        async_await: true,
-        async_stats: true,
-      });
-      if (output.type !== "async") {
-        throw new Error("output: not an async task");
-      }
-      set_job_info(output);
-      return output;
-    } catch (err) {
-      if (err === TIMEOUT_CALLING_PROJECT) {
-        // this will be fine, hopefully. We continue trying to get a reply.
-        await new Promise((done) => setTimeout(done, 100));
-      } else {
-        throw new Error(
-          "Unable to complete compilation. Check the project and try again...",
-        );
-      }
-    }
-  }
 }
 
 /**

src/packages/frontend/frame-editors/latex-editor/latexmk.ts

@@ -8,13 +8,10 @@ Convert LaTeX file to PDF using latexmk.
 */
 
 import { exec } from "@cocalc/frontend/frame-editors/generic/client";
-import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project";
 import type { ExecOutput } from "@cocalc/util/db-schema/projects";
 import { change_filename_extension, path_split } from "@cocalc/util/misc";
 import { ExecuteCodeOutputAsync } from "@cocalc/util/types/execute-code";
-import { TIMEOUT_LATEX_JOB_S } from "./constants";
-import { BuildLog } from "./types";
-import { gatherJobInfo, pdf_path } from "./util";
+import { pdf_path, runJob } from "./util";
 
 export async function latexmk(
   project_id: string,
@@ -38,59 +35,17 @@ export async function latexmk(
     status([command].concat(args).join(" "));
   }
 
-  const job_info = await exec({
-    bash: true, // we use ulimit so that the timeout on the backend is *enforced* via ulimit!!
-    timeout: TIMEOUT_LATEX_JOB_S,
+  // Step 1: Wait for the launched job to finish
+  const output = await runJob({
+    project_id,
     command,
     args,
-    project_id,
-    path: head,
-    err_on_exit: false,
+    rundir: head,
     aggregate: time,
-    async_call: true,
+    set_job_info,
   });
 
-  // Step 1: Wait for the launched job to finish
-  let output: BuildLog;
-  if (job_info.type !== "async") {
-    output = job_info;
-  } else {
-    set_job_info(job_info);
-    gatherJobInfo(project_id, job_info, set_job_info);
-
-    if (typeof job_info.pid !== "number") {
-      throw new Error("Unable to spawn LaTeX compile job.");
-    }
-
-    while (true) {
-      try {
-        output = await exec({
-          project_id,
-          async_get: job_info.job_id,
-          async_await: true,
-          async_stats: true,
-        });
-        // console.log("LaTeX/latexmk: got output=", output);
-        if (output.type !== "async") {
-          throw new Error("not an async job");
-        }
-        set_job_info(output);
-        break;
-      } catch (err) {
-        // console.log("latexmk/while err=", err);
-        if (err === TIMEOUT_CALLING_PROJECT) {
-          // this will be fine, hopefully. We continue trying to get a reply
-          await new Promise((done) => setTimeout(done, 100));
-        } else {
-          throw new Error(
-            "Unable to complete compilation. Check the project and try again...",
-          );
-        }
-      }
-    }
-  }
-
-  // Step 2: do a copy operation
+  // Step 2: do a copy operation, if we run this in an output_directory (somewhere in /tmp)
   if (output_directory != null) {
     // We use cp instead of `ln -sf` so the file persists after project restart.
     // Using a symlink would be faster and more efficient *while editing*,

src/packages/frontend/frame-editors/latex-editor/pythontex.ts

@@ -8,17 +8,11 @@ Run PythonTeX
 */
 
 import { parse_path } from "@cocalc/frontend/frame-editors/frame-tree/util";
-import {
-  exec,
-  ExecOutput,
-} from "@cocalc/frontend/frame-editors/generic/client";
-// import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project";
-import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project";
+import { ExecOutput } from "@cocalc/frontend/frame-editors/generic/client";
 import { ExecuteCodeOutputAsync } from "@cocalc/util/types/execute-code";
-import { TIMEOUT_LATEX_JOB_S } from "./constants";
 import { Error as ErrorLog, ProcessedLatexLog } from "./latex-log-parser";
 import { BuildLog } from "./types";
-import { gatherJobInfo } from "./util";
+import { runJob } from "./util";
 
 // command documentation
 //
@@ -47,50 +41,16 @@ export async function pythontex(
   const command = `$(which {pythontex3,pythontex} | head -1) ${args}`;
   status(`pythontex[3] ${args}`);
   const aggregate = time && !force ? { value: time } : undefined;
-  const job_info = await exec({
-    timeout: TIMEOUT_LATEX_JOB_S,
-    bash: true, // timeout is enforced by ulimit
-    command,
-    env: { MPLBACKEND: "Agg" }, // for python plots -- https://github.com/sagemathinc/cocalc/issues/4203
-    project_id: project_id,
-    path: output_directory || directory,
-    err_on_exit: false,
+
+  return runJob({
+    project_id,
     aggregate,
-    async_call: true,
+    command,
+    rundir: output_directory || directory,
+    set_job_info,
+    // for python plots -- https://github.com/sagemathinc/cocalc/issues/4203
+    env: { MPLBACKEND: "Agg" },
   });
-
-  if (job_info.type !== "async") {
-    // this is not an async job. This could happen for old projects.
-    return job_info;
-  }
-
-  set_job_info(job_info);
-  gatherJobInfo(project_id, job_info, set_job_info);
-
-  while (true) {
-    try {
-      const output = await exec({
-        project_id,
-        async_get: job_info.job_id,
-        async_await: true,
-        async_stats: true,
-      });
-      if (output.type !== "async") {
-        throw new Error("output type is not async exec");
-      }
-      set_job_info(output);
-      return output;
-    } catch (err) {
-      if (err === TIMEOUT_CALLING_PROJECT) {
-        // this will be fine, hopefully. We continue trying to get a reply.
-        await new Promise((done) => setTimeout(done, 100));
-      } else {
-        throw new Error(
-          "Unable to complete compilation. Check the project and try again...",
-        );
-      }
-    }
-  }
 }
 
 /*

src/packages/frontend/frame-editors/latex-editor/sagetex.ts

@@ -14,12 +14,10 @@ import {
   exec,
   ExecOutput,
 } from "@cocalc/frontend/frame-editors/generic/client";
-import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project";
 import { ExecuteCodeOutputAsync } from "@cocalc/util/types/execute-code";
-import { TIMEOUT_LATEX_JOB_S } from "./constants";
 import { Error as ErrorLog, ProcessedLatexLog } from "./latex-log-parser";
 import { BuildLog } from "./types";
-import { gatherJobInfo } from "./util";
+import { runJob } from "./util";
 
 function sagetex_file(base: string): string {
   return base + ".sagetex.sage";
@@ -58,50 +56,15 @@ export async function sagetex(
   const { base, directory } = parse_path(path); // base, directory, filename
   const s = sagetex_file(base);
   status(`sage ${s}`);
-  const job_info = await exec({
-    timeout: TIMEOUT_LATEX_JOB_S,
-    bash: true, // so timeout is enforced by ulimit
+
+  return runJob({
+    project_id,
     command: "sage",
     args: [s],
-    project_id: project_id,
-    path: output_directory || directory,
-    err_on_exit: false,
+    set_job_info,
+    rundir: output_directory || directory,
     aggregate: hash ? { value: hash } : undefined,
-    async_call: true,
   });
-
-  if (job_info.type !== "async") {
-    // this is not an async job. This could happen for old projects.
-    return job_info;
-  }
-
-  set_job_info(job_info);
-  gatherJobInfo(project_id, job_info, set_job_info);
-
-  while (true) {
-    try {
-      const output = await exec({
-        project_id,
-        async_get: job_info.job_id,
-        async_await: true,
-        async_stats: true,
-      });
-      if (output.type !== "async") {
-        throw new Error("output type is not async exec");
-      }
-      set_job_info(output);
-      return output;
-    } catch (err) {
-      if (err === TIMEOUT_CALLING_PROJECT) {
-        // this will be fine, hopefully. We continue trying to get a reply.
-        await new Promise((done) => setTimeout(done, 100));
-      } else {
-        throw new Error(
-          "Unable to complete compilation. Check the project and try again...",
-        );
-      }
-    }
-  }
 }
 
 /* example error

src/packages/frontend/frame-editors/latex-editor/util.ts

@@ -5,9 +5,17 @@
 
 // data and functions specific to the latex editor.
 
-import { exec } from "@cocalc/frontend/frame-editors/generic/client";
+import {
+  exec,
+  ExecOpts,
+  ExecOutput,
+} from "@cocalc/frontend/frame-editors/generic/client";
 import { separate_file_extension } from "@cocalc/util/misc";
 import { ExecuteCodeOutputAsync } from "@cocalc/util/types/execute-code";
+// import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project";
+import { TIMEOUT_CALLING_PROJECT } from "@cocalc/util/consts/project";
+import { ExecOptsBlocking } from "@cocalc/util/db-schema/projects";
+import { TIMEOUT_LATEX_JOB_S } from "./constants";
 
 export function pdf_path(path: string): string {
   // if it is already a pdf, don't change the upper/lower casing -- #4562
@@ -92,3 +100,73 @@ export async function gatherJobInfo(
     return;
   }
 }
+
+interface RunJobOpts {
+  aggregate: ExecOptsBlocking["aggregate"];
+  args?: string[];
+  command: string;
+  env?: { [key: string]: string };
+  project_id: string;
+  rundir: string; // a directory! (output_directory if in /tmp, or the directory of the file's path)
+  set_job_info: (info: ExecuteCodeOutputAsync) => void;
+  timeout?: number;
+}
+
+export async function runJob(opts: RunJobOpts): Promise<ExecOutput> {
+  const { aggregate, args, command, env, project_id, rundir, set_job_info } =
+    opts;
+
+  const haveArgs = Array.isArray(args);
+
+  const job: ExecOpts = {
+    aggregate,
+    args,
+    async_call: true,
+    bash: !haveArgs,
+    command,
+    env,
+    err_on_exit: false,
+    path: rundir,
+    project_id,
+    timeout: TIMEOUT_LATEX_JOB_S,
+  };
+
+  const job_info = await exec(job);
+
+  if (job_info.type !== "async") {
+    // this is not an async job. This could happen for old projects.
+    return job_info;
+  }
+
+  if (typeof job_info.pid !== "number") {
+    throw new Error("Unable to spawn compile job.");
+  }
+
+  set_job_info(job_info);
+  gatherJobInfo(project_id, job_info, set_job_info);
+
+  while (true) {
+    try {
+      const output = await exec({
+        project_id,
+        async_get: job_info.job_id,
+        async_await: true,
+        async_stats: true,
+      });
+      if (output.type !== "async") {
+        throw new Error("output type is not async exec");
+      }
+      set_job_info(output);
+      return output;
+    } catch (err) {
+      if (err === TIMEOUT_CALLING_PROJECT) {
+        // this will be fine, hopefully. We continue trying to get a reply.
+        await new Promise((done) => setTimeout(done, 100));
+      } else {
+        throw new Error(
+          "Unable to complete compilation. Check the project and try again...",
+        );
+      }
+    }
+  }
+}