Don't cancel aborted jobs immediately (#11375)

* Don't cancel aborted jobs immediately

Rather than cancelling Futures that capture jobs' logic,
this change introduces a two-level system:

- interrupt all jobs softly via ThreadInterrupted at safepoints
- if safepoint is not executed within some time period or it is
  but the job is still not cancelled, trigger a hard-interrupt
  by cancelling the job explicitly, if possible

Closes #11084.

* Only cancel Future when you mean it

Soft-cancelling a future only to later call it with `mayInterrupt` set
to `true` has no effect in the latter case.
Changed the logic so that interrupting a Future will really enforce it.

Ocassionally some commands should not attempt to run soft cancellations
- we know they will re-execute the program.

* Replace Thread.sleep with Future.get

No while loops etc, it's much easier to reason about what is soft and
hard interrupt supposed to do.

* Better comments/logs

* nit

* PR review

* Make test more robust

Author: hubertp

engine/language-server/src/main/scala/org/enso/languageserver/requesthandler/visualization/ModifyVisualizationHandler.scala

@@ -12,6 +12,7 @@ import org.enso.languageserver.runtime.{
 }
 import org.enso.languageserver.util.UnhandledLogging
 
+import java.util.UUID
 import scala.concurrent.duration.FiniteDuration
 
 /** A request handler for `executionContext/modifyVisualization` commands.
@@ -41,16 +42,23 @@ class ModifyVisualizationHandler(
       )
       val cancellable =
         context.system.scheduler.scheduleOnce(timeout, self, RequestTimeout)
-      context.become(responseStage(id, sender(), cancellable))
+      context.become(
+        responseStage(id, params.visualizationId, sender(), cancellable)
+      )
   }
 
   private def responseStage(
     id: Id,
+    visualizationID: UUID,
     replyTo: ActorRef,
     cancellable: Cancellable
   ): Receive = {
     case RequestTimeout =>
-      logger.error("Request [{}] timed out.", id)
+      logger.error(
+        "Request [{}] timed out for visualization {}.",
+        id,
+        visualizationID
+      )
       replyTo ! ResponseError(Some(id), Errors.RequestTimeout)
       context.stop(self)
 

engine/runtime-instrument-common/src/main/java/org/enso/interpreter/instrument/command/SetExecutionEnvironmentCommand.java

@@ -52,7 +52,9 @@ private void setExecutionEnvironment(
               if (!oldEnvironmentName.equals(executionEnvironment.name())) {
                 ctx.jobControlPlane()
                     .abortJobs(
-                        contextId, "set execution environment to " + executionEnvironment.name());
+                        contextId,
+                        "set execution environment to " + executionEnvironment.name(),
+                        false);
                 ctx.locking()
                     .withWriteCompilationLock(
                         this.getClass(),

engine/runtime-instrument-common/src/main/java/org/enso/interpreter/instrument/execution/JobControlPlane.java

@@ -21,11 +21,15 @@ public interface JobControlPlane {
    * Aborts jobs that relates to the specified execution context.
    *
    * @param contextId an identifier of a context
+   * @param reason reason for aborting job(s)
+   * @param softAbortFirst true if ongoing jobs should be aborted with safepoints first, even if
+   *     marked as interruptible
    * @param classOf abort jobs of a given class only. If empty all jobs for the given context are
    *     aborted
    */
   @SuppressWarnings("unchecked")
-  void abortJobs(UUID contextId, String reason, Class<? extends Job<?>>... classOf);
+  void abortJobs(
+      UUID contextId, String reason, boolean softAbortFirst, Class<? extends Job<?>>... classOf);
 
   /**
    * Aborts jobs that relate to the specified execution context.

engine/runtime-instrument-common/src/main/java/org/enso/interpreter/instrument/job/ExecuteExpressionJob.java

@@ -32,7 +32,7 @@ public ExecuteExpressionJob(
   }
 
   @Override
-  public Executable run(RuntimeContext ctx) {
+  public Executable runImpl(RuntimeContext ctx) {
     return ctx.locking()
         .withContextLock(
             ctx.locking().getOrCreateContextLock(contextId),

engine/runtime-instrument-common/src/main/java/org/enso/interpreter/instrument/job/SerializeModuleJob.java

@@ -18,7 +18,7 @@ public SerializeModuleJob(QualifiedName moduleName) {
   }
 
   @Override
-  public Void run(RuntimeContext ctx) {
+  public Void runImpl(RuntimeContext ctx) {
     var ensoContext = ctx.executionService().getContext();
     var compiler = ensoContext.getCompiler();
     boolean useGlobalCacheLocations = ensoContext.isUseGlobalCache();

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/command/DestroyContextCmd.scala

@@ -34,7 +34,7 @@ class DestroyContextCmd(
   }
 
   private def removeContext()(implicit ctx: RuntimeContext): Unit = {
-    ctx.jobControlPlane.abortJobs(request.contextId, "destroy context")
+    ctx.jobControlPlane.abortJobs(request.contextId, "destroy context", false)
     val contextLock = ctx.locking.getOrCreateContextLock(request.contextId)
     try {
       ctx.locking.withContextLock(

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/command/InterruptContextCmd.scala

@@ -23,7 +23,11 @@ class InterruptContextCmd(
   ): Future[Unit] =
     if (doesContextExist) {
       Future {
-        ctx.jobControlPlane.abortJobs(request.contextId, "interrupt context")
+        ctx.jobControlPlane.abortJobs(
+          request.contextId,
+          "interrupt context",
+          false
+        )
         reply(Api.InterruptContextResponse(request.contextId))
       }
     } else {

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/command/PopContextCmd.scala

@@ -43,7 +43,7 @@ class PopContextCmd(
     ec: ExecutionContext
   ): Future[Unit] =
     Future {
-      ctx.jobControlPlane.abortJobs(request.contextId, "pop context")
+      ctx.jobControlPlane.abortJobs(request.contextId, "pop context", false)
       val maybeTopItem = ctx.contextManager.pop(request.contextId)
       if (maybeTopItem.isDefined) {
         reply(Api.PopContextResponse(request.contextId))

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/command/PushContextCmd.scala

@@ -46,7 +46,7 @@ class PushContextCmd(
     ec: ExecutionContext
   ): Future[Boolean] =
     Future {
-      ctx.jobControlPlane.abortJobs(request.contextId, "push context")
+      ctx.jobControlPlane.abortJobs(request.contextId, "push context", false)
       val stack = ctx.contextManager.getStack(request.contextId)
       val pushed = request.stackItem match {
         case _: Api.StackItem.ExplicitCall if stack.isEmpty =>

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/command/RecomputeContextCmd.scala

@@ -42,7 +42,11 @@ class RecomputeContextCmd(
     ec: ExecutionContext
   ): Future[Boolean] = {
     Future {
-      ctx.jobControlPlane.abortJobs(request.contextId, "recompute context")
+      ctx.jobControlPlane.abortJobs(
+        request.contextId,
+        "recompute context",
+        false
+      )
       val stack = ctx.contextManager.getStack(request.contextId)
       if (stack.isEmpty) {
         reply(Api.EmptyStackError(request.contextId))