From 58bbd85e488f94e2e7a7eedacfb9a544ec163278 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Feb 2025 13:55:42 +0100 Subject: [PATCH 01/16] Generalize enso_cloud log API access --- .../enso/base/enso_cloud/audit/AuditLog.java | 17 +- .../enso_cloud/audit/AuditLogApiAccess.java | 261 +----------------- .../enso_cloud/audit/AuditLogMessage.java | 3 +- .../base/enso_cloud/logging/LogApiAccess.java | 230 +++++++++++++++ .../enso/base/enso_cloud/logging/LogJob.java | 30 ++ .../{audit => logging}/LogJobsQueue.java | 13 +- .../base/enso_cloud/logging/LogMessage.java | 8 + .../enso_cloud/logging/RequestConfig.java | 17 ++ .../logging/RequestFailureException.java | 7 + 9 files changed, 319 insertions(+), 267 deletions(-) create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java rename std-bits/base/src/main/java/org/enso/base/enso_cloud/{audit => logging}/LogJobsQueue.java (68%) create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java index 9e3800fd2c90..1823a6f41096 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java @@ -13,16 +13,27 @@ * the meantime, all waiting messages (up to some limit) will be sent in a single request. */ public final class AuditLog { + private AuditLog() {} + + private static AuditLogApiAccess apiAccess; + + private static AuditLogApiAccess apiAccess() { + if (apiAccess == null) { + apiAccess = new AuditLogApiAccess(); + } + return apiAccess; + } + /** Schedules the log message to be sent in the next batch, and returns immediately. */ public static void logAsync(String type, String message, ObjectNode metadata) { var event = new AuditLogMessage(type, message, metadata); - AuditLogApiAccess.INSTANCE.logWithoutConfirmation(event); + apiAccess().logWithoutConfirmation(event); } /** Schedules the log message to be sent in the next batch, and waits until it has been sent. */ public static void logSynchronously(String type, String message, ObjectNode metadata) { var event = new AuditLogMessage(type, message, metadata); - Future future = AuditLogApiAccess.INSTANCE.logWithConfirmation(event); + Future future = apiAccess().logWithConfirmation(event); try { future.get(); } catch (ExecutionException | InterruptedException e) { @@ -37,6 +48,6 @@ public AuditLogError(String message, Throwable cause) { } public static void resetCache() { - AuditLogApiAccess.INSTANCE.resetCache(); + apiAccess().resetCache(); } } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java index 372ad014ef78..b6161ead4368 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java @@ -1,268 +1,17 @@ package org.enso.base.enso_cloud.audit; -import java.io.IOException; import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; -import org.enso.base.enso_cloud.AuthenticationProvider; import org.enso.base.enso_cloud.CloudAPI; +import org.enso.base.enso_cloud.logging.LogApiAccess; /** * Gives access to the low-level log event API in the Cloud and manages asynchronously submitting * the logs. */ -class AuditLogApiAccess { - private static final Logger logger = Logger.getLogger(AuditLogApiAccess.class.getName()); +final class AuditLogApiAccess extends LogApiAccess { - /** - * We still want to limit the batch size to some reasonable number - sending too many logs in one - * request could also be problematic. - */ - private static final int MAX_BATCH_SIZE = 100; - - private static final int MAX_RETRIES = 5; - - public static AuditLogApiAccess INSTANCE = new AuditLogApiAccess(); - - private HttpClient httpClient; - private final LogJobsQueue logQueue = new LogJobsQueue(); - private final ThreadPoolExecutor backgroundThreadService; - - private AuditLogApiAccess() { - // We set-up a thread 'pool' that will contain at most one thread. - // If the thread is idle for 60 seconds, it will be shut down. - backgroundThreadService = - new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); - } - - public Future logWithConfirmation(LogMessage message) { - var currentRequestConfig = getRequestConfig(); - CompletableFuture completionNotification = new CompletableFuture<>(); - enqueueJob(new LogJob(message, completionNotification, currentRequestConfig)); - return completionNotification; - } - - public void logWithoutConfirmation(LogMessage message) { - var currentRequestConfig = getRequestConfig(); - enqueueJob(new LogJob(message, null, currentRequestConfig)); - } - - private void enqueueJob(LogJob job) { - int queuedJobs = logQueue.enqueue(job); - if (queuedJobs == 1 && backgroundThreadService.getQueue().isEmpty()) { - // If we are the first message in the queue, we need to start the background thread. - // It is possible that a job was already running, but adding a new one will not hurt - once - // the queue is empty, the currently running job will finish and any additional jobs will also - // terminate immediately. - backgroundThreadService.execute(this::logThreadEntryPoint); - } - - /* - * Liveness is guaranteed, because the queue size always increments exactly by 1, - * so `enqueue` returns 1 if and only if the queue was empty beforehand. - * - * If the queue was empty before adding a message, we always schedule a `logThreadEntryPoint` to run, - * unless it was already pending on the job queue. - * - * Any running `logThreadEntryPoint` will not finish until the queue is empty. - * So after every append, either a job is already running or scheduled to be run. - */ - } - - /** Runs as long as there are any pending log messages queued and sends them in batches. */ - private void logThreadEntryPoint() { - while (true) { - List pendingMessages = logQueue.popEnqueuedJobs(MAX_BATCH_SIZE); - if (pendingMessages.isEmpty()) { - // If there are no more pending messages, we can stop the thread for now. - // If during this teardown a new message is added, it will see no elements on `logQueue` and - // thus, - // `logQueue.enqueue` will return 1, thus ensuring that at least one new job is scheduled. - return; - } - - var batchesByConfig = splitMessagesByConfig(pendingMessages); - for (var batch : batchesByConfig) { - sendBatch(batch); - } - } - } - - /** - * Sends a batch of log messages. - * - *

The batch must not be empty and all messages must share the same request config. - */ - private void sendBatch(List batch) { - assert !batch.isEmpty() : "The batch must not be empty."; - // We use the request config from the first message - all messages in the batch should have the - // same request config. - var requestConfig = batch.get(0).requestConfig(); - assert requestConfig != null - : "The request configuration must be set before building a request."; - assert batch.stream().allMatch(job -> job.requestConfig().equals(requestConfig)) - : "All messages in a batch must have the same request configuration."; - - try { - var request = buildRequest(requestConfig, batch); - sendLogRequest(request, MAX_RETRIES); - notifyJobsAboutSuccess(batch); - } catch (RequestFailureException e) { - notifyJobsAboutFailure(batch, e); - } - } - - /** - * Only during testing, it is possible to encounter pending messages with different request - * configs (when the config changes between tests). To send each message where it is intended, we - * split up the batch by the config. - */ - Collection> splitMessagesByConfig(List messages) { - HashMap> hashMap = new HashMap<>(); - for (var message : messages) { - var list = hashMap.computeIfAbsent(message.requestConfig(), k -> new ArrayList<>()); - list.add(message); - } - - return hashMap.values(); - } - - private void notifyJobsAboutSuccess(List jobs) { - for (var job : jobs) { - if (job.completionNotification() != null) { - job.completionNotification().complete(null); - } - } - } - - private void notifyJobsAboutFailure(List jobs, RequestFailureException e) { - for (var job : jobs) { - if (job.completionNotification() != null) { - job.completionNotification().completeExceptionally(e); - } - } - } - - private HttpRequest buildRequest(RequestConfig requestConfig, List messages) { - assert requestConfig != null - : "The request configuration must be set before building a request."; - var payload = buildPayload(messages); - return HttpRequest.newBuilder() - .uri(requestConfig.apiUri) - .header("Authorization", "Bearer " + requestConfig.accessToken) - .POST(HttpRequest.BodyPublishers.ofString(payload, StandardCharsets.UTF_8)) - .build(); - } - - private String buildPayload(List messages) { - var payload = new StringBuilder(); - payload.append("{\"logs\": ["); - for (var message : messages) { - payload.append(message.message().payload()).append(","); - } - - // Remove the trailing comma. - payload.deleteCharAt(payload.length() - 1); - payload.append("]}"); - return payload.toString(); - } - - private RequestConfig cachedRequestConfig = null; - - /** - * Builds a request configuration based on runtime information. - * - *

This method must be called from the main thread. - * - *

The same instance is returned every time after the first call, unless the caches were - * flushed (which is mostly used in tests). - */ - private RequestConfig getRequestConfig() { - if (cachedRequestConfig != null) { - return cachedRequestConfig; - } - - var uri = URI.create(CloudAPI.getAPIRootURI() + "logs"); - var config = new RequestConfig(uri, AuthenticationProvider.getAccessToken()); - cachedRequestConfig = config; - return config; - } - - /** - * Contains information needed to build a request to the Cloud Logs API. - * - *

This information must be gathered on the main Enso thread, as only there we have access to - * the {@link AuthenticationProvider}. - * - *

We associate an instance with every message to be sent. When sending multiple messages in a - * batch, we will use the config from one of them. This should not matter as in normal operations - * the configs will be the same, they only change during testing. Tests should this into account, - * by sending the last message in synchronous mode. - */ - private record RequestConfig(URI apiUri, String accessToken) {} - - private void sendLogRequest(HttpRequest request, int retryCount) throws RequestFailureException { - try { - try { - if (httpClient == null) { - httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build(); - } - HttpResponse response = - httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() < 200 || response.statusCode() >= 300) { - throw new RequestFailureException( - "Unexpected status code: " + response.statusCode() + " " + response.body(), null); - } - } catch (IOException | InterruptedException e) { - // Promote a checked exception to a runtime exception to simplify the code. - var errorMessage = e.getMessage() != null ? e.getMessage() : e.toString(); - throw new RequestFailureException("Failed to send log messages: " + errorMessage, e); - } - } catch (RequestFailureException e) { - if (retryCount < 0) { - logger.severe("Failed to send log messages after retrying: " + e.getMessage()); - throw e; - } else { - logger.warning("Exception when sending log messages: " + e.getMessage() + ". Retrying..."); - sendLogRequest(request, retryCount - 1); - } - } - } - - interface LogMessage { - String payload(); - } - - static class RequestFailureException extends RuntimeException { - public RequestFailureException(String message, Throwable cause) { - super(message, cause); - } - } - - /** - * A record that represents a single log to be sent. - * - *

It may contain the `completionNotification` future that will be completed when the log is - * sent. If no-one is listening for confirmation, that field will be `null`. - */ - record LogJob( - LogMessage message, - CompletableFuture completionNotification, - RequestConfig requestConfig) {} - - void resetCache() { - cachedRequestConfig = null; + @Override + public URI endpoint() { + return URI.create(CloudAPI.getAPIRootURI() + "logs"); } } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java index 7273661744cd..da59f1d27021 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java @@ -7,8 +7,9 @@ import java.util.Objects; import org.enso.base.CurrentEnsoProject; import org.enso.base.enso_cloud.CloudAPI; +import org.enso.base.enso_cloud.logging.LogMessage; -class AuditLogMessage implements AuditLogApiAccess.LogMessage { +class AuditLogMessage implements LogMessage { /** * A reserved field that is currently added by the cloud backend. Duplicating it will lead to diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java new file mode 100644 index 000000000000..4e343e9ce951 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java @@ -0,0 +1,230 @@ +package org.enso.base.enso_cloud.logging; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import org.enso.base.enso_cloud.AuthenticationProvider; + +/** + * Gives access to the low-level log event API in the Cloud and manages asynchronously submitting + * the logs. + */ +public abstract class LogApiAccess { + /** + * We still want to limit the batch size to some reasonable number - sending too many logs in one + * request could also be problematic. + */ + private static final int MAX_BATCH_SIZE = 100; + + private static final int MAX_RETRIES = 5; + private static final Logger LOGGER = Logger.getLogger(LogApiAccess.class.getName()); + + private HttpClient httpClient; + private final LogJobsQueue logQueue = new LogJobsQueue(); + private final ThreadPoolExecutor backgroundThreadService; + private RequestConfig cachedRequestConfig = null; + + protected LogApiAccess() { + // We set-up a thread 'pool' that will contain at most one thread. + // If the thread is idle for 60 seconds, it will be shut down. + backgroundThreadService = + new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); + } + + public abstract URI endpoint(); + + public Future logWithConfirmation(LogMessage message) { + var currentRequestConfig = getRequestConfig(); + CompletableFuture completionNotification = new CompletableFuture<>(); + enqueueJob(new LogJob(message, completionNotification, currentRequestConfig)); + return completionNotification; + } + + public void logWithoutConfirmation(LogMessage message) { + var currentRequestConfig = getRequestConfig(); + enqueueJob(new LogJob(message, null, currentRequestConfig)); + } + + public void resetCache() { + cachedRequestConfig = null; + } + + private void enqueueJob(LogJob job) { + int queuedJobs = logQueue.enqueue(job); + if (queuedJobs == 1 && backgroundThreadService.getQueue().isEmpty()) { + // If we are the first message in the queue, we need to start the background thread. + // It is possible that a job was already running, but adding a new one will not hurt - once + // the queue is empty, the currently running job will finish and any additional jobs will also + // terminate immediately. + backgroundThreadService.execute(this::logThreadEntryPoint); + } + + /* + * Liveness is guaranteed, because the queue size always increments exactly by 1, + * so `enqueue` returns 1 if and only if the queue was empty beforehand. + * + * If the queue was empty before adding a message, we always schedule a `logThreadEntryPoint` to run, + * unless it was already pending on the job queue. + * + * Any running `logThreadEntryPoint` will not finish until the queue is empty. + * So after every append, either a job is already running or scheduled to be run. + */ + } + + /** Runs as long as there are any pending log messages queued and sends them in batches. */ + private void logThreadEntryPoint() { + while (true) { + List pendingMessages = logQueue.popEnqueuedJobs(MAX_BATCH_SIZE); + if (pendingMessages.isEmpty()) { + // If there are no more pending messages, we can stop the thread for now. + // If during this teardown a new message is added, it will see no elements on `logQueue` and + // thus, + // `logQueue.enqueue` will return 1, thus ensuring that at least one new job is scheduled. + return; + } + + var batchesByConfig = splitMessagesByConfig(pendingMessages); + for (var batch : batchesByConfig) { + sendBatch(batch); + } + } + } + + /** + * Sends a batch of log messages. + * + *

The batch must not be empty and all messages must share the same request config. + */ + private void sendBatch(List batch) { + assert !batch.isEmpty() : "The batch must not be empty."; + // We use the request config from the first message - all messages in the batch should have the + // same request config. + var requestConfig = batch.get(0).getRequestConfig(); + assert requestConfig != null + : "The request configuration must be set before building a request."; + assert batch.stream().allMatch(job -> job.getRequestConfig().equals(requestConfig)) + : "All messages in a batch must have the same request configuration."; + + try { + var request = buildRequest(requestConfig, batch); + sendLogRequest(request, MAX_RETRIES); + notifyJobsAboutSuccess(batch); + } catch (RequestFailureException e) { + notifyJobsAboutFailure(batch, e); + } + } + + /** + * Only during testing, it is possible to encounter pending messages with different request + * configs (when the config changes between tests). To send each message where it is intended, we + * split up the batch by the config. + */ + private Collection> splitMessagesByConfig(List messages) { + HashMap> hashMap = new HashMap<>(); + for (var message : messages) { + var list = hashMap.computeIfAbsent(message.getRequestConfig(), k -> new ArrayList<>()); + list.add(message); + } + + return hashMap.values(); + } + + private void notifyJobsAboutSuccess(List jobs) { + for (var job : jobs) { + if (job.getCompletionNotification() != null) { + job.getCompletionNotification().complete(null); + } + } + } + + private void notifyJobsAboutFailure(List jobs, RequestFailureException e) { + for (var job : jobs) { + if (job.getCompletionNotification() != null) { + job.getCompletionNotification().completeExceptionally(e); + } + } + } + + private HttpRequest buildRequest(RequestConfig requestConfig, List messages) { + assert requestConfig != null + : "The request configuration must be set before building a request."; + var payload = buildPayload(messages); + return HttpRequest.newBuilder() + .uri(requestConfig.apiUri()) + .header("Authorization", "Bearer " + requestConfig.accessToken()) + .POST(HttpRequest.BodyPublishers.ofString(payload, StandardCharsets.UTF_8)) + .build(); + } + + private String buildPayload(List messages) { + var payload = new StringBuilder(); + payload.append("{\"logs\": ["); + for (var message : messages) { + payload.append(message.getLogMessage().payload()).append(","); + } + + // Remove the trailing comma. + payload.deleteCharAt(payload.length() - 1); + payload.append("]}"); + return payload.toString(); + } + + /** + * Builds a request configuration based on runtime information. + * + *

This method must be called from the main thread. + * + *

The same instance is returned every time after the first call, unless the caches were + * flushed (which is mostly used in tests). + */ + private RequestConfig getRequestConfig() { + if (cachedRequestConfig != null) { + return cachedRequestConfig; + } + var uri = endpoint(); + var config = new RequestConfig(uri, AuthenticationProvider.getAccessToken()); + cachedRequestConfig = config; + return config; + } + + private void sendLogRequest(HttpRequest request, int retryCount) throws RequestFailureException { + try { + try { + if (httpClient == null) { + httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build(); + } + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() < 200 || response.statusCode() >= 300) { + throw new RequestFailureException( + "Unexpected status code: " + response.statusCode() + " " + response.body(), null); + } + } catch (IOException | InterruptedException e) { + // Promote a checked exception to a runtime exception to simplify the code. + var errorMessage = e.getMessage() != null ? e.getMessage() : e.toString(); + throw new RequestFailureException("Failed to send log messages: " + errorMessage, e); + } + } catch (RequestFailureException e) { + if (retryCount < 0) { + LOGGER.severe("Failed to send log messages after retrying: " + e.getMessage()); + throw e; + } else { + LOGGER.warning("Exception when sending log messages: " + e.getMessage() + ". Retrying..."); + sendLogRequest(request, retryCount - 1); + } + } + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java new file mode 100644 index 000000000000..c2e6ab2c7793 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java @@ -0,0 +1,30 @@ +package org.enso.base.enso_cloud.logging; + +import java.util.concurrent.CompletableFuture; + +public final class LogJob { + private final LogMessage logMessage; + private final CompletableFuture completionNotification; + private final RequestConfig requestConfig; + + public LogJob( + LogMessage logMessage, + CompletableFuture completionNotification, + RequestConfig requestConfig) { + this.logMessage = logMessage; + this.completionNotification = completionNotification; + this.requestConfig = requestConfig; + } + + public LogMessage getLogMessage() { + return logMessage; + } + + public CompletableFuture getCompletionNotification() { + return completionNotification; + } + + public RequestConfig getRequestConfig() { + return requestConfig; + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/LogJobsQueue.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java similarity index 68% rename from std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/LogJobsQueue.java rename to std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java index 22861df02ae8..25d553753db6 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/LogJobsQueue.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java @@ -1,16 +1,15 @@ -package org.enso.base.enso_cloud.audit; +package org.enso.base.enso_cloud.logging; import java.util.ArrayList; import java.util.Deque; import java.util.LinkedList; import java.util.List; -class LogJobsQueue { - - private final Deque queue = new LinkedList<>(); +public final class LogJobsQueue { + private final Deque queue = new LinkedList<>(); /** Enqueues a log message to be sent and returns the number of messages in the queue. */ - synchronized int enqueue(AuditLogApiAccess.LogJob job) { + synchronized int enqueue(LogJob job) { int previousSize = queue.size(); queue.addLast(job); int newSize = queue.size(); @@ -20,14 +19,14 @@ synchronized int enqueue(AuditLogApiAccess.LogJob job) { } /** Removes and returns up to {@code limit} enqueued jobs. */ - synchronized List popEnqueuedJobs(int limit) { + synchronized List popEnqueuedJobs(int limit) { assert limit > 0; if (queue.isEmpty()) { return List.of(); } int n = Math.min(limit, queue.size()); - List result = new ArrayList<>(n); + List result = new ArrayList<>(n); for (int i = 0; i < n; i++) { result.add(queue.removeFirst()); } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java new file mode 100644 index 000000000000..dd0c1f9bc2b5 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java @@ -0,0 +1,8 @@ +package org.enso.base.enso_cloud.logging; + +public interface LogMessage { + /** + * @return JSON string representation of the log message. + */ + String payload(); +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java new file mode 100644 index 000000000000..da33fab7ec5d --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java @@ -0,0 +1,17 @@ +package org.enso.base.enso_cloud.logging; + +import java.net.URI; +import org.enso.base.enso_cloud.AuthenticationProvider; + +/** + * Contains information needed to build a request to the Cloud Logs API. + * + *

This information must be gathered on the main Enso thread, as only there we have access to the + * {@link AuthenticationProvider}. + * + *

We associate an instance with every message to be sent. When sending multiple messages in a + * batch, we will use the config from one of them. This should not matter as in normal operations + * the configs will be the same, they only change during testing. Tests should this into account, by + * sending the last message in synchronous mode. + */ +public record RequestConfig(URI apiUri, String accessToken) {} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java new file mode 100644 index 000000000000..d26407fd5aa8 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java @@ -0,0 +1,7 @@ +package org.enso.base.enso_cloud.logging; + +public final class RequestFailureException extends RuntimeException { + public RequestFailureException(String message, Throwable cause) { + super(message, cause); + } +} From 5e98e7b3d59ae2efb49d2d4f6a31eeee12c3b2b3 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Feb 2025 17:46:15 +0100 Subject: [PATCH 02/16] No point in making uri abstract. There is just a single endpoint for logging in cloud. --- .../enso/base/enso_cloud/audit/AuditLog.java | 16 ++++------------ .../enso_cloud/audit/AuditLogApiAccess.java | 17 ----------------- .../base/enso_cloud/logging/LogApiAccess.java | 10 +++++----- 3 files changed, 9 insertions(+), 34 deletions(-) delete mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java index 1823a6f41096..90c820b2f01d 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import org.enso.base.enso_cloud.logging.LogApiAccess; /** * The high-level API for logging audit events. @@ -15,25 +16,16 @@ public final class AuditLog { private AuditLog() {} - private static AuditLogApiAccess apiAccess; - - private static AuditLogApiAccess apiAccess() { - if (apiAccess == null) { - apiAccess = new AuditLogApiAccess(); - } - return apiAccess; - } - /** Schedules the log message to be sent in the next batch, and returns immediately. */ public static void logAsync(String type, String message, ObjectNode metadata) { var event = new AuditLogMessage(type, message, metadata); - apiAccess().logWithoutConfirmation(event); + LogApiAccess.INSTANCE.logWithoutConfirmation(event); } /** Schedules the log message to be sent in the next batch, and waits until it has been sent. */ public static void logSynchronously(String type, String message, ObjectNode metadata) { var event = new AuditLogMessage(type, message, metadata); - Future future = apiAccess().logWithConfirmation(event); + Future future = LogApiAccess.INSTANCE.logWithConfirmation(event); try { future.get(); } catch (ExecutionException | InterruptedException e) { @@ -48,6 +40,6 @@ public AuditLogError(String message, Throwable cause) { } public static void resetCache() { - apiAccess().resetCache(); + LogApiAccess.INSTANCE.resetCache(); } } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java deleted file mode 100644 index b6161ead4368..000000000000 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.enso.base.enso_cloud.audit; - -import java.net.URI; -import org.enso.base.enso_cloud.CloudAPI; -import org.enso.base.enso_cloud.logging.LogApiAccess; - -/** - * Gives access to the low-level log event API in the Cloud and manages asynchronously submitting - * the logs. - */ -final class AuditLogApiAccess extends LogApiAccess { - - @Override - public URI endpoint() { - return URI.create(CloudAPI.getAPIRootURI() + "logs"); - } -} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java index 4e343e9ce951..31129315bd87 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java @@ -17,12 +17,13 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import org.enso.base.enso_cloud.AuthenticationProvider; +import org.enso.base.enso_cloud.CloudAPI; /** * Gives access to the low-level log event API in the Cloud and manages asynchronously submitting * the logs. */ -public abstract class LogApiAccess { +public final class LogApiAccess { /** * We still want to limit the batch size to some reasonable number - sending too many logs in one * request could also be problematic. @@ -31,21 +32,20 @@ public abstract class LogApiAccess { private static final int MAX_RETRIES = 5; private static final Logger LOGGER = Logger.getLogger(LogApiAccess.class.getName()); + public static final LogApiAccess INSTANCE = new LogApiAccess(); private HttpClient httpClient; private final LogJobsQueue logQueue = new LogJobsQueue(); private final ThreadPoolExecutor backgroundThreadService; private RequestConfig cachedRequestConfig = null; - protected LogApiAccess() { + private LogApiAccess() { // We set-up a thread 'pool' that will contain at most one thread. // If the thread is idle for 60 seconds, it will be shut down. backgroundThreadService = new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); } - public abstract URI endpoint(); - public Future logWithConfirmation(LogMessage message) { var currentRequestConfig = getRequestConfig(); CompletableFuture completionNotification = new CompletableFuture<>(); @@ -194,7 +194,7 @@ private RequestConfig getRequestConfig() { if (cachedRequestConfig != null) { return cachedRequestConfig; } - var uri = endpoint(); + var uri = URI.create(CloudAPI.getAPIRootURI() + "logs"); var config = new RequestConfig(uri, AuthenticationProvider.getAccessToken()); cachedRequestConfig = config; return config; From 0e2f2e7ad9d5befbf5cf230c3980e79114fc52c4 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Feb 2025 19:15:58 +0100 Subject: [PATCH 03/16] Move most functionality from AuditLogMessage to the base class --- .../enso/base/enso_cloud/audit/AuditLog.java | 4 +- .../enso_cloud/audit/AuditLogMessage.java | 73 ++------------ .../base/enso_cloud/logging/LogMessage.java | 99 ++++++++++++++++++- 3 files changed, 108 insertions(+), 68 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java index 90c820b2f01d..6b6289a8b751 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java @@ -18,13 +18,13 @@ private AuditLog() {} /** Schedules the log message to be sent in the next batch, and returns immediately. */ public static void logAsync(String type, String message, ObjectNode metadata) { - var event = new AuditLogMessage(type, message, metadata); + var event = AuditLogMessage.create(type, message, metadata); LogApiAccess.INSTANCE.logWithoutConfirmation(event); } /** Schedules the log message to be sent in the next batch, and waits until it has been sent. */ public static void logSynchronously(String type, String message, ObjectNode metadata) { - var event = new AuditLogMessage(type, message, metadata); + var event = AuditLogMessage.create(type, message, metadata); Future future = LogApiAccess.INSTANCE.logWithConfirmation(event); try { future.get(); diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java index da59f1d27021..e51e05afdd12 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java @@ -1,82 +1,29 @@ package org.enso.base.enso_cloud.audit; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.NullNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import java.util.Objects; -import org.enso.base.CurrentEnsoProject; -import org.enso.base.enso_cloud.CloudAPI; import org.enso.base.enso_cloud.logging.LogMessage; -class AuditLogMessage implements LogMessage { - - /** - * A reserved field that is currently added by the cloud backend. Duplicating it will lead to - * internal server errors and log messages being discarded. - */ - private static final String RESERVED_TYPE = "type"; +final class AuditLogMessage extends LogMessage { private static final String OPERATION = "operation"; - private static final String PROJECT_NAME = "projectName"; - private static final String PROJECT_ID = "projectId"; - private static final String PROJECT_SESSION_ID = "projectSessionId"; - private static final String LOCAL_TIMESTAMP = "localTimestamp"; - - private final String projectId; - private final String projectName; - private final String operation; - private final String message; - private final ObjectNode metadata; - - public AuditLogMessage(String operation, String message, ObjectNode metadata) { - this.operation = Objects.requireNonNull(operation); - this.message = Objects.requireNonNull(message); - this.metadata = Objects.requireNonNull(metadata); - checkNoRestrictedField(metadata, RESERVED_TYPE); - checkNoRestrictedField(metadata, OPERATION); - checkNoRestrictedField(metadata, PROJECT_NAME); - checkNoRestrictedField(metadata, PROJECT_SESSION_ID); - checkNoRestrictedField(metadata, LOCAL_TIMESTAMP); - this.projectId = CloudAPI.getCloudProjectId(); - - var currentProject = CurrentEnsoProject.get(); - this.projectName = currentProject == null ? null : currentProject.fullName(); + private AuditLogMessage(String message, ObjectNode metadata) { + super(message, metadata); } - private static void checkNoRestrictedField(ObjectNode metadata, String fieldName) { - if (metadata.has(fieldName)) { - throw new IllegalArgumentException( - "Metadata cannot contain a field named '" + fieldName + "'"); - } - } - - private ObjectNode computedMetadata() { + public static AuditLogMessage create(String operation, String message, ObjectNode metadata) { + Objects.requireNonNull(operation); + Objects.requireNonNull(message); + Objects.requireNonNull(metadata); var copy = metadata.deepCopy(); copy.set(OPERATION, TextNode.valueOf(operation)); - - // The project name may be null if a script is run outside a project. - if (projectName != null) { - copy.set(PROJECT_NAME, TextNode.valueOf(projectName)); - } - - String projectSessionId = CloudAPI.getCloudSessionId(); - if (projectSessionId != null) { - copy.set(PROJECT_SESSION_ID, TextNode.valueOf(projectSessionId)); - } - - return copy; + return new AuditLogMessage(message, copy); } @Override - public String payload() { - var payload = new ObjectNode(JsonNodeFactory.instance); - payload.set("message", TextNode.valueOf(message)); - payload.set( - PROJECT_ID, projectId == null ? NullNode.getInstance() : TextNode.valueOf(projectId)); - payload.set("metadata", computedMetadata()); - payload.set("kind", TextNode.valueOf("Lib")); - return payload.toString(); + protected String kind() { + return "Lib"; } } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java index dd0c1f9bc2b5..cce1d1c259ce 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java @@ -1,8 +1,101 @@ package org.enso.base.enso_cloud.logging; -public interface LogMessage { +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import java.util.Objects; +import org.enso.base.CurrentEnsoProject; +import org.enso.base.enso_cloud.CloudAPI; + +/** Base class for log messages that are passed to the OpenSearch cloud endpoint. */ +public abstract class LogMessage { + /** + * A reserved field that is currently added by the cloud backend. Duplicating it will lead to + * internal server errors and log messages being discarded. + */ + private static final String RESERVED_TYPE = "type"; + + private static final String PROJECT_NAME = "projectName"; + private static final String PROJECT_ID = "projectId"; + private static final String PROJECT_SESSION_ID = "projectSessionId"; + private static final String LOCAL_TIMESTAMP = "localTimestamp"; + + private final String message; + private final ObjectNode extraMetadata; + private final String projectId; + private final String projectName; + /** - * @return JSON string representation of the log message. + * @param message + * @param extraMetadata Optional additional metadata to include in the log message. May be null */ - String payload(); + protected LogMessage(String message, ObjectNode extraMetadata) { + this.message = Objects.requireNonNull(message); + this.extraMetadata = extraMetadata; + this.projectId = CloudAPI.getCloudProjectId(); + var currentProject = CurrentEnsoProject.get(); + this.projectName = currentProject == null ? null : currentProject.fullName(); + if (extraMetadata != null) { + checkNoRestrictedFields(extraMetadata); + } + } + + private static void checkNoRestrictedField(ObjectNode metadata, String fieldName) { + if (metadata.has(fieldName)) { + throw new IllegalArgumentException( + "Metadata cannot contain a field named '" + fieldName + "'. Metadata was: " + metadata); + } + } + + private static void checkNoRestrictedFields(ObjectNode metadata) { + checkNoRestrictedField(metadata, RESERVED_TYPE); + checkNoRestrictedField(metadata, LOCAL_TIMESTAMP); + checkNoRestrictedField(metadata, PROJECT_NAME); + checkNoRestrictedField(metadata, PROJECT_ID); + } + + private ObjectNode computedMetadata() { + var copy = new ObjectNode(JsonNodeFactory.instance); + + // The project name may be null if a script is run outside a project. + if (projectName != null) { + copy.set(PROJECT_NAME, TextNode.valueOf(projectName)); + } + + String projectSessionId = CloudAPI.getCloudSessionId(); + if (projectSessionId != null) { + copy.set(PROJECT_SESSION_ID, TextNode.valueOf(projectSessionId)); + } + + if (extraMetadata != null) { + extraMetadata + .fields() + .forEachRemaining( + entry -> { + copy.set(entry.getKey(), entry.getValue()); + }); + } + + return copy; + } + + public String payload() { + var payload = new ObjectNode(JsonNodeFactory.instance); + payload.set("message", TextNode.valueOf(message)); + payload.set( + PROJECT_ID, projectId == null ? NullNode.getInstance() : TextNode.valueOf(projectId)); + if (projectName != null) { + payload.set(PROJECT_NAME, TextNode.valueOf(projectName)); + } + String projectSessionId = CloudAPI.getCloudSessionId(); + if (projectSessionId != null) { + payload.set(PROJECT_SESSION_ID, TextNode.valueOf(projectSessionId)); + } + payload.set("metadata", computedMetadata()); + payload.set("kind", TextNode.valueOf(kind())); + return payload.toString(); + } + + protected abstract String kind(); } From e150c322598989363f1f0f3162d7a07b5580d6f7 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Feb 2025 19:34:59 +0100 Subject: [PATCH 04/16] Introduce TelemetryLog --- .../Standard/Base/0.0.0-dev/src/Logging.enso | 29 +++++++++++++++++++ .../enso_cloud/telemetry/TelemetryLog.java | 19 ++++++++++++ .../telemetry/TelemetryLogMessage.java | 27 +++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLog.java create mode 100644 std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso index 7fcc81c65ec1..5b4b085102bb 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso @@ -1,10 +1,16 @@ import project.Any.Any +import project.Data.Dictionary.Dictionary +import project.Data.Json.JS_Object import project.Data.Text.Text import project.Data.Vector.Vector import project.Data.Numbers.Integer +import project.Errors.Illegal_Argument import project.Meta import project.Nothing.Nothing +import project.Panic.Panic +polyglot java import org.enso.base.enso_cloud.telemetry.TelemetryLog +polyglot java import org.enso.base.enso_cloud.telemetry.TelemetryLog.TelemetryLogError polyglot java import org.slf4j.LoggerFactory ## PRIVATE @@ -83,6 +89,29 @@ type Progress ## Simple name of the progress to_text self = "Progress" + +type Telemetry + logger_root = "org.enso.telemetry" + + log self name:Text message:Text (metadata:Dictionary Text Any = Dictionary.empty) = + logger_name = Telemetry.logger_root + "." + name + metadata_js = JS_Object.from_pairs <| metadata.to_vector + Standard.Base.IO.println <| "Telemetry.log: name=" + name + ", message=" + message + ", metadata_js=" + metadata_js.to_text + Illegal_Argument.handle_java_exception <| + Telemetry_Log_Error.handle_java_exception <| + TelemetryLog.logAsync logger_name message metadata_js + + +type Telemetry_Log_Error + private Error message:Text cause:Any + + private handle_java_exception = + on_error caught_panic = + cause = caught_panic.payload + Panic.throw (Telemetry_Log_Error.Error cause.getMessage cause) + Panic.catch TelemetryLogError handler=on_error + + ## PRIVATE type Log_Level ## Finest (Trace) level log message. diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLog.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLog.java new file mode 100644 index 000000000000..2348ed486d3c --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLog.java @@ -0,0 +1,19 @@ +package org.enso.base.enso_cloud.telemetry; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.enso.base.enso_cloud.logging.LogApiAccess; + +public final class TelemetryLog { + private TelemetryLog() {} + + public static void logAsync(String loggerName, String message, ObjectNode metadata) { + var event = TelemetryLogMessage.create(loggerName, message, metadata); + LogApiAccess.INSTANCE.logWithoutConfirmation(event); + } + + public static class TelemetryLogError extends RuntimeException { + public TelemetryLogError(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java new file mode 100644 index 000000000000..a30cf44eecce --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java @@ -0,0 +1,27 @@ +package org.enso.base.enso_cloud.telemetry; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import java.util.Objects; +import org.enso.base.enso_cloud.logging.LogMessage; + +public final class TelemetryLogMessage extends LogMessage { + + private TelemetryLogMessage(String message, ObjectNode extraMeta) { + super(message, extraMeta); + } + + public static TelemetryLogMessage create(String loggerName, String message, ObjectNode metadata) { + Objects.requireNonNull(loggerName); + Objects.requireNonNull(message); + Objects.requireNonNull(metadata); + var copy = metadata.deepCopy(); + copy.set("loggerName", TextNode.valueOf(loggerName)); + return new TelemetryLogMessage(message, copy); + } + + @Override + protected String kind() { + return "Telemetry"; + } +} From 296b9267279d683c14ee950e9195e7af123c7856 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Feb 2025 19:47:37 +0100 Subject: [PATCH 05/16] Add Telemetry_Log_Spec test --- .../Standard/Base/0.0.0-dev/src/Logging.enso | 6 +- .../src/Network/Enso_Cloud/Main.enso | 2 + .../Enso_Cloud/Telemetry_Log_Spec.enso | 59 +++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso index 5b4b085102bb..a48e6661de5c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso @@ -4,7 +4,7 @@ import project.Data.Json.JS_Object import project.Data.Text.Text import project.Data.Vector.Vector import project.Data.Numbers.Integer -import project.Errors.Illegal_Argument +import project.Errors.Illegal_Argument.Illegal_Argument import project.Meta import project.Nothing.Nothing import project.Panic.Panic @@ -96,10 +96,10 @@ type Telemetry log self name:Text message:Text (metadata:Dictionary Text Any = Dictionary.empty) = logger_name = Telemetry.logger_root + "." + name metadata_js = JS_Object.from_pairs <| metadata.to_vector - Standard.Base.IO.println <| "Telemetry.log: name=" + name + ", message=" + message + ", metadata_js=" + metadata_js.to_text + Standard.Base.IO.println <| "Telemetry.log: name=" + logger_name + ", message=" + message + ", metadata_js=" + metadata_js.to_text Illegal_Argument.handle_java_exception <| Telemetry_Log_Error.handle_java_exception <| - TelemetryLog.logAsync logger_name message metadata_js + TelemetryLog.logAsync logger_name message metadata_js.object_node type Telemetry_Log_Error diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Main.enso b/test/Base_Tests/src/Network/Enso_Cloud/Main.enso index 4db6f27e95f5..e817bbb761e8 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Main.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Main.enso @@ -8,6 +8,7 @@ import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup import project.Network.Enso_Cloud.Enso_Cloud_Spec import project.Network.Enso_Cloud.Enso_File_Spec import project.Network.Enso_Cloud.Secrets_Spec +import project.Network.Enso_Cloud.Telemetry_Log_Spec add_specs suite_builder (setup : Cloud_Tests_Setup = Cloud_Tests_Setup.prepare) = Enso_Cloud_Spec.add_specs suite_builder setup @@ -15,6 +16,7 @@ add_specs suite_builder (setup : Cloud_Tests_Setup = Cloud_Tests_Setup.prepare) Secrets_Spec.add_specs suite_builder setup Cloud_Data_Link_Spec.add_specs suite_builder setup Audit_Log_Spec.add_specs suite_builder + Telemetry_Log_Spec.add_specs suite_builder main filter=Nothing = setup = Cloud_Tests_Setup.prepare diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso new file mode 100644 index 000000000000..2f0a79874212 --- /dev/null +++ b/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso @@ -0,0 +1,59 @@ +from Standard.Base import all +import Standard.Base.Logging.Telemetry +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.Errors.Time_Error.Time_Error +import Standard.Base.Enso_Cloud.Errors.Enso_Cloud_Error +from Standard.Base.Enso_Cloud.Public_Utils import get_optional_field, get_required_field, cloud_http_request_for_test + +from Standard.Test import all +import Standard.Test.Test_Environment + +import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup + + +add_specs suite_builder = + always_run_on_mock = True + setup = if always_run_on_mock then Cloud_Tests_Setup.prepare_mock_setup else Cloud_Tests_Setup.prepare + suite_builder.group "Telemetry logging" pending=setup.pending group_builder-> + group_builder.specify "send and receive simple log event" <| + setup.with_prepared_environment <| + random_payload = "payload-" + Random.uuid + Telemetry.log "TestTelemetry" "Message" (Dictionary.from_vector [["my_field", random_payload]]) . should_succeed + + my_event = Test.with_retries <| + event = get_telemetry_log_events . find ev-> (ev.metadata.get "my_field") == random_payload + event.should_succeed + event + + my_event.metadata.get "loggerName" . should_equal "org.enso.telemetry.TestTelemetry" + my_event.metadata.get "projectName" . should_equal "enso_dev.Base_Tests" + + my_event.message . should_equal "Message" + my_event.user_email . should_equal Enso_User.current.email + + +private get_telemetry_log_events -> Vector Telemetry_Log_Event = + json = cloud_http_request_for_test HTTP_Method.Get "log_events" + events_json = get_required_field "events" json + events_json.map Telemetry_Log_Event.from_json + + +type Telemetry_Log_Event + private Value organization_id:Text user_email:Text timestamp:Date_Time|Nothing metadata:JS_Object message:Text project_id:Text|Nothing + + private from_json json = + organization_id = get_required_field "organizationId" json expected_type=Text + user_email = get_required_field "userEmail" json expected_type=Text + timestamp_text = get_optional_field "timestamp" json expected_type=Text + timestamp = timestamp_text.if_not_nothing <| Date_Time.parse timestamp_text . catch Time_Error error-> + Error.throw (Enso_Cloud_Error.Invalid_Response_Payload "Invalid timestamp format in audit log event: "+error.to_display_text) + metadata = get_required_field "metadata" json + message = get_required_field "message" json expected_type=Text + project_id = get_optional_field "projectId" json expected_type=Text + Telemetry_Log_Event.Value organization_id user_email timestamp metadata message project_id + + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter From d485a521d2be870cf4e907c1662b41ad9788526c Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Feb 2025 19:53:08 +0100 Subject: [PATCH 06/16] Handle null EnsoProject --- .../base/src/main/java/org/enso/base/CurrentEnsoProject.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/std-bits/base/src/main/java/org/enso/base/CurrentEnsoProject.java b/std-bits/base/src/main/java/org/enso/base/CurrentEnsoProject.java index 236daa2ab4a4..156ae4523cf5 100644 --- a/std-bits/base/src/main/java/org/enso/base/CurrentEnsoProject.java +++ b/std-bits/base/src/main/java/org/enso/base/CurrentEnsoProject.java @@ -24,7 +24,8 @@ public static CurrentEnsoProject get() { EnsoMeta.callStaticModuleMethod("Standard.Base.Meta.Enso_Project", "enso_project"); Value namespace = invokeMember("namespace", ensoProject); Value name = invokeMember("name", ensoProject); - Value rootPath = invokeMember("path", invokeMember("root", ensoProject)); + Value root = invokeMember("root", ensoProject); + Value rootPath = root != null ? invokeMember("path", root) : null; if (namespace == null || name == null || rootPath == null) { cached = null; } else { From 0d7ce6b3737f1250155a327f5df01f18ff1da308 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Feb 2025 19:59:36 +0100 Subject: [PATCH 07/16] Add some telemetry to Data.read --- .../lib/Standard/Base/0.0.0-dev/src/Data.enso | 11 +++++++++-- .../org/enso/base/enso_cloud/logging/LogMessage.java | 5 +++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index 19d94f79c8f0..147cc7574b45 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -1,5 +1,6 @@ import project.Any.Any import project.Data.Download.Download_Mode.Download_Mode +import project.Data.Dictionary.Dictionary import project.Data.Pair.Pair import project.Data.Read.Many_Files_List.Many_Files_List import project.Data.Read.Return_As.Return_As @@ -16,6 +17,7 @@ import project.Errors.File_Error.File_Error import project.Errors.Illegal_Argument.Illegal_Argument import project.Errors.Problem_Behavior.Problem_Behavior import project.Internal.Data_Read_Helpers +import project.Logging.Telemetry import project.Meta import project.Network.HTTP.Cache_Policy.Cache_Policy import project.Network.HTTP.Header.Header @@ -91,11 +93,16 @@ read : Text | URI | File -> File_Format -> Problem_Behavior -> Any ! File_Error read path=(Missing_Argument.throw "path") format=Auto_Detect (on_problems : Problem_Behavior = ..Report_Warning) = case path of _ : Text -> if Data_Read_Helpers.looks_like_uri path then Data_Read_Helpers.fetch_following_data_links path format=format else read (File.new path) format on_problems - uri : URI -> fetch uri format=format + uri : URI -> + response = fetch uri format=format + Telemetry.log "Data.read" "fetching from URI" (Dictionary.from_vector [["content_length", response.content_length]]) + response _ -> file_obj = File.new path if file_obj.is_directory then Error.throw (Illegal_Argument.Error "Cannot `read` a directory, use `Data.list`.") else - file_obj.read format on_problems + file_content = file_obj.read format on_problems + Telemetry.log "Data.read" "read from file" (Dictionary.from_vector [["path", file_obj.path]]) + file_content ## ALIAS load, open GROUP Input diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java index cce1d1c259ce..a253ec45e5e5 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java @@ -98,4 +98,9 @@ public String payload() { } protected abstract String kind(); + + @Override + public String toString() { + return payload(); + } } From 5300f3592d45637709d033bd5fa4549c7abf31be Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Feb 2025 19:59:57 +0100 Subject: [PATCH 08/16] Remove IO.println from Logging --- distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso | 1 - 1 file changed, 1 deletion(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso index a48e6661de5c..9ffe0c455bad 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso @@ -96,7 +96,6 @@ type Telemetry log self name:Text message:Text (metadata:Dictionary Text Any = Dictionary.empty) = logger_name = Telemetry.logger_root + "." + name metadata_js = JS_Object.from_pairs <| metadata.to_vector - Standard.Base.IO.println <| "Telemetry.log: name=" + logger_name + ", message=" + message + ", metadata_js=" + metadata_js.to_text Illegal_Argument.handle_java_exception <| Telemetry_Log_Error.handle_java_exception <| TelemetryLog.logAsync logger_name message metadata_js.object_node From 51db71e1fc1e0d26e5f881d4e46a4f606d4d8094 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 4 Mar 2025 11:43:04 +0100 Subject: [PATCH 09/16] [WIP] Add more logging --- .../java/org/enso/base/enso_cloud/logging/LogApiAccess.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java index 31129315bd87..d03b366a1aaf 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java @@ -162,6 +162,7 @@ private HttpRequest buildRequest(RequestConfig requestConfig, List messa assert requestConfig != null : "The request configuration must be set before building a request."; var payload = buildPayload(messages); + LOGGER.warning(() -> "Constructed HttpRequest with payload " + payload + ". Requestconfig: " + requestConfig); return HttpRequest.newBuilder() .uri(requestConfig.apiUri()) .header("Authorization", "Bearer " + requestConfig.accessToken()) @@ -219,7 +220,7 @@ private void sendLogRequest(HttpRequest request, int retryCount) throws RequestF } } catch (RequestFailureException e) { if (retryCount < 0) { - LOGGER.severe("Failed to send log messages after retrying: " + e.getMessage()); + LOGGER.severe("Failed to send log messages after retrying: " + e.getMessage() + ". The request was: " + request); throw e; } else { LOGGER.warning("Exception when sending log messages: " + e.getMessage() + ". Retrying..."); From c540239727c9f4e779b23ea4e32c68c116c388b0 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 5 Mar 2025 14:32:54 +0100 Subject: [PATCH 10/16] Revert "[WIP] Add more logging" This reverts commit 51db71e1fc1e0d26e5f881d4e46a4f606d4d8094. --- .../java/org/enso/base/enso_cloud/logging/LogApiAccess.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java index d03b366a1aaf..31129315bd87 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java @@ -162,7 +162,6 @@ private HttpRequest buildRequest(RequestConfig requestConfig, List messa assert requestConfig != null : "The request configuration must be set before building a request."; var payload = buildPayload(messages); - LOGGER.warning(() -> "Constructed HttpRequest with payload " + payload + ". Requestconfig: " + requestConfig); return HttpRequest.newBuilder() .uri(requestConfig.apiUri()) .header("Authorization", "Bearer " + requestConfig.accessToken()) @@ -220,7 +219,7 @@ private void sendLogRequest(HttpRequest request, int retryCount) throws RequestF } } catch (RequestFailureException e) { if (retryCount < 0) { - LOGGER.severe("Failed to send log messages after retrying: " + e.getMessage() + ". The request was: " + request); + LOGGER.severe("Failed to send log messages after retrying: " + e.getMessage()); throw e; } else { LOGGER.warning("Exception when sending log messages: " + e.getMessage() + ". Retrying..."); From 6ecdd77b5b24a00ce7839b8921f752d804dd0597 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 6 Mar 2025 12:16:16 +0100 Subject: [PATCH 11/16] LogMessage may have extraM { + meta.set(entry.getKey(), entry.getValue()); + }); + return meta; + } } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java index a253ec45e5e5..ed5c060c2ac2 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java @@ -22,23 +22,16 @@ public abstract class LogMessage { private static final String LOCAL_TIMESTAMP = "localTimestamp"; private final String message; - private final ObjectNode extraMetadata; private final String projectId; private final String projectName; + private final String projectSessionId; - /** - * @param message - * @param extraMetadata Optional additional metadata to include in the log message. May be null - */ - protected LogMessage(String message, ObjectNode extraMetadata) { + protected LogMessage(String message) { this.message = Objects.requireNonNull(message); - this.extraMetadata = extraMetadata; this.projectId = CloudAPI.getCloudProjectId(); var currentProject = CurrentEnsoProject.get(); this.projectName = currentProject == null ? null : currentProject.fullName(); - if (extraMetadata != null) { - checkNoRestrictedFields(extraMetadata); - } + this.projectSessionId = CloudAPI.getCloudSessionId(); } private static void checkNoRestrictedField(ObjectNode metadata, String fieldName) { @@ -48,7 +41,7 @@ private static void checkNoRestrictedField(ObjectNode metadata, String fieldName } } - private static void checkNoRestrictedFields(ObjectNode metadata) { + protected static void checkNoRestrictedFields(ObjectNode metadata) { checkNoRestrictedField(metadata, RESERVED_TYPE); checkNoRestrictedField(metadata, LOCAL_TIMESTAMP); checkNoRestrictedField(metadata, PROJECT_NAME); @@ -56,49 +49,52 @@ private static void checkNoRestrictedFields(ObjectNode metadata) { } private ObjectNode computedMetadata() { - var copy = new ObjectNode(JsonNodeFactory.instance); - + var meta = new ObjectNode(JsonNodeFactory.instance); + meta.set(PROJECT_ID, projectId == null ? NullNode.getInstance() : TextNode.valueOf(projectId)); // The project name may be null if a script is run outside a project. if (projectName != null) { - copy.set(PROJECT_NAME, TextNode.valueOf(projectName)); + meta.set(PROJECT_NAME, TextNode.valueOf(projectName)); } - String projectSessionId = CloudAPI.getCloudSessionId(); if (projectSessionId != null) { - copy.set(PROJECT_SESSION_ID, TextNode.valueOf(projectSessionId)); + meta.set(PROJECT_SESSION_ID, TextNode.valueOf(projectSessionId)); } - if (extraMetadata != null) { - extraMetadata + if (extraMetadata() != null) { + extraMetadata() .fields() .forEachRemaining( entry -> { - copy.set(entry.getKey(), entry.getValue()); + meta.set(entry.getKey(), entry.getValue()); }); } - - return copy; + return meta; } public String payload() { var payload = new ObjectNode(JsonNodeFactory.instance); payload.set("message", TextNode.valueOf(message)); - payload.set( - PROJECT_ID, projectId == null ? NullNode.getInstance() : TextNode.valueOf(projectId)); - if (projectName != null) { - payload.set(PROJECT_NAME, TextNode.valueOf(projectName)); - } - String projectSessionId = CloudAPI.getCloudSessionId(); - if (projectSessionId != null) { - payload.set(PROJECT_SESSION_ID, TextNode.valueOf(projectSessionId)); - } payload.set("metadata", computedMetadata()); payload.set("kind", TextNode.valueOf(kind())); + if (extraPayload() != null) { + extraPayload() + .fields() + .forEachRemaining( + entry -> { + payload.set(entry.getKey(), entry.getValue()); + }); + } return payload.toString(); } protected abstract String kind(); + /** Returns optional JSON object that will be appended to the payload. May return null. */ + protected abstract ObjectNode extraPayload(); + + /** Returns optional JSON object that will be appended to the metadata. May return null. */ + protected abstract ObjectNode extraMetadata(); + @Override public String toString() { return payload(); diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java index a30cf44eecce..8f9e5e099f31 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java @@ -1,5 +1,6 @@ package org.enso.base.enso_cloud.telemetry; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import java.util.Objects; @@ -7,21 +8,37 @@ public final class TelemetryLogMessage extends LogMessage { - private TelemetryLogMessage(String message, ObjectNode extraMeta) { - super(message, extraMeta); + private final String loggerName; + private final ObjectNode extraMeta; + + private TelemetryLogMessage(String message, String loggerName, ObjectNode extraMeta) { + super(message); + this.loggerName = loggerName; + this.extraMeta = extraMeta; } public static TelemetryLogMessage create(String loggerName, String message, ObjectNode metadata) { Objects.requireNonNull(loggerName); Objects.requireNonNull(message); Objects.requireNonNull(metadata); - var copy = metadata.deepCopy(); - copy.set("loggerName", TextNode.valueOf(loggerName)); - return new TelemetryLogMessage(message, copy); + return new TelemetryLogMessage(message, loggerName, metadata); } @Override protected String kind() { return "Telemetry"; } + + @Override + protected ObjectNode extraPayload() { + return null; + } + + @Override + protected ObjectNode extraMetadata() { + var meta = new ObjectNode(JsonNodeFactory.instance); + meta.set("loggerName", TextNode.valueOf(loggerName)); + extraMeta.fields().forEachRemaining(entry -> meta.set(entry.getKey(), entry.getValue())); + return meta; + } } From 963f0889beb31180cbf7b4e89b8572d0dc2ccf19 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 6 Mar 2025 19:27:25 +0100 Subject: [PATCH 12/16] Move Telemetry to private module --- .../lib/Standard/Base/0.0.0-dev/src/Data.enso | 2 +- .../0.0.0-dev/src/Internal/Telemetry.enso | 41 +++++++++++++++++++ .../Standard/Base/0.0.0-dev/src/Logging.enso | 28 ------------- .../Enso_Cloud/Telemetry_Log_Spec.enso | 5 ++- 4 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Telemetry.enso diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index 147cc7574b45..4a2c4e706ccc 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -17,7 +17,7 @@ import project.Errors.File_Error.File_Error import project.Errors.Illegal_Argument.Illegal_Argument import project.Errors.Problem_Behavior.Problem_Behavior import project.Internal.Data_Read_Helpers -import project.Logging.Telemetry +import project.Internal.Telemetry.Telemetry import project.Meta import project.Network.HTTP.Cache_Policy.Cache_Policy import project.Network.HTTP.Header.Header diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Telemetry.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Telemetry.enso new file mode 100644 index 000000000000..df5fdf6002f9 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Telemetry.enso @@ -0,0 +1,41 @@ +private + +import project.Any.Any +import project.Data.Dictionary.Dictionary +import project.Data.Json.JS_Object +import project.Data.Text.Text +import project.Errors.Illegal_Argument.Illegal_Argument +import project.Panic.Panic + +polyglot java import org.enso.base.enso_cloud.telemetry.TelemetryLog +polyglot java import org.enso.base.enso_cloud.telemetry.TelemetryLog.TelemetryLogError + +## Heavily inspired by `Standard.Base.Enso_Cloud.Internal.Audit_Log`. +type Telemetry + logger_root = "org.enso.telemetry" + + ## Reports an event to the telemetry log. + The event is submitted asynchronously. + + Arguments: + - name: Name of the telemetry logger + - message: The message associated with the event. + - metadata: Additional metadata to include with the event. + Note that there are some _restricted_ keys that are added automatically. + See `Standard.Base.Enso_Cloud.Internal.Audit_Log`. + log self name:Text message:Text (metadata:Dictionary Text Any = Dictionary.empty) = + logger_name = Telemetry.logger_root + "." + name + metadata_js = JS_Object.from_pairs <| metadata.to_vector + Illegal_Argument.handle_java_exception <| + Telemetry_Log_Error.handle_java_exception <| + TelemetryLog.logAsync logger_name message metadata_js.object_node + + +type Telemetry_Log_Error + private Error message:Text cause:Any + + private handle_java_exception = + on_error caught_panic = + cause = caught_panic.payload + Panic.throw (Telemetry_Log_Error.Error cause.getMessage cause) + Panic.catch TelemetryLogError handler=on_error diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso index 9ffe0c455bad..7fcc81c65ec1 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Logging.enso @@ -1,16 +1,10 @@ import project.Any.Any -import project.Data.Dictionary.Dictionary -import project.Data.Json.JS_Object import project.Data.Text.Text import project.Data.Vector.Vector import project.Data.Numbers.Integer -import project.Errors.Illegal_Argument.Illegal_Argument import project.Meta import project.Nothing.Nothing -import project.Panic.Panic -polyglot java import org.enso.base.enso_cloud.telemetry.TelemetryLog -polyglot java import org.enso.base.enso_cloud.telemetry.TelemetryLog.TelemetryLogError polyglot java import org.slf4j.LoggerFactory ## PRIVATE @@ -89,28 +83,6 @@ type Progress ## Simple name of the progress to_text self = "Progress" - -type Telemetry - logger_root = "org.enso.telemetry" - - log self name:Text message:Text (metadata:Dictionary Text Any = Dictionary.empty) = - logger_name = Telemetry.logger_root + "." + name - metadata_js = JS_Object.from_pairs <| metadata.to_vector - Illegal_Argument.handle_java_exception <| - Telemetry_Log_Error.handle_java_exception <| - TelemetryLog.logAsync logger_name message metadata_js.object_node - - -type Telemetry_Log_Error - private Error message:Text cause:Any - - private handle_java_exception = - on_error caught_panic = - cause = caught_panic.payload - Panic.throw (Telemetry_Log_Error.Error cause.getMessage cause) - Panic.catch TelemetryLogError handler=on_error - - ## PRIVATE type Log_Level ## Finest (Trace) level log message. diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso index 2f0a79874212..f59116c04a0f 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso @@ -1,8 +1,8 @@ from Standard.Base import all -import Standard.Base.Logging.Telemetry import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Errors.Time_Error.Time_Error import Standard.Base.Enso_Cloud.Errors.Enso_Cloud_Error +import Standard.Base.Internal.Telemetry.Telemetry from Standard.Base.Enso_Cloud.Public_Utils import get_optional_field, get_required_field, cloud_http_request_for_test from Standard.Test import all @@ -10,7 +10,8 @@ import Standard.Test.Test_Environment import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup - +## Heavily inspired by `Audit_Log_Spec`. + TODO[pm]: Should be removed once telemetry is implemented as a slf4j logger appender. add_specs suite_builder = always_run_on_mock = True setup = if always_run_on_mock then Cloud_Tests_Setup.prepare_mock_setup else Cloud_Tests_Setup.prepare From 19212cff10fcb37c42a0e45cd0350ef62c7a9ea7 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 6 Mar 2025 19:31:56 +0100 Subject: [PATCH 13/16] Move Telemetry_Log_Spec to Base_Internal_Tests --- test/Base_Internal_Tests/src/Main.enso | 2 ++ .../src}/Telemetry_Log_Spec.enso | 2 +- test/Base_Tests/src/Network/Enso_Cloud/Main.enso | 2 -- 3 files changed, 3 insertions(+), 3 deletions(-) rename test/{Base_Tests/src/Network/Enso_Cloud => Base_Internal_Tests/src}/Telemetry_Log_Spec.enso (97%) diff --git a/test/Base_Internal_Tests/src/Main.enso b/test/Base_Internal_Tests/src/Main.enso index ead1b1c88be0..83737ff0d72f 100644 --- a/test/Base_Internal_Tests/src/Main.enso +++ b/test/Base_Internal_Tests/src/Main.enso @@ -8,6 +8,7 @@ import project.Comparator_Spec import project.Decimal_Constructor_Spec import project.Grapheme_Spec import project.Github_Annotations_Spec +import project.Telemetry_Log_Spec main filter=Nothing = suite = Test.build suite_builder-> @@ -17,5 +18,6 @@ main filter=Nothing = Input_Output_Spec.add_specs suite_builder Instrumentor_Spec.add_specs suite_builder Github_Annotations_Spec.add_specs suite_builder + Telemetry_Log_Spec.add_specs suite_builder suite.run_with_filter filter diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso b/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso similarity index 97% rename from test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso rename to test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso index f59116c04a0f..9fbfec54c9c1 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Telemetry_Log_Spec.enso +++ b/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso @@ -8,7 +8,7 @@ from Standard.Base.Enso_Cloud.Public_Utils import get_optional_field, get_requir from Standard.Test import all import Standard.Test.Test_Environment -import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup +import enso_dev.Base_Tests.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup ## Heavily inspired by `Audit_Log_Spec`. TODO[pm]: Should be removed once telemetry is implemented as a slf4j logger appender. diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Main.enso b/test/Base_Tests/src/Network/Enso_Cloud/Main.enso index e817bbb761e8..4db6f27e95f5 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Main.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Main.enso @@ -8,7 +8,6 @@ import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup import project.Network.Enso_Cloud.Enso_Cloud_Spec import project.Network.Enso_Cloud.Enso_File_Spec import project.Network.Enso_Cloud.Secrets_Spec -import project.Network.Enso_Cloud.Telemetry_Log_Spec add_specs suite_builder (setup : Cloud_Tests_Setup = Cloud_Tests_Setup.prepare) = Enso_Cloud_Spec.add_specs suite_builder setup @@ -16,7 +15,6 @@ add_specs suite_builder (setup : Cloud_Tests_Setup = Cloud_Tests_Setup.prepare) Secrets_Spec.add_specs suite_builder setup Cloud_Data_Link_Spec.add_specs suite_builder setup Audit_Log_Spec.add_specs suite_builder - Telemetry_Log_Spec.add_specs suite_builder main filter=Nothing = setup = Cloud_Tests_Setup.prepare From 039b27949d88dee15ef785f348fff2c4fe329043 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 6 Mar 2025 19:40:34 +0100 Subject: [PATCH 14/16] Make classes package-private --- .../main/java/org/enso/base/enso_cloud/logging/LogJob.java | 2 +- .../java/org/enso/base/enso_cloud/logging/LogJobsQueue.java | 2 +- .../java/org/enso/base/enso_cloud/logging/RequestConfig.java | 2 +- .../enso/base/enso_cloud/logging/RequestFailureException.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java index c2e6ab2c7793..a978381c08a8 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java @@ -2,7 +2,7 @@ import java.util.concurrent.CompletableFuture; -public final class LogJob { +final class LogJob { private final LogMessage logMessage; private final CompletableFuture completionNotification; private final RequestConfig requestConfig; diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java index 25d553753db6..bdfc3def9d68 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java @@ -5,7 +5,7 @@ import java.util.LinkedList; import java.util.List; -public final class LogJobsQueue { +final class LogJobsQueue { private final Deque queue = new LinkedList<>(); /** Enqueues a log message to be sent and returns the number of messages in the queue. */ diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java index da33fab7ec5d..9c0a8c9efaf1 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java @@ -14,4 +14,4 @@ * the configs will be the same, they only change during testing. Tests should this into account, by * sending the last message in synchronous mode. */ -public record RequestConfig(URI apiUri, String accessToken) {} +record RequestConfig(URI apiUri, String accessToken) {} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java index d26407fd5aa8..a1ee175528cc 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java @@ -1,7 +1,7 @@ package org.enso.base.enso_cloud.logging; -public final class RequestFailureException extends RuntimeException { - public RequestFailureException(String message, Throwable cause) { +final class RequestFailureException extends RuntimeException { + RequestFailureException(String message, Throwable cause) { super(message, cause); } } From 18748322188b1b52bef457e7e0a51b938e6272ae Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 6 Mar 2025 19:44:35 +0100 Subject: [PATCH 15/16] Mock PostLogHandler treats projectId as optional field --- .../main/java/org/enso/shttp/cloud_mock/EventsService.java | 4 ++++ .../java/org/enso/shttp/cloud_mock/PostLogHandler.java | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/EventsService.java b/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/EventsService.java index 084777edfa27..46f395d673b1 100644 --- a/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/EventsService.java +++ b/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/EventsService.java @@ -5,6 +5,10 @@ import java.util.List; public class EventsService { + + /** + * @param projectId May be null + */ public record LogEvent( String organizationId, String userEmail, diff --git a/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/PostLogHandler.java b/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/PostLogHandler.java index fc8a20a30096..7b3ec413c2df 100644 --- a/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/PostLogHandler.java +++ b/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/PostLogHandler.java @@ -94,7 +94,12 @@ private EventsService.LogEvent parseLogEvent(JsonNode json) { String userEmail = usersService.currentUserEmail(); String timestamp = ZonedDateTime.now().withZoneSameInstant(ZoneId.of("UTC")).toString(); JsonNode metadata = json.get("metadata"); - String projectId = json.get("projectId").asText(); + String projectId; + if (json.has("projectId")) { + projectId = json.get("projectId").asText(); + } else { + projectId = null; + } return new EventsService.LogEvent( organizationId, userEmail, timestamp, metadata, message, projectId); } From 965443f7ae441e25a1341f3cffc0352e0e554d35 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 6 Mar 2025 19:44:47 +0100 Subject: [PATCH 16/16] Update project name in Telemetry_Log_Spec --- test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso b/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso index 9fbfec54c9c1..b34b0d73f956 100644 --- a/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso +++ b/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso @@ -27,7 +27,7 @@ add_specs suite_builder = event my_event.metadata.get "loggerName" . should_equal "org.enso.telemetry.TestTelemetry" - my_event.metadata.get "projectName" . should_equal "enso_dev.Base_Tests" + my_event.metadata.get "projectName" . should_equal "enso_dev.Base_Internal_Tests" my_event.message . should_equal "Message" my_event.user_email . should_equal Enso_User.current.email