diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index 19d94f79c8f0..4a2c4e706ccc 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -1,5 +1,6 @@ import project.Any.Any import project.Data.Download.Download_Mode.Download_Mode +import project.Data.Dictionary.Dictionary import project.Data.Pair.Pair import project.Data.Read.Many_Files_List.Many_Files_List import project.Data.Read.Return_As.Return_As @@ -16,6 +17,7 @@ import project.Errors.File_Error.File_Error import project.Errors.Illegal_Argument.Illegal_Argument import project.Errors.Problem_Behavior.Problem_Behavior import project.Internal.Data_Read_Helpers +import project.Internal.Telemetry.Telemetry import project.Meta import project.Network.HTTP.Cache_Policy.Cache_Policy import project.Network.HTTP.Header.Header @@ -91,11 +93,16 @@ read : Text | URI | File -> File_Format -> Problem_Behavior -> Any ! File_Error read path=(Missing_Argument.throw "path") format=Auto_Detect (on_problems : Problem_Behavior = ..Report_Warning) = case path of _ : Text -> if Data_Read_Helpers.looks_like_uri path then Data_Read_Helpers.fetch_following_data_links path format=format else read (File.new path) format on_problems - uri : URI -> fetch uri format=format + uri : URI -> + response = fetch uri format=format + Telemetry.log "Data.read" "fetching from URI" (Dictionary.from_vector [["content_length", response.content_length]]) + response _ -> file_obj = File.new path if file_obj.is_directory then Error.throw (Illegal_Argument.Error "Cannot `read` a directory, use `Data.list`.") else - file_obj.read format on_problems + file_content = file_obj.read format on_problems + Telemetry.log "Data.read" "read from file" (Dictionary.from_vector [["path", file_obj.path]]) + file_content ## ALIAS load, open GROUP Input diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Telemetry.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Telemetry.enso new file mode 100644 index 000000000000..df5fdf6002f9 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Telemetry.enso @@ -0,0 +1,41 @@ +private + +import project.Any.Any +import project.Data.Dictionary.Dictionary +import project.Data.Json.JS_Object +import project.Data.Text.Text +import project.Errors.Illegal_Argument.Illegal_Argument +import project.Panic.Panic + +polyglot java import org.enso.base.enso_cloud.telemetry.TelemetryLog +polyglot java import org.enso.base.enso_cloud.telemetry.TelemetryLog.TelemetryLogError + +## Heavily inspired by `Standard.Base.Enso_Cloud.Internal.Audit_Log`. +type Telemetry + logger_root = "org.enso.telemetry" + + ## Reports an event to the telemetry log. + The event is submitted asynchronously. + + Arguments: + - name: Name of the telemetry logger + - message: The message associated with the event. + - metadata: Additional metadata to include with the event. + Note that there are some _restricted_ keys that are added automatically. + See `Standard.Base.Enso_Cloud.Internal.Audit_Log`. + log self name:Text message:Text (metadata:Dictionary Text Any = Dictionary.empty) = + logger_name = Telemetry.logger_root + "." + name + metadata_js = JS_Object.from_pairs <| metadata.to_vector + Illegal_Argument.handle_java_exception <| + Telemetry_Log_Error.handle_java_exception <| + TelemetryLog.logAsync logger_name message metadata_js.object_node + + +type Telemetry_Log_Error + private Error message:Text cause:Any + + private handle_java_exception = + on_error caught_panic = + cause = caught_panic.payload + Panic.throw (Telemetry_Log_Error.Error cause.getMessage cause) + Panic.catch TelemetryLogError handler=on_error diff --git a/std-bits/base/src/main/java/org/enso/base/CurrentEnsoProject.java b/std-bits/base/src/main/java/org/enso/base/CurrentEnsoProject.java index 236daa2ab4a4..156ae4523cf5 100644 --- a/std-bits/base/src/main/java/org/enso/base/CurrentEnsoProject.java +++ b/std-bits/base/src/main/java/org/enso/base/CurrentEnsoProject.java @@ -24,7 +24,8 @@ public static CurrentEnsoProject get() { EnsoMeta.callStaticModuleMethod("Standard.Base.Meta.Enso_Project", "enso_project"); Value namespace = invokeMember("namespace", ensoProject); Value name = invokeMember("name", ensoProject); - Value rootPath = invokeMember("path", invokeMember("root", ensoProject)); + Value root = invokeMember("root", ensoProject); + Value rootPath = root != null ? invokeMember("path", root) : null; if (namespace == null || name == null || rootPath == null) { cached = null; } else { diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java index 9e3800fd2c90..6b6289a8b751 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLog.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import org.enso.base.enso_cloud.logging.LogApiAccess; /** * The high-level API for logging audit events. @@ -13,16 +14,18 @@ * the meantime, all waiting messages (up to some limit) will be sent in a single request. */ public final class AuditLog { + private AuditLog() {} + /** Schedules the log message to be sent in the next batch, and returns immediately. */ public static void logAsync(String type, String message, ObjectNode metadata) { - var event = new AuditLogMessage(type, message, metadata); - AuditLogApiAccess.INSTANCE.logWithoutConfirmation(event); + var event = AuditLogMessage.create(type, message, metadata); + LogApiAccess.INSTANCE.logWithoutConfirmation(event); } /** Schedules the log message to be sent in the next batch, and waits until it has been sent. */ public static void logSynchronously(String type, String message, ObjectNode metadata) { - var event = new AuditLogMessage(type, message, metadata); - Future future = AuditLogApiAccess.INSTANCE.logWithConfirmation(event); + var event = AuditLogMessage.create(type, message, metadata); + Future future = LogApiAccess.INSTANCE.logWithConfirmation(event); try { future.get(); } catch (ExecutionException | InterruptedException e) { @@ -37,6 +40,6 @@ public AuditLogError(String message, Throwable cause) { } public static void resetCache() { - AuditLogApiAccess.INSTANCE.resetCache(); + LogApiAccess.INSTANCE.resetCache(); } } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java index 7273661744cd..488bb2295567 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogMessage.java @@ -5,77 +5,56 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import java.util.Objects; -import org.enso.base.CurrentEnsoProject; import org.enso.base.enso_cloud.CloudAPI; +import org.enso.base.enso_cloud.logging.LogMessage; -class AuditLogMessage implements AuditLogApiAccess.LogMessage { - - /** - * A reserved field that is currently added by the cloud backend. Duplicating it will lead to - * internal server errors and log messages being discarded. - */ - private static final String RESERVED_TYPE = "type"; +final class AuditLogMessage extends LogMessage { private static final String OPERATION = "operation"; - private static final String PROJECT_NAME = "projectName"; private static final String PROJECT_ID = "projectId"; - private static final String PROJECT_SESSION_ID = "projectSessionId"; - private static final String LOCAL_TIMESTAMP = "localTimestamp"; - - private final String projectId; - private final String projectName; private final String operation; - private final String message; private final ObjectNode metadata; + private final String projectId; - public AuditLogMessage(String operation, String message, ObjectNode metadata) { - this.operation = Objects.requireNonNull(operation); - this.message = Objects.requireNonNull(message); - this.metadata = Objects.requireNonNull(metadata); - checkNoRestrictedField(metadata, RESERVED_TYPE); - checkNoRestrictedField(metadata, OPERATION); - checkNoRestrictedField(metadata, PROJECT_NAME); - checkNoRestrictedField(metadata, PROJECT_SESSION_ID); - checkNoRestrictedField(metadata, LOCAL_TIMESTAMP); - - this.projectId = CloudAPI.getCloudProjectId(); - - var currentProject = CurrentEnsoProject.get(); - this.projectName = currentProject == null ? null : currentProject.fullName(); + private AuditLogMessage(String message, String operation, ObjectNode metadata, String projectId) { + super(message); + this.operation = operation; + this.metadata = metadata; + this.projectId = projectId; } - private static void checkNoRestrictedField(ObjectNode metadata, String fieldName) { - if (metadata.has(fieldName)) { - throw new IllegalArgumentException( - "Metadata cannot contain a field named '" + fieldName + "'"); - } + public static AuditLogMessage create(String operation, String message, ObjectNode metadata) { + Objects.requireNonNull(operation); + Objects.requireNonNull(message); + Objects.requireNonNull(metadata); + checkNoRestrictedFields(metadata); + var projectId = CloudAPI.getCloudProjectId(); + return new AuditLogMessage(message, operation, metadata, projectId); } - private ObjectNode computedMetadata() { - var copy = metadata.deepCopy(); - copy.set(OPERATION, TextNode.valueOf(operation)); - - // The project name may be null if a script is run outside a project. - if (projectName != null) { - copy.set(PROJECT_NAME, TextNode.valueOf(projectName)); - } - - String projectSessionId = CloudAPI.getCloudSessionId(); - if (projectSessionId != null) { - copy.set(PROJECT_SESSION_ID, TextNode.valueOf(projectSessionId)); - } - - return copy; + @Override + protected String kind() { + return "Lib"; } @Override - public String payload() { + protected ObjectNode extraPayload() { var payload = new ObjectNode(JsonNodeFactory.instance); - payload.set("message", TextNode.valueOf(message)); payload.set( PROJECT_ID, projectId == null ? NullNode.getInstance() : TextNode.valueOf(projectId)); - payload.set("metadata", computedMetadata()); - payload.set("kind", TextNode.valueOf("Lib")); - return payload.toString(); + return payload; + } + + @Override + protected ObjectNode extraMetadata() { + var meta = new ObjectNode(JsonNodeFactory.instance); + meta.set(OPERATION, TextNode.valueOf(operation)); + metadata + .fields() + .forEachRemaining( + entry -> { + meta.set(entry.getKey(), entry.getValue()); + }); + return meta; } } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java similarity index 77% rename from std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java rename to std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java index 372ad014ef78..31129315bd87 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/AuditLogApiAccess.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogApiAccess.java @@ -1,4 +1,4 @@ -package org.enso.base.enso_cloud.audit; +package org.enso.base.enso_cloud.logging; import java.io.IOException; import java.net.URI; @@ -23,9 +23,7 @@ * Gives access to the low-level log event API in the Cloud and manages asynchronously submitting * the logs. */ -class AuditLogApiAccess { - private static final Logger logger = Logger.getLogger(AuditLogApiAccess.class.getName()); - +public final class LogApiAccess { /** * We still want to limit the batch size to some reasonable number - sending too many logs in one * request could also be problematic. @@ -33,14 +31,15 @@ class AuditLogApiAccess { private static final int MAX_BATCH_SIZE = 100; private static final int MAX_RETRIES = 5; - - public static AuditLogApiAccess INSTANCE = new AuditLogApiAccess(); + private static final Logger LOGGER = Logger.getLogger(LogApiAccess.class.getName()); + public static final LogApiAccess INSTANCE = new LogApiAccess(); private HttpClient httpClient; private final LogJobsQueue logQueue = new LogJobsQueue(); private final ThreadPoolExecutor backgroundThreadService; + private RequestConfig cachedRequestConfig = null; - private AuditLogApiAccess() { + private LogApiAccess() { // We set-up a thread 'pool' that will contain at most one thread. // If the thread is idle for 60 seconds, it will be shut down. backgroundThreadService = @@ -59,6 +58,10 @@ public void logWithoutConfirmation(LogMessage message) { enqueueJob(new LogJob(message, null, currentRequestConfig)); } + public void resetCache() { + cachedRequestConfig = null; + } + private void enqueueJob(LogJob job) { int queuedJobs = logQueue.enqueue(job); if (queuedJobs == 1 && backgroundThreadService.getQueue().isEmpty()) { @@ -109,10 +112,10 @@ private void sendBatch(List batch) { assert !batch.isEmpty() : "The batch must not be empty."; // We use the request config from the first message - all messages in the batch should have the // same request config. - var requestConfig = batch.get(0).requestConfig(); + var requestConfig = batch.get(0).getRequestConfig(); assert requestConfig != null : "The request configuration must be set before building a request."; - assert batch.stream().allMatch(job -> job.requestConfig().equals(requestConfig)) + assert batch.stream().allMatch(job -> job.getRequestConfig().equals(requestConfig)) : "All messages in a batch must have the same request configuration."; try { @@ -129,10 +132,10 @@ private void sendBatch(List batch) { * configs (when the config changes between tests). To send each message where it is intended, we * split up the batch by the config. */ - Collection> splitMessagesByConfig(List messages) { + private Collection> splitMessagesByConfig(List messages) { HashMap> hashMap = new HashMap<>(); for (var message : messages) { - var list = hashMap.computeIfAbsent(message.requestConfig(), k -> new ArrayList<>()); + var list = hashMap.computeIfAbsent(message.getRequestConfig(), k -> new ArrayList<>()); list.add(message); } @@ -141,16 +144,16 @@ Collection> splitMessagesByConfig(List messages) { private void notifyJobsAboutSuccess(List jobs) { for (var job : jobs) { - if (job.completionNotification() != null) { - job.completionNotification().complete(null); + if (job.getCompletionNotification() != null) { + job.getCompletionNotification().complete(null); } } } private void notifyJobsAboutFailure(List jobs, RequestFailureException e) { for (var job : jobs) { - if (job.completionNotification() != null) { - job.completionNotification().completeExceptionally(e); + if (job.getCompletionNotification() != null) { + job.getCompletionNotification().completeExceptionally(e); } } } @@ -160,8 +163,8 @@ private HttpRequest buildRequest(RequestConfig requestConfig, List messa : "The request configuration must be set before building a request."; var payload = buildPayload(messages); return HttpRequest.newBuilder() - .uri(requestConfig.apiUri) - .header("Authorization", "Bearer " + requestConfig.accessToken) + .uri(requestConfig.apiUri()) + .header("Authorization", "Bearer " + requestConfig.accessToken()) .POST(HttpRequest.BodyPublishers.ofString(payload, StandardCharsets.UTF_8)) .build(); } @@ -170,7 +173,7 @@ private String buildPayload(List messages) { var payload = new StringBuilder(); payload.append("{\"logs\": ["); for (var message : messages) { - payload.append(message.message().payload()).append(","); + payload.append(message.getLogMessage().payload()).append(","); } // Remove the trailing comma. @@ -179,8 +182,6 @@ private String buildPayload(List messages) { return payload.toString(); } - private RequestConfig cachedRequestConfig = null; - /** * Builds a request configuration based on runtime information. * @@ -193,26 +194,12 @@ private RequestConfig getRequestConfig() { if (cachedRequestConfig != null) { return cachedRequestConfig; } - var uri = URI.create(CloudAPI.getAPIRootURI() + "logs"); var config = new RequestConfig(uri, AuthenticationProvider.getAccessToken()); cachedRequestConfig = config; return config; } - /** - * Contains information needed to build a request to the Cloud Logs API. - * - *

This information must be gathered on the main Enso thread, as only there we have access to - * the {@link AuthenticationProvider}. - * - *

We associate an instance with every message to be sent. When sending multiple messages in a - * batch, we will use the config from one of them. This should not matter as in normal operations - * the configs will be the same, they only change during testing. Tests should this into account, - * by sending the last message in synchronous mode. - */ - private record RequestConfig(URI apiUri, String accessToken) {} - private void sendLogRequest(HttpRequest request, int retryCount) throws RequestFailureException { try { try { @@ -232,37 +219,12 @@ private void sendLogRequest(HttpRequest request, int retryCount) throws RequestF } } catch (RequestFailureException e) { if (retryCount < 0) { - logger.severe("Failed to send log messages after retrying: " + e.getMessage()); + LOGGER.severe("Failed to send log messages after retrying: " + e.getMessage()); throw e; } else { - logger.warning("Exception when sending log messages: " + e.getMessage() + ". Retrying..."); + LOGGER.warning("Exception when sending log messages: " + e.getMessage() + ". Retrying..."); sendLogRequest(request, retryCount - 1); } } } - - interface LogMessage { - String payload(); - } - - static class RequestFailureException extends RuntimeException { - public RequestFailureException(String message, Throwable cause) { - super(message, cause); - } - } - - /** - * A record that represents a single log to be sent. - * - *

It may contain the `completionNotification` future that will be completed when the log is - * sent. If no-one is listening for confirmation, that field will be `null`. - */ - record LogJob( - LogMessage message, - CompletableFuture completionNotification, - RequestConfig requestConfig) {} - - void resetCache() { - cachedRequestConfig = null; - } } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java new file mode 100644 index 000000000000..a978381c08a8 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJob.java @@ -0,0 +1,30 @@ +package org.enso.base.enso_cloud.logging; + +import java.util.concurrent.CompletableFuture; + +final class LogJob { + private final LogMessage logMessage; + private final CompletableFuture completionNotification; + private final RequestConfig requestConfig; + + public LogJob( + LogMessage logMessage, + CompletableFuture completionNotification, + RequestConfig requestConfig) { + this.logMessage = logMessage; + this.completionNotification = completionNotification; + this.requestConfig = requestConfig; + } + + public LogMessage getLogMessage() { + return logMessage; + } + + public CompletableFuture getCompletionNotification() { + return completionNotification; + } + + public RequestConfig getRequestConfig() { + return requestConfig; + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/LogJobsQueue.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java similarity index 68% rename from std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/LogJobsQueue.java rename to std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java index 22861df02ae8..bdfc3def9d68 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/audit/LogJobsQueue.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogJobsQueue.java @@ -1,16 +1,15 @@ -package org.enso.base.enso_cloud.audit; +package org.enso.base.enso_cloud.logging; import java.util.ArrayList; import java.util.Deque; import java.util.LinkedList; import java.util.List; -class LogJobsQueue { - - private final Deque queue = new LinkedList<>(); +final class LogJobsQueue { + private final Deque queue = new LinkedList<>(); /** Enqueues a log message to be sent and returns the number of messages in the queue. */ - synchronized int enqueue(AuditLogApiAccess.LogJob job) { + synchronized int enqueue(LogJob job) { int previousSize = queue.size(); queue.addLast(job); int newSize = queue.size(); @@ -20,14 +19,14 @@ synchronized int enqueue(AuditLogApiAccess.LogJob job) { } /** Removes and returns up to {@code limit} enqueued jobs. */ - synchronized List popEnqueuedJobs(int limit) { + synchronized List popEnqueuedJobs(int limit) { assert limit > 0; if (queue.isEmpty()) { return List.of(); } int n = Math.min(limit, queue.size()); - List result = new ArrayList<>(n); + List result = new ArrayList<>(n); for (int i = 0; i < n; i++) { result.add(queue.removeFirst()); } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java new file mode 100644 index 000000000000..ed5c060c2ac2 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/LogMessage.java @@ -0,0 +1,102 @@ +package org.enso.base.enso_cloud.logging; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import java.util.Objects; +import org.enso.base.CurrentEnsoProject; +import org.enso.base.enso_cloud.CloudAPI; + +/** Base class for log messages that are passed to the OpenSearch cloud endpoint. */ +public abstract class LogMessage { + /** + * A reserved field that is currently added by the cloud backend. Duplicating it will lead to + * internal server errors and log messages being discarded. + */ + private static final String RESERVED_TYPE = "type"; + + private static final String PROJECT_NAME = "projectName"; + private static final String PROJECT_ID = "projectId"; + private static final String PROJECT_SESSION_ID = "projectSessionId"; + private static final String LOCAL_TIMESTAMP = "localTimestamp"; + + private final String message; + private final String projectId; + private final String projectName; + private final String projectSessionId; + + protected LogMessage(String message) { + this.message = Objects.requireNonNull(message); + this.projectId = CloudAPI.getCloudProjectId(); + var currentProject = CurrentEnsoProject.get(); + this.projectName = currentProject == null ? null : currentProject.fullName(); + this.projectSessionId = CloudAPI.getCloudSessionId(); + } + + private static void checkNoRestrictedField(ObjectNode metadata, String fieldName) { + if (metadata.has(fieldName)) { + throw new IllegalArgumentException( + "Metadata cannot contain a field named '" + fieldName + "'. Metadata was: " + metadata); + } + } + + protected static void checkNoRestrictedFields(ObjectNode metadata) { + checkNoRestrictedField(metadata, RESERVED_TYPE); + checkNoRestrictedField(metadata, LOCAL_TIMESTAMP); + checkNoRestrictedField(metadata, PROJECT_NAME); + checkNoRestrictedField(metadata, PROJECT_ID); + } + + private ObjectNode computedMetadata() { + var meta = new ObjectNode(JsonNodeFactory.instance); + meta.set(PROJECT_ID, projectId == null ? NullNode.getInstance() : TextNode.valueOf(projectId)); + // The project name may be null if a script is run outside a project. + if (projectName != null) { + meta.set(PROJECT_NAME, TextNode.valueOf(projectName)); + } + + if (projectSessionId != null) { + meta.set(PROJECT_SESSION_ID, TextNode.valueOf(projectSessionId)); + } + + if (extraMetadata() != null) { + extraMetadata() + .fields() + .forEachRemaining( + entry -> { + meta.set(entry.getKey(), entry.getValue()); + }); + } + return meta; + } + + public String payload() { + var payload = new ObjectNode(JsonNodeFactory.instance); + payload.set("message", TextNode.valueOf(message)); + payload.set("metadata", computedMetadata()); + payload.set("kind", TextNode.valueOf(kind())); + if (extraPayload() != null) { + extraPayload() + .fields() + .forEachRemaining( + entry -> { + payload.set(entry.getKey(), entry.getValue()); + }); + } + return payload.toString(); + } + + protected abstract String kind(); + + /** Returns optional JSON object that will be appended to the payload. May return null. */ + protected abstract ObjectNode extraPayload(); + + /** Returns optional JSON object that will be appended to the metadata. May return null. */ + protected abstract ObjectNode extraMetadata(); + + @Override + public String toString() { + return payload(); + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java new file mode 100644 index 000000000000..9c0a8c9efaf1 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestConfig.java @@ -0,0 +1,17 @@ +package org.enso.base.enso_cloud.logging; + +import java.net.URI; +import org.enso.base.enso_cloud.AuthenticationProvider; + +/** + * Contains information needed to build a request to the Cloud Logs API. + * + *

This information must be gathered on the main Enso thread, as only there we have access to the + * {@link AuthenticationProvider}. + * + *

We associate an instance with every message to be sent. When sending multiple messages in a + * batch, we will use the config from one of them. This should not matter as in normal operations + * the configs will be the same, they only change during testing. Tests should this into account, by + * sending the last message in synchronous mode. + */ +record RequestConfig(URI apiUri, String accessToken) {} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java new file mode 100644 index 000000000000..a1ee175528cc --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/logging/RequestFailureException.java @@ -0,0 +1,7 @@ +package org.enso.base.enso_cloud.logging; + +final class RequestFailureException extends RuntimeException { + RequestFailureException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLog.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLog.java new file mode 100644 index 000000000000..2348ed486d3c --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLog.java @@ -0,0 +1,19 @@ +package org.enso.base.enso_cloud.telemetry; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.enso.base.enso_cloud.logging.LogApiAccess; + +public final class TelemetryLog { + private TelemetryLog() {} + + public static void logAsync(String loggerName, String message, ObjectNode metadata) { + var event = TelemetryLogMessage.create(loggerName, message, metadata); + LogApiAccess.INSTANCE.logWithoutConfirmation(event); + } + + public static class TelemetryLogError extends RuntimeException { + public TelemetryLogError(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java new file mode 100644 index 000000000000..8f9e5e099f31 --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/telemetry/TelemetryLogMessage.java @@ -0,0 +1,44 @@ +package org.enso.base.enso_cloud.telemetry; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import java.util.Objects; +import org.enso.base.enso_cloud.logging.LogMessage; + +public final class TelemetryLogMessage extends LogMessage { + + private final String loggerName; + private final ObjectNode extraMeta; + + private TelemetryLogMessage(String message, String loggerName, ObjectNode extraMeta) { + super(message); + this.loggerName = loggerName; + this.extraMeta = extraMeta; + } + + public static TelemetryLogMessage create(String loggerName, String message, ObjectNode metadata) { + Objects.requireNonNull(loggerName); + Objects.requireNonNull(message); + Objects.requireNonNull(metadata); + return new TelemetryLogMessage(message, loggerName, metadata); + } + + @Override + protected String kind() { + return "Telemetry"; + } + + @Override + protected ObjectNode extraPayload() { + return null; + } + + @Override + protected ObjectNode extraMetadata() { + var meta = new ObjectNode(JsonNodeFactory.instance); + meta.set("loggerName", TextNode.valueOf(loggerName)); + extraMeta.fields().forEachRemaining(entry -> meta.set(entry.getKey(), entry.getValue())); + return meta; + } +} diff --git a/test/Base_Internal_Tests/src/Main.enso b/test/Base_Internal_Tests/src/Main.enso index ead1b1c88be0..83737ff0d72f 100644 --- a/test/Base_Internal_Tests/src/Main.enso +++ b/test/Base_Internal_Tests/src/Main.enso @@ -8,6 +8,7 @@ import project.Comparator_Spec import project.Decimal_Constructor_Spec import project.Grapheme_Spec import project.Github_Annotations_Spec +import project.Telemetry_Log_Spec main filter=Nothing = suite = Test.build suite_builder-> @@ -17,5 +18,6 @@ main filter=Nothing = Input_Output_Spec.add_specs suite_builder Instrumentor_Spec.add_specs suite_builder Github_Annotations_Spec.add_specs suite_builder + Telemetry_Log_Spec.add_specs suite_builder suite.run_with_filter filter diff --git a/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso b/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso new file mode 100644 index 000000000000..b34b0d73f956 --- /dev/null +++ b/test/Base_Internal_Tests/src/Telemetry_Log_Spec.enso @@ -0,0 +1,60 @@ +from Standard.Base import all +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.Errors.Time_Error.Time_Error +import Standard.Base.Enso_Cloud.Errors.Enso_Cloud_Error +import Standard.Base.Internal.Telemetry.Telemetry +from Standard.Base.Enso_Cloud.Public_Utils import get_optional_field, get_required_field, cloud_http_request_for_test + +from Standard.Test import all +import Standard.Test.Test_Environment + +import enso_dev.Base_Tests.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup + +## Heavily inspired by `Audit_Log_Spec`. + TODO[pm]: Should be removed once telemetry is implemented as a slf4j logger appender. +add_specs suite_builder = + always_run_on_mock = True + setup = if always_run_on_mock then Cloud_Tests_Setup.prepare_mock_setup else Cloud_Tests_Setup.prepare + suite_builder.group "Telemetry logging" pending=setup.pending group_builder-> + group_builder.specify "send and receive simple log event" <| + setup.with_prepared_environment <| + random_payload = "payload-" + Random.uuid + Telemetry.log "TestTelemetry" "Message" (Dictionary.from_vector [["my_field", random_payload]]) . should_succeed + + my_event = Test.with_retries <| + event = get_telemetry_log_events . find ev-> (ev.metadata.get "my_field") == random_payload + event.should_succeed + event + + my_event.metadata.get "loggerName" . should_equal "org.enso.telemetry.TestTelemetry" + my_event.metadata.get "projectName" . should_equal "enso_dev.Base_Internal_Tests" + + my_event.message . should_equal "Message" + my_event.user_email . should_equal Enso_User.current.email + + +private get_telemetry_log_events -> Vector Telemetry_Log_Event = + json = cloud_http_request_for_test HTTP_Method.Get "log_events" + events_json = get_required_field "events" json + events_json.map Telemetry_Log_Event.from_json + + +type Telemetry_Log_Event + private Value organization_id:Text user_email:Text timestamp:Date_Time|Nothing metadata:JS_Object message:Text project_id:Text|Nothing + + private from_json json = + organization_id = get_required_field "organizationId" json expected_type=Text + user_email = get_required_field "userEmail" json expected_type=Text + timestamp_text = get_optional_field "timestamp" json expected_type=Text + timestamp = timestamp_text.if_not_nothing <| Date_Time.parse timestamp_text . catch Time_Error error-> + Error.throw (Enso_Cloud_Error.Invalid_Response_Payload "Invalid timestamp format in audit log event: "+error.to_display_text) + metadata = get_required_field "metadata" json + message = get_required_field "message" json expected_type=Text + project_id = get_optional_field "projectId" json expected_type=Text + Telemetry_Log_Event.Value organization_id user_email timestamp metadata message project_id + + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter diff --git a/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/EventsService.java b/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/EventsService.java index 084777edfa27..46f395d673b1 100644 --- a/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/EventsService.java +++ b/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/EventsService.java @@ -5,6 +5,10 @@ import java.util.List; public class EventsService { + + /** + * @param projectId May be null + */ public record LogEvent( String organizationId, String userEmail, diff --git a/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/PostLogHandler.java b/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/PostLogHandler.java index fc8a20a30096..7b3ec413c2df 100644 --- a/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/PostLogHandler.java +++ b/tools/http-test-helper/src/main/java/org/enso/shttp/cloud_mock/PostLogHandler.java @@ -94,7 +94,12 @@ private EventsService.LogEvent parseLogEvent(JsonNode json) { String userEmail = usersService.currentUserEmail(); String timestamp = ZonedDateTime.now().withZoneSameInstant(ZoneId.of("UTC")).toString(); JsonNode metadata = json.get("metadata"); - String projectId = json.get("projectId").asText(); + String projectId; + if (json.has("projectId")) { + projectId = json.get("projectId").asText(); + } else { + projectId = null; + } return new EventsService.LogEvent( organizationId, userEmail, timestamp, metadata, message, projectId); }