From 463b75391a4fd4eb56078361be40c68ba67f7eb2 Mon Sep 17 00:00:00 2001 From: "Alvey, Travis" Date: Tue, 30 Dec 2014 22:21:53 +0000 Subject: [PATCH 1/4] Simple refactoring of the JSONEventLayoutV1 format method to separate the creation of the Logstash event and the formatting operation. The benefit is subclasses of JSONEventLayoutV1 can create the Logstash event, add any specific details to the event, and then get the formatted string. --- .../java/net/logstash/log4j/JSONEventLayoutV1.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java index aaf3228..a3051b8 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java @@ -60,6 +60,12 @@ public JSONEventLayoutV1(boolean locationInfo) { } public String format(LoggingEvent loggingEvent) { + JSONObject lsEvent = createLogstashEvent(loggingEvent); + + return lsEvent.toString() + "\n"; + } + + protected JSONObject createLogstashEvent(LoggingEvent loggingEvent) { threadName = loggingEvent.getThreadName(); timestamp = loggingEvent.getTimeStamp(); exceptionInformation = new HashMap(); @@ -82,7 +88,7 @@ public String format(LoggingEvent loggingEvent) { */ if (getUserFields() != null) { String userFlds = getUserFields(); - LogLog.debug("["+whoami+"] Got user data from log4j property: "+ userFlds); + LogLog.debug("[" + whoami + "] Got user data from log4j property: " + userFlds); addUserFields(userFlds); } @@ -134,7 +140,7 @@ public String format(LoggingEvent loggingEvent) { addEventData("level", loggingEvent.getLevel().toString()); addEventData("thread_name", threadName); - return logstashEvent.toString() + "\n"; + return logstashEvent; } public boolean ignoresThrowable() { From c9ca077fe556482ad9d0c5081423e22d5f1dc483 Mon Sep 17 00:00:00 2001 From: "Alvey, Travis" Date: Tue, 30 Dec 2014 22:24:06 +0000 Subject: [PATCH 2/4] Adding fork description --- FORK.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 FORK.md diff --git a/FORK.md b/FORK.md new file mode 100644 index 0000000..4b251cf --- /dev/null +++ b/FORK.md @@ -0,0 +1,3 @@ +Simple refactoring of the JSONEventLayoutV1 format method to separate the creation of the Logstash event and the formatting operation. +The benefit is subclasses of JSONEventLayoutV1 can create the Logstash event, add any specific details to the event, and +then get the formatted string. \ No newline at end of file From 56cf8de564a19968b96a31780f3f22b114ccb57d Mon Sep 17 00:00:00 2001 From: "Alvey, Travis" Date: Wed, 21 Jan 2015 22:45:05 +0000 Subject: [PATCH 3/4] Added a new Layout that allows for renaming output field names --- .../net/logstash/log4j/JSONEventLayoutV2.java | 262 ++++++++++++++++++ .../fieldnames/LogstashCommonFieldNames.java | 47 ++++ .../log4j/fieldnames/LogstashFieldNames.java | 243 ++++++++++++++++ 3 files changed, 552 insertions(+) create mode 100644 src/main/java/net/logstash/log4j/JSONEventLayoutV2.java create mode 100644 src/main/java/net/logstash/log4j/fieldnames/LogstashCommonFieldNames.java create mode 100644 src/main/java/net/logstash/log4j/fieldnames/LogstashFieldNames.java diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV2.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV2.java new file mode 100644 index 0000000..b150ed7 --- /dev/null +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV2.java @@ -0,0 +1,262 @@ +package net.logstash.log4j; + + +import net.logstash.log4j.data.HostData; +import net.logstash.log4j.fieldnames.LogstashFieldNames; +import net.minidev.json.JSONObject; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.time.FastDateFormat; +import org.apache.log4j.Layout; +import org.apache.log4j.helpers.LogLog; +import org.apache.log4j.helpers.OnlyOnceErrorHandler; +import org.apache.log4j.spi.*; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; + + +/** + * Log4j JSON Layout providing mutable output field names + * + * Based upon the similar field name solution for logback found here + * https://github.com/logstash/logstash-logback-encoder + * + * Also allows for "flattening" of the output structure, removing any nested structures + */ +public class JSONEventLayoutV2 extends Layout { + + protected ErrorHandler errorHandler = new OnlyOnceErrorHandler(); + + private LogstashFieldNames fieldNames = new LogstashFieldNames(); + private boolean locationInfo = true; + private String customUserFields; + private boolean ignoreThrowable = false; + private String hostname = new HostData().getHostName(); + private static Integer version = 1; + + public static final TimeZone UTC = TimeZone.getTimeZone("UTC"); + public static final FastDateFormat ISO_DATETIME_TIME_ZONE_FORMAT_WITH_MILLIS = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", UTC); + public static final String ADDITIONAL_DATA_PROPERTY = "net.logstash.log4j.JSONEventLayoutV2.UserFields"; + + public static String dateFormat(long timestamp) { + return ISO_DATETIME_TIME_ZONE_FORMAT_WITH_MILLIS.format(timestamp); + } + + public JSONEventLayoutV2(boolean isLocationInfo) { + locationInfo = isLocationInfo; + } + + public String format(LoggingEvent loggingEvent) { + JSONObject lsEvent = createLogstashEvent(loggingEvent); + + return lsEvent.toString() + "\n"; + } + + protected JSONObject createLogstashEvent(LoggingEvent loggingEvent) { + String threadName = loggingEvent.getThreadName(); + Long timestamp = loggingEvent.getTimeStamp(); + + Map mdc = loggingEvent.getProperties(); + String ndc = loggingEvent.getNDC(); + + JSONObject logstashEvent = new JSONObject(); + String whoami = this.getClass().getSimpleName(); + + /** + * All v1 of the event format requires is + * "@timestamp" and "@version" + * Every other field is arbitrary + */ + addEventData(logstashEvent, fieldNames.getVersion(), version); + addEventData(logstashEvent, fieldNames.getTimestamp(), dateFormat(timestamp)); + + /** + * Extract and add fields from log4j config, if defined + */ + if (getUserFields() != null) { + String userFlds = getUserFields(); + LogLog.debug("[" + whoami + "] Got user data from log4j property: " + userFlds); + addUserFields(logstashEvent, userFlds); + } + + /** + * Extract fields from system properties, if defined + * Note that CLI props will override conflicts with log4j config + */ + if (System.getProperty(ADDITIONAL_DATA_PROPERTY) != null) { + if (getUserFields() != null) { + LogLog.warn("[" + whoami + "] Loading UserFields from command-line. This will override any UserFields set in the log4j configuration file"); + } + String userFieldsProperty = System.getProperty(ADDITIONAL_DATA_PROPERTY); + LogLog.debug("[" + whoami + "] Got user data from system property: " + userFieldsProperty); + addUserFields(logstashEvent, userFieldsProperty); + } + + /** + * Now we start injecting our own stuff. + */ + addEventData(logstashEvent, fieldNames.getHostName(), hostname); + addEventData(logstashEvent, fieldNames.getMessage(), loggingEvent.getRenderedMessage()); + + if (loggingEvent.getThrowableInformation() != null) { + final ThrowableInformation throwableInformation = loggingEvent.getThrowableInformation(); + + HashMap exceptionInformation = new HashMap(); + if (throwableInformation.getThrowable().getClass().getCanonicalName() != null) { + exceptionInformation.put(fieldNames.getExceptionClass(), throwableInformation.getThrowable().getClass().getCanonicalName()); + } + if (throwableInformation.getThrowable().getMessage() != null) { + exceptionInformation.put(fieldNames.getExceptionMessage(), throwableInformation.getThrowable().getMessage()); + } + if (throwableInformation.getThrowableStrRep() != null) { + String stackTrace = StringUtils.join(throwableInformation.getThrowableStrRep(), "\n"); + exceptionInformation.put(fieldNames.getStackTrace(), stackTrace); + } + if (fieldNames.getException() != null) { + addEventData(logstashEvent, fieldNames.getException(), exceptionInformation); + } else { + addEventData(logstashEvent, exceptionInformation); + } + + } + + if (getLocationInfo()) { + LocationInfo info = loggingEvent.getLocationInformation(); + Map locMap = new HashMap(); + + addEventData(locMap, fieldNames.getCallerFile(), info.getFileName()); + addEventData(locMap, fieldNames.getCallerLine(), info.getLineNumber()); + addEventData(locMap, fieldNames.getCallerClass(), info.getClassName()); + addEventData(locMap, fieldNames.getCallerMethod(), info.getMethodName()); + + if (fieldNames.getCaller() != null) { + addEventData(logstashEvent, fieldNames.getCaller(), locMap); + } else { + addEventData(logstashEvent, locMap); + } + + /* addEventData(logstashEvent, fieldNames.getCallerFile(), info.getFileName()); + addEventData(logstashEvent, fieldNames.getCallerLine(), info.getLineNumber()); + addEventData(logstashEvent, fieldNames.getCallerClass(), info.getClassName()); + addEventData(logstashEvent, fieldNames.getCallerMethod(), info.getMethodName());*/ + } + + addEventData(logstashEvent, fieldNames.getLogger(), loggingEvent.getLoggerName()); + + + if (fieldNames.getMdc() != null) { + addEventData(logstashEvent, fieldNames.getMdc(), mdc); + + } else { + addEventData(logstashEvent, mdc); + } + + + addEventData(logstashEvent, fieldNames.getNdc(), ndc); + addEventData(logstashEvent, fieldNames.getLevel(), loggingEvent.getLevel().toString()); + addEventData(logstashEvent, fieldNames.getThread(), threadName); + + return logstashEvent; + } + + private void addEventData(JSONObject logstashEvent, Map map) { + Set entries = map.entrySet(); + for (Map.Entry entry : entries) { + String key = entry.getKey().toString(); + Object value = entry.getValue(); + addEventData(logstashEvent, key, value); + } + } + + private void addEventData(JSONObject logstashEvent, String keyName, Object keyVal) { + if (keyVal != null && keyName != null) { + logstashEvent.put(keyName, keyVal); + } + } + + private void addEventData(Map map, String keyName, Object keyVal) { + if (keyVal != null && keyName != null) { + map.put(keyName, keyVal); + } + } + + //TODO: This should be just using a JSON string instead of comma separated "name:value" pairs + private void addUserFields(JSONObject logstashEvent, String data) { + if (data != null) { + String[] pairs = data.split(","); + for (String pair : pairs) { + String[] userField = pair.split(":", 2); + if (userField[0] != null) { + String key = userField[0]; + String val = userField[1]; + addEventData(logstashEvent, key, val); + } + } + } + } + + + public LogstashFieldNames getFieldNames() { + return fieldNames; + } + + public void setFieldNames(LogstashFieldNames fieldNames) { + this.fieldNames = fieldNames; + } + + public void setFieldsClassName(String fieldsClassName) { + try { + Class clazz = Class.forName(fieldsClassName); + Object o = clazz.newInstance(); + if (o instanceof LogstashFieldNames) { + setFieldNames((LogstashFieldNames) o); + } else { + errorHandler.error("Class for " + fieldsClassName + " is not a valid type for defining field names. Will use default field names"); + } + + } catch (Exception e) { + errorHandler.error("Failed to load class for FieldNames " + fieldsClassName, e, ErrorCode.GENERIC_FAILURE); + } + } + + public void setFlattenOutput(boolean isFlatten) { + fieldNames.setFlattenOutput(isFlatten); + } + + public boolean ignoresThrowable() { + return ignoreThrowable; + } + + /** + * Query whether log messages include location information. + * + * @return true if location information is included in log messages, false otherwise. + */ + public boolean getLocationInfo() { + return locationInfo; + } + + /** + * Set whether log messages should include location information. + * + * @param locationInfo true if location information should be included, false otherwise. + */ + public void setLocationInfo(boolean locationInfo) { + this.locationInfo = locationInfo; + } + + public String getUserFields() { + return customUserFields; + } + + public void setUserFields(String userFields) { + this.customUserFields = userFields; + } + + public void activateOptions() { + + //activeIgnoreThrowable = ignoreThrowable; + } +} diff --git a/src/main/java/net/logstash/log4j/fieldnames/LogstashCommonFieldNames.java b/src/main/java/net/logstash/log4j/fieldnames/LogstashCommonFieldNames.java new file mode 100644 index 0000000..fcfc4d1 --- /dev/null +++ b/src/main/java/net/logstash/log4j/fieldnames/LogstashCommonFieldNames.java @@ -0,0 +1,47 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.logstash.log4j.fieldnames; + +/** + * Common field names + */ +public abstract class LogstashCommonFieldNames { + private String timestamp = "@timestamp"; + private String version = "@version"; + private String message = "message"; + + public String getTimestamp() { + return timestamp; + } + + public void setTimestamp(String timestamp) { + this.timestamp = timestamp; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/src/main/java/net/logstash/log4j/fieldnames/LogstashFieldNames.java b/src/main/java/net/logstash/log4j/fieldnames/LogstashFieldNames.java new file mode 100644 index 0000000..b15a025 --- /dev/null +++ b/src/main/java/net/logstash/log4j/fieldnames/LogstashFieldNames.java @@ -0,0 +1,243 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.logstash.log4j.fieldnames; + +/** + * Names of standard fields that appear in the JSON output. + * + * Based upn the similar solution for logback + * https://github.com/logstash/logstash-logback-encoder/blob/master/src/main/java/net/logstash/logback/fieldnames/LogstashFieldNames.java + */ +public class LogstashFieldNames extends LogstashCommonFieldNames { + + private String logger = "loggername"; + private String thread = "threadname"; + private String level = "level"; + private String levelValue = "levelvalue"; + + private String callerClass = "classname"; + private String callerMethod = "methodname"; + private String callerFile = "filename"; + private String callerLine = "linenumber"; + private String stackTrace = "stacktrace"; + private String tags = "tags"; + private String ndc = "ndc"; + + private String hostName = "hostname"; + private String exceptionClass = "exceptionclass"; + private String exceptionMessage = "exceptionmessage"; + + //IF we populate these, the output will create nested data for these names + private String exception; + private String caller; + private String mdc; + private String context; + + public static final String EXCEPTION_DEFAULT = "exception"; + public static final String CALLER_DEFAULT = "caller"; + public static final String MDC_DEFAULT = "mdc"; + public static final String CONTEXT_DEFAULT = "context"; + + + public void setFlattenOutput(Boolean isFlatten) { + if (isFlatten) { + setException(null); + setCaller(null); + setMdc(null); + setContext(null); + } else { + String exception = getException() != null ? getException() : EXCEPTION_DEFAULT; + String caller = getCaller() != null ? getCaller() : CALLER_DEFAULT; + String mdc = getMdc() != null ? getMdc() : MDC_DEFAULT; + String context = getContext() != null ? getContext() : CONTEXT_DEFAULT; + setException(exception); + setCaller(caller); + setMdc(mdc); + setContext(context); + } + } + + public String getLogger() { + return logger; + } + + public void setLogger(String logger) { + this.logger = logger; + } + + public String getThread() { + return thread; + } + + public void setThread(String thread) { + this.thread = thread; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + public String getLevelValue() { + return levelValue; + } + + public void setLevelValue(String levelValue) { + this.levelValue = levelValue; + } + + /** + * The name of the caller object field. + *

+ * If this returns null, then the caller data fields will be written inline at the root level of the JSON event output (e.g. as a sibling to all the other fields in this class). + *

+ * If this returns non-null, then the caller data fields will be written inside an object with field name returned by this method + */ + public String getCaller() { + return caller; + } + + public void setCaller(String caller) { + this.caller = caller; + } + + public String getCallerClass() { + return callerClass; + } + + public void setCallerClass(String callerClass) { + this.callerClass = callerClass; + } + + public String getCallerMethod() { + return callerMethod; + } + + public void setCallerMethod(String callerMethod) { + this.callerMethod = callerMethod; + } + + public String getCallerFile() { + return callerFile; + } + + public void setCallerFile(String callerFile) { + this.callerFile = callerFile; + } + + public String getCallerLine() { + return callerLine; + } + + public void setCallerLine(String callerLine) { + this.callerLine = callerLine; + } + + public String getStackTrace() { + return stackTrace; + } + + public void setStackTrace(String stackTrace) { + this.stackTrace = stackTrace; + } + + public String getTags() { + return tags; + } + + public void setTags(String tags) { + this.tags = tags; + } + + /** + * The name of the mdc object field. + *

+ * If this returns null, then the mdc fields will be written inline at the root level of the JSON event output (e.g. as a sibling to all the other fields in this class). + *

+ * If this returns non-null, then the mdc fields will be written inside an object with field name returned by this method + */ + public String getMdc() { + return mdc; + } + + public void setMdc(String mdc) { + this.mdc = mdc; + } + + /** + * The name of the context object field. + *

+ * If this returns null, then the context fields will be written inline at the root level of the JSON event output (e.g. as a sibling to all the other fields in this class). + *

+ * If this returns non-null, then the context fields will be written inside an object with field name returned by this method + */ + public String getContext() { + return context; + } + + public void setContext(String context) { + this.context = context; + } + + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public String getExceptionClass() { + return exceptionClass; + } + + public void setExceptionClass(String exceptionClass) { + this.exceptionClass = exceptionClass; + } + + public String getExceptionMessage() { + return exceptionMessage; + } + + public void setExceptionMessage(String exceptionMessage) { + this.exceptionMessage = exceptionMessage; + } + + /** + * The name of the exception object field. + *

+ * If this returns null, then the context fields will be written inline at the root level of the JSON event output (e.g. as a sibling to all the other fields in this class). + *

+ * If this returns non-null, then the context fields will be written inside an object with field name returned by this method + */ + public String getException() { + return exception; + } + + public void setException(String exception) { + this.exception = exception; + } + + + public String getNdc() { + return ndc; + } + + public void setNdc(String ndc) { + this.ndc = ndc; + } +} From bcbaeb506ce61ba4fb7df9db8c8b2013f66ab27e Mon Sep 17 00:00:00 2001 From: "Alvey, Travis" Date: Thu, 12 Feb 2015 21:45:40 +0000 Subject: [PATCH 4/4] some simple refactoring and added some tests --- FORK.md | 4 +- pom.xml | 4 +- .../net/logstash/log4j/IJSONEventLayout.java | 10 + .../net/logstash/log4j/JSONEventLayoutV1.java | 8 +- .../net/logstash/log4j/JSONEventLayoutV2.java | 15 +- .../fieldnames/LogstashCommonFieldNames.java | 13 ++ .../log4j/fieldnames/LogstashFieldNames.java | 27 ++- .../logstash/log4j/JSONEventLayoutV1Test.java | 191 ++++++++++++++---- .../net/logstash/log4j/MockAppenderV1.java | 4 +- 9 files changed, 221 insertions(+), 55 deletions(-) create mode 100644 src/main/java/net/logstash/log4j/IJSONEventLayout.java diff --git a/FORK.md b/FORK.md index 4b251cf..a07c9b0 100644 --- a/FORK.md +++ b/FORK.md @@ -1,3 +1,5 @@ Simple refactoring of the JSONEventLayoutV1 format method to separate the creation of the Logstash event and the formatting operation. The benefit is subclasses of JSONEventLayoutV1 can create the Logstash event, add any specific details to the event, and -then get the formatted string. \ No newline at end of file +then get the formatted string. + +Also, added the ability to customize the output field names. \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1dc589b..f8b4ca4 100644 --- a/pom.xml +++ b/pom.xml @@ -66,8 +66,8 @@ maven-compiler-plugin 2.3.2 - 1.5 - 1.5 + 1.7 + 1.7 diff --git a/src/main/java/net/logstash/log4j/IJSONEventLayout.java b/src/main/java/net/logstash/log4j/IJSONEventLayout.java new file mode 100644 index 0000000..74554ab --- /dev/null +++ b/src/main/java/net/logstash/log4j/IJSONEventLayout.java @@ -0,0 +1,10 @@ +package net.logstash.log4j; + + +public interface IJSONEventLayout { + + public abstract String getUserFields(); + public abstract void setUserFields(String userFields); + public abstract boolean getLocationInfo(); + public abstract void setLocationInfo(boolean locationInfo); +} diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java index a3051b8..c2c7cdc 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java @@ -14,7 +14,7 @@ import java.util.Map; import java.util.TimeZone; -public class JSONEventLayoutV1 extends Layout { +public class JSONEventLayoutV1 extends Layout implements IJSONEventLayout { private boolean locationInfo = false; private String customUserFields; @@ -152,6 +152,7 @@ public boolean ignoresThrowable() { * * @return true if location information is included in log messages, false otherwise. */ + @Override public boolean getLocationInfo() { return locationInfo; } @@ -161,11 +162,15 @@ public boolean getLocationInfo() { * * @param locationInfo true if location information should be included, false otherwise. */ + @Override public void setLocationInfo(boolean locationInfo) { this.locationInfo = locationInfo; } + @Override public String getUserFields() { return customUserFields; } + + @Override public void setUserFields(String userFields) { this.customUserFields = userFields; } public void activateOptions() { @@ -185,6 +190,7 @@ private void addUserFields(String data) { } } } + private void addEventData(String keyname, Object keyval) { if (null != keyval) { logstashEvent.put(keyname, keyval); diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV2.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV2.java index b150ed7..14acce2 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV2.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV2.java @@ -19,13 +19,13 @@ /** * Log4j JSON Layout providing mutable output field names - * + *

* Based upon the similar field name solution for logback found here * https://github.com/logstash/logstash-logback-encoder - * + *

* Also allows for "flattening" of the output structure, removing any nested structures */ -public class JSONEventLayoutV2 extends Layout { +public class JSONEventLayoutV2 extends Layout implements IJSONEventLayout { protected ErrorHandler errorHandler = new OnlyOnceErrorHandler(); @@ -44,6 +44,10 @@ public static String dateFormat(long timestamp) { return ISO_DATETIME_TIME_ZONE_FORMAT_WITH_MILLIS.format(timestamp); } + public JSONEventLayoutV2() { + this(true); + } + public JSONEventLayoutV2(boolean isLocationInfo) { locationInfo = isLocationInfo; } @@ -225,6 +229,7 @@ public void setFlattenOutput(boolean isFlatten) { fieldNames.setFlattenOutput(isFlatten); } + @Override public boolean ignoresThrowable() { return ignoreThrowable; } @@ -234,6 +239,7 @@ public boolean ignoresThrowable() { * * @return true if location information is included in log messages, false otherwise. */ + @Override public boolean getLocationInfo() { return locationInfo; } @@ -243,14 +249,17 @@ public boolean getLocationInfo() { * * @param locationInfo true if location information should be included, false otherwise. */ + @Override public void setLocationInfo(boolean locationInfo) { this.locationInfo = locationInfo; } + @Override public String getUserFields() { return customUserFields; } + @Override public void setUserFields(String userFields) { this.customUserFields = userFields; } diff --git a/src/main/java/net/logstash/log4j/fieldnames/LogstashCommonFieldNames.java b/src/main/java/net/logstash/log4j/fieldnames/LogstashCommonFieldNames.java index fcfc4d1..fc2ebad 100644 --- a/src/main/java/net/logstash/log4j/fieldnames/LogstashCommonFieldNames.java +++ b/src/main/java/net/logstash/log4j/fieldnames/LogstashCommonFieldNames.java @@ -13,6 +13,9 @@ */ package net.logstash.log4j.fieldnames; +import java.util.ArrayList; +import java.util.List; + /** * Common field names */ @@ -44,4 +47,14 @@ public String getMessage() { public void setMessage(String message) { this.message = message; } + + public List listCommonNames() { + List namesList = new ArrayList<>(); + + namesList.add(getTimestamp()); + namesList.add(getMessage()); + namesList.add(getVersion()); + + return namesList; + } } diff --git a/src/main/java/net/logstash/log4j/fieldnames/LogstashFieldNames.java b/src/main/java/net/logstash/log4j/fieldnames/LogstashFieldNames.java index b15a025..4a65e8b 100644 --- a/src/main/java/net/logstash/log4j/fieldnames/LogstashFieldNames.java +++ b/src/main/java/net/logstash/log4j/fieldnames/LogstashFieldNames.java @@ -13,6 +13,9 @@ */ package net.logstash.log4j.fieldnames; +import java.util.ArrayList; +import java.util.List; + /** * Names of standard fields that appear in the JSON output. * @@ -24,7 +27,7 @@ public class LogstashFieldNames extends LogstashCommonFieldNames { private String logger = "loggername"; private String thread = "threadname"; private String level = "level"; - private String levelValue = "levelvalue"; + //private String levelValue = "levelvalue"; private String callerClass = "classname"; private String callerMethod = "methodname"; @@ -92,6 +95,7 @@ public void setLevel(String level) { this.level = level; } + /** public String getLevelValue() { return levelValue; } @@ -99,7 +103,7 @@ public String getLevelValue() { public void setLevelValue(String levelValue) { this.levelValue = levelValue; } - + **/ /** * The name of the caller object field. *

@@ -240,4 +244,23 @@ public String getNdc() { public void setNdc(String ndc) { this.ndc = ndc; } + + + public List listNames() { + List namesList = new ArrayList<>(); + + namesList.addAll(super.listCommonNames()); + namesList.add(getLogger()); + namesList.add(getThread()); + namesList.add(getLevel()); + namesList.add(getCallerClass()); + namesList.add(getCallerMethod()); + namesList.add(getCallerFile()); + namesList.add(getCallerLine()); + namesList.add(getStackTrace()); + namesList.add(getTags()); + namesList.add(getNdc()); + + return namesList; + } } diff --git a/src/test/java/net/logstash/log4j/JSONEventLayoutV1Test.java b/src/test/java/net/logstash/log4j/JSONEventLayoutV1Test.java index 96ad821..22aab20 100644 --- a/src/test/java/net/logstash/log4j/JSONEventLayoutV1Test.java +++ b/src/test/java/net/logstash/log4j/JSONEventLayoutV1Test.java @@ -1,18 +1,29 @@ package net.logstash.log4j; import junit.framework.Assert; +import net.logstash.log4j.fieldnames.LogstashCommonFieldNames; +import net.logstash.log4j.fieldnames.LogstashFieldNames; import net.minidev.json.JSONObject; import net.minidev.json.JSONValue; import org.apache.log4j.*; -import org.apache.log4j.or.ObjectRenderer; import org.junit.After; import org.junit.Before; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; + + +/* TODO: I made modifications so this test class would cover both V1 and V2 -- The intent being to use Junit's "Parameterized" + TODO: functionality to run the full suite of tests, once for each layout. Unfortunately, to keep true to the original tests + TODO: this required a bunch of "instanceof" conditionals - not ideal. So, going forward, it would be preferable to + TODO: refactor and clean this up. Maybe it makes more sense to just separate the test classes into V1 and V2 + */ /** * Created with IntelliJ IDEA. @@ -21,14 +32,14 @@ * Time: 12:07 AM * To change this template use File | Settings | File Templates. */ +@RunWith(Parameterized.class) public class JSONEventLayoutV1Test { - static Logger logger; - static MockAppenderV1 appender; - static MockAppenderV1 userFieldsAppender; - static JSONEventLayoutV1 userFieldsLayout; - static final String userFieldsSingle = new String("field1:value1"); - static final String userFieldsMulti = new String("field2:value2,field3:value3"); - static final String userFieldsSingleProperty = new String("field1:propval1"); + Logger logger; + MockAppenderV1 appender; + + static final String userFieldsSingle = "field1:value1"; + static final String userFieldsMulti = "field2:value2,field3:value3"; + static final String userFieldsSingleProperty = "field1:propval1"; static final String[] logstashFields = new String[]{ "message", @@ -37,9 +48,28 @@ public class JSONEventLayoutV1Test { "@version" }; - @BeforeClass - public static void setupTestAppender() { - appender = new MockAppenderV1(new JSONEventLayoutV1()); + private Layout jsonLayout; + + @Parameterized.Parameters + public static java.util.Collection data() { + + Layout[] layout1Array = new Layout[]{new JSONEventLayoutV1()}; + Layout[] layout2Array = new Layout[]{new JSONEventLayoutV2()}; + List list = new ArrayList(); + list.add(layout1Array); + list.add(layout2Array); + return list; + + } + + public JSONEventLayoutV1Test(Layout layout) { + jsonLayout = layout; + } + + @Before + public void setupTestAppender() { + + appender = new MockAppenderV1(jsonLayout); logger = Logger.getRootLogger(); appender.setThreshold(Level.TRACE); appender.setName("mockappenderv1"); @@ -63,20 +93,25 @@ public void testJSONEventLayoutIsJSON() { @Test public void testJSONEventLayoutHasUserFieldsFromProps() { - System.setProperty(JSONEventLayoutV1.ADDITIONAL_DATA_PROPERTY, userFieldsSingleProperty); + String additionalDataProperty = JSONEventLayoutV1.ADDITIONAL_DATA_PROPERTY; + if (appender.getLayout() instanceof JSONEventLayoutV2) { + JSONEventLayoutV2 layout = (JSONEventLayoutV2) appender.getLayout(); + additionalDataProperty = layout.ADDITIONAL_DATA_PROPERTY; + } + System.setProperty(additionalDataProperty, userFieldsSingleProperty); logger.info("this is an info message with user fields"); String message = appender.getMessages()[0]; Assert.assertTrue("Event is not valid JSON", JSONValue.isValidJsonStrict(message)); Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertTrue("Event does not contain field 'field1'" , jsonObject.containsKey("field1")); + Assert.assertTrue("Event does not contain field 'field1'", jsonObject.containsKey("field1")); Assert.assertEquals("Event does not contain value 'value1'", "propval1", jsonObject.get("field1")); - System.clearProperty(JSONEventLayoutV1.ADDITIONAL_DATA_PROPERTY); + System.clearProperty(additionalDataProperty); } @Test public void testJSONEventLayoutHasUserFieldsFromConfig() { - JSONEventLayoutV1 layout = (JSONEventLayoutV1) appender.getLayout(); + IJSONEventLayout layout = getJsonEventLayout(); String prevUserData = layout.getUserFields(); layout.setUserFields(userFieldsSingle); @@ -85,15 +120,16 @@ public void testJSONEventLayoutHasUserFieldsFromConfig() { Assert.assertTrue("Event is not valid JSON", JSONValue.isValidJsonStrict(message)); Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertTrue("Event does not contain field 'field1'" , jsonObject.containsKey("field1")); + Assert.assertTrue("Event does not contain field 'field1'", jsonObject.containsKey("field1")); Assert.assertEquals("Event does not contain value 'value1'", "value1", jsonObject.get("field1")); layout.setUserFields(prevUserData); } + @Test public void testJSONEventLayoutUserFieldsMulti() { - JSONEventLayoutV1 layout = (JSONEventLayoutV1) appender.getLayout(); + IJSONEventLayout layout = getJsonEventLayout(); String prevUserData = layout.getUserFields(); layout.setUserFields(userFieldsMulti); @@ -102,9 +138,9 @@ public void testJSONEventLayoutUserFieldsMulti() { Assert.assertTrue("Event is not valid JSON", JSONValue.isValidJsonStrict(message)); Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertTrue("Event does not contain field 'field2'" , jsonObject.containsKey("field2")); + Assert.assertTrue("Event does not contain field 'field2'", jsonObject.containsKey("field2")); Assert.assertEquals("Event does not contain value 'value2'", "value2", jsonObject.get("field2")); - Assert.assertTrue("Event does not contain field 'field3'" , jsonObject.containsKey("field3")); + Assert.assertTrue("Event does not contain field 'field3'", jsonObject.containsKey("field3")); Assert.assertEquals("Event does not contain value 'value3'", "value3", jsonObject.get("field3")); layout.setUserFields(prevUserData); @@ -112,11 +148,16 @@ public void testJSONEventLayoutUserFieldsMulti() { @Test public void testJSONEventLayoutUserFieldsPropOverride() { + String additionalDataProperty = JSONEventLayoutV1.ADDITIONAL_DATA_PROPERTY; + if (appender.getLayout() instanceof JSONEventLayoutV2) { + JSONEventLayoutV2 layout = (JSONEventLayoutV2) appender.getLayout(); + additionalDataProperty = layout.ADDITIONAL_DATA_PROPERTY; + } // set the property first - System.setProperty(JSONEventLayoutV1.ADDITIONAL_DATA_PROPERTY, userFieldsSingleProperty); + System.setProperty(additionalDataProperty, userFieldsSingleProperty); // set the config values - JSONEventLayoutV1 layout = (JSONEventLayoutV1) appender.getLayout(); + IJSONEventLayout layout = getJsonEventLayout(); String prevUserData = layout.getUserFields(); layout.setUserFields(userFieldsSingle); @@ -125,11 +166,11 @@ public void testJSONEventLayoutUserFieldsPropOverride() { Assert.assertTrue("Event is not valid JSON", JSONValue.isValidJsonStrict(message)); Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertTrue("Event does not contain field 'field1'" , jsonObject.containsKey("field1")); + Assert.assertTrue("Event does not contain field 'field1'", jsonObject.containsKey("field1")); Assert.assertEquals("Event does not contain value 'propval1'", "propval1", jsonObject.get("field1")); layout.setUserFields(prevUserData); - System.clearProperty(JSONEventLayoutV1.ADDITIONAL_DATA_PROPERTY); + System.clearProperty(additionalDataProperty); } @@ -139,7 +180,15 @@ public void testJSONEventLayoutHasKeys() { String message = appender.getMessages()[0]; Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - for (String fieldName : logstashFields) { + + List fieldNames = Arrays.asList(logstashFields); + if (appender.getLayout() instanceof JSONEventLayoutV2) { + JSONEventLayoutV2 layout = (JSONEventLayoutV2) appender.getLayout(); + LogstashCommonFieldNames commonFieldNames = layout.getFieldNames(); + fieldNames = commonFieldNames.listCommonNames(); + } + + for (String fieldName : fieldNames) { Assert.assertTrue("Event does not contain field: " + fieldName, jsonObject.containsKey(fieldName)); } } @@ -158,30 +207,45 @@ public void testJSONEventLayoutHasNDC() { @Test public void testJSONEventLayoutHasMDC() { + MDC.put("foo", "bar"); logger.warn("I should have MDC data in my log"); String message = appender.getMessages()[0]; Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - JSONObject mdc = (JSONObject) jsonObject.get("mdc"); - Assert.assertEquals("MDC is wrong","bar", mdc.get("foo")); + if (appender.getLayout() instanceof JSONEventLayoutV2) { + //flattened by default + Assert.assertEquals("MDC is wrong", "bar", jsonObject.get("foo")); + } else { + JSONObject mdc = (JSONObject) jsonObject.get("mdc"); + Assert.assertEquals("MDC is wrong", "bar", mdc.get("foo")); + } } @Test public void testJSONEventLayoutHasNestedMDC() { HashMap nestedMdc = new HashMap(); - nestedMdc.put("bar","baz"); - MDC.put("foo",nestedMdc); + nestedMdc.put("bar", "baz"); + MDC.put("foo", nestedMdc); logger.warn("I should have nested MDC data in my log"); String message = appender.getMessages()[0]; Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - JSONObject mdc = (JSONObject) jsonObject.get("mdc"); - JSONObject nested = (JSONObject) mdc.get("foo"); - Assert.assertTrue("Event is missing foo key", mdc.containsKey("foo")); - Assert.assertEquals("Nested MDC data is wrong", "baz", nested.get("bar")); + if (appender.getLayout() instanceof JSONEventLayoutV2) { + //flattened by default + Assert.assertTrue("Event is missing foo key", jsonObject.containsKey("foo")); + JSONObject nested = (JSONObject) jsonObject.get("foo"); + Assert.assertEquals("Nested MDC data is wrong", "baz", nested.get("bar")); + + } else { + + JSONObject mdc = (JSONObject) jsonObject.get("mdc"); + JSONObject nested = (JSONObject) mdc.get("foo"); + Assert.assertTrue("Event is missing foo key", mdc.containsKey("foo")); + Assert.assertEquals("Nested MDC data is wrong", "baz", nested.get("bar")); + } } @Test @@ -191,10 +255,17 @@ public void testJSONEventLayoutExceptions() { String message = appender.getMessages()[0]; Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - JSONObject exceptionInformation = (JSONObject) jsonObject.get("exception"); - Assert.assertEquals("Exception class missing", "java.lang.IllegalArgumentException", exceptionInformation.get("exception_class")); - Assert.assertEquals("Exception exception message", exceptionMessage, exceptionInformation.get("exception_message")); + if (appender.getLayout() instanceof JSONEventLayoutV2) { + //flattened + JSONEventLayoutV2 layout = (JSONEventLayoutV2) appender.getLayout(); + Assert.assertEquals("Exception class missing", "java.lang.IllegalArgumentException", jsonObject.get(layout.getFieldNames().getExceptionClass())); + Assert.assertEquals("Exception exception message", exceptionMessage, jsonObject.get(layout.getFieldNames().getExceptionMessage())); + } else { + JSONObject exceptionInformation = (JSONObject) jsonObject.get("exception"); + Assert.assertEquals("Exception class missing", "java.lang.IllegalArgumentException", exceptionInformation.get("exception_class")); + Assert.assertEquals("Exception exception message", exceptionMessage, exceptionInformation.get("exception_message")); + } } @Test @@ -204,7 +275,13 @@ public void testJSONEventLayoutHasClassName() { Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertEquals("Logged class does not match", this.getClass().getCanonicalName().toString(), jsonObject.get("class")); + String nameOfValueToGet = "class"; + if (appender.getLayout() instanceof JSONEventLayoutV2) { + JSONEventLayoutV2 layout = (JSONEventLayoutV2) appender.getLayout(); + nameOfValueToGet = layout.getFieldNames().getCallerClass(); + } + + Assert.assertEquals("Logged class does not match", this.getClass().getCanonicalName().toString(), jsonObject.get(nameOfValueToGet)); } @Test @@ -214,16 +291,30 @@ public void testJSONEventHasFileName() { Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertNotNull("File value is missing", jsonObject.get("file")); + String nameOfValueToGet = "file"; + if (appender.getLayout() instanceof JSONEventLayoutV2) { + JSONEventLayoutV2 layout = (JSONEventLayoutV2) appender.getLayout(); + nameOfValueToGet = layout.getFieldNames().getCallerFile(); + } + + Assert.assertNotNull("File value is missing", jsonObject.get(nameOfValueToGet)); } + @Test public void testJSONEventHasLoggerName() { logger.warn("whoami"); String message = appender.getMessages()[0]; Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertNotNull("LoggerName value is missing", jsonObject.get("logger_name")); + + String nameOfValueToGet = "logger_name"; + if (appender.getLayout() instanceof JSONEventLayoutV2) { + JSONEventLayoutV2 layout = (JSONEventLayoutV2) appender.getLayout(); + nameOfValueToGet = layout.getFieldNames().getLogger(); + } + + Assert.assertNotNull("LoggerName value is missing", jsonObject.get(nameOfValueToGet)); } @Test @@ -232,12 +323,19 @@ public void testJSONEventHasThreadName() { String message = appender.getMessages()[0]; Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertNotNull("ThreadName value is missing", jsonObject.get("thread_name")); + + String nameOfValueToGet = "thread_name"; + if (appender.getLayout() instanceof JSONEventLayoutV2) { + JSONEventLayoutV2 layout = (JSONEventLayoutV2) appender.getLayout(); + nameOfValueToGet = layout.getFieldNames().getLogger(); + } + + Assert.assertNotNull("ThreadName value is missing", jsonObject.get(nameOfValueToGet)); } @Test public void testJSONEventLayoutNoLocationInfo() { - JSONEventLayoutV1 layout = (JSONEventLayoutV1) appender.getLayout(); + IJSONEventLayout layout = getJsonEventLayout(); boolean prevLocationInfo = layout.getLocationInfo(); layout.setLocationInfo(false); @@ -259,7 +357,7 @@ public void testJSONEventLayoutNoLocationInfo() { @Test @Ignore public void measureJSONEventLayoutLocationInfoPerformance() { - JSONEventLayoutV1 layout = (JSONEventLayoutV1) appender.getLayout(); + IJSONEventLayout layout = getJsonEventLayout(); boolean locationInfo = layout.getLocationInfo(); int iterations = 100000; long start, stop; @@ -289,6 +387,11 @@ public void measureJSONEventLayoutLocationInfoPerformance() { @Test public void testDateFormat() { long timestamp = 1364844991207L; - Assert.assertEquals("format does not produce expected output", "2013-04-01T19:36:31.207Z", JSONEventLayoutV1.dateFormat(timestamp)); + Assert.assertEquals("format does not produce expected output", "2013-04-01T19:36:31.207Z", JSONEventLayoutV2.dateFormat(timestamp)); + } + + protected IJSONEventLayout getJsonEventLayout() { + return (IJSONEventLayout) appender.getLayout(); } + } diff --git a/src/test/java/net/logstash/log4j/MockAppenderV1.java b/src/test/java/net/logstash/log4j/MockAppenderV1.java index 5ebe656..9ae9c3b 100644 --- a/src/test/java/net/logstash/log4j/MockAppenderV1.java +++ b/src/test/java/net/logstash/log4j/MockAppenderV1.java @@ -9,7 +9,7 @@ public class MockAppenderV1 extends AppenderSkeleton { - private static List messages = new ArrayList(); + private List messages = new ArrayList(); public MockAppenderV1(Layout layout){ this.layout = layout; @@ -27,7 +27,7 @@ public boolean requiresLayout(){ return true; } - public static String[] getMessages() { + public String[] getMessages() { return (String[]) messages.toArray(new String[messages.size()]); }