From ff4958954ac86e759ed68fa81fe2afeb792183f2 Mon Sep 17 00:00:00 2001 From: Joydeep Sinha Date: Tue, 1 Dec 2020 14:44:35 -0800 Subject: [PATCH 1/3] Changing metricsDB provider while executing an integTest Today the gauntlet only let's you provide the metricsDB table at the top of a test class or at the top of a test method. But sometimes during the test test execution, one might want to change the instance of metricsDB. This is made possible by this patch. In order to do that, You create a new inner class (doesn't have to be inner) with an appropriate name. Add, the metrics details as you would if you were to do it as part of the main test class itself. @AMetric(name = CPU_Utilization.class, dimensionNames = {SHARDID_VALUE, INDEX_NAME_VALUE, OPERATION_VALUE, SHARD_ROLE_VALUE}, tables = { @ATable(hostTag = HostTag.DATA_0, tuple = { @ATuple(dimensionValues = {"0", "logs", "bulk", "p"}, sum = 0.0, avg = 0.0, min = 0.0, max = 111.0), @ATuple(dimensionValues = {"1", "logs", "bulk", "r"}, sum = 0.0, avg = 0.0, min = 0.0, max = 88.0), @ATuple(dimensionValues = {"2", "logs", "bulk", "p"}, sum = 0.0, avg = 0.0, min = 0.0, max = 11.0) } ), } ) public static class MetricsAfterEventX { } then from your test method, you can do something like this: @Test @AExpect( what = AExpect.Type.REST_API, on = HostTag.DATA_0, validator = UpdatedMetricsPocValidator.class, forRca = RcaItPocDedicated.SimpleAnalysisGraphForDedicated.ClusterRca.class) public void testMetricsUpdated() throws Exception { // Initial logic, uses the Class default metricsDB choice. // ... api.updateMetrics(MetricsAfterEventX.class); // Other test logic follows. } --- CONTRIBUTING.md | 2 +- docs/{rcait.md => gauntlet.md} | 2 +- .../rca/framework/api/Metric.java | 1 + .../rca/integTests/framework/api/TestApi.java | 32 +++++++++++++------ .../tests/poc/RcaItPocSingleNode.java | 2 ++ 5 files changed, 27 insertions(+), 12 deletions(-) rename docs/{rcait.md => gauntlet.md} (99%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 586c02d8f..925ddefaf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ e.g. ### Adding your own tests - You can add a unit tests using the junit framework - There is also a mechanism to add integration tests for the RCA framework. For details, please - see [here](docs/rcait.md). + see [here](docs/gauntlet.md). Before submitting the PR, we request that you also run ```shell script diff --git a/docs/rcait.md b/docs/gauntlet.md similarity index 99% rename from docs/rcait.md rename to docs/gauntlet.md index 0e5713803..9a5724eec 100644 --- a/docs/rcait.md +++ b/docs/gauntlet.md @@ -1,4 +1,4 @@ -# Rca Integration test framework +# Gauntlet: Rca Integration test framework ## Scope To be able to test scenarios where multiple RCA Schedulers are running on different hosts of a diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/Metric.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/Metric.java index f0223f93a..9070d137d 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/Metric.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/Metric.java @@ -71,6 +71,7 @@ public MetricFlowUnit gather(Queryable queryable) { } try { Result result = queryable.queryMetrics(db, name); + LOG.info(result); return new MetricFlowUnit(queryable.getDBTimestamp(db), result); } catch (DataAccessException dex) { // This can happen if the RCA started querying for metrics before the Reader obtained them. diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/framework/api/TestApi.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/framework/api/TestApi.java index 6bb03e813..b93e207e1 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/framework/api/TestApi.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/framework/api/TestApi.java @@ -1,23 +1,27 @@ package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.api; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.Cluster; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.AMetric; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.configs.HostTag; import com.google.gson.JsonElement; import com.google.gson.JsonParser; import java.util.Map; import java.util.Objects; import javax.annotation.Nonnull; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jooq.Record; import org.jooq.Result; /** - * This is API class whose object is injected into each of the test methods in case test class - * declares a @{code setTestApi(final TestApi api)}. + * This is API class whose object is injected into each of the test methods in case test class declares a @{code setTestApi(final TestApi + * api)}. */ public class TestApi { + private static final Logger LOG = LogManager.getLogger(TestApi.class); + /** - * An instance of the cluster object to get access to the nodes to query various data to see - * that the tests have the desired results. + * An instance of the cluster object to get access to the nodes to query various data to see that the tests have the desired results. */ private final Cluster cluster; @@ -34,28 +38,36 @@ public Object constructObjectFromDBOnHost(HostTag hostTag, Class clz) thr } /** - * This let's you make a REST request to the REST endpoint of a particular host identified by - * the host tag. + * This let's you make a REST request to the REST endpoint of a particular host identified by the host tag. * * @param params the key value map that is passes as the request parameter. * @param hostByTag The host whose rest endpoint we will hit. * @return The response serialized as a String. */ public JsonElement getRestResponse(@Nonnull final String queryUrl, - @Nonnull final Map params, - HostTag hostByTag) { + @Nonnull final Map params, + HostTag hostByTag) { Objects.requireNonNull(params); JsonElement json = JsonParser.parseString(cluster.getRcaRestResponse(queryUrl, params, hostByTag)); return json; } /** - * This can be used to get all the records from all the tables in a host. This can be used for validation of - * what gets persisted in the rca.sqlite tables. + * This can be used to get all the records from all the tables in a host. This can be used for validation of what gets persisted in the + * rca.sqlite tables. + * * @param hostTag The host whose rca.sqlite will be queried. * @return A list of all the data stored in all the tables in the particular host. */ public Map> getRecordsForAllTablesOnHost(HostTag hostTag) { return cluster.getRecordsForAllTablesOnHost(hostTag); } + + public void updateMetrics(Class clz) throws Exception { + if (clz.isAnnotationPresent(AMetric.Metrics.class) || clz.isAnnotationPresent(AMetric.class)) { + cluster.updateMetricsDB(clz.getAnnotationsByType(AMetric.class)); + cluster.stopRcaScheduler(); + cluster.startRcaScheduler(); + } + } } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/poc/RcaItPocSingleNode.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/poc/RcaItPocSingleNode.java index 32cf092bb..c912b1787 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/poc/RcaItPocSingleNode.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/poc/RcaItPocSingleNode.java @@ -19,12 +19,14 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ARcaGraph; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ATable; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ATuple; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.api.IValidator; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.api.TestApi; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.configs.ClusterType; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.configs.HostTag; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.runners.RcaItNotEncryptedRunner; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.runners.RcaItRunnerBase; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.poc.validator.PocValidator; +import com.google.gson.JsonElement; import java.util.Arrays; import java.util.Collections; import org.apache.logging.log4j.LogManager; From 68236715a19b34666310bbb4fc753356c84d0873 Mon Sep 17 00:00:00 2001 From: Joydeep Sinha Date: Tue, 1 Dec 2020 15:01:56 -0800 Subject: [PATCH 2/3] remove the debug log --- .../performanceanalyzer/rca/framework/api/Metric.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/Metric.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/Metric.java index 9070d137d..f0223f93a 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/Metric.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/Metric.java @@ -71,7 +71,6 @@ public MetricFlowUnit gather(Queryable queryable) { } try { Result result = queryable.queryMetrics(db, name); - LOG.info(result); return new MetricFlowUnit(queryable.getDBTimestamp(db), result); } catch (DataAccessException dex) { // This can happen if the RCA started querying for metrics before the Reader obtained them. From 112c89ce0486de31d4c7f0d4b514dd8e017b166a Mon Sep 17 00:00:00 2001 From: Joydeep Sinha Date: Tue, 1 Dec 2020 15:05:31 -0800 Subject: [PATCH 3/3] added some more comments --- .../rca/integTests/framework/api/TestApi.java | 8 ++++++++ .../rca/integTests/tests/poc/RcaItPocSingleNode.java | 3 --- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/framework/api/TestApi.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/framework/api/TestApi.java index b93e207e1..8b879a04b 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/framework/api/TestApi.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/framework/api/TestApi.java @@ -63,9 +63,17 @@ public Map> getRecordsForAllTablesOnHost(HostTag hostTag) return cluster.getRecordsForAllTablesOnHost(hostTag); } + /** + * This API let's a gauntlet test writer swap out the metricsDB for a new one. + * + * @param clz The class whose AMetric@ should be used to replace it + * @throws Exception Throws Exception + */ public void updateMetrics(Class clz) throws Exception { if (clz.isAnnotationPresent(AMetric.Metrics.class) || clz.isAnnotationPresent(AMetric.class)) { cluster.updateMetricsDB(clz.getAnnotationsByType(AMetric.class)); + + // The scheduler needs to be restarted to pick this change up. cluster.stopRcaScheduler(); cluster.startRcaScheduler(); } diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/poc/RcaItPocSingleNode.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/poc/RcaItPocSingleNode.java index c912b1787..d48c1c13a 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/poc/RcaItPocSingleNode.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/integTests/tests/poc/RcaItPocSingleNode.java @@ -19,14 +19,11 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ARcaGraph; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ATable; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.annotations.ATuple; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.api.IValidator; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.api.TestApi; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.configs.ClusterType; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.configs.HostTag; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.runners.RcaItNotEncryptedRunner; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.runners.RcaItRunnerBase; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.tests.poc.validator.PocValidator; -import com.google.gson.JsonElement; import java.util.Arrays; import java.util.Collections; import org.apache.logging.log4j.LogManager;