[Backport 2.x] Adding security enabled integration tests (#400) (#403)

Adding security enabled integration tests (#400) * Adding intiial security integration tests, addin test security workflow * updating set up to v4 * Fixing run docker image task * Fixing pull and run docket * Fixing pull and run docket * Testing integ test if security is not available * Removing non-security integ test from workflow * test * test * Removing docker -ps * Pulling in secuirty as a zipArchive dependency, installed and configured only for security y enabled clusters * fixing ci * using v1 * Addressing PR comments, using security.emabled system property instead * Adding remaining read access role tests * spotless * Addressing PR comments, adding full access tests, fixing create workflow bug * Added more APIs to full access client test * updating DEVELOPER_GUIDE * Updating developer guide, adding back ML Commons security system indices to security plugin configuration --------- Signed-off-by: Joshua Palis <[email protected]>
opensearch-project · Jan 13, 2024 · 9e5b25f · 9e5b25f
1 parent d3e0e64
commit 9e5b25f
Show file tree

Hide file tree

Showing 11 changed files with 604 additions and 237 deletions.
diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml
@@ -0,0 +1,43 @@
+name: Security test workflow for Flow Framework
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  integ-test-with-security-linux:
+    strategy:
+      matrix:
+        java: [11, 17, 21]
+
+    name: Run Security Integration Tests on Linux
+    runs-on: ubuntu-latest
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      - name: Checkout Flow Framework
+        uses: actions/checkout@v1
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+      - name: Run tests
+        # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true"
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -37,8 +37,13 @@ This package uses the [Gradle](https://docs.gradle.org/current/userguide/usergui
 
 1. `./gradlew check` builds and tests.
 2. `./gradlew :run` installs and runs ML-Commons and Flow Framework Plugins into a local cluster
-3. `./gradlew spotlessApply` formats code. And/or import formatting rules in [formatterConfig.xml](formatter/formatterConfig.xml) with IDE.
-4. `./gradlew test` to run the complete test suite.
+3. `./gradlew run -Dsecurity.enabled=true` installs, configures and runs ML-Commons, Flow Framework and Security Plugins into a local cluster
+4. `./gradlew spotlessApply` formats code. And/or import formatting rules in [formatterConfig.xml](formatter/formatterConfig.xml) with IDE.
+5. `./gradlew test` to run the complete test suite.
+6. `./gradlew integTest` to run only the non-security enabled integration tests
+7. `./gradlew integTest -Dsecurity.enabled=true` to run only the security enabled integration tests
+6. `./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster` to run only the non-security enabled integration tests on a remote cluster
+7. `./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity.enabled=true` to run only the security enabled integration tests on a remote cluster
 
 #### Building from the IDE
 

diff --git a/build.gradle b/build.gradle
@@ -1,6 +1,48 @@
 import java.nio.file.Files
+import org.opensearch.gradle.testclusters.OpenSearchCluster
 import org.opensearch.gradle.test.RestIntegTestTask
 import java.util.concurrent.Callable
+import java.nio.file.Paths
+
+buildscript {
+    ext {
+        opensearch_version = System.getProperty("opensearch.version", "2.12.0-SNAPSHOT")
+        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
+        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
+        version_tokens = opensearch_version.tokenize('-')
+        opensearch_build = version_tokens[0] + '.0'
+        plugin_no_snapshot = opensearch_build
+        if (buildVersionQualifier) {
+            opensearch_build += "-${buildVersionQualifier}"
+            plugin_no_snapshot += "-${buildVersionQualifier}"
+        }
+        if (isSnapshot) {
+            opensearch_build += "-SNAPSHOT"
+        }
+        opensearch_group = "org.opensearch"
+        opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
+        System.setProperty('tests.security.manager', 'false')
+        common_utils_version = System.getProperty("common_utils.version", opensearch_build)
+    }
+
+    repositories {
+        mavenLocal()
+        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
+        mavenCentral()
+        maven { url "https://plugins.gradle.org/m2/" }
+        maven { url 'https://jitpack.io' }
+    }
+
+    dependencies {
+        classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
+        classpath "com.diffplug.spotless:spotless-plugin-gradle:6.23.3"
+        classpath "com.github.form-com.diff-coverage-gradle:diff-coverage:0.9.5"
+    }
+}
+
+plugins {
+    id "de.undercouch.download" version "5.3.0"
+}
 
 apply plugin: 'java'
 apply plugin: 'idea'
@@ -39,42 +81,6 @@ thirdPartyAudit.enabled = false
 // No need to validate pom, as we do not upload to maven/sonatype
 validateNebulaPom.enabled = false
 
-buildscript {
-    ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.12.0-SNAPSHOT")
-        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
-        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        version_tokens = opensearch_version.tokenize('-')
-        opensearch_build = version_tokens[0] + '.0'
-        plugin_no_snapshot = opensearch_build
-        if (buildVersionQualifier) {
-            opensearch_build += "-${buildVersionQualifier}"
-            plugin_no_snapshot += "-${buildVersionQualifier}"
-        }
-        if (isSnapshot) {
-            opensearch_build += "-SNAPSHOT"
-        }
-        opensearch_group = "org.opensearch"
-        opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
-        System.setProperty('tests.security.manager', 'false')
-        common_utils_version = System.getProperty("common_utils.version", opensearch_build)
-    }
-
-    repositories {
-        mavenLocal()
-        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
-        mavenCentral()
-        maven { url "https://plugins.gradle.org/m2/" }
-        maven { url 'https://jitpack.io' }
-    }
-
-    dependencies {
-        classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
-        classpath "com.diffplug.spotless:spotless-plugin-gradle:6.23.3"
-        classpath "com.github.form-com.diff-coverage-gradle:diff-coverage:0.9.5"
-    }
-}
-
 allprojects {
     // Default to the apache license
     project.ext.licenseName = 'The Apache Software License, Version 2.0'
@@ -155,6 +161,7 @@ dependencies {
 
     // ZipArchive dependencies used for integration tests
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
+    zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"
 
     configurations.all {
         resolutionStrategy {
@@ -170,6 +177,81 @@ def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absolu
 opensearch_tmp_dir.mkdirs()
 def _numNodes = findProperty('numNodes') as Integer ?: 1
 
+ext{
+
+    configureSecurityPlugin = { OpenSearchCluster cluster ->
+
+        // Retrieve Security Plugin Zip from zipArchive
+        configurations.zipArchive.asFileTree.each {
+            if(it.name.contains("opensearch-security")) {
+                cluster.plugin(provider(new Callable<RegularFile>(){
+                        @Override
+                        RegularFile call() throws Exception {
+                            return new RegularFile() {
+                                @Override
+                                File getAsFile() {
+                                    return it
+                                }
+                            }
+                        }
+                    })
+                )
+            }
+        }
+
+        cluster.getNodes().forEach { node ->
+            var creds = node.getCredentials()
+            if (creds.isEmpty()) {
+                creds.add(Map.of('username', 'admin', 'password', 'admin'))
+            } else {
+                creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
+            }
+        }
+
+        // Config below including files are copied from security demo configuration
+        ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file ->
+            File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
+            download.run {
+                src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
+                dest local
+                overwrite false
+            }
+            cluster.extraConfigFile(file, local)
+        }
+
+        // This configuration is copied from the security plugins demo install:
+        // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
+        cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+        cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+        cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+        cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+        cluster.setting("plugins.security.ssl.http.enabled", "true")
+        cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+        cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+        cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+        cluster.setting("plugins.security.allow_unsafe_democertificates", "true")
+        cluster.setting("plugins.security.allow_default_init_securityindex", "true")
+        cluster.setting("plugins.security.unsupported.inject_user.enabled", "true")
+
+        cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
+        cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
+        cluster.setting('plugins.security.system_indices.enabled', "true")
+        cluster.setting('plugins.security.system_indices.indices', '[' +
+                '".plugins-ml-config", ' +
+                '".plugins-ml-connector", ' +
+                '".plugins-ml-model-group", ' +
+                '".plugins-ml-model", ".plugins-ml-task", ' +
+                '".plugins-ml-conversation-meta", ' +
+                '".plugins-ml-conversation-interactions", ' +
+                '".plugins-flow-framework-config", ' +
+                '".plugins-flow-framework-templates", ' +
+                '".plugins-flow-framework-state"' +
+                ']'
+        )
+        cluster.setSecure(true)
+    }
+}
+
 test {
     include '**/*Tests.class'
 }
@@ -197,9 +279,18 @@ integTest {
     systemProperty 'tests.security.manager', 'false'
     systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
     systemProperty('project.root', project.rootDir.absolutePath)
-    systemProperty "https", System.getProperty("https")
-    systemProperty "user", System.getProperty("user")
-    systemProperty "password", System.getProperty("password")
+    systemProperty 'security.enabled', System.getProperty('security.enabled')
+    var is_https = System.getProperty('https')
+    var user = System.getProperty('user')
+    var password = System.getProperty('password')
+    if (System.getProperty('security.enabled') != null) {
+        is_https = is_https == null ? 'true' : is_https
+        user = user == null ? 'admin' : user
+        password = password == null ? 'admin' : password
+    }
+    systemProperty('https', is_https)
+    systemProperty('user', user)
+    systemProperty('password', password)
 
     // Only rest case can run with remote cluster
     if (System.getProperty("tests.rest.cluster") != null) {
@@ -208,6 +299,20 @@ integTest {
         }
     }
 
+    // Exclude integration tests that require security plugin
+    if (System.getProperty("security.enabled") == null || System.getProperty("security.enabled") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkSecureRestApiIT"
+        }
+    }
+
+    // Include only secure integration tests in security enabled clusters
+    if (System.getProperty("security.enabled") != null && System.getProperty("security.enabled") == "true") {
+        filter {
+            includeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkSecureRestApiIT"
+            excludeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkRestApiIT"
+        }
+    }
 
     // doFirst delays this block until execution time
     doFirst {
@@ -234,19 +339,27 @@ integTest {
 testClusters.integTest {
     testDistribution = "ARCHIVE"
 
-    // Installs all registered zipArchive dependencies on integTest cluster nodes
+    // Optionally install security
+    if (System.getProperty("security.enabled") != null && System.getProperty("security.enabled") == "true") {
+        configureSecurityPlugin(testClusters.integTest)
+    }
+
+    // Installs all registered zipArchive dependencies on integTest cluster nodes except security
     configurations.zipArchive.asFileTree.each {
-        plugin(provider(new Callable<RegularFile>(){
-            @Override
-            RegularFile call() throws Exception {
-                return new RegularFile() {
+        if(!it.name.contains("opensearch-security")) {
+            plugin(provider(new Callable<RegularFile>(){
                     @Override
-                    File getAsFile() {
-                        return it
+                    RegularFile call() throws Exception {
+                        return new RegularFile() {
+                            @Override
+                            File getAsFile() {
+                                return it
+                            }
+                        }
                     }
-                }
-            }
-        }))
+                })
+            )
+        }
     }
 
     // Install Flow Framwork Plugin on integTest cluster nodes
@@ -271,10 +384,17 @@ testClusters.integTest {
 task integTestRemote(type: RestIntegTestTask) {
     testClassesDirs = sourceSets.test.output.classesDirs
     classpath = sourceSets.test.runtimeClasspath
-
-    systemProperty "https", System.getProperty("https")
-    systemProperty "user", System.getProperty("user")
-    systemProperty "password", System.getProperty("password")
+    var is_https = System.getProperty('https')
+    var user = System.getProperty('user')
+    var password = System.getProperty('password')
+    if (System.getProperty('security.enabled') != null) {
+        is_https = is_https == null ? 'true' : is_https
+        user = user == null ? 'admin' : user
+        password = password == null ? 'admin' : password
+    }
+    systemProperty('https', is_https)
+    systemProperty('user', user)
+    systemProperty('password', password)
     systemProperty 'cluster.number_of_nodes', "${_numNodes}"
     systemProperty 'tests.security.manager', 'false'
 
@@ -284,6 +404,21 @@ task integTestRemote(type: RestIntegTestTask) {
             includeTestsMatching "org.opensearch.flowframework.rest.*IT"
         }
     }
+
+    // Exclude integration tests that require security plugin
+    if (System.getProperty("https") == null || System.getProperty("https") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkSecureRestApiIT"
+        }
+    }
+
+    // Include only secure integration tests in security enabled clusters
+    if (System.getProperty("https") != null && System.getProperty("https") == "true") {
+        filter {
+            includeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkSecureRestApiIT"
+            excludeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkRestApiIT"
+        }
+    }
 }
 
 // Automatically sets up the integration test cluster locally

diff --git a/src/main/java/org/opensearch/flowframework/rest/RestCreateWorkflowAction.java b/src/main/java/org/opensearch/flowframework/rest/RestCreateWorkflowAction.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.ExceptionsHelper;
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.rest.RestStatus;
@@ -95,11 +96,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
                 channel.sendResponse(new BytesRestResponse(RestStatus.CREATED, builder));
             }, exception -> {
                 try {
-                    FlowFrameworkException ex = (FlowFrameworkException) exception;
+                    FlowFrameworkException ex = exception instanceof FlowFrameworkException
+                        ? (FlowFrameworkException) exception
+                        : new FlowFrameworkException(exception.getMessage(), ExceptionsHelper.status(exception));
                     XContentBuilder exceptionBuilder = ex.toXContent(channel.newErrorBuilder(), ToXContent.EMPTY_PARAMS);
                     channel.sendResponse(new BytesRestResponse(ex.getRestStatus(), exceptionBuilder));
                 } catch (IOException e) {
-                    logger.error("Failed to send back create workflow exception", e);
+                    logger.error("Failed to send back provision workflow exception", e);
+                    channel.sendResponse(new BytesRestResponse(ExceptionsHelper.status(e), e.getMessage()));
                 }
             }));
         } catch (Exception e) {