forked from apache/kyuubi
-
Notifications
You must be signed in to change notification settings - Fork 3
[ADH-8475] Support engine profiles #24
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
tigrulya-exe
wants to merge
2
commits into
develop/4.3.0/1.11.1.1
Choose a base branch
from
feature/1.11/ADH-8475
base: develop/4.3.0/1.11.1.1
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
127 changes: 127 additions & 0 deletions
127
kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/EngineProfileGroup.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.kyuubi.client.api.v1.dto; | ||
|
|
||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import java.util.Objects; | ||
| import org.apache.commons.lang3.builder.ReflectionToStringBuilder; | ||
| import org.apache.commons.lang3.builder.ToStringStyle; | ||
|
|
||
| /** Active engine instances grouped by engine profile name and engine type. */ | ||
| public class EngineProfileGroup { | ||
|
|
||
| private String profile; | ||
| private String engineType; | ||
| private String version; | ||
| private int instanceCount; | ||
| private String status; | ||
| private List<Engine> engines; | ||
|
|
||
| public EngineProfileGroup() {} | ||
|
|
||
| public EngineProfileGroup( | ||
| String profile, | ||
| String engineType, | ||
| String version, | ||
| int instanceCount, | ||
| String status, | ||
| List<Engine> engines) { | ||
| this.profile = profile; | ||
| this.engineType = engineType; | ||
| this.version = version; | ||
| this.instanceCount = instanceCount; | ||
| this.status = status; | ||
| this.engines = engines; | ||
| } | ||
|
|
||
| public String getProfile() { | ||
| return profile; | ||
| } | ||
|
|
||
| public void setProfile(String profile) { | ||
| this.profile = profile; | ||
| } | ||
|
|
||
| public String getEngineType() { | ||
| return engineType; | ||
| } | ||
|
|
||
| public void setEngineType(String engineType) { | ||
| this.engineType = engineType; | ||
| } | ||
|
|
||
| public String getVersion() { | ||
| return version; | ||
| } | ||
|
|
||
| public void setVersion(String version) { | ||
| this.version = version; | ||
| } | ||
|
|
||
| public int getInstanceCount() { | ||
| return instanceCount; | ||
| } | ||
|
|
||
| public void setInstanceCount(int instanceCount) { | ||
| this.instanceCount = instanceCount; | ||
| } | ||
|
|
||
| public String getStatus() { | ||
| return status; | ||
| } | ||
|
|
||
| public void setStatus(String status) { | ||
| this.status = status; | ||
| } | ||
|
|
||
| public List<Engine> getEngines() { | ||
| if (null == engines) { | ||
| return Collections.emptyList(); | ||
| } | ||
| return engines; | ||
| } | ||
|
|
||
| public void setEngines(List<Engine> engines) { | ||
| this.engines = engines; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object o) { | ||
| if (this == o) return true; | ||
| if (o == null || getClass() != o.getClass()) return false; | ||
| EngineProfileGroup that = (EngineProfileGroup) o; | ||
| return getInstanceCount() == that.getInstanceCount() | ||
| && Objects.equals(getProfile(), that.getProfile()) | ||
| && Objects.equals(getEngineType(), that.getEngineType()) | ||
| && Objects.equals(getVersion(), that.getVersion()) | ||
| && Objects.equals(getStatus(), that.getStatus()) | ||
| && Objects.equals(getEngines(), that.getEngines()); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash( | ||
| getProfile(), getEngineType(), getVersion(), getInstanceCount(), getStatus(), getEngines()); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); | ||
| } | ||
| } |
32 changes: 32 additions & 0 deletions
32
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineProfile.scala
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.kyuubi.engine | ||
|
|
||
| /** | ||
| * A materialized named engine profile, declared under | ||
| * `kyuubi.engine.profile.<name>.*`. The profile bundles, for a single named engine | ||
| * configuration, the engine type, environment variables, Kyuubi session variables and | ||
| * engine-native configs - already mapped to their effective Kyuubi config keys. | ||
| * | ||
| * All [[conf]] keys are applied to a session as defaults: any conflicting option the client | ||
| * supplies (e.g. via the JDBC url) overrides the profile. | ||
| * | ||
| * @param name the profile name | ||
| * @param conf the materialized engine-conf key-value pairs | ||
| */ | ||
| case class EngineProfile(name: String, conf: Map[String, String]) |
124 changes: 124 additions & 0 deletions
124
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineProfileRegistry.scala
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.kyuubi.engine | ||
|
|
||
| import java.util.Locale | ||
|
|
||
| import scala.collection.mutable | ||
|
|
||
| import org.apache.kyuubi.config.KyuubiConf | ||
| import org.apache.kyuubi.config.KyuubiConf._ | ||
|
|
||
| /** | ||
| * An immutable registry of all [[EngineProfile]]s declared under `kyuubi.engine.profile.<name>.*`. | ||
| * | ||
| * This owns the engine-profile configuration logic: reading the profile declarations from a | ||
| * [[KyuubiConf]] and materializing them into [[EngineProfile]]s. Engine profiles are loaded from | ||
| * `kyuubi-defaults.conf` at startup and never change at runtime, so [[EngineProfileRegistry.apply]] | ||
| * materializes them once (validating each) and the result is reused for every session open and | ||
| * REST listing, rather than being re-parsed each time. | ||
| */ | ||
| class EngineProfileRegistry private (profiles: Map[String, EngineProfile]) { | ||
|
|
||
| /** Names of all declared engine profiles. */ | ||
| def names: Set[String] = profiles.keySet | ||
|
|
||
| /** The materialized profile for `name`, or None if no such profile is declared. */ | ||
| def get(name: String): Option[EngineProfile] = profiles.get(name) | ||
| } | ||
|
|
||
| object EngineProfileRegistry { | ||
|
|
||
| // The profile declaration prefix, e.g. `kyuubi.engine.profile`. | ||
| private val PROFILE_PREFIX = ENGINE_PROFILE.key | ||
|
|
||
| /** | ||
| * Materialize all engine profiles declared in `conf` into an immutable registry. Fails fast with | ||
| * [[IllegalArgumentException]] if any profile has an invalid engine type or unknown bucket. | ||
| */ | ||
| def apply(conf: KyuubiConf): EngineProfileRegistry = { | ||
| val profiles = profileNames(conf) | ||
| .map(name => name -> buildProfile(conf, name)) | ||
| .toMap | ||
| new EngineProfileRegistry(profiles) | ||
| } | ||
|
|
||
| /** Names of all engine profiles declared under `kyuubi.engine.profile.<name>.*`. */ | ||
| private def profileNames(conf: KyuubiConf): Set[String] = { | ||
| val prefix = s"$PROFILE_PREFIX." | ||
| conf.getAll.keys.collect { | ||
| case k if k.startsWith(prefix) => k.stripPrefix(prefix).split("\\.", 2).head | ||
| }.toSet | ||
| } | ||
|
|
||
| /** | ||
| * Materialize one named engine profile into effective engine-conf keys. Fails fast with | ||
| * [[IllegalArgumentException]] on an invalid engine type or an unknown profile bucket. | ||
| * | ||
| * Bucket mapping for a `kyuubi.engine.profile.<name>.<bucket>[.<rest>]` key: | ||
| * - `type` -> `kyuubi.engine.type` (validated against [[EngineType]]) | ||
| * - `env.<VAR>` -> `kyuubi.engineEnv.<VAR>` | ||
| * - `session.<rest>` -> `kyuubi.session.<rest>` (Kyuubi session variable) | ||
| * - `conf.<rest>` -> `<rest>` verbatim (engine-native property) | ||
| * | ||
| * All materialized keys are applied to a session as defaults; conflicting client conf wins. | ||
| */ | ||
| private def buildProfile(conf: KyuubiConf, name: String): EngineProfile = { | ||
| val raw = conf.getAllWithPrefix(s"$PROFILE_PREFIX.$name", "") | ||
| val profileConf = mutable.Map[String, String]() | ||
| raw.foreach { case (suffix, value) => | ||
| val parts = suffix.split("\\.", 2) | ||
| parts.head match { | ||
| case "type" => | ||
| require( | ||
| parts.length == 1, | ||
| s"Invalid engine profile key '$PROFILE_PREFIX.$name.$suffix':" + | ||
| s" 'type' does not accept a sub-key.") | ||
| val engineType = value.toUpperCase(Locale.ROOT) | ||
| require( | ||
| EngineType.values.exists(_.toString == engineType), | ||
| s"Invalid engine type '$value' in profile '$name', expected one of" + | ||
| s" ${EngineType.values.mkString("[", ", ", "]")}.") | ||
| profileConf += (ENGINE_TYPE.key -> engineType) | ||
| case "env" => | ||
| require( | ||
| parts.length == 2, | ||
| s"Invalid engine profile key '$PROFILE_PREFIX.$name.$suffix':" + | ||
| s" 'env' requires an environment variable name, e.g. env.SPARK_HOME.") | ||
| profileConf += (s"$KYUUBI_ENGINE_ENV_PREFIX.${parts(1)}" -> value) | ||
| case "session" => | ||
| require( | ||
| parts.length == 2, | ||
| s"Invalid engine profile key '$PROFILE_PREFIX.$name.$suffix':" + | ||
| s" 'session' requires a Kyuubi session config key.") | ||
| profileConf += (s"kyuubi.session.${parts(1)}" -> value) | ||
| case "conf" => | ||
| require( | ||
| parts.length == 2, | ||
| s"Invalid engine profile key '$PROFILE_PREFIX.$name.$suffix':" + | ||
| s" 'conf' requires an engine config key.") | ||
| profileConf += (parts(1) -> value) | ||
| case bucket => | ||
| throw new IllegalArgumentException( | ||
| s"Unknown engine profile bucket '$bucket' in key '$PROFILE_PREFIX.$name.$suffix'." + | ||
| s" Expected one of: type, env.<VAR>, session.<key>, conf.<key>.") | ||
| } | ||
| } | ||
| EngineProfile(name, profileConf.toMap) | ||
| } | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.