diff --git a/assembly/pom.xml b/assembly/pom.xml index 6263ee4932d51..772e0f1c2f490 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index bd2f88b0ee8ed..3cc3bae48d528 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 85a4631729dc1..413deda400692 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index bca0537e4d7b4..fbe0bb1d64071 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index a0b7e88e6c935..c5a90affce5a7 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 613561b92a954..25119c85db780 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index c749387db4975..eb0c99e9416ea 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index e0d48291afa2f..c4c13ec4400c3 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index dd4fa242f3064..5e8c54b72f3f3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 99735631d7b78..127e59d0aed0a 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -85,7 +85,7 @@ GIT_REF=${GIT_REF:-master} RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/spark" GPG="gpg -u $GPG_KEY --no-tty --batch --pinentry-mode loopback" -NEXUS_ROOT=https://repository.apache.org/service/local/staging +NEXUS_ROOT=https://jfrog.fkinternal.com/artifactory/maven_virtual/ NEXUS_PROFILE=d63f592e7eac0 # Profile for Spark staging uploads BASE_DIR=$(pwd) diff --git a/dev/create-release/vote.tmpl b/dev/create-release/vote.tmpl index aa22a9c516535..1072c288ed328 100644 --- a/dev/create-release/vote.tmpl +++ b/dev/create-release/vote.tmpl @@ -18,7 +18,7 @@ Signatures used for Spark RCs can be found in this file: https://dist.apache.org/repos/dist/dev/spark/KEYS The staging repository for this release can be found at: -https://repository.apache.org/content/repositories/orgapachespark-{repo_id}/ +https://jfrog.fkinternal.com/artifactory/maven_virtual/ The documentation corresponding to this release can be found at: https://dist.apache.org/repos/dist/dev/spark/{tag}-docs/ diff --git a/examples/pom.xml b/examples/pom.xml index c9bcfe5d9e753..768fc045150c3 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index c678e0c689a05..8639af95a7505 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 0a3e6837421bd..afac1b9120460 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index 0376d0672dd92..764c8b01b9a72 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 79778fb4ea066..fd9abddefc878 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml index bdae71dd9a5d3..5e4fe347020c7 100644 --- a/external/kafka-0-10-token-provider/pom.xml +++ b/external/kafka-0-10-token-provider/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index 77aa1b5afba1f..f4239457385b8 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 655ec2680fcb5..46492edcfc36d 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 7d27794b982da..bef1960aa89e5 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index facef36532c25..b9bb17987efe8 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index ed81876459a84..6a8f0dc2ee6df 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index d66aae994f71d..98abfdd50b366 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 2097cbd9890cb..45416ffaac684 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index efd5ddaa1db05..1e6e3f5665147 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 5ee35f537d5f5..52ac1e7f7e34f 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/pom.xml b/pom.xml index 3f114fd1a3f8d..fc8fb37b05673 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 pom Spark Project Parent POM http://spark.apache.org/ @@ -279,6 +279,14 @@ + flipkart-libs-snapshots-local + https://jfrog.fkinternal.com/artifactory/maven_virtual + + + flipkart-libs-release-local + https://jfrog.fkinternal.com/artifactory/maven_virtual + + central Maven Repository https://repo.maven.apache.org/maven2 @@ -306,6 +314,14 @@ false + + flipkart-libs-snapshots-local + https://jfrog.fkinternal.com/artifactory/maven_virtual + + + flipkart-libs-release-local + https://jfrog.fkinternal.com/artifactory/maven_virtual + central https://repo.maven.apache.org/maven2 @@ -3316,8 +3332,8 @@ snapshots-and-staging - https://repository.apache.org/content/groups/staging/ - https://repository.apache.org/content/repositories/snapshots/ + https://jfrog.fkinternal.com/artifactory/maven_virtual + https://jfrog.fkinternal.com/artifactory/maven_virtual/ @@ -3383,4 +3399,16 @@ + + + fk-jfrog-snapshot + libs-snapshot + https://jfrog.fkinternal.com/artifactory/maven_virtual + + + fk-jfrog-release + libs-rel + https://jfrog.fkinternal.com/artifactory/maven_virtual + + diff --git a/repl/pom.xml b/repl/pom.xml index fbe6a84ad922f..8731ca01dbc5e 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 5e997b217644a..f0c43c6d5ce4c 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 9e8963d66d964..4072941fe8e65 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 946416e32cf56..e32db53cfa6ca 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index b2c635ca78543..4fa081d09ebe6 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 618faef2d58b3..af2d4af929447 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -54,6 +54,7 @@ import org.apache.spark.deploy.{SparkApplication, SparkHadoopUtil} import org.apache.spark.deploy.security.HadoopDelegationTokenManager import org.apache.spark.deploy.yarn.ResourceRequestHelper._ import org.apache.spark.deploy.yarn.config._ +import org.apache.spark.deploy.yarn.propertymodifier.QueueEnforcer import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ import org.apache.spark.internal.config.Python._ @@ -241,6 +242,39 @@ private[spark] class Client( cleanupStagingDirInternal() } + private def getInitiator: String = { + var user: String = null + try { + user = JobInitiatorFetcher.getJobInitiator + } catch { + case e: Throwable => + throw new RuntimeException("User couldn't be set. ", e) + } + user + } + + @throws[IOException] + private def getEnforcedQueue(queue: String): String = { + log.info("Input queue name: " + queue) + val queueEnforcerClassName: String = sparkConf.get(QUEUE_ENFORCER_CLASS) + + if (queueEnforcerClassName == null) { + throw new RuntimeException("Queue enforcer class is null.") + } + + var queueEnforcer: QueueEnforcer = null + try {queueEnforcer = Utils.classForName(queueEnforcerClassName) + .newInstance.asInstanceOf[QueueEnforcer] + } catch { + case e: Throwable => + throw new RuntimeException("Couldn't enforce queue. ", e) + } + log.info("queue enforcer class: " + queueEnforcerClassName) + val enforcedQueue: String = queueEnforcer.getEnforcedQueue(queue, getInitiator, sparkConf) + log.info("Enforced queue name: " + enforcedQueue) + enforcedQueue + } + /** * Set up the context for submitting our ApplicationMaster. * This uses the YarnClientApplication not available in the Yarn alpha API. @@ -260,7 +294,8 @@ private[spark] class Client( logDebug(s"AM resources: $amResources") val appContext = newApp.getApplicationSubmissionContext appContext.setApplicationName(sparkConf.get("spark.app.name", "Spark")) - appContext.setQueue(sparkConf.get(QUEUE_NAME)) + log.info("setting queue") + appContext.setQueue(getEnforcedQueue(sparkConf.get(QUEUE_NAME))) appContext.setAMContainerSpec(containerContext) appContext.setApplicationType(sparkConf.get(APPLICATION_TYPE)) @@ -1267,6 +1302,21 @@ private[spark] class Client( } +private object JobInitiatorFetcher { + val SUDO_USER_PROP_KEY = "SUDO_USER" + val USER = "USER" + + @throws[IOException] + @throws[InterruptedException] + def getJobInitiator: String = { + val sudoUser = Option(System.getenv.get(SUDO_USER_PROP_KEY)) + val user = Option(System.getenv.get(USER)) + if (sudoUser.isDefined) return sudoUser.get + else if (user.isDefined) return user.get + null + } +} + private object Client extends Logging { // Alias for the user jar diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala index 89a4af2d2a741..14baa9df0b70d 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala @@ -101,6 +101,11 @@ package object config extends Logging { .stringConf .createWithDefault("default") + private[spark] val QUEUE_ENFORCER_CLASS = ConfigBuilder("spark.queue.enforcer.class") + .doc("Fully Qualified class name for concrete impl to be used to enforce queue") + .stringConf + .createWithDefault(null) + private[spark] val HISTORY_SERVER_ADDRESS = ConfigBuilder("spark.yarn.historyServer.address") .version("1.0.0") .stringConf diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/QueueEnforcer.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/QueueEnforcer.scala new file mode 100644 index 0000000000000..53c2bc156d584 --- /dev/null +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/QueueEnforcer.scala @@ -0,0 +1,24 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.spark.deploy.yarn.propertymodifier + +import org.apache.spark.SparkConf + +trait QueueEnforcer { + def getEnforcedQueue(existingQueue: String, initiator: String, conf: SparkConf): String +} \ No newline at end of file diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/ReturnExistingQueue.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/ReturnExistingQueue.scala new file mode 100644 index 0000000000000..a32be552a112a --- /dev/null +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/propertymodifier/ReturnExistingQueue.scala @@ -0,0 +1,25 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.spark.deploy.yarn.propertymodifier + +import org.apache.spark.SparkConf + +class ReturnExistingQueue extends QueueEnforcer { + override def getEnforcedQueue(existingQueue: String, initiator: String, conf: SparkConf): + String = existingQueue +} \ No newline at end of file diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 08bb504da9c87..eb735635fffe1 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index a4043d9cb3a7b..fe53f13c1dea4 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index addb2dca0c463..cd0ce129090ae 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index b07e34f1cf86e..8eaae41005993 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index e64e269cb39d7..d53c33c955788 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 3b2314dc8d370..56f50ec258f92 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.2 + 3.1.2.fdp.1 ../pom.xml