CLOUDSTACK-10012: Migrate to Embedded Jetty

- Migrate to embedded Jetty server.
- Improve ServerDaemon implementation.
- Introduce a new server.properties file for easier configuration.
- Have a single /etc/default/cloudstack-management to configure env.
- Reduce shaded jar file, removing unnecessary dependencies.
- Upgrade to Spring 5.x, upgrade several jar dependencies.
- Does not shade and include mysql-connector, used from classpath instead.
- Upgrade and use bountcastle as a separate un-shaded jar dependency.
- Remove tomcat related configuration and files.
- Have both embedded UI assets in uber jar and separate webapp directory.
- Refactor systemd and init scripts, cleanup packaging.
- Made cloudstack-setup-databases faster, using `urandom`.
- Remove unmaintained distro packagings.
- Moves creation and usage of server keystore in CA manager, this
  deprecates the need to create/store cloud.jks in conf folder and
  the db.cloud.keyStorePassphrase in db.properties file. This also
  remove the need of the --keystore-passphrase in the
  cloudstack-setup-encryption script.
- GZip contents dynamically in embedded Jetty

Signed-off-by: Rohit Yadav <[email protected]>

Author: rohityadavcloud

INSTALL.md

@@ -15,7 +15,7 @@ was tested against a CentOS 6.5 x86_64 setup.
 Install tools and dependencies used for development:
 
     $ yum install git ant ant-devel java-1.6.0-openjdk java-1.6.0-openjdk-devel
-    mysql mysql-server tomcat6 mkisofs gcc python MySQL-python openssh-clients wget
+    mysql mysql-server mkisofs gcc python MySQL-python openssh-clients wget
 
     # yum -y update
     # yum -y install java-1.7.0-openjdk
@@ -124,7 +124,6 @@ To create debs install the following extra packages:
 
     # apt-get -y install python-mysqldb
     # apt-get -y install debhelper
-    # apt-get -y install tomcat6
 
 Then:
 
@@ -138,7 +137,6 @@ All the deb packages will be located one level down.
 To create rpms, install the following extra packages:
 
     # yum -y install rpm-build
-    # yum -y install tomcat6
     # yum -y install ws-commons-util
     # yum -y instal gcc
     # yum -y install glibc-devel

README.md

@@ -42,7 +42,7 @@ For more information on Apache CloudStack, please visit the [website](http://clo
 
 Apache CloudStack project uses Git. The official Git repository is at:
 
-    https://git-wip-us.apache.org/repos/asf/cloudstack.git
+    https://gitbox.apache.org/repos/asf/cloudstack.git
 
 And a mirror is hosted on Github:
 

build/replace.properties

@@ -27,4 +27,4 @@ AGENTLOG=logs/agent.log
 MSMNTDIR=/mnt
 COMPONENTS-SPEC=components.xml
 REMOTEHOST=localhost
-COMMONLIBDIR=client/target/cloud-client-ui-4.7.0/WEB-INF/lib/
+COMMONLIBDIR=client/target/common/

client/WEB-INF/web.xml

@@ -19,23 +19,20 @@
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
     version="2.5">
-   
-    <context-param>  
-      <param-name>log4jConfigLocation</param-name>  
-      <param-value>classpath:log4j-cloud.xml</param-value>  
-    </context-param>  
-    <listener>  
-      <listener-class>org.springframework.web.util.Log4jConfigListener</listener-class>  
-    </listener>  
-    
-	<listener>
-      <listener-class>org.apache.cloudstack.spring.module.web.CloudStackContextLoaderListener</listener-class>
-	</listener>
+
     <context-param>
-        <param-name>contextConfigLocation</param-name>    
-        <param-value>classpath:META-INF/cloudstack/webApplicationContext.xml</param-value>    
+        <param-name>log4jConfigLocation</param-name>
+        <param-value>classpath:log4j-cloud.xml</param-value>
     </context-param>
-          
+
+    <listener>
+        <listener-class>org.apache.cloudstack.spring.module.web.CloudStackContextLoaderListener</listener-class>
+    </listener>
+    <context-param>
+        <param-name>contextConfigLocation</param-name>
+        <param-value>classpath:META-INF/cloudstack/webApplicationContext.xml</param-value>
+    </context-param>
+
     <servlet>
         <servlet-name>cloudStartupServlet</servlet-name>
         <servlet-class>com.cloud.servlet.CloudStartupServlet</servlet-class>
@@ -47,40 +44,26 @@
         <servlet-class>com.cloud.api.ApiServlet</servlet-class>
         <load-on-startup>5</load-on-startup>
     </servlet>
-    
+
     <servlet>
         <servlet-name>consoleServlet</servlet-name>
         <servlet-class>com.cloud.servlet.ConsoleProxyServlet</servlet-class>
         <load-on-startup>6</load-on-startup>
     </servlet>
-	
 
-    <servlet>
-        <servlet-name>staticResources</servlet-name>
-        <servlet-class>com.cloud.servlet.StaticResourceServlet</servlet-class>
-    </servlet>
- 
     <servlet-mapping>
          <servlet-name>apiServlet</servlet-name>
          <url-pattern>/api/*</url-pattern>
     </servlet-mapping>
-    
-    <servlet-mapping>   
-         <servlet-name>consoleServlet</servlet-name>   
-         <url-pattern>/console</url-pattern> 
-    </servlet-mapping>
-	
 
     <servlet-mapping>
-        <servlet-name>staticResources</servlet-name>
-        <url-pattern>*.css</url-pattern>
-        <url-pattern>*.html</url-pattern>
-        <url-pattern>*.js</url-pattern>
+         <servlet-name>consoleServlet</servlet-name>
+         <url-pattern>/console</url-pattern>
     </servlet-mapping>
 
     <error-page>
-      <exception-type>java.lang.Exception</exception-type>
-      <location>/error.html</location>
+        <exception-type>java.lang.Exception</exception-type>
+        <location>/error.html</location>
     </error-page>
 
 </web-app>

client/bindir/cloud-setup-management.in

@@ -29,13 +29,13 @@ if __name__ == '__main__':
 
     parser = OptionParser()
     parser.add_option("--https", action="store_true", dest="https", help="Enable HTTPs connection of management server")
-    parser.add_option("--tomcat7", action="store_true", dest="tomcat7", help="Use Tomcat7 configuration files in Management Server")
+    parser.add_option("--tomcat7", action="store_true", dest="tomcat7", help="Depreciated option, don't use it")
     parser.add_option("--no-start", action="store_true", dest="nostart", help="Do not start management server after successful configuration")
     (options, args) = parser.parse_args()
     if options.https:
         glbEnv.svrMode = "HttpsServer"
     if options.tomcat7:
-        glbEnv.svrConf = "Tomcat7"
+        print "The --tomcat7 option is deprecated, CloudStack now uses embedded Jetty server."
     if options.nostart:
         glbEnv.noStart = True
 

client/tomcatconf/commons-logging.properties.in client/conf/commons-logging.properties.in

@@ -17,9 +17,9 @@
 # under the License.
 #
 
-# This is the logging properties that goes to the war, there are two logging conf kept at the 
+# This is the logging properties that goes to the war, there are two logging conf kept at the
 # svn, one for developement (one at src/test-resources) and other for producation
- 
+
 # Uncomment the next line to disable all logging.
 #org.apache.commons.logging.Log=org.apache.commons.logging.impl.NoOpLog
 

client/tomcatconf/db.properties.in client/conf/db.properties.in

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,7 +16,7 @@
 # under the License.
 
 # management server clustering parameters, change cluster.node.IP to the machine IP address
-# in which the management server(Tomcat) is running
+# in which the management server is running
 cluster.node.IP=127.0.0.1
 cluster.servlet.port=9090
 region.id=1
@@ -47,7 +47,6 @@ db.cloud.keyStore=
 db.cloud.keyStorePassword=
 db.cloud.trustStore=
 db.cloud.trustStorePassword=
-db.cloud.keyStorePassphrase=vmops.com
 
 # Encryption Settings
 db.cloud.encryption.type=none

client/tomcatconf/ehcache.xml.in client/conf/ehcache.xml.in

@@ -23,7 +23,7 @@ under the License.
     CacheManager Configuration
     ==========================
     An ehcache.xml corresponds to a single CacheManager.
-    
+
     See instructions below or the ehcache schema (ehcache.xsd) on how to configure.
 
     System property tokens can be specified in this file which are replaced when the configuration is loaded.
@@ -143,7 +143,7 @@ under the License.
 
     The hostname is the hostname of the remote CacheManager peer. The port is the listening
     port of the RMICacheManagerPeerListener of the remote CacheManager peer.
-    
+
     Configuring JGroups replication:
     <cacheManagerPeerProviderFactory class="net.sf.ehcache.distribution.jgroups.JGroupsCacheManagerPeerProviderFactory"
                                      properties="connect=UDP(mcast_addr=231.12.21.132;mcast_port=45566;ip_ttl=32;
@@ -160,7 +160,7 @@ under the License.
             />
      The only property necessay is the connect String used by jgroups to configure itself. Refer to the Jgroups documentation for explanation
      of all the protocols. The example above uses UDP multicast. If the connect property is not specified the default JGroups connection will be
-     used.       
+     used.
 
     -->
     <cacheManagerPeerProviderFactory
@@ -266,7 +266,7 @@ under the License.
     to this area and then asynchronously written to disk. The default size is 30MB.
     Each spool buffer is used only by its cache. If you get OutOfMemory errors consider
     lowering this value. To improve DiskStore performance consider increasing it. Trace level
-    logging in the DiskStore will show if put back ups are occurring. 
+    logging in the DiskStore will show if put back ups are occurring.
 
     memoryStoreEvictionPolicy:
     Policy would be enforced upon reaching the maxElementsInMemory limit. Default
@@ -332,9 +332,9 @@ under the License.
 	For the Jgroups replication this is done with:
 	<cacheEventListenerFactory class="net.sf.ehcache.distribution.jgroups.JGroupsCacheReplicatorFactory"
                             properties="replicateAsynchronously=true, replicatePuts=true,
- 							replicateUpdates=true, replicateUpdatesViaCopy=false,
- 							replicateRemovals=true,asynchronousReplicationIntervalMillis=1000"/>
-    This listener supports the same property than the RMICacheReplicationFactory. 
+							replicateUpdates=true, replicateUpdatesViaCopy=false,
+							replicateRemovals=true,asynchronousReplicationIntervalMillis=1000"/>
+    This listener supports the same property than the RMICacheReplicationFactory.
 
     Cluster Bootstrapping
 
@@ -435,7 +435,7 @@ under the License.
     disk cache, which in this configuration will go to wherever java.io.tmp is
     defined on your system. On a standard Linux system this will be /tmp"
     -->
-    <!-- 
+    <!--
     <cache name="sampleCache1"
            maxElementsInMemory="10000"
            maxElementsOnDisk="1000"
@@ -455,7 +455,7 @@ under the License.
     is also the maximum cache size. Note that when a cache is eternal, timeToLive and
     timeToIdle are not used and do not need to be specified.
     -->
-    <!-- 
+    <!--
     <cache name="sampleCache2"
            maxElementsInMemory="1000"
            eternal="true"
@@ -470,7 +470,7 @@ under the License.
     persistent between cache and VM restarts. The disk expiry thread interval is set to 10
     minutes, overriding the default of 2 minutes.
     -->
-    <!-- 
+    <!--
     <cache name="sampleCache3"
            maxElementsInMemory="500"
            eternal="false"
@@ -489,7 +489,7 @@ under the License.
     This cache replicates using defaults.
     It also bootstraps from the cluster, using default properties.
     -->
-    <!-- 
+    <!--
     <cache name="sampleDistributedCache1"
            maxElementsInMemory="10"
            eternal="false"
@@ -509,7 +509,7 @@ under the License.
     This cache replicates using specific properties.
     It only replicates updates and does so synchronously via copy
     -->
-    <!-- 
+    <!--
     <cache name="sampleDistributedCache2"
            maxElementsInMemory="10"
            eternal="false"
@@ -529,7 +529,7 @@ under the License.
     This cache replicates using defaults except that the asynchronous replication
     interval is set to 200ms.
     -->
-    <!-- 
+    <!--
     <cache name="sampleDistributedCache3"
            maxElementsInMemory="10"
            eternal="false"

client/tomcatconf/environment.properties.in client/conf/environment.properties.in

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

client/tomcatconf/java.security.ciphers.in client/conf/java.security.ciphers.in

@@ -38,7 +38,7 @@ under the License.
          <param name="ConversionPattern" value="%d{ISO8601} %-5p [%c{1.}] (%t:%x) (logid:%X{logcontextid}) %m%n"/>
       </layout>
    </appender>
-   
+
    <appender name="APISERVER" class="org.apache.log4j.rolling.RollingFileAppender">
       <param name="Append" value="true"/>
       <param name="Threshold" value="DEBUG"/>
@@ -111,19 +111,19 @@ under the License.
    <category name="com.cloud">
      <priority value="DEBUG"/>
    </category>
-   
+
    <category name="org.apache.cloudstack">
      <priority value="DEBUG"/>
    </category>
-   
+
    <category name="org.apache.cloudstack">
       <priority value="DEBUG"/>
    </category>
 
    <category name="com.cloud.utils.nio">
      <priority value="INFO"/>
    </category>
-   
+
    <!-- Limit the org.apache category to INFO as its DEBUG is verbose -->
    <category name="org.apache">
       <priority value="INFO"/>
@@ -136,19 +136,19 @@ under the License.
    <category name="org.apache.cloudstack.api.command">
       <priority value="TRACE"/>
    </category>
-   
+
    <category name="org">
       <priority value="INFO"/>
    </category>
-   
+
    <category name="org.springframework">
       <priority value="WARN"/>
    </category>
-   
+
    <category name="org.apache.cloudstack.spring.module.context.ResourceApplicationContext">
       <priority value="WARN"/>
    </category>
-   
+
    <category name="net">
      <priority value="INFO"/>
    </category>

client/tomcatconf/log4j-cloud.xml.in client/conf/log4j-cloud.xml.in

@@ -1,5 +1,3 @@
-#!/bin/sh
-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -17,4 +15,27 @@
 # specific language governing permissions and limitations
 # under the License.
 
-cat /usr/share/cloudstack-common/scripts/gitrev.txt
+# The binding interface for the management server
+bind.interface=
+
+# The service context path where URL requests should be served
+context.path=/client
+
+# The HTTP port to be used by the management server
+http.port=8080
+
+# Options to configure and enable HTTPS on management server
+#
+# For management server to pickup these configuration settings, the configured
+# keystore file should exists and be readable by the management server.
+https.enable=false
+https.port=8443
+# The keystore and manager passwords are assumed to be same.
+https.keystore=/etc/cloudstack/management/cloud.jks
+https.keystore.password=vmops.com
+
+# The path to webapp directory
+webapp.dir=/usr/share/cloudstack-management/webapp
+
+# The path to access log file
+access.log=/var/log/cloudstack/management/access.log