Support for adding resources for SDC (#50)

start.py

@@ -87,6 +87,7 @@
 NAVIGATOR_METASERVER_POSTGRESQL_PASSWORD = '5lgfqqyGWP'
 NAVIGATOR_POSTGRESQL_PORT = 7432
 SDC_PORT = 18630
+SDC_RESOURCES_DIRECTORY = '/var/lib/sdc/resources'  # Default path for $SDC_RESOURCES.
 
 logger = logging.getLogger('clusterdock.{}'.format(__name__))
 
@@ -152,6 +153,16 @@ def main(args):
         for node in nodes:
             node.volumes.append(sdc_parcel_image)
 
+        # Volume mount SDC resources.
+        # e.g. If /home/ubuntu/protobuf is passed, then it gets mounted to SDC_RESOURCES_DIRECTORY/protobuf on SDC node
+        if args.sdc_resources_directory:
+            sdc_resources_directory_path = os.path.realpath(os.path.expanduser(args.sdc_resources_directory))
+            sdc_resources_basename = os.path.basename(sdc_resources_directory_path)
+            sdc_resources_mount_point = '{}/{}'.format(SDC_RESOURCES_DIRECTORY, sdc_resources_basename)
+            logger.debug('Volume mounting resources from %s to %s ...',
+                         sdc_resources_directory_path, sdc_resources_mount_point)
+            primary_node.volumes.append({sdc_resources_directory_path: sdc_resources_mount_point})
+
     if args.kerberos:
         dir = '{}/kerberos'.format(args.clusterdock_config_directory)
         kerberos_config_host_dir = os.path.realpath(os.path.expanduser(dir))
@@ -383,7 +394,8 @@ def cm_server_not_dead(primary_node):
         logger.info('Configure Cloudera Manager for Kerberos ...')
         _configure_cm_for_kerberos(deployment, cluster, args.kerberos_ticket_lifetime)
 
-    _configure_for_streamsets_before_start(deployment, cluster_name=DEFAULT_CLUSTER_NAME)
+    _configure_for_streamsets_before_start(deployment, cluster, cluster_name=DEFAULT_CLUSTER_NAME,
+                                           sdc_resources_directory=args.sdc_resources_directory)
     logger.info('Deploying client config ...')
     cm_cluster.deploy_client_config()
 
@@ -1286,7 +1298,11 @@ def _configure_after_start(deployment, cluster_name, cluster, quiet, kerberos_pr
             _execute_commands_against_kerberized_hdfs(cluster, dir_commands, quiet)
 
 
-def _configure_for_streamsets_before_start(deployment, cluster_name):
+def _configure_for_streamsets_before_start(deployment, cluster, cluster_name, sdc_resources_directory):
+    if sdc_resources_directory:
+        logger.info('Setting correct permissions for sdc resources recursively ...')
+        cluster.primary_node.execute('chown -R sdc:sdc {}'.format(os.path.dirname(SDC_RESOURCES_DIRECTORY)))
+
     logger.info('Adding HDFS proxy user ...')
     for service in deployment.get_cluster_services(cluster_name=cluster_name):
         if service['type'] == 'HDFS':

topology.yaml

@@ -79,6 +79,9 @@ start args:
     --sdc-version:
         help: If specified, the version of StreamSets Data Collector to install
         metavar: ver
+    --sdc-resources-directory:
+        help: If specified, a directory containing resources files to mount into the StreamSets Data Collector resources
+        metavar: dir
     --single-node:
         action: store_true
         help: If specified, a single-node deployment will be started