Merge pull request kubernetes#55334 from Cynerva/gkk/worker-extra-args

Automatic merge from submit-queue (batch tested with PRs 55044, 55257, 55334). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add extra-args configs to kubernetes-worker charm

**What this PR does / why we need it**:

This adds `kubelet-extra-args` and `proxy-extra-args` config options to the kubernetes-worker charm, which allow operators to pass arbitrary configuraton to the kubelet and kube-proxy services.

This also fixes a bug where the equivalent config options on kubernetes-master (`api-extra-args`, `controller-manager-extra-args`, `scheduler-extra-args`) don't remove arguments from the services when they are removed from the config.

**Release note**:

```release-note
Add extra-args configs to kubernetes-worker charm
```

Author: Kubernetes Submit Queue

cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/flagmanager.py

@@ -43,7 +43,6 @@
 from charms.reactive.helpers import data_changed, any_file_changed
 from charms.kubernetes.common import get_version
 from charms.kubernetes.common import retry
-from charms.kubernetes.flagmanager import FlagManager
 
 from charms.layer import tls_client
 
@@ -172,11 +171,6 @@ def migrate_from_pre_snaps():
             hookenv.log("Removing file: " + file)
             os.remove(file)
 
-    # clear the flag managers
-    FlagManager('kube-apiserver').destroy_all()
-    FlagManager('kube-controller-manager').destroy_all()
-    FlagManager('kube-scheduler').destroy_all()
-
 
 def install_snaps():
     channel = hookenv.config('channel')
@@ -228,15 +222,10 @@ def configure_cni(cni):
 @when_not('authentication.setup')
 def setup_leader_authentication():
     '''Setup basic authentication and token access for the cluster.'''
-    api_opts = FlagManager('kube-apiserver')
-    controller_opts = FlagManager('kube-controller-manager')
-
     service_key = '/root/cdk/serviceaccount.key'
     basic_auth = '/root/cdk/basic_auth.csv'
     known_tokens = '/root/cdk/known_tokens.csv'
 
-    api_opts.add('basic-auth-file', basic_auth)
-    api_opts.add('token-auth-file', known_tokens)
     hookenv.status_set('maintenance', 'Rendering authentication templates.')
 
     keys = [service_key, basic_auth, known_tokens]
@@ -257,9 +246,6 @@ def setup_leader_authentication():
             check_call(cmd)
         remove_state('reconfigure.authentication.setup')
 
-    api_opts.add('service-account-key-file', service_key)
-    controller_opts.add('service-account-private-key-file', service_key)
-
     # read service account key for syndication
     leader_data = {}
     for f in [known_tokens, basic_auth, service_key]:
@@ -294,13 +280,6 @@ def setup_non_leader_authentication():
         return
 
     hookenv.status_set('maintenance', 'Rendering authentication templates.')
-    api_opts = FlagManager('kube-apiserver')
-    api_opts.add('basic-auth-file', basic_auth)
-    api_opts.add('token-auth-file', known_tokens)
-    api_opts.add('service-account-key-file', service_key)
-
-    controller_opts = FlagManager('kube-controller-manager')
-    controller_opts.add('service-account-private-key-file', service_key)
 
     remove_state('kubernetes-master.components.started')
     set_state('authentication.setup')
@@ -400,7 +379,7 @@ def start_master(etcd):
     handle_etcd_relation(etcd)
 
     # Add CLI options to all components
-    configure_apiserver()
+    configure_apiserver(etcd)
     configure_controller_manager()
     configure_scheduler()
 
@@ -768,8 +747,9 @@ def on_config_allow_privileged_change():
 
 @when('config.changed.api-extra-args')
 @when('kubernetes-master.components.started')
-def on_config_api_extra_args_change():
-    configure_apiserver()
+@when('etcd.available')
+def on_config_api_extra_args_change(etcd):
+    configure_apiserver(etcd)
 
 
 @when('config.changed.controller-manager-extra-args')
@@ -957,9 +937,9 @@ def get_kubernetes_service_ip():
 def handle_etcd_relation(reldata):
     ''' Save the client credentials and set appropriate daemon flags when
     etcd declares itself as available'''
-    connection_string = reldata.get_connection_string()
     # Define where the etcd tls files will be kept.
     etcd_dir = '/root/cdk/etcd'
+
     # Create paths to the etcd client ca, key, and cert file locations.
     ca = os.path.join(etcd_dir, 'client-ca.pem')
     key = os.path.join(etcd_dir, 'client-key.pem')
@@ -968,85 +948,45 @@ def handle_etcd_relation(reldata):
     # Save the client credentials (in relation data) to the paths provided.
     reldata.save_client_credentials(key, cert, ca)
 
-    api_opts = FlagManager('kube-apiserver')
 
-    # Never use stale data, always prefer whats coming in during context
-    # building. if its stale, its because whats in unitdata is stale
-    data = api_opts.data
-    if data.get('etcd-servers-strict') or data.get('etcd-servers'):
-        api_opts.destroy('etcd-cafile')
-        api_opts.destroy('etcd-keyfile')
-        api_opts.destroy('etcd-certfile')
-        api_opts.destroy('etcd-servers', strict=True)
-        api_opts.destroy('etcd-servers')
+def parse_extra_args(config_key):
+    elements = hookenv.config().get(config_key, '').split()
+    args = {}
+
+    for element in elements:
+        if '=' in element:
+            key, _, value = element.partition('=')
+            args[key] = value
+        else:
+            args[element] = 'true'
 
-    # Set the apiserver flags in the options manager
-    api_opts.add('etcd-cafile', ca)
-    api_opts.add('etcd-keyfile', key)
-    api_opts.add('etcd-certfile', cert)
-    api_opts.add('etcd-servers', connection_string, strict=True)
+    return args
 
 
-def get_config_args(key):
+def configure_kubernetes_service(service, base_args, extra_args_key):
     db = unitdata.kv()
-    old_config_args = db.get(key, [])
-    # We have to convert them to tuples becuase we use sets
-    old_config_args = [tuple(i) for i in old_config_args]
-    new_config_args = []
-    new_config_arg_names = []
-    for arg in hookenv.config().get(key, '').split():
-        new_config_arg_names.append(arg.split('=', 1)[0])
-        if len(arg.split('=', 1)) == 1:  # handle flags ie. --profiling
-            new_config_args.append(tuple([arg, 'true']))
-        else:
-            new_config_args.append(tuple(arg.split('=', 1)))
-
-    hookenv.log('Handling "%s" option.' % key)
-    hookenv.log('Old arguments: {}'.format(old_config_args))
-    hookenv.log('New arguments: {}'.format(new_config_args))
-    if set(new_config_args) == set(old_config_args):
-        return (new_config_args, [])
-    # Store new args
-    db.set(key, new_config_args)
-    to_add = set(new_config_args)
-    to_remove = set(old_config_args) - set(new_config_args)
-    # Extract option names only
-    to_remove = [i[0] for i in to_remove if i[0] not in new_config_arg_names]
-    return (to_add, to_remove)
 
+    prev_args_key = 'kubernetes-master.prev_args.' + service
+    prev_args = db.get(prev_args_key) or {}
 
-def configure_kubernetes_service(service, base_args, extra_args_key):
-    # Handle api-extra-args config option
-    to_add, to_remove = get_config_args(extra_args_key)
-
-    flag_manager = FlagManager(service)
-
-    # Remove arguments that are no longer provided as config option
-    # this allows them to be reverted to charm defaults
-    for arg in to_remove:
-        hookenv.log('Removing option: {}'.format(arg))
-        flag_manager.destroy(arg)
-        # We need to "unset" options by setting their value to "null" string
-        cmd = ['snap', 'set', service, '{}=null'.format(arg)]
-        check_call(cmd)
+    extra_args = parse_extra_args(extra_args_key)
 
-    # Add base arguments
+    args = {}
+    for arg in prev_args:
+        # remove previous args by setting to null
+        args[arg] = 'null'
     for k, v in base_args.items():
-        flag_manager.add(k, v, strict=True)
-
-    # Add operator-provided arguments, this allows operators
-    # to override defaults
-    for arg in to_add:
-        hookenv.log('Adding option: {} {}'.format(arg[0], arg[1]))
-        # Make sure old value is gone
-        flag_manager.destroy(arg[0])
-        flag_manager.add(arg[0], arg[1], strict=True)
+        args[k] = v
+    for k, v in extra_args.items():
+        args[k] = v
 
-    cmd = ['snap', 'set', service] + flag_manager.to_s().split(' ')
+    cmd = ['snap', 'set', service] + ['%s=%s' % item for item in args.items()]
     check_call(cmd)
 
+    db.set(prev_args_key, args)
 
-def configure_apiserver():
+
+def configure_apiserver(etcd):
     api_opts = {}
 
     # Get the tls paths from the layer data.
@@ -1078,6 +1018,20 @@ def configure_apiserver():
     api_opts['insecure-port'] = '8080'
     api_opts['storage-backend'] = 'etcd2'  # FIXME: add etcd3 support
 
+    api_opts['basic-auth-file'] = '/root/cdk/basic_auth.csv'
+    api_opts['token-auth-file'] = '/root/cdk/known_tokens.csv'
+    api_opts['service-account-key-file'] = '/root/cdk/serviceaccount.key'
+
+    etcd_dir = '/root/cdk/etcd'
+    etcd_ca = os.path.join(etcd_dir, 'client-ca.pem')
+    etcd_key = os.path.join(etcd_dir, 'client-key.pem')
+    etcd_cert = os.path.join(etcd_dir, 'client-cert.pem')
+
+    api_opts['etcd-cafile'] = etcd_ca
+    api_opts['etcd-keyfile'] = etcd_key
+    api_opts['etcd-certfile'] = etcd_cert
+    api_opts['etcd-servers'] = etcd.get_connection_string()
+
     admission_control = [
         'Initializers',
         'NamespaceLifecycle',
@@ -1120,6 +1074,9 @@ def configure_controller_manager():
     controller_opts['logtostderr'] = 'true'
     controller_opts['master'] = 'http://127.0.0.1:8080'
 
+    controller_opts['service-account-private-key-file'] = \
+        '/root/cdk/serviceaccount.key'
+
     configure_kubernetes_service('kube-controller-manager', controller_opts,
                                  'controller-manager-extra-args')