Tarantool config storage (#4056)

Author: andreyaksenov

doc/code_snippets/snippets/centralized_config/README.md

@@ -0,0 +1,3 @@
+# Centralized configuration storages
+
+Sample applications demonstrating how to store configuration data in one place using Tarantool or etcd-based storage. Learn more at [Centralized configuration storages](https://www.tarantool.io/en/doc/latest/concepts/configuration/configuration_etcd/).

doc/code_snippets/snippets/centralized_config/etcd_config_storage.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# 1. Remove the 'default.etcd' directory to reset etcd to initial state.
+# 2. Start etcd by executing the 'etcd' command.
+# 3. Execute this script to enable authentication.
+etcdctl user add root:topsecret
+etcdctl role add myapp_config_manager
+etcdctl role grant-permission myapp_config_manager --prefix=true readwrite /myapp/
+etcdctl user add sampleuser:123456
+etcdctl user grant-role sampleuser myapp_config_manager
+etcdctl auth enable

doc/code_snippets/snippets/centralized_config/instances.enabled/config_etcd/README.md

@@ -0,0 +1,13 @@
+# etcd configuration storage
+
+A sample application demonstrating how to obtain a cluster's configuration from the etcd-based configuration storage.
+
+## Running
+
+Before running this sample, start etcd and enable authentication by executing [etcd_config_storage.sh](../../etcd_config_storage.sh).
+
+To start all instances, execute the following command in the [centralized_config](../../../centralized_config) directory:
+
+```console
+$ tt start config_etcd
+```

doc/code_snippets/snippets/centralized_config/instances.enabled/config_etcd/config.yaml

@@ -0,0 +1,10 @@
+config:
+  etcd:
+    endpoints:
+    - http://localhost:2379
+    prefix: /myapp
+    username: sampleuser
+    password: '123456'
+    http:
+      request:
+        timeout: 3

doc/code_snippets/snippets/config/instances.enabled/etcd/instances.yml renamed to doc/code_snippets/snippets/centralized_config/instances.enabled/config_etcd/instances.yml

@@ -0,0 +1,13 @@
+# Tarantool configuration storage
+
+A sample application demonstrating how to obtain a cluster's configuration from the Tarantool-based configuration storage.
+
+## Running
+
+Before running this sample, start a Tarantool-based configuration storage: [tarantool_config_storage](../tarantool_config_storage).
+
+To start all instances, execute the following command in the [centralized_config](../../../centralized_config) directory:
+
+```console
+$ tt start config_storage
+```

doc/code_snippets/snippets/centralized_config/instances.enabled/config_storage/README.md

@@ -0,0 +1,19 @@
+config:
+  storage:
+    endpoints:
+      - uri: '127.0.0.1:4401'
+        login: sampleuser
+        password: '123456'
+      - uri: '127.0.0.1:4402'
+        login: sampleuser
+        password: '123456'
+      - uri: '127.0.0.1:4403'
+        login: sampleuser
+        password: '123456'
+    prefix: /myapp
+    timeout: 3
+    reconnect_after: 5
+
+# Watch key changes
+app:
+  file: 'myapp.lua'

doc/code_snippets/snippets/centralized_config/instances.enabled/config_storage/config.yaml

@@ -0,0 +1,6 @@
+net_box = require('net.box')
+local conn = net_box.connect('127.0.0.1:4401')
+local log = require('log')
+conn:watch('config.storage:/myapp/config/all', function(key, value)
+    log.info("Configuration stored by the '/myapp/config/all' key is changed")
+end)

doc/code_snippets/snippets/config/instances.enabled/etcd_full/instances.yml renamed to doc/code_snippets/snippets/centralized_config/instances.enabled/config_storage/instances.yml

@@ -0,0 +1,11 @@
+# Setting up Tarantool configuration storage
+
+A sample application demonstrating how to set up Tarantool configuration storage.
+
+## Running
+
+To start all instances of the configuration storage, execute the following command in the [centralized_config](../../../centralized_config) directory:
+
+```console
+$ tt start tarantool_config_storage
+```

doc/code_snippets/snippets/centralized_config/instances.enabled/config_storage/myapp.lua

@@ -0,0 +1,49 @@
+credentials:
+  users:
+    sampleuser:
+      password: '123456'
+      privileges:
+      - permissions: [ read, write ]
+        spaces: [ config_storage, config_storage_meta ]
+      - permissions: [ execute ]
+        universe: true
+    replicator:
+      password: 'topsecret'
+      roles: [ replication ]
+
+iproto:
+  advertise:
+    peer:
+      login: replicator
+
+replication:
+  failover: election
+
+database:
+  use_mvcc_engine: true
+
+groups:
+  group001:
+    replicasets:
+      replicaset001:
+        roles: [ config.storage ]
+        roles_cfg:
+          config.storage:
+            status_check_interval: 3
+        instances:
+          instance001:
+            iproto:
+              listen:
+              - uri: '127.0.0.1:4401'
+          instance002:
+            iproto:
+              listen:
+              - uri: '127.0.0.1:4402'
+          instance003:
+            iproto:
+              listen:
+              - uri: '127.0.0.1:4403'
+
+# Interact with config storage
+app:
+  file: 'myapp.lua'

doc/code_snippets/snippets/centralized_config/instances.enabled/tarantool_config_storage/README.md

@@ -0,0 +1,3 @@
+instance001:
+instance002:
+instance003:

doc/code_snippets/snippets/centralized_config/instances.enabled/tarantool_config_storage/config.yaml

@@ -0,0 +1,37 @@
+function put_config()
+    local fio = require('fio')
+    local cluster_config_handle = fio.open('../../source.yaml')
+    local cluster_config = cluster_config_handle:read()
+    local response = config.storage.put('/myapp/config/all', cluster_config)
+    cluster_config_handle:close()
+    return response
+end
+
+function get_config_by_path()
+    local response = config.storage.get('/myapp/config/all')
+    return response
+end
+
+function get_config_by_prefix()
+    local response = config.storage.get('/myapp/')
+    return response
+end
+
+function make_txn_request()
+    local response = config.storage.txn({
+        predicates = { { 'value', '==', 'v0', '/myapp/config/all' } },
+        on_success = { { 'put', '/myapp/config/all', 'v1' } },
+        on_failure = { { 'get', '/myapp/config/all' } }
+    })
+    return response
+end
+
+function delete_config()
+    local response = config.storage.delete('/myapp/config/all')
+    return response
+end
+
+function delete_all_configs()
+    local response = config.storage.delete('/')
+    return response
+end

doc/code_snippets/snippets/centralized_config/instances.enabled/tarantool_config_storage/instances.yml

@@ -0,0 +1,34 @@
+# A sample cluster config used to demonstrate a centralized configuration storage.
+# Learn more: https://www.tarantool.io/en/doc/latest/concepts/configuration/configuration_etcd.
+credentials:
+  users:
+    replicator:
+      password: 'topsecret'
+      roles: [replication]
+
+iproto:
+  advertise:
+    peer:
+      login: replicator
+
+replication:
+  failover: manual
+
+groups:
+  group001:
+    replicasets:
+      replicaset001:
+        leader: instance001
+        instances:
+          instance001:
+            iproto:
+              listen:
+              - uri: '127.0.0.1:3301'
+          instance002:
+            iproto:
+              listen:
+              - uri: '127.0.0.1:3302'
+          instance003:
+            iproto:
+              listen:
+              - uri: '127.0.0.1:3303'

doc/code_snippets/snippets/centralized_config/instances.enabled/tarantool_config_storage/myapp.lua

@@ -0,0 +1,54 @@
+modules:
+  # Directory where the external modules are stored.
+  directory: modules
+
+env:
+  # Restart instance on failure.
+  restart_on_failure: false
+
+  # Directory that stores binary files.
+  bin_dir: bin
+
+  # Directory that stores Tarantool header files.
+  inc_dir: include
+
+  # Path to directory that stores all applications.
+  # The directory can also contain symbolic links to applications.
+  instances_enabled: instances.enabled
+
+  # Tarantoolctl artifacts layout compatibility: if set to true tt will not create application
+  # sub-directories for control socket, pid files, log files, etc.. Data files (wal, vinyl,
+  # snap) and multi-instance applications are not affected by this option.
+  tarantoolctl_layout: false
+
+app:
+  # Directory that stores various instance runtime
+  # artifacts like console socket, PID file, etc.
+  run_dir: var/run
+
+  # Directory that stores log files.
+  log_dir: var/log
+
+  # Directory where write-ahead log (.xlog) files are stored.
+  wal_dir: var/lib
+
+  # Directory where memtx stores snapshot (.snap) files.
+  memtx_dir: var/lib
+
+  # Directory where vinyl files or subdirectories will be stored.
+  vinyl_dir: var/lib
+
+# Path to file with credentials for downloading Tarantool Enterprise Edition.
+# credential_path: /path/to/file
+ee:
+  credential_path: 
+
+templates:
+  # The path to templates search directory.
+  - path: templates
+
+repo:
+  # Directory where local rocks files could be found.
+  rocks: 
+  # Directory that stores installation files.
+  distfiles: distfiles

doc/code_snippets/snippets/centralized_config/source.yaml

@@ -11,7 +11,7 @@ There are two approaches to configuring Tarantool:
 *   *Since version 3.0*: In the YAML format.
 
     YAML configuration allows you to provide the full cluster topology and specify all configuration options.
-    You can use local configuration in a YAML file for each instance or store configuration data in one reliable place using :ref:`etcd <configuration_etcd_overview>`.
+    You can use local configuration in a YAML file for each instance or store configuration data in a reliable :ref:`centralized storage <configuration_etcd_overview>`.
 
 *   *In version 2.11 and earlier*: :ref:`In code <configuration_code>` using the ``box.cfg`` API.
 
@@ -327,25 +327,29 @@ Below are a few examples that show how to set environment variables of different
 
 .. _configuration_etcd_overview:
 
-Configuration in etcd
-~~~~~~~~~~~~~~~~~~~~~
+Centralized configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ..  include:: /concepts/configuration/configuration_etcd.rst
-    :start-after: ee_note_etcd_start
-    :end-before: ee_note_etcd_end
+    :start-after: ee_note_centralized_config_start
+    :end-before: ee_note_centralized_config_end
+
 
-Tarantool enables you to store configuration data in one reliable place using `etcd <https://etcd.io/>`_.
+Tarantool enables you to store configuration data in one place using a Tarantool or etcd-based storage.
 To achieve this, you need to:
 
-1.   Provide a local YAML configuration with an etcd endpoint address and key prefix in the ``config`` section:
+1.  Set up a centralized configuration storage.
+
+2.  Publish a cluster's configuration to the storage.
 
-    ..  literalinclude:: /code_snippets/snippets/config/instances.enabled/etcd/config.yaml
+3.  Configure a connection to the storage by providing a local YAML configuration with an endpoint address and key prefix in the ``config`` section:
+
+    ..  literalinclude:: /code_snippets/snippets/centralized_config/instances.enabled/config_etcd/config.yaml
         :language: yaml
+        :end-at: prefix: /myapp
         :dedent:
 
-2.   Publish a cluster's configuration to an etcd server.
-
-Learn more from the following guide: :ref:`Storing configuration in etcd <configuration_etcd>`.
+Learn more from the following guide: :ref:`configuration_etcd`.
 
 
 ..  _configuration_precedence:
@@ -357,7 +361,7 @@ Tarantool configuration options are applied from multiple sources with the follo
 
 -   `TT_*` :ref:`environment variables <configuration_environment_variable>`.
 -   Configuration from a :ref:`local YAML file <configuration_file>`.
--   :ref:`Centralized configuration <configuration_etcd_overview>` stored in etcd.
+-   :ref:`Centralized configuration <configuration_etcd_overview>`.
 -   `TT_*_DEFAULT` :ref:`environment variables <configuration_environment_variable>`.
 
 If the same option is defined in two or more locations, the option with the highest precedence is applied.