Feature: Watchdog with Reloading

HISTORY.md

@@ -10,6 +10,8 @@
 * Fix exception handling and propagation in failure situations.
 * Add feature to support dashboard builders, in the spirit of
   dashboard-as-code.
+* Add watchdog feature, monitoring the input dashboard for changes on
+  disk, and re-uploading it, when changed.
 
 ## 0.2.0 (2022-02-05)
 * Migrated from grafana_api to grafana_client

README.md

@@ -13,6 +13,9 @@ _Export and import Grafana dashboards using the [Grafana HTTP API] and
   - Supported builders are [grafana-dashboard], [grafanalib], and
     any other executable program which emits Grafana Dashboard JSON
     on STDOUT.
+- Watchdog: For a maximum of authoring and editing efficiency, the
+  watchdog monitors the input dashboard for changes on disk, and
+  re-uploads it to the Grafana API, when changed.
 - Remove dashboards.
 
 
@@ -67,6 +70,12 @@ when a dashboard with the same name already exists in the same folder.
 grafana-import import --overwrite -i gd-prometheus.py
 ```
 
+### Import using reloading
+Watch the input dashboard for changes on disk, and re-upload it, when changed.
+```shell
+grafana-import import --overwrite --reload -i gd-prometheus.py
+```
+
 ### Export
 Export the dashboard titled `my-first-dashboard` to the default export directory.
 ```bash
@@ -205,10 +214,12 @@ optional arguments:
                         path to the dashboard file to import into Grafana.
   -o, --overwrite       if a dashboard with same name exists in folder,
                         overwrite it with this new one.
+  -r, --reload          Watch the input dashboard for changes on disk, and
+                        re-upload it, when changed.
   -p, --pretty          use JSON indentation when exporting or extraction of
                         dashboards.
   -v, --verbose         verbose mode; display log message to stdout.
-  -V, --version         display program version and exit..
+  -V, --version         display program version and exit.
 
 ```
 

docs/backlog.md

@@ -1,5 +1,7 @@
 # grafana-import backlog
 
+## Iteration +1
+- Print dashboard URL after uploading
 - `grafana-dashboard` offers the option to convert Grafana JSON
   back to Python code. It should be used on the `export` subsystem,
   to provide an alternative output format.

grafana_import/cli.py

@@ -9,6 +9,9 @@
     V 0.0.0 - 2021/03/15 - JF. PIK - initial version
 
 """
+import logging
+from pathlib import Path
+
 #***********************************************************************************************
 #
 #
@@ -24,8 +27,9 @@
 
 import grafana_client.client as GrafanaApi
 import grafana_import.grafana as Grafana
+from grafana_import.service import watchdog_service
 
-from grafana_import.util import load_yaml_config, grafana_settings, read_dashboard_file
+from grafana_import.util import load_yaml_config, grafana_settings, read_dashboard_file, setup_logging
 
 #******************************************************************************************
 config = None
@@ -83,9 +87,15 @@ def __repr__(self):
            obj[attr] = val
     return json.dumps(obj)
 
+
+logger = logging.getLogger(__name__)
+
 
 #***********************************************************************************************
 def main():
+
+   setup_logging()
+
    #******************************************************************
    # get command line arguments
 
@@ -123,6 +133,11 @@ def main():
          , default=False
 			, help='if a dashboard with same name exists in same folder, overwrite it with this new one.')
 
+   parser.add_argument('-r', '--reload'
+                       , action='store_true'
+                       , default=False
+                       , help='Watch the input dashboard for changes on disk, and re-upload it, when changed.')
+
    parser.add_argument('-p', '--pretty'
 			, action='store_true'
 			, help='use JSON indentation when exporting or extraction of dashboards.')
@@ -224,27 +239,40 @@ def main():
             import_path = os.path.join(import_path, config['general']['imports_path'] )
       import_file = os.path.join(import_path, import_file)
 
-      try:
-         dash = read_dashboard_file(import_file)
-      except Exception as ex:
-         print(f"ERROR: Failed to import dashboard from: {import_file}. Reason: {ex}")
-         sys.exit(1)
+      def process_dashboard():
+         try:
+            dash = read_dashboard_file(import_file)
+         except Exception as ex:
+            msg = f"Failed to load dashboard from: {import_file}. Reason: {ex}"
+            logger.exception(msg)
+            raise IOError(msg)
+
+         try:
+            res = grafana_api.import_dashboard( dash )
+         except GrafanaApi.GrafanaClientError as ex:
+            msg = f"Failed to upload dashboard to Grafana. Reason: {ex}"
+            logger.exception(msg)
+            raise IOError(msg)
+
+         title = dash['title']
+         folder_name = grafana_api.grafana_folder
+         if res:
+            logger.info(f"Dashboard '{title}' imported into folder '{folder_name}'")
+         else:
+            msg = f"Failed to import dashboard into Grafana. title={title}, folder={folder_name}"
+            logger.error(msg)
+            raise IOError(msg)
 
       try:
-         res = grafana_api.import_dashboard( dash )
-      except GrafanaApi.GrafanaClientError as exp:
-         print('ERROR: {0}.'.format(exp))
-         print("maybe you want to set --overwrite option.")
+         process_dashboard()
+      except:
          sys.exit(1)
 
-      title = dash['title']
-      folder_name = grafana_api.grafana_folder
-      if res:
-         print(f"OK: Dashboard '{title}' imported into folder '{folder_name}'")
-         sys.exit(0)
-      else:
-         print(f"KO: Dashboard '{title}' not imported into folder '{folder_name}'")
-         sys.exit(1)
+      if args.reload:
+         watchdog_service(import_file, process_dashboard)
+
+      sys.exit(0)
+
 
    #*******************************************************************************
    elif args.action == 'remove':

grafana_import/service.py

@@ -0,0 +1,50 @@
+import logging
+import time
+import typing as t
+from pathlib import Path
+
+from watchdog.events import FileSystemEvent, PatternMatchingEventHandler
+from watchdog.observers import Observer
+
+logger = logging.getLogger(__name__)
+
+
+class SingleFileModifiedHandler(PatternMatchingEventHandler):
+
+    def __init__(
+        self,
+        *args,
+        action: t.Union[t.Callable, None] = None,
+        **kwargs,
+    ):
+        self.action = action
+        super().__init__(*args, **kwargs)
+
+    def on_modified(self, event: FileSystemEvent) -> None:
+        super().on_modified(event)
+        logger.info(f"File was modified: {event.src_path}")
+        try:
+            self.action and self.action()
+            logger.debug(f"File processed successfully: {event.src_path}")
+        except Exception:
+            logger.exception(f"Processing file failed: {event.src_path}")
+
+
+def watchdog_service(path: Path, action: t.Union[t.Callable, None] = None):
+    """
+    https://python-watchdog.readthedocs.io/en/stable/quickstart.html
+    """
+
+    import_file = Path(path).absolute()
+    import_path = import_file.parent
+
+    event_handler = SingleFileModifiedHandler(action=action, patterns=[import_file.name], ignore_directories=True)
+    observer = Observer()
+    observer.schedule(event_handler, str(import_path), recursive=False)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    finally:
+        observer.stop()
+        observer.join()

grafana_import/util.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import os
 import shlex
 import subprocess
@@ -12,6 +13,11 @@
 SettingsType = t.Dict[str, t.Union[str, int, bool]]
 
 
+def setup_logging(level=logging.INFO, verbose: bool = False):
+    log_format = "%(asctime)-15s [%(name)-26s] %(levelname)-8s: %(message)s"
+    logging.basicConfig(format=log_format, stream=sys.stderr, level=level)
+
+
 def load_yaml_config(config_file: str) -> ConfigType:
     """
     Load configuration file in YAML format from disk.

setup.py

@@ -9,6 +9,7 @@
     'grafana-client<5',
     'jinja2<4',
     'pyyaml<7',
+    'watchdog<5',
 ]
 
 extras = {

tests/conftest.py

@@ -9,7 +9,6 @@
 from grafana_import.util import grafana_settings, load_yaml_config
 from tests.util import mock_grafana_health, mock_grafana_search
 
-
 if sys.version_info < (3, 9):
     from importlib_resources import files
 else:

tests/test_cli.py

@@ -21,7 +21,7 @@ def get_settings_arg(use_settings: bool = True):
 
 
 @pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"])
-def test_import_dashboard_success(mocked_grafana, mocked_responses, tmp_path, capsys, use_settings):
+def test_import_dashboard_success(mocked_grafana, mocked_responses, tmp_path, caplog, use_settings):
     """
     Verify "import dashboard" works.
     """
@@ -42,8 +42,7 @@ def test_import_dashboard_success(mocked_grafana, mocked_responses, tmp_path, ca
         main()
     assert ex.match("0")
 
-    out, err = capsys.readouterr()
-    assert "OK: Dashboard 'Dashboard One' imported into folder 'General'" in out
+    assert "Dashboard 'Dashboard One' imported into folder 'General'" in caplog.messages
 
 
 @pytest.mark.parametrize("use_settings", [True, False], ids=["config-yes", "config-no"])