Table refresh with ntie

README.rst

@@ -147,9 +147,9 @@ All config options can be overruled by environment variables. Prefix with
 options. It is also possible to override logging by setting "HOWITZ_LOGGING" to
 a string of a python dict but we do not recommend it, use a config file instead.
 
-Poll interval for events table can be changed by adding for example ``poll_interval = 30`` to
-the ``[howitz]``-section or setting the environment variable ``HOWITZ_POLL_INTERVAL`` to a new value.
-Poll interval values represented seconds and must be integers. The default value is ``60`` seconds.
+Refresh interval for events table can be changed by adding for example ``refresh_interval = 10`` to
+the ``[howitz]``-section or setting the environment variable ``HOWITZ_REFRESH_INTERVAL`` to a new value.
+Refresh interval values represented seconds and must be integers. The default value is ``5`` seconds.
 
 Debugging can be turned on either by adding ``DEBUG = true`` to the
 ``[flask]``-section or setting the environment variable ``HOWITZ_DEBUG`` to ``1``.

dev-howitz.toml

@@ -5,7 +5,7 @@ DEBUG = true
 [howitz]
 storage = "./howitz.sqlite3"
 devmode = true
-poll_interval = 30
+refresh_interval = 10
 timezone='LOCAL'
 
 [zino.connections.default]

requirements-frozen.txt

@@ -48,5 +48,5 @@ werkzeug==2.3.8
     #   flask
     #   flask-login
     #   howitz (pyproject.toml)
-zinolib==1.0.1
+zinolib==1.0.4
     # via howitz (pyproject.toml)

src/howitz/__init__.py

@@ -17,7 +17,6 @@
 from howitz.users.commands import user_cli
 from zinolib.controllers.zino1 import Zino1EventManager
 
-
 __all__ = ["create_app"]
 
 

src/howitz/config/models.py

@@ -28,11 +28,11 @@ class DevStorageConfig(StorageConfig):
 
 class HowitzConfig(ServerConfig, StorageConfig):
     devmode: bool = Literal[False]
-    poll_interval: int = 60
+    refresh_interval: int = 5
     timezone: str = DEFAULT_TIMEZONE
 
 
 class DevHowitzConfig(DevServerConfig, DevStorageConfig):
     devmode: bool = Literal[True]
-    poll_interval: int = 30
+    refresh_interval: int = 5
     timezone: str = DEFAULT_TIMEZONE

src/howitz/endpoints.py

@@ -17,7 +17,7 @@
 from datetime import datetime, timezone
 
 from werkzeug.exceptions import BadRequest
-from zinolib.controllers.zino1 import Zino1EventManager, RetryError, EventClosedError
+from zinolib.controllers.zino1 import Zino1EventManager, RetryError, EventClosedError, UpdateHandler
 from zinolib.event_types import Event, AdmState, PortState, BFDState, ReachabilityState, LogEntry, HistoryEntry
 from zinolib.compat import StrEnum
 from zinolib.ritz import NotConnectedError, AuthenticationError
@@ -55,6 +55,10 @@ def auth_handler(username, password):
             current_app.logger.info('Authenticated in Zino %s', current_app.event_manager.is_authenticated)
 
         if current_app.event_manager.is_authenticated:  # is zino authenticated
+            current_app.updater = UpdateHandler(current_app.event_manager, autoremove=current_app.zino_config.autoremove)
+            current_app.updater.connect()
+            current_app.logger.debug('UpdateHandler %s', current_app.updater)
+
             current_app.logger.debug('User is Zino authenticated %s', current_app.event_manager.is_authenticated)
             current_app.logger.debug('HOWITZ CONFIG %s', current_app.howitz_config)
             login_user(user, remember=True)
@@ -63,6 +67,7 @@ def auth_handler(username, password):
             session["expanded_events"] = {}
             session["errors"] = {}
             session["not_connected_counter"] = 0
+            session["event_ids"] = []
             return user
 
         raise AuthenticationError('Unexpected error on Zino authentication')
@@ -79,12 +84,20 @@ def logout_handler():
         session.pop('selected_events', [])
         session.pop('errors', {})
         session.pop('not_connected_counter', 0)
+        session.pop('event_ids', [])
         current_app.logger.info("Logged out successfully.")
 
 
 def get_current_events():
     try:
         current_app.event_manager.get_events()
+    except RetryError as retryErr:  # Intermittent error in Zino
+        current_app.logger.exception('RetryError when fetching current events %s', retryErr)
+        try:
+            current_app.event_manager.get_events()
+        except RetryError as retryErr:  # Intermittent error in Zino
+            current_app.logger.exception('RetryError when fetching current events after retry, %s', retryErr)
+            raise
     except NotConnectedError as notConnErr:
         if session["not_connected_counter"] > 1:  # This error is not intermittent - increase counter and handle
             current_app.logger.exception('Recurrent NotConnectedError %s', notConnErr)
@@ -104,47 +117,71 @@ def get_current_events():
                                                       events[k].updated,
                                                   ), reverse=True)}
 
+    # Save current events' IDs
+    session["event_ids"] = list(events_sorted.keys())
+    session.modified = True
+
     table_events = []
     for c in events_sorted.values():
-        table_events.append(create_table_event(c))
+        table_events.append(create_table_event(c, expanded=str(c.id) in session["expanded_events"],
+                                               selected=str(c.id) in session["selected_events"]))
 
     current_app.logger.debug('TABLE EVENTS %s', table_events[0])
 
     return table_events
 
 
-def poll_current_events():
-    try:
-        current_app.event_manager.get_events()
-    except NotConnectedError as notConnErr:
-        if session["not_connected_counter"] > 1:  # This error is not intermittent - increase counter and handle
-            current_app.logger.exception('Recurrent NotConnectedError %s', notConnErr)
-            session["not_connected_counter"] += 1
-            raise
-        else:  # This error is intermittent - increase counter and retry
-            current_app.logger.exception('Intermittent NotConnectedError %s', notConnErr)
-            session["not_connected_counter"] += 1
-            current_app.event_manager.get_events()
-            pass
-
-    events = current_app.event_manager.events
+def update_events():
+    updated_ids = set()
 
-    events_sorted = {k: events[k] for k in sorted(events,
-                                                  key=lambda k: (
-                                                      0 if events[k].adm_state == AdmState.IGNORED else 1,
-                                                      events[k].updated,
-                                                  ), reverse=True)}
+    while True:
+        try:
+            updated = current_app.updater.get_event_update()
+        except RetryError as retryErr:  # Intermittent error in Zino
+            current_app.logger.exception('RetryError when NTIE refreshing current events %s', retryErr)
+            try:
+                updated = current_app.updater.get_event_update()
+            except RetryError as retryErr:  # Intermittent error in Zino
+                current_app.logger.exception('RetryError when NTIE refreshing current events after retry, %s', retryErr)
+                raise
+        if not updated:
+            break
+        updated_ids.add(updated)
+
+    return updated_ids
+
+
+def refresh_current_events():
+    event_ids = update_events()
+    current_app.logger.debug('UPDATED EVENT IDS %s', event_ids)
+
+    removed_events = []
+    modified_events = []
+    added_events = []
+    removed = current_app.event_manager.removed_ids
+    existing = session["event_ids"]
+    for i in event_ids:
+        if i in removed:
+            removed_events.append(i)
+            existing.remove(i)
+        elif i not in existing:
+            c = current_app.event_manager.create_event_from_id(int(i))
+            added_events.append(create_table_event(c, expanded=False, selected=False))
+            existing.insert(0, int(i))
+        else:
+            c = current_app.event_manager.create_event_from_id(int(i))
+            modified_events.append(create_table_event(c,
+                                                      expanded=str(c.id) in session["expanded_events"],
+                                                      selected=str(c.id) in session["selected_events"]))
 
-    poll_events = []
-    for c in events_sorted.values():
-        poll_events.append(create_polled_event(create_table_event(c), expanded=str(c.id) in session["expanded_events"],
-                                               selected=str(c.id) in session["selected_events"]))
+    session["event_ids"] = existing
+    session.modified = True
 
-    return poll_events
+    return removed_events, modified_events, added_events
 
 
 # todo remove all use of helpers from curitz
-def create_table_event(event):
+def create_table_event(event, expanded=False, selected=False):
     common = {}
 
     try:
@@ -164,23 +201,18 @@ def create_table_event(event):
         raise
 
     common.update(vars(event))
-
-    return common
-
-
-def create_polled_event(table_event, expanded=False, selected=False):
-    poll_event = {
-        "event": table_event
+    table_event = {
+        "event": common
     }
     if expanded:
-        poll_event["event_attr"], poll_event["event_logs"], poll_event["event_history"], poll_event["event_msgs"] = (
-            get_event_details(int(table_event["id"])))
-        poll_event["expanded"] = expanded
+        table_event["event_attr"], table_event["event_logs"], table_event["event_history"], table_event["event_msgs"] = (
+            get_event_details(int(event.id)))
+        table_event["expanded"] = expanded
 
     if selected:
-        poll_event["selected"] = selected
+        table_event["selected"] = selected
 
-    return poll_event
+    return table_event
 
 
 # fixme implementation copied from curitz
@@ -208,7 +240,11 @@ def get_event_attributes(id, res_format=dict):
         event = current_app.event_manager.create_event_from_id(int(id))
     except RetryError as retryErr:  # Intermittent error in Zino
         current_app.logger.exception('RetryError when fetching event attributes %s', retryErr)
-        raise
+        try:
+            event = current_app.event_manager.create_event_from_id(int(id))
+        except RetryError as retryErr:  # Intermittent error in Zino
+            current_app.logger.exception('RetryError when fetching event attributes after retry, %s', retryErr)
+            raise
 
     event_dict = vars(event)
     attr_list = [f"{k}:{v}" for k, v in event_dict.items()]
@@ -250,7 +286,12 @@ def get_event_details(id):
         format_dt_event_attrs(event_attr)
     except RetryError as retryErr:  # Intermittent error in Zino
         current_app.logger.exception('RetryError when fetching event details %s', retryErr)
-        raise
+        try:
+            event_attr = vars(current_app.event_manager.create_event_from_id(int(id)))
+            format_dt_event_attrs(event_attr)
+        except RetryError as retryErr:  # Intermittent error in Zino
+            current_app.logger.exception('RetryError when fetching event details after retry, %s', retryErr)
+            raise
 
     event_logs = current_app.event_manager.get_log_for_id(int(id))
     event_history = current_app.event_manager.get_history_for_id(int(id))
@@ -275,7 +316,8 @@ def footer():
     elif not tz == DEFAULT_TIMEZONE:  # Fall back to default if invalid value is provided
         tz = f"{DEFAULT_TIMEZONE} (default)"
 
-    return render_template('/components/footer/footer-info.html', poll_interval=current_app.howitz_config["poll_interval"],
+    return render_template('/components/footer/footer-info.html',
+                           refresh_interval=current_app.howitz_config["refresh_interval"],
                            timezone=tz)
 
 
@@ -328,7 +370,7 @@ def auth():
 @main.route('/events-table.html')
 def events_table():
     return render_template('/components/table/events-table.html',
-                           poll_interval=current_app.howitz_config["poll_interval"])
+                           refresh_interval=current_app.howitz_config["refresh_interval"])
 
 
 @main.route('/get_events')
@@ -338,11 +380,12 @@ def get_events():
     return render_template('/components/table/event-rows.html', event_list=table_events)
 
 
-@main.route('/poll_events')
-def poll_events():
-    poll_events_list = poll_current_events()
+@main.route('/refresh_events')
+def refresh_events():
+    removed_events, modified_events, added_events = refresh_current_events()
 
-    return render_template('/components/poll/poll-rows.html', poll_event_list=poll_events_list)
+    return render_template('/responses/updated-rows.html', modified_event_list=modified_events,
+                           removed_event_list=removed_events, added_event_list=added_events)
 
 
 @main.route('/events/<event_id>/expand_row', methods=["GET"])
@@ -367,7 +410,7 @@ def expand_event_row(event_id):
         except RetryError as retryErr:  # Intermittent error in Zino
             current_app.logger.exception('RetryError on row expand after retry, %s', retryErr)
             raise
-    event = create_table_event(eventobj)
+    event = create_table_event(eventobj)["event"]
 
     return render_template('/components/row/expanded-row.html', event=event, id=event_id, event_attr=event_attr,
                            event_logs=event_logs,
@@ -396,7 +439,7 @@ def collapse_event_row(event_id):
         except RetryError as retryErr:  # Intermittent error in Zino
             current_app.logger.exception('RetryError on row collapse %s', retryErr)
             raise
-    event = create_table_event(eventobj)
+    event = create_table_event(eventobj)["event"]
 
     return render_template('/responses/collapse-row.html', event=event, id=event_id,
                            is_selected=str(event_id) in selected_events)
@@ -425,7 +468,7 @@ def update_event_status(event_id):
             add_history_res = current_app.event_manager.add_history_entry_for_id(event_id, new_history)
 
         event_attr, event_logs, event_history, event_msgs = get_event_details(event_id)
-        event = create_table_event(current_app.event_manager.create_event_from_id(event_id))
+        event = create_table_event(current_app.event_manager.create_event_from_id(event_id))["event"]
 
         return render_template('/responses/update-event-response.html', event=event, id=event_id, event_attr=event_attr,
                                event_logs=event_logs,
@@ -466,8 +509,8 @@ def bulk_update_events_status():
     current_app.logger.debug("SELECTED EVENTS %s", session["selected_events"])
 
     # Rerender whole events table
-    poll_events_list = poll_current_events()  # Calling poll events method is needed to preserve info about which events are expanded
-    return render_template('/responses/bulk-update-events-status.html', poll_event_list=poll_events_list)
+    event_list = get_current_events()
+    return render_template('/responses/bulk-update-events-status.html', event_list=event_list)
 
 
 @main.route('/show_update_status_modal', methods=['GET'])

src/howitz/templates/components/accordion/event-messages.html

@@ -3,32 +3,24 @@
     <ul
             id="ol-event-{{ id }}"
             class="relative border-l border-gray-200 dark:border-gray-700">
+        {% for msg in event_msgs %}
+            <li class="mb-3 ml-4">
 
-        {% if event_msgs|length>0 %}
+                <div class="absolute w-3 h-3  rounded-full mt-1.5 -left-1.5 border border-white dark:border-gray-900 {% if msg.user %} bg-orange-950 dark:bg-orange-400 {% else %} bg-blue-950 dark:bg-blue-400 {% endif %}"></div>
+                <h3 class="mb-1 text-xs font-normal leading-none text-gray-400">
+                    {{ msg.user }}
+                </h3>
+                <div
+                        class="flex gap-0 flex-wrap"
+                >
+                    <time class="mr-2 text-sm font-semibold text-white shrink-0">{{ msg.date }}</time>
+                    <p class="mb-2 text-sm font-normal text-gray-400 break-words">{{ msg.log }}</p>
+                </div>
 
-            {% for msg in event_msgs %}
+            </li>
 
 
-                <li class="mb-3 ml-4">
-
-                    <div class="absolute w-3 h-3  rounded-full mt-1.5 -left-1.5 border border-white dark:border-gray-900 {% if msg.user %} bg-orange-950 dark:bg-orange-400 {% else %} bg-blue-950 dark:bg-blue-400 {% endif %}"></div>
-                    <h3 class="mb-1 text-xs font-normal leading-none text-gray-400">
-                        {{ msg.user }}
-                    </h3>
-                    <div
-                            class="flex gap-0 flex-wrap"
-                    >
-                        <time class="mr-2 text-sm font-semibold text-white shrink-0">{{ msg.date }}</time>
-                        <p class="mb-2 text-sm font-normal text-gray-400 break-words">{{ msg.log }}</p>
-                    </div>
-
-                </li>
-
-
-            {% endfor %}
-
-        {% endif %}
-
+        {% endfor %}
     </ul>
 
 </div>

src/howitz/templates/components/footer/footer-info.html

@@ -1,5 +1,5 @@
 <p class="p-2 text-white text-semibold inline-block">
-    Updating every {{ poll_interval }}s.
+    Updating every {{ refresh_interval }}s.
 </p>
 <p class="p-2 text-white text-semibold inline-block">
     Configured timezone is {{ timezone }}.