feat: periodically update arduino library index

conf.py

@@ -20,6 +20,7 @@ class Settings(BaseSettings):
     code_cache_duration: int = 3600
     max_library_caches: int = 50
     library_cache_duration: int = 24 * 3600
+    library_index_refresh_interval: int = 3600
 
     # Max number of concurrent compile tasks
     max_concurrent_tasks: int = 10

deps/lifespan.py → deps/tasks.py

@@ -6,11 +6,19 @@
 
 from conf import settings
 from deps.logs import logger
+from deps.utils import repeat_every
 
 
 @asynccontextmanager
-async def lifespan(_app: FastAPI):
-    """At startup, update the Arduino library index"""
+async def startup(_app: FastAPI) -> None:
+    """Startup context manager"""
+    await refresh_library_index()
+    yield
+
+
+@repeat_every(seconds=settings.library_index_refresh_interval, logger=logger)
+async def refresh_library_index():
+    """Update the Arduino library index"""
     logger.info("Updating library index...")
     installer = await asyncio.create_subprocess_exec(
         settings.arduino_cli_path,
@@ -24,4 +32,3 @@ async def lifespan(_app: FastAPI):
         raise EnvironmentError(
             f"Failed to update library index: {stderr.decode() + stdout.decode()}"
         )
-    yield

deps/utils.py

@@ -0,0 +1,87 @@
+""" Tool to repeatedly call a function, copied from fastapi_utils"""
+import asyncio
+import logging
+from asyncio import ensure_future
+from functools import wraps
+from traceback import format_exception
+from typing import Any, Callable, Coroutine, Optional, Union
+
+from starlette.concurrency import run_in_threadpool
+
+NoArgsNoReturnFuncT = Callable[[], None]
+NoArgsNoReturnAsyncFuncT = Callable[[], Coroutine[Any, Any, None]]
+NoArgsNoReturnDecorator = Callable[
+    [Union[NoArgsNoReturnFuncT, NoArgsNoReturnAsyncFuncT]], NoArgsNoReturnAsyncFuncT
+]
+
+
+def repeat_every(
+    *,
+    seconds: float,
+    wait_first: bool = False,
+    logger: Optional[logging.Logger] = None,
+    raise_exceptions: bool = False,
+    max_repetitions: Optional[int] = None,
+) -> NoArgsNoReturnDecorator:
+    """
+    This function returns a decorator that modifies a function so it is periodically re-executed after its first call.
+
+    The function it decorates should accept no arguments and return nothing. If necessary, this can be accomplished
+    by using `functools.partial` or otherwise wrapping the target function prior to decoration.
+
+    Parameters
+    ----------
+    seconds: float
+        The number of seconds to wait between repeated calls
+    wait_first: bool (default False)
+        If True, the function will wait for a single period before the first call
+    logger: Optional[logging.Logger] (default None)
+        The logger to use to log any exceptions raised by calls to the decorated function.
+        If not provided, exceptions will not be logged by this function (though they may be handled by the event loop).
+    raise_exceptions: bool (default False)
+        If True, errors raised by the decorated function will be raised to the event loop's exception handler.
+        Note that if an error is raised, the repeated execution will stop.
+        Otherwise, exceptions are just logged and the execution continues to repeat.
+        See https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.set_exception_handler for more info.
+    max_repetitions: Optional[int] (default None)
+        The maximum number of times to call the repeated function. If `None`, the function is repeated forever.
+    """
+
+    def decorator(
+        func: Union[NoArgsNoReturnAsyncFuncT, NoArgsNoReturnFuncT]
+    ) -> NoArgsNoReturnAsyncFuncT:
+        """
+        Converts the decorated function into a repeated, periodically-called version of itself.
+        """
+        is_coroutine = asyncio.iscoroutinefunction(func)
+
+        @wraps(func)
+        async def wrapped() -> None:
+            repetitions = 0
+
+            async def loop() -> None:
+                nonlocal repetitions
+                if wait_first:
+                    await asyncio.sleep(seconds)
+                while max_repetitions is None or repetitions < max_repetitions:
+                    try:
+                        if is_coroutine:
+                            await func()  # type: ignore
+                        else:
+                            await run_in_threadpool(func)
+                        repetitions += 1
+                    except Exception as exc:  # pylint: disable=broad-except
+                        if logger is not None:
+                            formatted_exception = "".join(
+                                format_exception(type(exc), exc, exc.__traceback__)
+                            )
+                            logger.error(formatted_exception)
+                        if raise_exceptions:
+                            raise exc
+                    await asyncio.sleep(seconds)
+
+            ensure_future(loop())
+
+        return wrapped
+
+    return decorator

main.py

@@ -11,12 +11,12 @@
 
 from conf import settings
 from deps.cache import code_cache, get_code_cache_key, library_cache
-from deps.lifespan import lifespan
 from deps.logs import logger
 from deps.session import Session, sessions
+from deps.tasks import startup
 from models import Sketch, Library, PythonProgram
 
-app = FastAPI(lifespan=lifespan)
+app = FastAPI(lifespan=startup)
 app.add_middleware(
     CORSMiddleware,
     allow_origins=settings.cors_origins,

models.py

@@ -4,7 +4,7 @@
 from pydantic import BaseModel, Field
 
 # Regex match to (hopefully) prevent weird CLI injection issues
-Library = Annotated[str, Field(pattern=r"^[a-zA-Z0-9_ @]*$")]
+Library = Annotated[str, Field(pattern=r"^[a-zA-Z0-9_ \.@]*$")]
 
 
 class Sketch(BaseModel):

requirements.txt

@@ -1,7 +1,7 @@
-fastapi==0.104.1
+fastapi==0.109.0
 aiofiles==23.2.1
-uvicorn==0.24.0.post1
-pydantic==2.5.2
+uvicorn==0.27.0
+pydantic==2.5.3
 pydantic-settings==2.1.0
 cachetools==5.3.2
 python-minifier==2.9.0