diff --git a/.circleci/config.yml b/.circleci/config.yml index a54b6cbe4e..e8c17a2df9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -227,6 +227,61 @@ jobs: - store_artifacts: path: /tmp/dash_artifacts + test-312-async: &test + working_directory: ~/dash + docker: + - image: cimg/python:3.12.1-browsers + auth: + username: dashautomation + password: $DASH_PAT_DOCKERHUB + environment: + PERCY_ENABLE: 1 + PERCY_PARALLEL_TOTAL: -1 + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: True + PYVERSION: python312 + REDIS_URL: redis://localhost:6379 + - image: cimg/redis:6.2.6 + auth: + username: dashautomation + password: $DASH_PAT_DOCKERHUB + parallelism: 3 + steps: + - checkout: + path: ~/dash + - run: + name: Add chrome keys & update. + command: | + wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add - + sudo apt-get update + - run: echo $PYVERSION > ver.txt + - run: cat requirements/*.txt > requirements-all.txt + - restore_cache: + key: dep-{{ checksum ".circleci/config.yml" }}-{{ checksum "ver.txt" }}-{{ checksum "requirements-all.txt" }} + - browser-tools/install-browser-tools: + chrome-version: 120.0.6099.71 + install-firefox: false + install-geckodriver: false + - attach_workspace: + at: ~/dash + - run: + name: ️️🏗️ Install package + command: | + . venv/bin/activate + npm ci + pip install dash-package/dash-package.tar.gz[async,ci,dev,testing,celery,diskcache] --progress-bar off + pip list | grep dash + - run: + name: 🧪 Run Integration Tests + command: | + . venv/bin/activate + npm run citest.integration + - store_artifacts: + path: test-reports + - store_test_results: + path: test-reports + - store_artifacts: + path: /tmp/dash_artifacts + test-38: <<: *test docker: @@ -604,6 +659,9 @@ workflows: - test-312: requires: - install-dependencies-312 + - test-312-async: + requires: + - install-dependencies-312 - test-38: requires: - install-dependencies-38 diff --git a/dash/_callback.py b/dash/_callback.py index 12d8463eae..37a3c84584 100644 --- a/dash/_callback.py +++ b/dash/_callback.py @@ -5,6 +5,7 @@ from typing import Callable, Optional, Any, List, Tuple, Union +import asyncio import flask from .dependencies import ( @@ -43,6 +44,16 @@ from ._callback_context import context_value +async def _async_invoke_callback( + func, *args, **kwargs +): # used to mark the frame for the debugger + # Check if the function is a coroutine function + if asyncio.iscoroutinefunction(func): + return await func(*args, **kwargs) # %% callback invoked %% + # If the function is not a coroutine, call it directly + return func(*args, **kwargs) # %% callback invoked %% + + def _invoke_callback(func, *args, **kwargs): # used to mark the frame for the debugger return func(*args, **kwargs) # %% callback invoked %% @@ -309,6 +320,267 @@ def _set_side_update(ctx, response) -> bool: return False +def _initialize_context(args, kwargs, inputs_state_indices, has_output, insert_output): + """Initialize context and validate output specifications.""" + app = kwargs.pop("app", None) + output_spec = kwargs.pop("outputs_list") + callback_ctx = kwargs.pop("callback_context", AttributeDict({"updated_props": {}})) + context_value.set(callback_ctx) + original_packages = set(ComponentRegistry.registry) + + if has_output: + _validate.validate_output_spec(insert_output, output_spec, Output) + + func_args, func_kwargs = _validate.validate_and_group_input_args( + args, inputs_state_indices + ) + return ( + output_spec, + callback_ctx, + func_args, + func_kwargs, + app, + original_packages, + False, + ) + + +def _get_callback_manager(kwargs, background): + """Set up the background callback and manage jobs.""" + callback_manager = background.get( + "manager", kwargs.get("background_callback_manager", None) + ) + if background is not None: + if not callback_manager: + raise MissingLongCallbackManagerError( + "Running `background` callbacks requires a manager to be installed.\n" + "Available managers:\n" + "- Diskcache (`pip install dash[diskcache]`) to run callbacks in a separate Process" + " and store results on the local filesystem.\n" + "- Celery (`pip install dash[celery]`) to run callbacks in a celery worker" + " and store results on redis.\n" + ) + + old_job = flask.request.args.getlist("oldJob") + + if old_job: + for job in old_job: + callback_manager.terminate_job(job) + + return callback_manager + + +def _setup_background_callback( + kwargs, background, background_key, func, func_args, func_kwargs, callback_ctx +): + """Set up the background callback and manage jobs.""" + callback_manager = _get_callback_manager(kwargs, background) + + progress_outputs = background.get("progress") + + cache_ignore_triggered = background.get("cache_ignore_triggered", True) + + cache_key = callback_manager.build_cache_key( + func, + # Inputs provided as dict is kwargs. + func_args if func_args else func_kwargs, + background.get("cache_args_to_ignore", []), + None if cache_ignore_triggered else callback_ctx.get("triggered_inputs", []), + ) + + job_fn = callback_manager.func_registry.get(background_key) + + ctx_value = AttributeDict(**context_value.get()) + ctx_value.ignore_register_page = True + ctx_value.pop("background_callback_manager") + ctx_value.pop("dash_response") + + job = callback_manager.call_job_fn( + cache_key, + job_fn, + func_args if func_args else func_kwargs, + ctx_value, + ) + + data = { + "cacheKey": cache_key, + "job": job, + } + + cancel = background.get("cancel") + if cancel: + data["cancel"] = cancel + + progress_default = background.get("progressDefault") + if progress_default: + data["progressDefault"] = { + str(o): x for o, x in zip(progress_outputs, progress_default) + } + return to_json(data) + + +def _progress_background_callback(response, callback_manager, background): + progress_outputs = background.get("progress") + cache_key = flask.request.args.get("cacheKey") + + if progress_outputs: + # Get the progress before the result as it would be erased after the results. + progress = callback_manager.get_progress(cache_key) + if progress: + response["progress"] = { + str(x): progress[i] for i, x in enumerate(progress_outputs) + } + + +def _update_background_callback( + error_handler, callback_ctx, response, kwargs, background, multi +): + """Set up the background callback and manage jobs.""" + callback_manager = _get_callback_manager(kwargs, background) + + cache_key = flask.request.args.get("cacheKey") + job_id = flask.request.args.get("job") + + _progress_background_callback(response, callback_manager, background) + + output_value = callback_manager.get_result(cache_key, job_id) + + return _handle_rest_background_callback( + output_value, callback_manager, response, error_handler, callback_ctx, multi + ) + + +def _handle_rest_background_callback( + output_value, + callback_manager, + response, + error_handler, + callback_ctx, + multi, + has_update=False, +): + cache_key = flask.request.args.get("cacheKey") + job_id = flask.request.args.get("job") + # Must get job_running after get_result since get_results terminates it. + job_running = callback_manager.job_running(job_id) + if not job_running and output_value is callback_manager.UNDEFINED: + # Job canceled -> no output to close the loop. + output_value = NoUpdate() + + elif isinstance(output_value, dict) and "background_callback_error" in output_value: + error = output_value.get("background_callback_error", {}) + exc = BackgroundCallbackError( + f"An error occurred inside a background callback: {error['msg']}\n{error['tb']}" + ) + if error_handler: + output_value = error_handler(exc) + + if output_value is None: + output_value = NoUpdate() + # set_props from the error handler uses the original ctx + # instead of manager.get_updated_props since it runs in the + # request process. + has_update = ( + _set_side_update(callback_ctx, response) or output_value is not None + ) + else: + raise exc + + if job_running and output_value is not callback_manager.UNDEFINED: + # cached results. + callback_manager.terminate_job(job_id) + + if multi and isinstance(output_value, (list, tuple)): + output_value = [ + NoUpdate() if NoUpdate.is_no_update(r) else r for r in output_value + ] + updated_props = callback_manager.get_updated_props(cache_key) + if len(updated_props) > 0: + response["sideUpdate"] = updated_props + has_update = True + + if output_value is callback_manager.UNDEFINED: + return to_json(response), has_update, True + return output_value, has_update, False + + +# pylint: disable=too-many-branches +def _prepare_response( + output_value, + output_spec, + multi, + response, + callback_ctx, + app, + original_packages, + background, + has_update, + has_output, + output, + callback_id, + allow_dynamic_callbacks, +): + """Prepare the response object based on the callback output.""" + component_ids = collections.defaultdict(dict) + + if has_output: + if not multi: + output_value, output_spec = [output_value], [output_spec] + flat_output_values = output_value + else: + if isinstance(output_value, (list, tuple)): + # For multi-output, allow top-level collection to be + # list or tuple + output_value = list(output_value) + if NoUpdate.is_no_update(output_value): + flat_output_values = [output_value] + else: + # Flatten grouping and validate grouping structure + flat_output_values = flatten_grouping(output_value, output) + + if not NoUpdate.is_no_update(output_value): + _validate.validate_multi_return( + output_spec, flat_output_values, callback_id + ) + + for val, spec in zip(flat_output_values, output_spec): + if NoUpdate.is_no_update(val): + continue + for vali, speci in ( + zip(val, spec) if isinstance(spec, list) else [[val, spec]] + ): + if not NoUpdate.is_no_update(vali): + has_update = True + id_str = stringify_id(speci["id"]) + prop = clean_property_name(speci["property"]) + component_ids[id_str][prop] = vali + + else: + if output_value is not None: + raise InvalidCallbackReturnValue( + f"No-output callback received return value: {output_value}" + ) + + if not background: + has_update = _set_side_update(callback_ctx, response) or has_output + + if not has_update: + raise PreventUpdate + + if len(ComponentRegistry.registry) != len(original_packages): + diff_packages = list( + set(ComponentRegistry.registry).difference(original_packages) + ) + if not allow_dynamic_callbacks: + raise ImportedInsideCallbackError( + f"Component librar{'y' if len(diff_packages) == 1 else 'ies'} was imported during callback.\n" + "You can set `_allow_dynamic_callbacks` to allow for development purpose only." + ) + dist = app.get_dist(diff_packages) + response["dist"] = dist + return response.update({"response": component_ids}) + + # pylint: disable=too-many-branches,too-many-statements def register_callback( callback_list, callback_map, config_prevent_initial_callbacks, *_args, **_kwargs @@ -364,7 +636,6 @@ def register_callback( # pylint: disable=too-many-locals def wrap_func(func): - if background is not None: background_key = BaseBackgroundCallbackManager.register_func( func, @@ -374,227 +645,140 @@ def wrap_func(func): @wraps(func) def add_context(*args, **kwargs): - output_spec = kwargs.pop("outputs_list") - app_callback_manager = kwargs.pop("background_callback_manager", None) - - callback_ctx = kwargs.pop( - "callback_context", AttributeDict({"updated_props": {}}) - ) - app = kwargs.pop("app", None) - callback_manager = background and background.get( - "manager", app_callback_manager - ) + """Handles synchronous callbacks with context management.""" error_handler = on_error or kwargs.pop("app_on_error", None) - original_packages = set(ComponentRegistry.registry) - - if has_output: - _validate.validate_output_spec(insert_output, output_spec, Output) - context_value.set(callback_ctx) - - func_args, func_kwargs = _validate.validate_and_group_input_args( - args, inputs_state_indices + ( + output_spec, + callback_ctx, + func_args, + func_kwargs, + app, + original_packages, + has_update, + ) = _initialize_context( + args, kwargs, inputs_state_indices, has_output, insert_output ) response: dict = {"multi": True} - has_update = False - - if background is not None: - if not callback_manager: - raise MissingLongCallbackManagerError( - "Running `background` callbacks requires a manager to be installed.\n" - "Available managers:\n" - "- Diskcache (`pip install dash[diskcache]`) to run callbacks in a separate Process" - " and store results on the local filesystem.\n" - "- Celery (`pip install dash[celery]`) to run callbacks in a celery worker" - " and store results on redis.\n" - ) - - progress_outputs = background.get("progress") - cache_key = flask.request.args.get("cacheKey") - job_id = flask.request.args.get("job") - old_job = flask.request.args.getlist("oldJob") - - cache_ignore_triggered = background.get("cache_ignore_triggered", True) - - current_key = callback_manager.build_cache_key( - func, - # Inputs provided as dict is kwargs. - func_args if func_args else func_kwargs, - background.get("cache_args_to_ignore", []), - None - if cache_ignore_triggered - else callback_ctx.get("triggered_inputs", []), - ) - - if old_job: - for job in old_job: - callback_manager.terminate_job(job) - - if not cache_key: - cache_key = current_key - - job_fn = callback_manager.func_registry.get(background_key) - ctx_value = AttributeDict(**context_value.get()) - ctx_value.ignore_register_page = True - ctx_value.pop("background_callback_manager") - ctx_value.pop("dash_response") - - job = callback_manager.call_job_fn( - cache_key, - job_fn, - func_args if func_args else func_kwargs, - ctx_value, - ) - - data = { - "cacheKey": cache_key, - "job": job, - } - - cancel = background.get("cancel") - if cancel: - data["cancel"] = cancel - - progress_default = background.get("progressDefault") - if progress_default: - data["progressDefault"] = { - str(o): x - for o, x in zip(progress_outputs, progress_default) - } - return to_json(data) - if progress_outputs: - # Get the progress before the result as it would be erased after the results. - progress = callback_manager.get_progress(cache_key) - if progress: - response["progress"] = { - str(x): progress[i] for i, x in enumerate(progress_outputs) - } - - output_value = callback_manager.get_result(cache_key, job_id) - # Must get job_running after get_result since get_results terminates it. - job_running = callback_manager.job_running(job_id) - if not job_running and output_value is callback_manager.UNDEFINED: - # Job canceled -> no output to close the loop. - output_value = NoUpdate() - - elif ( - isinstance(output_value, dict) - and "background_callback_error" in output_value - ): - error = output_value.get("background_callback_error", {}) - exc = BackgroundCallbackError( - f"An error occurred inside a background callback: {error['msg']}\n{error['tb']}" - ) - if error_handler: - output_value = error_handler(exc) - - if output_value is None: - output_value = NoUpdate() - # set_props from the error handler uses the original ctx - # instead of manager.get_updated_props since it runs in the - # request process. - has_update = ( - _set_side_update(callback_ctx, response) - or output_value is not None + try: + if background is not None: + if not flask.request.args.get("cacheKey"): + return _setup_background_callback( + kwargs, + background, + background_key, + func, + func_args, + func_kwargs, + callback_ctx, ) - else: - raise exc - - if job_running and output_value is not callback_manager.UNDEFINED: - # cached results. - callback_manager.terminate_job(job_id) - - if multi and isinstance(output_value, (list, tuple)): - output_value = [ - NoUpdate() if NoUpdate.is_no_update(r) else r - for r in output_value - ] - updated_props = callback_manager.get_updated_props(cache_key) - if len(updated_props) > 0: - response["sideUpdate"] = updated_props - has_update = True - if output_value is callback_manager.UNDEFINED: - return to_json(response) - else: - try: + output_value, has_update, skip = _update_background_callback( + error_handler, callback_ctx, response, kwargs, background, multi + ) + if skip: + return output_value + else: output_value = _invoke_callback(func, *func_args, **func_kwargs) - except PreventUpdate as err: - raise err - except Exception as err: # pylint: disable=broad-exception-caught - if error_handler: - output_value = error_handler(err) - - # If the error returns nothing, automatically puts NoUpdate for response. - if output_value is None and has_output: - output_value = NoUpdate() - else: - raise err - - component_ids = collections.defaultdict(dict) - - if has_output: - if not multi: - output_value, output_spec = [output_value], [output_spec] - flat_output_values = output_value + except PreventUpdate: + raise + except Exception as err: # pylint: disable=broad-exception-caught + if error_handler: + output_value = error_handler(err) + if output_value is None and output_spec: + output_value = NoUpdate() else: - if isinstance(output_value, (list, tuple)): - # For multi-output, allow top-level collection to be - # list or tuple - output_value = list(output_value) - - if NoUpdate.is_no_update(output_value): - flat_output_values = [output_value] - else: - # Flatten grouping and validate grouping structure - flat_output_values = flatten_grouping(output_value, output) - - if not NoUpdate.is_no_update(output_value): - _validate.validate_multi_return( - output_spec, flat_output_values, callback_id - ) + raise err - for val, spec in zip(flat_output_values, output_spec): - if NoUpdate.is_no_update(val): - continue - for vali, speci in ( - zip(val, spec) if isinstance(spec, list) else [[val, spec]] - ): - if not NoUpdate.is_no_update(vali): - has_update = True - id_str = stringify_id(speci["id"]) - prop = clean_property_name(speci["property"]) - component_ids[id_str][prop] = vali - else: - if output_value is not None: - raise InvalidCallbackReturnValue( - f"No-output callback received return value: {output_value}" - ) - output_value = [] - flat_output_values = [] + _prepare_response( + output_value, + output_spec, + multi, + response, + callback_ctx, + app, + original_packages, + background, + has_update, + has_output, + output, + callback_id, + allow_dynamic_callbacks, + ) + try: + jsonResponse = to_json(response) + except TypeError: + _validate.fail_callback_output(output_value, output) + + return jsonResponse - if not background: - has_update = _set_side_update(callback_ctx, response) or has_update + @wraps(func) + async def async_add_context(*args, **kwargs): + """Handles async callbacks with context management.""" + error_handler = on_error or kwargs.pop("app_on_error", None) - if not has_update: - raise PreventUpdate + ( + output_spec, + callback_ctx, + func_args, + func_kwargs, + app, + original_packages, + has_update, + ) = _initialize_context( + args, kwargs, inputs_state_indices, has_output, insert_output + ) - response["response"] = component_ids + response: dict = {"multi": True} - if len(ComponentRegistry.registry) != len(original_packages): - diff_packages = list( - set(ComponentRegistry.registry).difference(original_packages) - ) - if not allow_dynamic_callbacks: - raise ImportedInsideCallbackError( - f"Component librar{'y' if len(diff_packages) == 1 else 'ies'} was imported during callback.\n" - "You can set `_allow_dynamic_callbacks` to allow for development purpose only." + try: + if background is not None: + if not flask.request.args.get("cacheKey"): + return _setup_background_callback( + kwargs, + background, + background_key, + func, + func_args, + func_kwargs, + callback_ctx, + ) + output_value, has_update, skip = _update_background_callback( + error_handler, callback_ctx, response, kwargs, background, multi + ) + if skip: + return output_value + else: + output_value = await _async_invoke_callback( + func, *func_args, **func_kwargs ) - dist = app.get_dist(diff_packages) - response["dist"] = dist + except PreventUpdate: + raise + except Exception as err: # pylint: disable=broad-exception-caught + if error_handler: + output_value = error_handler(err) + if output_value is None and output_spec: + output_value = NoUpdate() + else: + raise err + _prepare_response( + output_value, + output_spec, + multi, + response, + callback_ctx, + app, + original_packages, + background, + has_update, + has_output, + output, + callback_id, + allow_dynamic_callbacks, + ) try: jsonResponse = to_json(response) except TypeError: @@ -602,7 +786,10 @@ def add_context(*args, **kwargs): return jsonResponse - callback_map[callback_id]["callback"] = add_context + if asyncio.iscoroutinefunction(func): + callback_map[callback_id]["callback"] = async_add_context + else: + callback_map[callback_id]["callback"] = add_context return func diff --git a/dash/background_callback/managers/celery_manager.py b/dash/background_callback/managers/celery_manager.py index 66ca9de51a..5235b65a80 100644 --- a/dash/background_callback/managers/celery_manager.py +++ b/dash/background_callback/managers/celery_manager.py @@ -1,6 +1,8 @@ import json import traceback from contextvars import copy_context +import asyncio +from functools import partial from _plotly_utils.utils import PlotlyJSONEncoder @@ -16,7 +18,7 @@ class CeleryManager(BaseBackgroundCallbackManager): def __init__(self, celery_app, cache_by=None, expire=None): """ - Long callback manager that runs callback logic on a celery task queue, + Background callback manager that runs callback logic on a celery task queue, and stores results using a celery result backend. :param celery_app: @@ -40,7 +42,7 @@ def __init__(self, celery_app, cache_by=None, expire=None): except ImportError as missing_imports: raise ImportError( """\ -CeleryLongCallbackManager requires extra dependencies which can be installed doing +CeleryManager requires extra dependencies which can be installed doing $ pip install "dash[celery]"\n""" ) from missing_imports @@ -135,11 +137,13 @@ def get_updated_props(self, key): return json.loads(updated_props) -def _make_job_fn(fn, celery_app, progress, key): +def _make_job_fn(fn, celery_app, progress, key): # pylint: disable=too-many-statements cache = celery_app.backend @celery_app.task(name=f"background_callback_{key}") - def job_fn(result_key, progress_key, user_callback_args, context=None): + def job_fn( + result_key, progress_key, user_callback_args, context=None + ): # pylint: disable=too-many-statements def _set_progress(progress_value): if not isinstance(progress_value, (list, tuple)): progress_value = [progress_value] @@ -197,7 +201,59 @@ def run(): result_key, json.dumps(user_callback_output, cls=PlotlyJSONEncoder) ) - ctx.run(run) + async def async_run(): + c = AttributeDict(**context) + c.ignore_register_page = False + c.updated_props = ProxySetProps(_set_props) + context_value.set(c) + errored = False + try: + if isinstance(user_callback_args, dict): + user_callback_output = await fn( + *maybe_progress, **user_callback_args + ) + elif isinstance(user_callback_args, (list, tuple)): + user_callback_output = await fn( + *maybe_progress, *user_callback_args + ) + else: + user_callback_output = await fn(*maybe_progress, user_callback_args) + except PreventUpdate: + # Put NoUpdate dict directly to avoid circular imports. + errored = True + cache.set( + result_key, + json.dumps( + {"_dash_no_update": "_dash_no_update"}, cls=PlotlyJSONEncoder + ), + ) + except Exception as err: # pylint: disable=broad-except + errored = True + cache.set( + result_key, + json.dumps( + { + "background_callback_error": { + "msg": str(err), + "tb": traceback.format_exc(), + } + }, + ), + ) + + if asyncio.iscoroutine(user_callback_output): + user_callback_output = await user_callback_output + + if not errored: + cache.set( + result_key, json.dumps(user_callback_output, cls=PlotlyJSONEncoder) + ) + + if asyncio.iscoroutinefunction(fn): + func = partial(ctx.run, async_run) + asyncio.run(func()) + else: + ctx.run(run) return job_fn diff --git a/dash/background_callback/managers/diskcache_manager.py b/dash/background_callback/managers/diskcache_manager.py index 994da4c0ac..ed89e5df41 100644 --- a/dash/background_callback/managers/diskcache_manager.py +++ b/dash/background_callback/managers/diskcache_manager.py @@ -1,5 +1,7 @@ import traceback from contextvars import copy_context +import asyncio +from functools import partial from . import BaseBackgroundCallbackManager from .._proxy_set_props import ProxySetProps @@ -15,7 +17,7 @@ class DiskcacheManager(BaseBackgroundCallbackManager): def __init__(self, cache=None, cache_by=None, expire=None): """ - Long callback manager that runs callback logic in a subprocess and stores + Background callback manager that runs callback logic in a subprocess and stores results on disk using diskcache :param cache: @@ -38,7 +40,7 @@ def __init__(self, cache=None, cache_by=None, expire=None): except ImportError as missing_imports: raise ImportError( """\ -DiskcacheLongCallbackManager requires extra dependencies which can be installed doing +DiskcacheManager requires extra dependencies which can be installed doing $ pip install "dash[diskcache]"\n""" ) from missing_imports @@ -116,16 +118,52 @@ def clear_cache_entry(self, key): # noinspection PyUnresolvedReferences def call_job_fn(self, key, job_fn, args, context): + """ + Call the job function, supporting both sync and async jobs. + Args: + key: Cache key for the job. + job_fn: The job function to execute. + args: Arguments for the job function. + context: Context for the job. + Returns: + The PID of the spawned process or None for async execution. + """ # pylint: disable-next=import-outside-toplevel,no-name-in-module,import-error from multiprocess import Process # pylint: disable-next=not-callable - proc = Process( + process = Process( target=job_fn, args=(key, self._make_progress_key(key), args, context), ) - proc.start() - return proc.pid + process.start() + return process.pid + + @staticmethod + def _run_async_in_process(job_fn, key, args, context): + """ + Helper function to run an async job in a new process. + Args: + job_fn: The async job function. + key: Cache key for the job. + args: Arguments for the job function. + context: Context for the job. + """ + # Create a new event loop for the process + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Wrap the job function to include key and progress + async_job = partial(job_fn, key, args, context) + + try: + # Run the async job and wait for completion + loop.run_until_complete(async_job()) + except Exception as e: + # Handle errors, log them, and cache if necessary + raise Exception(str(e)) from e + finally: + loop.close() def get_progress(self, key): progress_key = self._make_progress_key(key) @@ -168,7 +206,9 @@ def get_updated_props(self, key): return result +# pylint: disable-next=too-many-statements def _make_job_fn(fn, cache, progress): + # pylint: disable-next=too-many-statements def job_fn(result_key, progress_key, user_callback_args, context): def _set_progress(progress_value): if not isinstance(progress_value, (list, tuple)): @@ -214,7 +254,47 @@ def run(): if not errored: cache.set(result_key, user_callback_output) - ctx.run(run) + async def async_run(): + c = AttributeDict(**context) + c.ignore_register_page = False + c.updated_props = ProxySetProps(_set_props) + context_value.set(c) + errored = False + try: + if isinstance(user_callback_args, dict): + user_callback_output = await fn( + *maybe_progress, **user_callback_args + ) + elif isinstance(user_callback_args, (list, tuple)): + user_callback_output = await fn( + *maybe_progress, *user_callback_args + ) + else: + user_callback_output = await fn(*maybe_progress, user_callback_args) + except PreventUpdate: + errored = True + cache.set(result_key, {"_dash_no_update": "_dash_no_update"}) + except Exception as err: # pylint: disable=broad-except + errored = True + cache.set( + result_key, + { + "background_callback_error": { + "msg": str(err), + "tb": traceback.format_exc(), + } + }, + ) + if asyncio.iscoroutine(user_callback_output): + user_callback_output = await user_callback_output + if not errored: + cache.set(result_key, user_callback_output) + + if asyncio.iscoroutinefunction(fn): + func = partial(ctx.run, async_run) + asyncio.run(func()) + else: + ctx.run(run) return job_fn diff --git a/dash/dash.py b/dash/dash.py index 068cfdc162..4045b3b920 100644 --- a/dash/dash.py +++ b/dash/dash.py @@ -20,6 +20,7 @@ from urllib.parse import urlparse from typing import Any, Callable, Dict, Optional, Union, Sequence +import asyncio import flask from importlib_metadata import version as _get_distribution_version @@ -163,6 +164,7 @@ def _get_traceback(secret, error: Exception): def _get_skip(error): from dash._callback import ( # pylint: disable=import-outside-toplevel _invoke_callback, + _async_invoke_callback, ) tb = error.__traceback__ @@ -170,7 +172,10 @@ def _get_skip(error): while tb.tb_next is not None: skip += 1 tb = tb.tb_next - if tb.tb_frame.f_code is _invoke_callback.__code__: + if tb.tb_frame.f_code in [ + _invoke_callback.__code__, + _async_invoke_callback.__code__, + ]: return skip return skip @@ -178,11 +183,15 @@ def _get_skip(error): def _do_skip(error): from dash._callback import ( # pylint: disable=import-outside-toplevel _invoke_callback, + _async_invoke_callback, ) tb = error.__traceback__ while tb.tb_next is not None: - if tb.tb_frame.f_code is _invoke_callback.__code__: + if tb.tb_frame.f_code in [ + _invoke_callback.__code__, + _async_invoke_callback.__code__, + ]: return tb.tb_next tb = tb.tb_next return error.__traceback__ @@ -206,6 +215,14 @@ def _do_skip(error): no_update = _callback.NoUpdate() # pylint: disable=protected-access +async def execute_async_function(func, *args, **kwargs): + # Check if the function is a coroutine function + if asyncio.iscoroutinefunction(func): + return await func(*args, **kwargs) + # If the function is not a coroutine, call it directly + return func(*args, **kwargs) + + # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments, too-many-locals class Dash(ObsoleteChecker): @@ -386,6 +403,10 @@ class Dash(ObsoleteChecker): an exception is raised. Receives the exception object as first argument. The callback_context can be used to access the original callback inputs, states and output. + + :param use_async: When True, the app will create async endpoints, as a dev, + they will be responsible for installing the `flask[async]` dependency. + :type use_async: boolean """ _plotlyjs_url: str @@ -430,8 +451,25 @@ def __init__( # pylint: disable=too-many-statements routing_callback_inputs: Optional[Dict[str, Union[Input, State]]] = None, description: Optional[str] = None, on_error: Optional[Callable[[Exception], Any]] = None, + use_async: Optional[bool] = None, **obsolete, ): + + if use_async is None: + try: + import asgiref # pylint: disable=unused-import, import-outside-toplevel # noqa + + use_async = True + except ImportError: + pass + elif use_async: + try: + import asgiref # pylint: disable=unused-import, import-outside-toplevel # noqa + except ImportError as exc: + raise Exception( + "You are trying to use dash[async] without having installed the requirements please install via: `pip install dash[async]`" + ) from exc + _validate.check_obsolete(obsolete) caller_name = None if name else get_caller_name() @@ -543,6 +581,7 @@ def __init__( # pylint: disable=too-many-statements self.validation_layout = None self._on_error = on_error self._extra_components = [] + self._use_async = use_async self._setup_dev_tools() self._hot_reload = AttributeDict( @@ -707,7 +746,10 @@ def _setup_routes(self): ) self._add_url("_dash-layout", self.serve_layout) self._add_url("_dash-dependencies", self.dependencies) - self._add_url("_dash-update-component", self.dispatch, ["POST"]) + if self._use_async: + self._add_url("_dash-update-component", self.async_dispatch, ["POST"]) + else: + self._add_url("_dash-update-component", self.dispatch, ["POST"]) self._add_url("_reload-hash", self.serve_reload_hash) self._add_url("_favicon.ico", self._serve_default_favicon) self._add_url("", self.index) @@ -1292,36 +1334,30 @@ def callback(self, *_args, **_kwargs) -> Callable[..., Any]: ) # pylint: disable=R0915 - def dispatch(self): - body = flask.request.get_json() - + def _initialize_context(self, body): + """Initialize the global context for the request.""" g = AttributeDict({}) - - g.inputs_list = inputs = body.get( # pylint: disable=assigning-non-slot - "inputs", [] - ) - g.states_list = state = body.get( # pylint: disable=assigning-non-slot - "state", [] - ) - output = body["output"] - outputs_list = body.get("outputs") - g.outputs_list = outputs_list # pylint: disable=assigning-non-slot - - g.input_values = ( # pylint: disable=assigning-non-slot - input_values - ) = inputs_to_dict(inputs) - g.state_values = inputs_to_dict(state) # pylint: disable=assigning-non-slot - changed_props = body.get("changedPropIds", []) - g.triggered_inputs = [ # pylint: disable=assigning-non-slot - {"prop_id": x, "value": input_values.get(x)} for x in changed_props + g.inputs_list = body.get("inputs", []) + g.states_list = body.get("state", []) + g.outputs_list = body.get("outputs", []) + g.input_values = inputs_to_dict(g.inputs_list) + g.state_values = inputs_to_dict(g.states_list) + g.triggered_inputs = [ + {"prop_id": x, "value": g.input_values.get(x)} + for x in body.get("changedPropIds", []) ] - - response = ( - g.dash_response # pylint: disable=assigning-non-slot - ) = flask.Response(mimetype="application/json") - - args = inputs_to_vals(inputs + state) - + g.dash_response = flask.Response(mimetype="application/json") + g.cookies = dict(**flask.request.cookies) + g.headers = dict(**flask.request.headers) + g.path = flask.request.full_path + g.remote = flask.request.remote_addr + g.origin = flask.request.origin + g.updated_props = {} + return g + + def _prepare_callback(self, g, body): + """Prepare callback-related data.""" + output = body["output"] try: cb = self.callback_map[output] func = cb["callback"] @@ -1332,89 +1368,114 @@ def dispatch(self): # Add args_grouping inputs_state_indices = cb["inputs_state_indices"] - inputs_state = inputs + state - inputs_state = convert_to_AttributeDict(inputs_state) + inputs_state = convert_to_AttributeDict(g.inputs_list + g.states_list) if cb.get("no_output"): - outputs_list = [] - elif not outputs_list: - # FIXME Old renderer support? + g.outputs_list = [] + elif not g.outputs_list: + # Legacy support for older renderers split_callback_id(output) - # update args_grouping attributes + # Update args_grouping attributes for s in inputs_state: # check for pattern matching: list of inputs or state if isinstance(s, list): for pattern_match_g in s: - update_args_group(pattern_match_g, changed_props) - update_args_group(s, changed_props) + update_args_group( + pattern_match_g, body.get("changedPropIds", []) + ) + update_args_group(s, body.get("changedPropIds", [])) - args_grouping = map_grouping( - lambda ind: inputs_state[ind], inputs_state_indices + g.args_grouping, g.using_args_grouping = self._prepare_grouping( + inputs_state, inputs_state_indices + ) + g.outputs_grouping, g.using_outputs_grouping = self._prepare_grouping( + g.outputs_list, cb.get("outputs_indices", []) ) + except KeyError as e: + raise KeyError(f"Callback function not found for output '{output}'.") from e + return func + + def _prepare_grouping(self, data_list, indices): + """Prepare grouping logic for inputs or outputs.""" + if not isinstance(data_list, list): + flat_data = [data_list] + else: + flat_data = data_list - g.args_grouping = args_grouping # pylint: disable=assigning-non-slot - g.using_args_grouping = ( # pylint: disable=assigning-non-slot - not isinstance(inputs_state_indices, int) - and ( - inputs_state_indices - != list(range(grouping_len(inputs_state_indices))) - ) + if len(flat_data) > 0: + grouping = map_grouping(lambda ind: flat_data[ind], indices) + using_grouping = not isinstance(indices, int) and indices != list( + range(grouping_len(indices)) ) + else: + grouping, using_grouping = [], False - # Add outputs_grouping - outputs_indices = cb["outputs_indices"] - if not isinstance(outputs_list, list): - flat_outputs = [outputs_list] - else: - flat_outputs = outputs_list + return grouping, using_grouping - if len(flat_outputs) > 0: - outputs_grouping = map_grouping( - lambda ind: flat_outputs[ind], outputs_indices - ) - g.outputs_grouping = ( - outputs_grouping # pylint: disable=assigning-non-slot - ) - g.using_outputs_grouping = ( # pylint: disable=assigning-non-slot - not isinstance(outputs_indices, int) - and outputs_indices != list(range(grouping_len(outputs_indices))) - ) - else: - g.outputs_grouping = [] - g.using_outputs_grouping = [] - g.updated_props = {} + def _execute_callback(self, func, args, outputs_list, g): + """Execute the callback with the prepared arguments.""" + g.cookies = dict(**flask.request.cookies) + g.headers = dict(**flask.request.headers) + g.path = flask.request.full_path + g.remote = flask.request.remote_addr + g.origin = flask.request.origin + g.custom_data = AttributeDict({}) - g.cookies = dict(**flask.request.cookies) - g.headers = dict(**flask.request.headers) - g.path = flask.request.full_path - g.remote = flask.request.remote_addr - g.origin = flask.request.origin - g.custom_data = AttributeDict({}) + for hook in self._hooks.get_hooks("custom_data"): + g.custom_data[hook.data["namespace"]] = hook(g) - for hook in self._hooks.get_hooks("custom_data"): - g.custom_data[hook.data["namespace"]] = hook(g) + # noinspection PyArgumentList + partial_func = functools.partial( + func, + *args, + outputs_list=outputs_list, + background_callback_manager=g.background_callback_manager, + callback_context=g, + app=self, + app_on_error=self._on_error, + app_use_async=self._use_async, + ) + return partial_func - except KeyError as missing_callback_function: - msg = f"Callback function not found for output '{output}', perhaps you forgot to prepend the '@'?" - raise KeyError(msg) from missing_callback_function + async def async_dispatch(self): + body = flask.request.get_json() + g = self._initialize_context(body) + func = self._prepare_callback(g, body) + args = inputs_to_vals(g.inputs_list + g.states_list) ctx = copy_context() - # noinspection PyArgumentList - response.set_data( - ctx.run( - functools.partial( - func, - *args, - outputs_list=outputs_list, - background_callback_manager=self._background_manager, - callback_context=g, - app=self, - app_on_error=self._on_error, - ) + partial_func = self._execute_callback(func, args, g.outputs_list, g) + if asyncio.iscoroutine(func): + response_data = await ctx.run(partial_func) + else: + response_data = ctx.run(partial_func) + + if asyncio.iscoroutine(response_data): + response_data = await response_data + + g.dash_response.set_data(response_data) + return g.dash_response + + def dispatch(self): + body = flask.request.get_json() + g = self._initialize_context(body) + func = self._prepare_callback(g, body) + args = inputs_to_vals(g.inputs_list + g.states_list) + + ctx = copy_context() + partial_func = self._execute_callback(func, args, g.outputs_list, g) + response_data = ctx.run(partial_func) + + if asyncio.iscoroutine(response_data): + raise Exception( + "You are trying to use a coroutine without dash[async]. " + "Please install the dependencies via `pip install dash[async]` and ensure " + "that `use_async=False` is not being passed to the app." ) - ) - return response + + g.dash_response.set_data(response_data) + return g.dash_response def _setup_server(self): if self._got_first_request["setup_server"]: @@ -2284,65 +2345,133 @@ def router(): } inputs.update(self.routing_callback_inputs) - @self.callback( - Output(_ID_CONTENT, "children"), - Output(_ID_STORE, "data"), - inputs=inputs, - prevent_initial_call=True, - ) - def update(pathname_, search_, **states): - """ - Updates dash.page_container layout on page navigation. - Updates the stored page title which will trigger the clientside callback to update the app title - """ + if self._use_async: - query_parameters = _parse_query_string(search_) - page, path_variables = _path_to_page( - self.strip_relative_path(pathname_) + @self.callback( + Output(_ID_CONTENT, "children"), + Output(_ID_STORE, "data"), + inputs=inputs, + prevent_initial_call=True, ) + async def update(pathname_, search_, **states): + """ + Updates dash.page_container layout on page navigation. + Updates the stored page title which will trigger the clientside callback to update the app title + """ + + query_parameters = _parse_query_string(search_) + page, path_variables = _path_to_page( + self.strip_relative_path(pathname_) + ) - # get layout - if page == {}: - for module, page in _pages.PAGE_REGISTRY.items(): - if module.split(".")[-1] == "not_found_404": - layout = page["layout"] - title = page["title"] - break + # get layout + if page == {}: + for module, page in _pages.PAGE_REGISTRY.items(): + if module.split(".")[-1] == "not_found_404": + layout = page["layout"] + title = page["title"] + break + else: + layout = html.H1("404 - Page not found") + title = self.title else: - layout = html.H1("404 - Page not found") - title = self.title - else: - layout = page.get("layout", "") - title = page["title"] - - if callable(layout): - layout = ( - layout(**path_variables, **query_parameters, **states) - if path_variables - else layout(**query_parameters, **states) - ) - if callable(title): - title = title(**path_variables) if path_variables else title() + layout = page.get("layout", "") + title = page["title"] - return layout, {"title": title} + if callable(layout): + layout = await execute_async_function( + layout, + **{**(path_variables or {}), **query_parameters, **states}, + ) + if callable(title): + title = await execute_async_function( + title, **(path_variables or {}) + ) - _validate.check_for_duplicate_pathnames(_pages.PAGE_REGISTRY) - _validate.validate_registry(_pages.PAGE_REGISTRY) + return layout, {"title": title} + + _validate.check_for_duplicate_pathnames(_pages.PAGE_REGISTRY) + _validate.validate_registry(_pages.PAGE_REGISTRY) + + # Set validation_layout + if not self.config.suppress_callback_exceptions: + self.validation_layout = html.Div( + [ + asyncio.run(execute_async_function(page["layout"])) + if callable(page["layout"]) + else page["layout"] + for page in _pages.PAGE_REGISTRY.values() + ] + + [ + # pylint: disable=not-callable + self.layout() + if callable(self.layout) + else self.layout + ] + ) + if _ID_CONTENT not in self.validation_layout: + raise Exception("`dash.page_container` not found in the layout") + else: - # Set validation_layout - if not self.config.suppress_callback_exceptions: - self.validation_layout = html.Div( - [ - page["layout"]() if callable(page["layout"]) else page["layout"] - for page in _pages.PAGE_REGISTRY.values() - ] - + [ - # pylint: disable=not-callable - self.layout() - if callable(self.layout) - else self.layout - ] + @self.callback( + Output(_ID_CONTENT, "children"), + Output(_ID_STORE, "data"), + inputs=inputs, + prevent_initial_call=True, ) + def update(pathname_, search_, **states): + """ + Updates dash.page_container layout on page navigation. + Updates the stored page title which will trigger the clientside callback to update the app title + """ + + query_parameters = _parse_query_string(search_) + page, path_variables = _path_to_page( + self.strip_relative_path(pathname_) + ) + + # get layout + if page == {}: + for module, page in _pages.PAGE_REGISTRY.items(): + if module.split(".")[-1] == "not_found_404": + layout = page["layout"] + title = page["title"] + break + else: + layout = html.H1("404 - Page not found") + title = self.title + else: + layout = page.get("layout", "") + title = page["title"] + + if callable(layout): + layout = layout( + **{**(path_variables or {}), **query_parameters, **states} + ) + if callable(title): + title = title(**(path_variables or {})) + + return layout, {"title": title} + + _validate.check_for_duplicate_pathnames(_pages.PAGE_REGISTRY) + _validate.validate_registry(_pages.PAGE_REGISTRY) + + # Set validation_layout + if not self.config.suppress_callback_exceptions: + self.validation_layout = html.Div( + [ + page["layout"]() + if callable(page["layout"]) + else page["layout"] + for page in _pages.PAGE_REGISTRY.values() + ] + + [ + # pylint: disable=not-callable + self.layout() + if callable(self.layout) + else self.layout + ] + ) if _ID_CONTENT not in self.validation_layout: raise Exception("`dash.page_container` not found in the layout") diff --git a/dash/testing/browser.py b/dash/testing/browser.py index c4fcb8fca8..0c55f7031a 100644 --- a/dash/testing/browser.py +++ b/dash/testing/browser.py @@ -159,6 +159,12 @@ def percy_snapshot( """ if widths is None: widths = [1280] + try: + import asgiref # pylint: disable=unused-import, import-outside-toplevel # noqa: F401, C0415 + + name += "_async" + except ImportError: + pass logger.info("taking snapshot name => %s", name) try: diff --git a/package.json b/package.json index cf5f5335a0..93e5ac1b6c 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "private::test.unit-dash": "pytest tests/unit", "private::test.unit-renderer": "cd dash/dash-renderer && npm run test", "private::test.unit-generation": "cd @plotly/dash-generator-test-component-typescript && npm ci && npm test", - "private::test.integration-dash": "TESTFILES=$(circleci tests glob \"tests/integration/**/test_*.py\" | circleci tests split --split-by=timings) && pytest --headless --nopercyfinalize --junitxml=test-reports/junit_intg.xml ${TESTFILES}", + "private::test.integration-dash": "TESTFILES=$(circleci tests glob \"tests/integration/**/test_*.py\" | circleci tests split --split-by=timings) && pytest --headless --nopercyfinalize --junitxml=test-reports/junit_intg.xml ${TESTFILES} && python rerun_failed_tests.py", "private::test.integration-dash-import": "cd tests/integration/dash && python dash_import_test.py", "cibuild": "run-s private::cibuild.*", "build": "run-s private::build.*", diff --git a/requirements/async.txt b/requirements/async.txt new file mode 100644 index 0000000000..fafa8e7e6e --- /dev/null +++ b/requirements/async.txt @@ -0,0 +1 @@ +flask[async] diff --git a/rerun_failed_tests.py b/rerun_failed_tests.py new file mode 100644 index 0000000000..9b2859eb57 --- /dev/null +++ b/rerun_failed_tests.py @@ -0,0 +1,25 @@ +import xml.etree.ElementTree as ET +import subprocess + +def parse_test_results(file_path): + tree = ET.parse(file_path) + root = tree.getroot() + failed_tests = [] + for testcase in root.iter('testcase'): + if testcase.find('failure') is not None: + failed_tests.append(testcase.get('name')) + return failed_tests + +def rerun_failed_tests(failed_tests): + if failed_tests: + print("Initial failed tests:", failed_tests) + failed_test_names = ' '.join(failed_tests) + result = subprocess.run(f'pytest --headless {failed_test_names}', shell=True, capture_output=True, text=True) + print(result.stdout) + print(result.stderr) + else: + print('All tests passed.') + +if __name__ == "__main__": + failed_tests = parse_test_results('test-reports/junit_intg.xml') + rerun_failed_tests(failed_tests) diff --git a/setup.py b/setup.py index ea616e2a18..7ed781c20d 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ def read_req_file(req_type): install_requires=read_req_file("install"), python_requires=">=3.8", extras_require={ + "async": read_req_file("async"), "ci": read_req_file("ci"), "dev": read_req_file("dev"), "testing": read_req_file("testing"), diff --git a/tests/integration/async_tests/__init__.py b/tests/integration/async_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/async_tests/app1_async.py b/tests/integration/async_tests/app1_async.py new file mode 100644 index 0000000000..daf76e885b --- /dev/null +++ b/tests/integration/async_tests/app1_async.py @@ -0,0 +1,31 @@ +from dash import Dash, Input, Output, dcc, html +import time + +from tests.integration.background_callback.utils import get_background_callback_manager + +background_callback_manager = get_background_callback_manager() +handle = background_callback_manager.handle + +app = Dash(__name__) +app.layout = html.Div( + [ + dcc.Input(id="input", value="initial value"), + html.Div(html.Div([1.5, None, "string", html.Div(id="output-1")])), + ] +) + + +@app.callback( + Output("output-1", "children"), + [Input("input", "value")], + interval=500, + manager=background_callback_manager, + background=True, +) +async def update_output(value): + time.sleep(0.1) + return value + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/tests/integration/async_tests/app_arbitrary_async.py b/tests/integration/async_tests/app_arbitrary_async.py new file mode 100644 index 0000000000..080859b22b --- /dev/null +++ b/tests/integration/async_tests/app_arbitrary_async.py @@ -0,0 +1,50 @@ +from dash import Dash, Input, Output, html, callback, set_props +import time + +from tests.integration.background_callback.utils import get_background_callback_manager + +background_callback_manager = get_background_callback_manager() +handle = background_callback_manager.handle + +app = Dash(__name__, background_callback_manager=background_callback_manager) +app.test_lock = lock = background_callback_manager.test_lock + +app.layout = html.Div( + [ + html.Button("start", id="start"), + html.Div(id="secondary"), + html.Div(id="no-output"), + html.Div("initial", id="output"), + html.Button("start-no-output", id="start-no-output"), + ] +) + + +@callback( + Output("output", "children"), + Input("start", "n_clicks"), + prevent_initial_call=True, + background=True, + interval=500, +) +async def on_click(_): + set_props("secondary", {"children": "first"}) + set_props("secondary", {"style": {"background": "red"}}) + time.sleep(2) + set_props("secondary", {"children": "second"}) + return "completed" + + +@callback( + Input("start-no-output", "n_clicks"), + prevent_initial_call=True, + background=True, +) +async def on_click(_): + set_props("no-output", {"children": "started"}) + time.sleep(2) + set_props("no-output", {"children": "completed"}) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/tests/integration/async_tests/conftest.py b/tests/integration/async_tests/conftest.py new file mode 100644 index 0000000000..b701eea91a --- /dev/null +++ b/tests/integration/async_tests/conftest.py @@ -0,0 +1,15 @@ +import os + +import pytest + + +if "REDIS_URL" in os.environ: + managers = ["celery", "diskcache"] +else: + print("Skipping celery tests because REDIS_URL is not defined") + managers = ["diskcache"] + + +@pytest.fixture(params=managers) +def manager(request): + return request.param diff --git a/tests/integration/async_tests/test_async_background_callbacks.py b/tests/integration/async_tests/test_async_background_callbacks.py new file mode 100644 index 0000000000..03526d7941 --- /dev/null +++ b/tests/integration/async_tests/test_async_background_callbacks.py @@ -0,0 +1,62 @@ +import sys + +import pytest +from flaky import flaky +from multiprocessing import Lock +from tests.integration.async_tests.utils import setup_background_callback_app +import time +from tests.utils import test_async + + +def test_001ab_arbitrary(dash_duo, manager): + if not test_async(): + return + with setup_background_callback_app(manager, "app_arbitrary_async") as app: + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#output", "initial") + # pause for sync + time.sleep(0.2) + dash_duo.find_element("#start").click() + + dash_duo.wait_for_text_to_equal("#secondary", "first") + dash_duo.wait_for_style_to_equal( + "#secondary", "background-color", "rgba(255, 0, 0, 1)" + ) + dash_duo.wait_for_text_to_equal("#output", "initial") + dash_duo.wait_for_text_to_equal("#secondary", "second") + dash_duo.wait_for_text_to_equal("#output", "completed") + + dash_duo.find_element("#start-no-output").click() + + dash_duo.wait_for_text_to_equal("#no-output", "started") + dash_duo.wait_for_text_to_equal("#no-output", "completed") + + +@pytest.mark.skipif( + sys.version_info < (3, 7), reason="Python 3.6 long callbacks tests hangs up" +) +@flaky(max_runs=3) +def test_002ab_basic(dash_duo, manager): + """ + Make sure that we settle to the correct final value when handling rapid inputs + """ + if not test_async(): + return + lock = Lock() + with setup_background_callback_app(manager, "app1_async") as app: + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal("#output-1", "initial value", 15) + input_ = dash_duo.find_element("#input") + # pause for sync + time.sleep(0.2) + dash_duo.clear_input(input_) + + for key in "hello world": + with lock: + input_.send_keys(key) + + dash_duo.wait_for_text_to_equal("#output-1", "hello world", 8) + + assert not dash_duo.redux_state_is_loading + assert dash_duo.get_logs() == [] diff --git a/tests/integration/async_tests/test_async_callbacks.py b/tests/integration/async_tests/test_async_callbacks.py new file mode 100644 index 0000000000..2314da16f0 --- /dev/null +++ b/tests/integration/async_tests/test_async_callbacks.py @@ -0,0 +1,968 @@ +import json +import os +from multiprocessing import Lock, Value +import pytest +import time + +import numpy as np +import werkzeug + +from dash_test_components import ( + AsyncComponent, + CollapseComponent, + DelayedEventComponent, + FragmentComponent, +) +from dash import ( + Dash, + Input, + Output, + State, + html, + dcc, + dash_table, + no_update, +) +from dash.exceptions import PreventUpdate +from tests.integration.utils import json_engine +from tests.utils import test_async + + +def test_async_cbsc001_simple_callback(dash_duo, *args): + if not test_async(): + return + lock = Lock() + + app = Dash(__name__) + app.layout = html.Div( + [ + dcc.Input(id="input", value="initial value"), + html.Div(html.Div([1.5, None, "string", html.Div(id="output-1")])), + ] + ) + call_count = Value("i", 0) + + @app.callback(Output("output-1", "children"), [Input("input", "value")]) + async def update_output(value): + with lock: + call_count.value = call_count.value + 1 + return value + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#output-1", "initial value") + + input_ = dash_duo.find_element("#input") + dash_duo.clear_input(input_) + + for key in "hello world": + with lock: + input_.send_keys(key) + + dash_duo.wait_for_text_to_equal("#output-1", "hello world") + + assert call_count.value == 2 + len("hello world"), "initial count + each key stroke" + + assert not dash_duo.redux_state_is_loading + + assert dash_duo.get_logs() == [] + + +def test_async_cbsc002_callbacks_generating_children(dash_duo, *args): + """Modify the DOM tree by adding new components in the callbacks.""" + if not test_async(): + return + # some components don't exist in the initial render + app = Dash(__name__, suppress_callback_exceptions=True) + app.layout = html.Div( + [dcc.Input(id="input", value="initial value"), html.Div(id="output")] + ) + + @app.callback(Output("output", "children"), [Input("input", "value")]) + async def pad_output(input): + return html.Div( + [ + dcc.Input(id="sub-input-1", value="sub input initial value"), + html.Div(id="sub-output-1"), + ] + ) + + call_count = Value("i", 0) + + @app.callback(Output("sub-output-1", "children"), [Input("sub-input-1", "value")]) + async def update_input(value): + call_count.value += 1 + return value + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#sub-output-1", "sub input initial value") + + assert call_count.value == 1, "called once at initial stage" + + pad_input, pad_div = dash_duo.dash_innerhtml_dom.select_one( + "#output > div" + ).contents + + assert ( + pad_input.attrs["value"] == "sub input initial value" + and pad_input.attrs["id"] == "sub-input-1" + ) + assert pad_input.name == "input" + + assert ( + pad_div.text == pad_input.attrs["value"] and pad_div.get("id") == "sub-output-1" + ), "the sub-output-1 content reflects to sub-input-1 value" + + paths = dash_duo.redux_state_paths + assert paths["objs"] == {} + assert paths["strs"] == { + "input": ["props", "children", 0], + "output": ["props", "children", 1], + "sub-input-1": [ + "props", + "children", + 1, + "props", + "children", + "props", + "children", + 0, + ], + "sub-output-1": [ + "props", + "children", + 1, + "props", + "children", + "props", + "children", + 1, + ], + }, "the paths should include these new output IDs" + + # editing the input should modify the sub output + dash_duo.find_element("#sub-input-1").send_keys("deadbeef") + + # the total updates is initial one + the text input changes + dash_duo.wait_for_text_to_equal( + "#sub-output-1", pad_input.attrs["value"] + "deadbeef" + ) + + assert not dash_duo.redux_state_is_loading, "loadingMap is empty" + + assert dash_duo.get_logs() == [], "console is clean" + + +def test_async_cbsc003_callback_with_unloaded_async_component(dash_duo, *args): + if not test_async(): + return + app = Dash() + app.layout = html.Div( + children=[ + dcc.Tabs( + children=[ + dcc.Tab( + children=[ + html.Button(id="btn", children="Update Input"), + html.Div(id="output", children=["Hello"]), + ] + ), + dcc.Tab(children=dash_table.DataTable(id="other-table")), + ] + ) + ] + ) + + @app.callback(Output("output", "children"), [Input("btn", "n_clicks")]) + async def update_out(n_clicks): + if n_clicks is None: + raise PreventUpdate + + return "Bye" + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#output", "Hello") + dash_duo.find_element("#btn").click() + dash_duo.wait_for_text_to_equal("#output", "Bye") + assert dash_duo.get_logs() == [] + + +def test_async_cbsc004_callback_using_unloaded_async_component(dash_duo, *args): + if not test_async(): + return + app = Dash() + app.layout = html.Div( + [ + dcc.Tabs( + [ + dcc.Tab("boo!"), + dcc.Tab( + dash_table.DataTable( + id="table", + columns=[{"id": "a", "name": "A"}], + data=[{"a": "b"}], + ) + ), + ] + ), + html.Button("Update Input", id="btn"), + html.Div("Hello", id="output"), + html.Div(id="output2"), + ] + ) + + @app.callback( + Output("output", "children"), + [Input("btn", "n_clicks")], + [State("table", "data")], + ) + async def update_out(n_clicks, data): + return json.dumps(data) + " - " + str(n_clicks) + + @app.callback( + Output("output2", "children"), + [Input("btn", "n_clicks")], + [State("table", "derived_viewport_data")], + ) + async def update_out2(n_clicks, data): + return json.dumps(data) + " - " + str(n_clicks) + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#output", '[{"a": "b"}] - None') + dash_duo.wait_for_text_to_equal("#output2", "null - None") + + dash_duo.find_element("#btn").click() + dash_duo.wait_for_text_to_equal("#output", '[{"a": "b"}] - 1') + dash_duo.wait_for_text_to_equal("#output2", "null - 1") + + dash_duo.find_element(".tab:not(.tab--selected)").click() + dash_duo.wait_for_text_to_equal("#table th", "A") + # table props are in state so no change yet + dash_duo.wait_for_text_to_equal("#output2", "null - 1") + + # repeat a few times, since one of the failure modes I saw during dev was + # intermittent - but predictably so? + for i in range(2, 10): + expected = '[{"a": "b"}] - ' + str(i) + dash_duo.find_element("#btn").click() + dash_duo.wait_for_text_to_equal("#output", expected) + # now derived props are available + dash_duo.wait_for_text_to_equal("#output2", expected) + + assert dash_duo.get_logs() == [] + + +@pytest.mark.parametrize("engine", ["json", "orjson"]) +def test_async_cbsc005_children_types(dash_duo, engine): + if not test_async(): + return + with json_engine(engine): + app = Dash() + app.layout = html.Div([html.Button(id="btn"), html.Div("init", id="out")]) + + outputs = [ + [None, ""], + ["a string", "a string"], + [123, "123"], + [123.45, "123.45"], + [[6, 7, 8], "678"], + [["a", "list", "of", "strings"], "alistofstrings"], + [["strings", 2, "numbers"], "strings2numbers"], + [["a string", html.Div("and a div")], "a string\nand a div"], + ] + + @app.callback(Output("out", "children"), [Input("btn", "n_clicks")]) + async def set_children(n): + if n is None or n > len(outputs): + return no_update + return outputs[n - 1][0] + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal("#out", "init") + + for children, text in outputs: + dash_duo.find_element("#btn").click() + dash_duo.wait_for_text_to_equal("#out", text) + + +@pytest.mark.parametrize("engine", ["json", "orjson"]) +def test_async_cbsc006_array_of_objects(dash_duo, engine): + if not test_async(): + return + with json_engine(engine): + app = Dash() + app.layout = html.Div( + [html.Button(id="btn"), dcc.Dropdown(id="dd"), html.Div(id="out")] + ) + + @app.callback(Output("dd", "options"), [Input("btn", "n_clicks")]) + async def set_options(n): + return [{"label": "opt{}".format(i), "value": i} for i in range(n or 0)] + + @app.callback(Output("out", "children"), [Input("dd", "options")]) + async def set_out(opts): + print(repr(opts)) + return len(opts) + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#out", "0") + for i in range(5): + dash_duo.find_element("#btn").click() + dash_duo.wait_for_text_to_equal("#out", str(i + 1)) + dash_duo.select_dcc_dropdown("#dd", "opt{}".format(i)) + + +@pytest.mark.xfail( + condition=werkzeug.__version__ in ("2.1.0", "2.1.1"), + reason="Bug with 204 and Transfer-Encoding", + strict=False, +) +@pytest.mark.parametrize("refresh", [False, True]) +def test_async_cbsc007_parallel_updates(refresh, dash_duo, *args): + # This is a funny case, that seems to mostly happen with dcc.Location + # but in principle could happen in other cases too: + # A callback chain (in this case the initial hydration) is set to update a + # value, but after that callback is queued and before it returns, that value + # is also set explicitly from the front end (in this case Location.pathname, + # which gets set in its componentDidMount during the render process, and + # callbacks are delayed until after rendering is finished because of the + # async table) + # At one point in the wildcard PR #1103, changing from requestQueue to + # pendingCallbacks, calling PreventUpdate in the callback would also skip + # any callbacks that depend on pathname, despite the new front-end-provided + # value. + if not test_async(): + return + app = Dash() + + app.layout = html.Div( + [ + dcc.Location(id="loc", refresh=refresh), + html.Button("Update path", id="btn"), + dash_table.DataTable(id="t", columns=[{"name": "a", "id": "a"}]), + html.Div(id="out"), + ] + ) + + @app.callback(Output("t", "data"), [Input("loc", "pathname")]) + async def set_data(path): + return [{"a": (path or repr(path)) + ":a"}] + + @app.callback( + Output("out", "children"), [Input("loc", "pathname"), Input("t", "data")] + ) + async def set_out(path, data): + return json.dumps(data) + " - " + (path or repr(path)) + + @app.callback(Output("loc", "pathname"), [Input("btn", "n_clicks")]) + async def set_path(n): + if not n: + raise PreventUpdate + + return "/{0}".format(n) + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#out", '[{"a": "/:a"}] - /') + dash_duo.find_element("#btn").click() + # the refresh=True case here is testing that we really do get the right + # pathname, not the prevented default value from the layout. + dash_duo.wait_for_text_to_equal("#out", '[{"a": "/1:a"}] - /1') + if not refresh: + dash_duo.find_element("#btn").click() + dash_duo.wait_for_text_to_equal("#out", '[{"a": "/2:a"}] - /2') + + +def test_async_cbsc008_wildcard_prop_callbacks(dash_duo, *args): + if not test_async(): + return + lock = Lock() + + app = Dash(__name__) + app.layout = html.Div( + [ + dcc.Input(id="input", value="initial value"), + html.Div( + html.Div( + [ + 1.5, + None, + "string", + html.Div( + id="output-1", + **{"data-cb": "initial value", "aria-cb": "initial value"}, + ), + ] + ) + ), + ] + ) + + input_call_count = Value("i", 0) + percy_enabled = Value("b", False) + + def snapshot(name): + percy_enabled.value = os.getenv("PERCY_ENABLE", "") != "" + dash_duo.percy_snapshot(name=name) + percy_enabled.value = False + + @app.callback(Output("output-1", "data-cb"), [Input("input", "value")]) + async def update_data(value): + with lock: + if not percy_enabled.value: + input_call_count.value += 1 + return value + + @app.callback(Output("output-1", "children"), [Input("output-1", "data-cb")]) + async def update_text(data): + return data + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal("#output-1", "initial value") + assert ( + dash_duo.find_element("#output-1").get_attribute("data-cb") == "initial value" + ) + + input1 = dash_duo.find_element("#input") + dash_duo.clear_input(input1) + + for key in "hello world": + with lock: + input1.send_keys(key) + + dash_duo.wait_for_text_to_equal("#output-1", "hello world") + assert dash_duo.find_element("#output-1").get_attribute("data-cb") == "hello world" + + # an initial call, one for clearing the input + # and one for each hello world character + assert input_call_count.value == 2 + len("hello world") + + assert dash_duo.get_logs() == [] + + +def test_async_cbsc009_callback_using_unloaded_async_component_and_graph( + dash_duo, *args +): + if not test_async(): + return + app = Dash(__name__) + app.layout = FragmentComponent( + [ + CollapseComponent([AsyncComponent(id="async", value="A")], id="collapse"), + html.Button("n", id="n"), + DelayedEventComponent(id="d"), + html.Div("Output init", id="output"), + ] + ) + + @app.callback( + Output("output", "children"), + Output("collapse", "display"), + Input("n", "n_clicks"), + Input("d", "n_clicks"), + Input("async", "value"), + ) + async def content(n, d, v): + return json.dumps([n, d, v]), (n or 0) > 1 + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#output", '[null, null, "A"]') + dash_duo.wait_for_element("#d").click() + + dash_duo.wait_for_text_to_equal("#output", '[null, 1, "A"]') + + dash_duo.wait_for_element("#n").click() + dash_duo.wait_for_text_to_equal("#output", '[1, 1, "A"]') + + dash_duo.wait_for_element("#d").click() + dash_duo.wait_for_text_to_equal("#output", '[1, 2, "A"]') + + dash_duo.wait_for_no_elements("#async") + + dash_duo.wait_for_element("#n").click() + dash_duo.wait_for_text_to_equal("#output", '[2, 2, "A"]') + dash_duo.wait_for_text_to_equal("#async", "A") + + assert dash_duo.get_logs() == [] + + +def test_async_cbsc010_event_properties(dash_duo, *args): + if not test_async(): + return + app = Dash(__name__) + app.layout = html.Div([html.Button("Click Me", id="button"), html.Div(id="output")]) + + call_count = Value("i", 0) + + @app.callback(Output("output", "children"), [Input("button", "n_clicks")]) + async def update_output(n_clicks): + if not n_clicks: + raise PreventUpdate + call_count.value += 1 + return "Click" + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal("#output", "") + assert call_count.value == 0 + + dash_duo.find_element("#button").click() + dash_duo.wait_for_text_to_equal("#output", "Click") + assert call_count.value == 1 + + +def test_async_cbsc011_one_call_for_multiple_outputs_initial(dash_duo, *args): + if not test_async(): + return + app = Dash(__name__) + call_count = Value("i", 0) + + app.layout = html.Div( + [ + html.Div( + [ + dcc.Input(value="Input {}".format(i), id="input-{}".format(i)) + for i in range(10) + ] + ), + html.Div(id="container"), + dcc.RadioItems(), + ] + ) + + @app.callback( + Output("container", "children"), + [Input("input-{}".format(i), "value") for i in range(10)], + ) + async def dynamic_output(*args): + call_count.value += 1 + return json.dumps(args) + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal("#input-9", "Input 9") + dash_duo.wait_for_contains_text("#container", "Input 9") + + assert call_count.value == 1 + inputs = [f'"Input {i}"' for i in range(10)] + expected = f'[{", ".join(inputs)}]' + dash_duo.wait_for_text_to_equal("#container", expected) + assert dash_duo.get_logs() == [] + + +def test_async_cbsc012_one_call_for_multiple_outputs_update(dash_duo, *args): + if not test_async(): + return + app = Dash(__name__, suppress_callback_exceptions=True) + call_count = Value("i", 0) + + app.layout = html.Div( + [ + html.Button(id="display-content", children="Display Content"), + html.Div(id="container"), + dcc.RadioItems(), + ] + ) + + @app.callback(Output("container", "children"), Input("display-content", "n_clicks")) + async def display_output(n_clicks): + if not n_clicks: + return "" + return html.Div( + [ + html.Div( + [ + dcc.Input(value="Input {}".format(i), id="input-{}".format(i)) + for i in range(10) + ] + ), + html.Div(id="dynamic-output"), + ] + ) + + @app.callback( + Output("dynamic-output", "children"), + [Input("input-{}".format(i), "value") for i in range(10)], + ) + async def dynamic_output(*args): + call_count.value += 1 + return json.dumps(args) + + dash_duo.start_server(app) + + dash_duo.find_element("#display-content").click() + + dash_duo.wait_for_text_to_equal("#input-9", "Input 9") + + ### order altered from the original, as these are non-blocking callbacks now + inputs = [f'"Input {i}"' for i in range(10)] + expected = f'[{", ".join(inputs)}]' + dash_duo.wait_for_text_to_equal("#dynamic-output", expected) + assert call_count.value == 1 + assert dash_duo.get_logs() == [] + + +def test_async_cbsc013_multi_output_out_of_order(dash_duo, *args): + if not test_async(): + return + app = Dash(__name__) + app.layout = html.Div( + [ + html.Button("Click", id="input", n_clicks=0), + html.Div(id="output1"), + html.Div(id="output2"), + ] + ) + + call_count = Value("i", 0) + lock = Lock() + + @app.callback( + Output("output1", "children"), + Output("output2", "children"), + Input("input", "n_clicks"), + ) + async def update_output(n_clicks): + call_count.value += 1 + if n_clicks == 1: + with lock: + pass + return n_clicks, n_clicks + 1 + + dash_duo.start_server(app) + + button = dash_duo.find_element("#input") + with lock: + button.click() + button.click() + + dash_duo.wait_for_text_to_equal("#output1", "2") + dash_duo.wait_for_text_to_equal("#output2", "3") + assert call_count.value == 3 + assert dash_duo.driver.execute_script("return !window.store.getState().isLoading;") + assert dash_duo.get_logs() == [] + + +def test_async_cbsc014_multiple_properties_update_at_same_time_on_same_component( + dash_duo, +): + if not test_async(): + return + call_count = Value("i", 0) + timestamp_1 = Value("d", -5) + timestamp_2 = Value("d", -5) + + app = Dash(__name__) + app.layout = html.Div( + [ + html.Div(id="container"), + html.Button("Click 1", id="button-1", n_clicks=0, n_clicks_timestamp=-1), + html.Button("Click 2", id="button-2", n_clicks=0, n_clicks_timestamp=-1), + ] + ) + + @app.callback( + Output("container", "children"), + Input("button-1", "n_clicks"), + Input("button-1", "n_clicks_timestamp"), + Input("button-2", "n_clicks"), + Input("button-2", "n_clicks_timestamp"), + ) + async def update_output(n1, t1, n2, t2): + call_count.value += 1 + timestamp_1.value = t1 + timestamp_2.value = t2 + return "{}, {}".format(n1, n2) + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#container", "0, 0") + assert timestamp_1.value == -1 + assert timestamp_2.value == -1 + assert call_count.value == 1 + + dash_duo.find_element("#button-1").click() + dash_duo.wait_for_text_to_equal("#container", "1, 0") + assert timestamp_1.value > ((time.time() - (24 * 60 * 60)) * 1000) + assert timestamp_2.value == -1 + assert call_count.value == 2 + prev_timestamp_1 = timestamp_1.value + + dash_duo.find_element("#button-2").click() + dash_duo.wait_for_text_to_equal("#container", "1, 1") + assert timestamp_1.value == prev_timestamp_1 + assert timestamp_2.value > ((time.time() - 24 * 60 * 60) * 1000) + assert call_count.value == 3 + prev_timestamp_2 = timestamp_2.value + + dash_duo.find_element("#button-2").click() + dash_duo.wait_for_text_to_equal("#container", "1, 2") + assert timestamp_1.value == prev_timestamp_1 + assert timestamp_2.value > prev_timestamp_2 + assert timestamp_2.value > timestamp_1.value + assert call_count.value == 4 + + +def test_async_cbsc015_input_output_callback(dash_duo, *args): + if not test_async(): + return + return + # disabled because this is trying to look at a synchronous lock in an async environment + # + # import asyncio + # lock = asyncio.Lock() + # + # call_count = Value("i", 0) + # + # app = Dash(__name__) + # app.layout = html.Div( + # [html.Div("0", id="input-text"), dcc.Input(id="input", type="number", value=0)] + # ) + # + # @app.callback( + # Output("input", "value"), + # Input("input", "value"), + # ) + # async def circular_output(v): + # ctx = callback_context + # if not ctx.triggered: + # value = v + # else: + # value = v + 1 + # return value + # + # @app.callback( + # Output("input-text", "children"), + # Input("input", "value"), + # ) + # async def follower_output(v): + # async with lock: + # call_count.value = call_count.value + 1 + # return str(v) + # + # dash_duo.start_server(app) + # + # input_ = dash_duo.find_element("#input") + # for key in "2": + # async with lock: + # input_.send_keys(key) + # + # dash_duo.wait_for_text_to_equal("#input-text", "3") + # + # assert call_count.value == 2, "initial + changed once" + # + # assert not dash_duo.redux_state_is_loading + # + # assert dash_duo.get_logs() == [] + + +def test_async_cbsc016_extra_components_callback(dash_duo, *args): + if not test_async(): + return + lock = Lock() + + app = Dash(__name__) + app._extra_components.append(dcc.Store(id="extra-store", data=123)) + + app.layout = html.Div( + [ + dcc.Input(id="input", value="initial value"), + html.Div(html.Div([1.5, None, "string", html.Div(id="output-1")])), + ] + ) + store_data = Value("i", 0) + + @app.callback( + Output("output-1", "children"), + [Input("input", "value"), Input("extra-store", "data")], + ) + async def update_output(value, data): + with lock: + store_data.value = data + return value + + dash_duo.start_server(app) + + dash_duo.wait_for_text_to_equal("#output-1", "initial value") + + input_ = dash_duo.find_element("#input") + dash_duo.clear_input(input_) + input_.send_keys("A") + + dash_duo.wait_for_text_to_equal("#output-1", "A") + + assert store_data.value == 123 + assert dash_duo.get_logs() == [] + + +def test_async_cbsc017_callback_directly_callable(): + if not test_async(): + return + ## unneeded + # app = Dash(__name__) + # app.layout = html.Div( + # [ + # dcc.Input(id="input", value="initial value"), + # html.Div(html.Div([1.5, None, "string", html.Div(id="output-1")])), + # ] + # ) + # + # @app.callback( + # Output("output-1", "children"), + # [Input("input", "value")], + # ) + # async def update_output(value): + # return f"returning {value}" + # + # assert update_output("my-value") == "returning my-value" + + +def test_async_cbsc018_callback_ndarray_output(dash_duo, *args): + if not test_async(): + return + app = Dash(__name__) + app.layout = html.Div([dcc.Store(id="output"), html.Button("click", id="clicker")]) + + @app.callback( + Output("output", "data"), + Input("clicker", "n_clicks"), + ) + async def on_click(_): + return np.array([[1, 2, 3], [4, 5, 6]], np.int32) + + dash_duo.start_server(app) + + assert dash_duo.get_logs() == [] + + +def test_async_cbsc019_callback_running(dash_duo, *args): + if not test_async(): + return + lock = Lock() + app = Dash(__name__) + + app.layout = html.Div( + [ + html.Div("off", id="running"), + html.Button("start", id="start"), + html.Div(id="output"), + ] + ) + + @app.callback( + Output("output", "children"), + Input("start", "n_clicks"), + running=[[Output("running", "children"), html.B("on", id="content"), "off"]], + prevent_initial_call=True, + ) + async def on_click(_): + with lock: + pass + return "done" + + dash_duo.start_server(app) + with lock: + dash_duo.find_element("#start").click() + dash_duo.wait_for_text_to_equal("#content", "on") + + dash_duo.wait_for_text_to_equal("#output", "done") + dash_duo.wait_for_text_to_equal("#running", "off") + + +def test_async_cbsc020_callback_running_non_existing_component(dash_duo, *args): + if not test_async(): + return + lock = Lock() + app = Dash(__name__, suppress_callback_exceptions=True) + + app.layout = html.Div( + [ + html.Button("start", id="start"), + html.Div(id="output"), + ] + ) + + @app.callback( + Output("output", "children"), + Input("start", "n_clicks"), + running=[ + [ + Output("non_existent_component", "children"), + html.B("on", id="content"), + "off", + ] + ], + prevent_initial_call=True, + ) + async def on_click(_): + with lock: + pass + return "done" + + dash_duo.start_server(app) + with lock: + dash_duo.find_element("#start").click() + + dash_duo.wait_for_text_to_equal("#output", "done") + + +def test_async_cbsc021_callback_running_non_existing_component(dash_duo, *args): + if not test_async(): + return + lock = Lock() + app = Dash(__name__) + + app.layout = html.Div( + [ + html.Button("start", id="start"), + html.Div(id="output"), + ] + ) + + @app.callback( + Output("output", "children"), + Input("start", "n_clicks"), + running=[ + [ + Output("non_existent_component", "children"), + html.B("on", id="content"), + "off", + ] + ], + prevent_initial_call=True, + ) + async def on_click(_): + with lock: + pass + return "done" + + dash_duo.start_server( + app, + debug=True, + use_reloader=False, + use_debugger=True, + dev_tools_hot_reload=False, + ) + with lock: + dash_duo.find_element("#start").click() + + dash_duo.wait_for_text_to_equal("#output", "done") + error_title = "ID running component not found in layout" + error_message = [ + "Component defined in running keyword not found in layout.", + 'Component id: "non_existent_component"', + "This ID was used in the callback(s) for Output(s):", + "output.children", + "You can suppress this exception by setting", + "`suppress_callback_exceptions=True`.", + ] + # The error should show twice, once for trying to set running on and once for + # turning it off. + dash_duo.wait_for_text_to_equal(dash_duo.devtools_error_count_locator, "2") + for error in dash_duo.find_elements(".dash-fe-error__title"): + assert error.text == error_title + for error_text in dash_duo.find_elements(".dash-backend-error"): + assert all(line in error_text for line in error_message) diff --git a/tests/integration/async_tests/utils.py b/tests/integration/async_tests/utils.py new file mode 100644 index 0000000000..ab2038f79f --- /dev/null +++ b/tests/integration/async_tests/utils.py @@ -0,0 +1,154 @@ +import os +import shutil +import subprocess +import tempfile +import time +from contextlib import contextmanager + +import psutil +import redis + +from dash.background_callback import DiskcacheManager + +manager = None + + +class TestDiskCacheManager(DiskcacheManager): + def __init__(self, cache=None, cache_by=None, expire=None): + super().__init__(cache=cache, cache_by=cache_by, expire=expire) + self.running_jobs = [] + + def call_job_fn( + self, + key, + job_fn, + args, + context, + ): + pid = super().call_job_fn(key, job_fn, args, context) + self.running_jobs.append(pid) + return pid + + +def get_background_callback_manager(): + """ + Get the long callback mangaer configured by environment variables + """ + if os.environ.get("LONG_CALLBACK_MANAGER", None) == "celery": + from dash.background_callback import CeleryManager + from celery import Celery + import redis + + celery_app = Celery( + __name__, + broker=os.environ.get("CELERY_BROKER"), + backend=os.environ.get("CELERY_BACKEND"), + ) + background_callback_manager = CeleryManager(celery_app) + redis_conn = redis.Redis(host="localhost", port=6379, db=1) + background_callback_manager.test_lock = redis_conn.lock("test-lock") + elif os.environ.get("LONG_CALLBACK_MANAGER", None) == "diskcache": + import diskcache + + cache = diskcache.Cache(os.environ.get("DISKCACHE_DIR")) + background_callback_manager = TestDiskCacheManager(cache) + background_callback_manager.test_lock = diskcache.Lock(cache, "test-lock") + else: + raise ValueError( + "Invalid long callback manager specified as LONG_CALLBACK_MANAGER " + "environment variable" + ) + + global manager + manager = background_callback_manager + + return background_callback_manager + + +def kill(proc_pid): + process = psutil.Process(proc_pid) + for proc in process.children(recursive=True): + proc.kill() + process.kill() + + +@contextmanager +def setup_background_callback_app(manager_name, app_name): + from dash.testing.application_runners import import_app + + if manager_name == "celery": + os.environ["LONG_CALLBACK_MANAGER"] = "celery" + redis_url = os.environ["REDIS_URL"].rstrip("/") + os.environ["CELERY_BROKER"] = f"{redis_url}/0" + os.environ["CELERY_BACKEND"] = f"{redis_url}/1" + + # Clear redis of cached values + redis_conn = redis.Redis(host="localhost", port=6379, db=1) + cache_keys = redis_conn.keys() + if cache_keys: + redis_conn.delete(*cache_keys) + + worker = subprocess.Popen( + [ + "celery", + "-A", + f"tests.integration.async_tests.{app_name}:handle", + "worker", + "-P", + "prefork", + "--concurrency", + "2", + "--loglevel=info", + ], + encoding="utf8", + preexec_fn=os.setpgrp, + stderr=subprocess.PIPE, + ) + # Wait for the worker to be ready, if you cancel before it is ready, the job + # will still be queued. + lines = [] + for line in iter(worker.stderr.readline, ""): + if "ready" in line: + break + lines.append(line) + else: + error = "\n".join(lines) + raise RuntimeError(f"celery failed to start: {error}") + + try: + yield import_app(f"tests.integration.async_tests.{app_name}") + finally: + # Interval may run one more time after settling on final app state + # Sleep for 1 interval of time + time.sleep(0.5) + os.environ.pop("LONG_CALLBACK_MANAGER") + os.environ.pop("CELERY_BROKER") + os.environ.pop("CELERY_BACKEND") + kill(worker.pid) + from dash import page_registry + + page_registry.clear() + + elif manager_name == "diskcache": + os.environ["LONG_CALLBACK_MANAGER"] = "diskcache" + cache_directory = tempfile.mkdtemp(prefix="lc-diskcache-") + print(cache_directory) + os.environ["DISKCACHE_DIR"] = cache_directory + try: + app = import_app(f"tests.integration.async_tests.{app_name}") + yield app + finally: + # Interval may run one more time after settling on final app state + # Sleep for a couple of intervals + time.sleep(2.0) + + if hasattr(manager, "running_jobs"): + for job in manager.running_jobs: + manager.terminate_job(job) + + shutil.rmtree(cache_directory, ignore_errors=True) + os.environ.pop("LONG_CALLBACK_MANAGER") + os.environ.pop("DISKCACHE_DIR") + from dash import page_registry + + page_registry.clear() diff --git a/tests/integration/devtools/test_devtools_error_handling.py b/tests/integration/devtools/test_devtools_error_handling.py index 1a2fcf6373..40d5731202 100644 --- a/tests/integration/devtools/test_devtools_error_handling.py +++ b/tests/integration/devtools/test_devtools_error_handling.py @@ -73,14 +73,14 @@ def test_dveh001_python_errors(dash_duo): assert "Special 2 clicks exception" in error0 assert "in bad_sub" not in error0 # dash and flask part of the traceback not included - assert "%% callback invoked %%" not in error0 + assert "dash.py" not in error0 assert "self.wsgi_app" not in error0 error1 = get_error_html(dash_duo, 1) assert "in update_output" in error1 assert "in bad_sub" in error1 assert "ZeroDivisionError" in error1 - assert "%% callback invoked %%" not in error1 + assert "dash.py" not in error1 assert "self.wsgi_app" not in error1 @@ -109,14 +109,14 @@ def test_dveh006_long_python_errors(dash_duo): assert "in bad_sub" not in error0 # dash and flask part of the traceback ARE included # since we set dev_tools_prune_errors=False - assert "%% callback invoked %%" in error0 + assert "dash.py" in error0 assert "self.wsgi_app" in error0 error1 = get_error_html(dash_duo, 1) assert "in update_output" in error1 assert "in bad_sub" in error1 assert "ZeroDivisionError" in error1 - assert "%% callback invoked %%" in error1 + assert "dash.py" in error1 assert "self.wsgi_app" in error1 diff --git a/tests/integration/renderer/test_request_hooks.py b/tests/integration/renderer/test_request_hooks.py index 7f707cf823..fe7d780c72 100644 --- a/tests/integration/renderer/test_request_hooks.py +++ b/tests/integration/renderer/test_request_hooks.py @@ -1,13 +1,10 @@ import json -import functools import flask import pytest - -from flaky import flaky +from multiprocessing import Value from dash import Dash, Output, Input, html, dcc from dash.types import RendererHooks -from werkzeug.exceptions import HTTPException def test_rdrh001_request_hooks(dash_duo): @@ -200,7 +197,7 @@ def update_output(value): assert dash_duo.get_logs() == [] -@flaky(max_runs=3) +# @flaky(max_runs=3) @pytest.mark.parametrize("expiry_code", [401, 400]) def test_rdrh003_refresh_jwt(expiry_code, dash_duo): app = Dash(__name__) @@ -244,61 +241,49 @@ def test_rdrh003_refresh_jwt(expiry_code, dash_duo): ] ) - @app.callback(Output("output-1", "children"), [Input("input", "value")]) + @app.callback( + Output("output-1", "children"), + [Input("input", "value")], + prevent_initial_call=True, + ) def update_output(value): + jwt_token.value = len(value) + 1 return value - required_jwt_len = 0 + jwt_token = Value("i", 0) # test with an auth layer that requires a JWT with a certain length - def protect_route(func): - @functools.wraps(func) - def wrap(*args, **kwargs): - try: - if flask.request.method == "OPTIONS": - return func(*args, **kwargs) - token = flask.request.headers.environ.get("HTTP_AUTHORIZATION") - if required_jwt_len and ( - not token or len(token) != required_jwt_len + len("Bearer ") - ): - # Read the data to prevent bug with base http server. - flask.request.get_json(silent=True) - flask.abort(expiry_code, description="JWT Expired " + str(token)) - except HTTPException as e: - return e - return func(*args, **kwargs) - - return wrap - - # wrap all API calls with auth. - for name, method in ( - (x, app.server.view_functions[x]) - for x in app.routes - if x in app.server.view_functions - ): - app.server.view_functions[name] = protect_route(method) + @app.server.before_request + def add_auth(): + if flask.request.method != "OPTIONS": + token = flask.request.headers.environ.get("HTTP_AUTHORIZATION") + if jwt_token.value and ( + not token or len(token) != jwt_token.value + len("Bearer ") + ): + # Read the data to prevent bug with base http server. + flask.request.get_json(silent=True) + flask.abort(expiry_code, description="JWT Expired " + str(token)) dash_duo.start_server(app) _in = dash_duo.find_element("#input") dash_duo.clear_input(_in) - required_jwt_len = 1 - - _in.send_keys("fired request") + dash_duo.wait_for_text_to_equal("#output-1", "") - dash_duo.wait_for_text_to_equal("#output-1", "fired request") + _in.send_keys(".") + dash_duo.wait_for_text_to_equal("#output-1", ".") dash_duo.wait_for_text_to_equal("#output-token", ".") - required_jwt_len = 2 - - dash_duo.clear_input(_in) - _in.send_keys("fired request again") - - dash_duo.wait_for_text_to_equal("#output-1", "fired request again") + _in.send_keys(".") + dash_duo.wait_for_text_to_equal("#output-1", "..") dash_duo.wait_for_text_to_equal("#output-token", "..") - assert len(dash_duo.get_logs()) == 2 + _in.send_keys(".") + dash_duo.wait_for_text_to_equal("#output-1", "...") + dash_duo.wait_for_text_to_equal("#output-token", "...") + + assert len(dash_duo.get_logs()) == 3 def test_rdrh004_layout_hooks(dash_duo): diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000000..4c1cc2e5e1 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,8 @@ +def test_async(): + try: + + import asgiref # pylint: disable=unused-import, # noqa: F401 + + return True + except ImportError: + return False