From 71701e19cea59c1bf069a77cba0ff64ed66c2766 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Mon, 10 Sep 2018 11:58:06 -0700 Subject: [PATCH 01/15] Add ClockEventLoop class with fixture and test (close #95) --- pytest_asyncio/plugin.py | 26 ++++++++++++++++++++++++++ tests/test_simple_35.py | 19 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index 1cf4b0b7..e3b9f85c 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -159,6 +159,23 @@ def pytest_runtest_setup(item): item.fixturenames.append(fixture) +class ClockEventLoop(asyncio.SelectorEventLoop): + _now = 0 + + def time(self): + return self._now + + def advance_time(self, amount): + # advance the clock and run the loop + self._now += amount + self._run_once() + + if amount > 0: + # Once advanced, new tasks may have just been scheduled for running + # in the next loop, advance once more to start these handlers + self._run_once() + + # maps marker to the name of the event loop fixture that will be available # to marked test functions _markers_2_fixtures = { @@ -174,6 +191,15 @@ def event_loop(request): loop.close() +@pytest.yield_fixture +def clock_event_loop(request): + """Create an instance of the default event loop for each test case.""" + loop = ClockEventLoop() + asyncio.get_event_loop_policy().set_event_loop(loop) + yield loop + loop.close() + + @pytest.fixture def unused_tcp_port(): """Find an unused localhost TCP port from 1024-65535 and return it.""" diff --git a/tests/test_simple_35.py b/tests/test_simple_35.py index 1e4d697c..40f0a620 100644 --- a/tests/test_simple_35.py +++ b/tests/test_simple_35.py @@ -86,3 +86,22 @@ async def test_asyncio_marker_method(self, event_loop): def test_async_close_loop(event_loop): event_loop.close() return 'ok' + + +def test_clock_loop(clock_event_loop): + assert clock_event_loop.time() == 0 + + async def foo(): + await asyncio.sleep(1) + + # create the task + task = clock_event_loop.create_task(foo()) + assert not task.done() + + # start the task + clock_event_loop.advance_time(0) + assert not task.done() + + # process the timeout + clock_event_loop.advance_time(1) + assert task.done() From 3d340374d7a92c5ebdbbade0acf43271ad42e89f Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Mon, 10 Sep 2018 12:01:24 -0700 Subject: [PATCH 02/15] add doc string to ClockEventLoop --- pytest_asyncio/plugin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index e3b9f85c..7ada1a4d 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -160,6 +160,9 @@ def pytest_runtest_setup(item): class ClockEventLoop(asyncio.SelectorEventLoop): + """ + A custom event loop that explicitly advances time when requested. + """ _now = 0 def time(self): From a723020e90d4ddf16e5f834fb0728e3e69fb0d58 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Fri, 4 Jan 2019 13:47:42 -0800 Subject: [PATCH 03/15] improve name of nap function in test_clock_loop --- tests/test_simple_35.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_simple_35.py b/tests/test_simple_35.py index 40f0a620..4bbfe797 100644 --- a/tests/test_simple_35.py +++ b/tests/test_simple_35.py @@ -91,11 +91,11 @@ def test_async_close_loop(event_loop): def test_clock_loop(clock_event_loop): assert clock_event_loop.time() == 0 - async def foo(): + async def short_nap(): await asyncio.sleep(1) # create the task - task = clock_event_loop.create_task(foo()) + task = clock_event_loop.create_task(short_nap()) assert not task.done() # start the task From 997afe20482e3e4eab47a1328d58c9a5ac8e54f8 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Fri, 4 Jan 2019 14:44:12 -0800 Subject: [PATCH 04/15] fix ClockEventLoop being a selector loop, instead use default of asyncio --- pytest_asyncio/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index 7ada1a4d..1308bd97 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -159,7 +159,7 @@ def pytest_runtest_setup(item): item.fixturenames.append(fixture) -class ClockEventLoop(asyncio.SelectorEventLoop): +class ClockEventLoop(asyncio.new_event_loop().__class__): """ A custom event loop that explicitly advances time when requested. """ From 6fbeb608ea94475008baa2444d1dd60a110d4fa5 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Fri, 4 Jan 2019 14:44:57 -0800 Subject: [PATCH 05/15] refactor ClockEventLoop to only advance time but otherwise act as standard loop improve documentation of ClockEventLoop --- pytest_asyncio/plugin.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index 1308bd97..e28ab691 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -161,22 +161,35 @@ def pytest_runtest_setup(item): class ClockEventLoop(asyncio.new_event_loop().__class__): """ - A custom event loop that explicitly advances time when requested. + A custom event loop that explicitly advances time when requested. Otherwise, + this event loop advances time as expected. """ - _now = 0 + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._offset = 0 def time(self): - return self._now - - def advance_time(self, amount): + """ + Return the time according the event loop's clock. + + This time is adjusted by the stored offset that allows for advancement + with `advance_time`. + """ + return super().time() + self._offset + + async def advance_time(self, seconds): + ''' + Advance time by a given offset in seconds. + ''' + if seconds < 0: + raise ValueError('cannot go backwards in time') # advance the clock and run the loop - self._now += amount - self._run_once() - - if amount > 0: - # Once advanced, new tasks may have just been scheduled for running - # in the next loop, advance once more to start these handlers - self._run_once() + self._offset += seconds + # Once advanced, new tasks may have just been scheduled for running + # in the next loop, advance once more to start these handlers + await asyncio.sleep(0) + await asyncio.sleep(0) + await asyncio.sleep(0) # maps marker to the name of the event loop fixture that will be available From 7d94f1425a291b71fdc6669190e4fbb96f639146 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Fri, 4 Jan 2019 14:46:01 -0800 Subject: [PATCH 06/15] add asyncio_clock mark so that coroutines can be run in a ClockEventLoop explicitly --- pytest_asyncio/plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index e28ab691..e75d9363 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -196,6 +196,7 @@ async def advance_time(self, seconds): # to marked test functions _markers_2_fixtures = { 'asyncio': 'event_loop', + 'asyncio_clock': 'clock_event_loop', } From 8841d15c562f551660869215462328d3e2bf59d2 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Fri, 4 Jan 2019 14:46:59 -0800 Subject: [PATCH 07/15] fix test_clock_advance_time for changes to ClockEventLoop interface --- tests/test_simple_35.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_simple_35.py b/tests/test_simple_35.py index 4bbfe797..18b8fc21 100644 --- a/tests/test_simple_35.py +++ b/tests/test_simple_35.py @@ -88,9 +88,12 @@ def test_async_close_loop(event_loop): return 'ok' -def test_clock_loop(clock_event_loop): - assert clock_event_loop.time() == 0 +@pytest.mark.asyncio_clock +async def test_clock_loop_advance_time(clock_event_loop): + """ + Test the sliding time event loop fixture + """ async def short_nap(): await asyncio.sleep(1) @@ -99,9 +102,9 @@ async def short_nap(): assert not task.done() # start the task - clock_event_loop.advance_time(0) + await clock_event_loop.advance_time(0) assert not task.done() # process the timeout - clock_event_loop.advance_time(1) + await clock_event_loop.advance_time(1) assert task.done() From 98c6bf88ce2c15361af3c5e3c11a89c28fc9e073 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Fri, 4 Jan 2019 14:47:25 -0800 Subject: [PATCH 08/15] add simple tests for asyncio_clock mark and clock_event_loop fixture --- tests/test_simple_35.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_simple_35.py b/tests/test_simple_35.py index 18b8fc21..05628e48 100644 --- a/tests/test_simple_35.py +++ b/tests/test_simple_35.py @@ -88,6 +88,24 @@ def test_async_close_loop(event_loop): return 'ok' +@pytest.mark.asyncio_clock +async def test_mark_asyncio_clock(): + """ + Test that coroutines marked with asyncio_clock are run with a ClockEventLoop + """ + import pytest_asyncio + assert isinstance(asyncio.get_event_loop(), pytest_asyncio.plugin.ClockEventLoop) + + +def test_clock_loop_loop_fixture(clock_event_loop): + """ + Test that the clock_event_loop fixture returns a proper instance of the loop + """ + import pytest_asyncio + assert isinstance(asyncio.get_event_loop(), pytest_asyncio.plugin.ClockEventLoop) + clock_event_loop.close() + return 'ok' + @pytest.mark.asyncio_clock async def test_clock_loop_advance_time(clock_event_loop): From ce6a2c7fe38fe4d362996d2ff130b2338703c5ad Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Fri, 4 Jan 2019 14:59:55 -0800 Subject: [PATCH 09/15] rearrange ClockEventLoop.advance_time so that extra iterations are not run unless needed --- pytest_asyncio/plugin.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index d8485f56..f9defce5 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -213,14 +213,20 @@ async def advance_time(self, seconds): ''' if seconds < 0: raise ValueError('cannot go backwards in time') - # advance the clock and run the loop + + # advance the clock by the given offset self._offset += seconds - # Once advanced, new tasks may have just been scheduled for running - # in the next loop, advance once more to start these handlers - await asyncio.sleep(0) - await asyncio.sleep(0) + + # ensure waiting callbacks are run before advancing the clock await asyncio.sleep(0) + if seconds > 0: + # Once the clock is adjusted, new tasks may have just been scheduled for running + # in the next pass through the event loop and advance again for the task + # that calls `advance_time` + await asyncio.sleep(0) + await asyncio.sleep(0) + # maps marker to the name of the event loop fixture that will be available # to marked test functions From 1a8b887b8a898f79da1bd01e8ce171c70e342e74 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Fri, 4 Jan 2019 15:36:54 -0800 Subject: [PATCH 10/15] calls to asyncio.sleep in ClockEventLoop force the sleeping loop --- pytest_asyncio/plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index f9defce5..079b5c28 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -218,14 +218,14 @@ async def advance_time(self, seconds): self._offset += seconds # ensure waiting callbacks are run before advancing the clock - await asyncio.sleep(0) + await asyncio.sleep(0, loop=self) if seconds > 0: # Once the clock is adjusted, new tasks may have just been scheduled for running # in the next pass through the event loop and advance again for the task # that calls `advance_time` - await asyncio.sleep(0) - await asyncio.sleep(0) + await asyncio.sleep(0, loop=self) + await asyncio.sleep(0, loop=self) # maps marker to the name of the event loop fixture that will be available From f3efb64e91499a25f4377e694fc566ee32399a98 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Mon, 7 Jan 2019 11:26:02 -0800 Subject: [PATCH 11/15] remove ValueError from ClockEventLoop.advance_time in favor of returnning immediately --- pytest_asyncio/plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index 079b5c28..f1c72c6a 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -212,7 +212,8 @@ async def advance_time(self, seconds): Advance time by a given offset in seconds. ''' if seconds < 0: - raise ValueError('cannot go backwards in time') + # cannot go backwards in time, so return immediately + return # advance the clock by the given offset self._offset += seconds From 8452d4184dd15a65a8f2906677e866013a446acf Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Tue, 8 Jan 2019 16:34:36 -0800 Subject: [PATCH 12/15] Fixes for given questions posed - ClockEventLoop is developed in a function to allow for later changes to event loop policies - rework `advance_time` method to better symbolize the needed loop iterations. This is marked by a change to a function that returns an awaitable. --- pytest_asyncio/plugin.py | 94 ++++++++++++++++++++++------------------ tests/test_simple_35.py | 14 +++--- 2 files changed, 59 insertions(+), 49 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index f1c72c6a..fe2c387a 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -189,46 +189,6 @@ def pytest_runtest_setup(item): ) -class ClockEventLoop(asyncio.new_event_loop().__class__): - """ - A custom event loop that explicitly advances time when requested. Otherwise, - this event loop advances time as expected. - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._offset = 0 - - def time(self): - """ - Return the time according the event loop's clock. - - This time is adjusted by the stored offset that allows for advancement - with `advance_time`. - """ - return super().time() + self._offset - - async def advance_time(self, seconds): - ''' - Advance time by a given offset in seconds. - ''' - if seconds < 0: - # cannot go backwards in time, so return immediately - return - - # advance the clock by the given offset - self._offset += seconds - - # ensure waiting callbacks are run before advancing the clock - await asyncio.sleep(0, loop=self) - - if seconds > 0: - # Once the clock is adjusted, new tasks may have just been scheduled for running - # in the next pass through the event loop and advance again for the task - # that calls `advance_time` - await asyncio.sleep(0, loop=self) - await asyncio.sleep(0, loop=self) - - # maps marker to the name of the event loop fixture that will be available # to marked test functions _markers_2_fixtures = { @@ -245,10 +205,62 @@ def event_loop(request): loop.close() +def _clock_event_loop_class(): + """ + Create a new class for ClockEventLoop based on the current + class-type produced by `asyncio.new_event_loop()`. This is important + for instances in which the enent-loop-policy has been changed. + """ + class ClockEventLoop(asyncio.new_event_loop().__class__): + """ + A custom event loop that explicitly advances time when requested. Otherwise, + this event loop advances time as expected. + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._offset = 0 + + def time(self): + """ + Return the time according the event loop's clock. + + This time is adjusted by the stored offset that allows for advancement + with `advance_time`. + """ + return super().time() + self._offset + + def advance_time(self, seconds): + ''' + Advance time by a given offset in seconds. Returns an awaitable + that will complete after all tasks scheduled for after advancement + of time are proceeding. + ''' + if seconds <= 0: + # cannot go backwards in time, so return after one iteration of a loop + return asyncio.sleep(0) + + # Add a task associated with iterating the currently "ready" tasks and handles + # + # NOTE: This can actually take place after the offset changed, but + # it is here to highlight that the loop is for currently ready + # items before offset is applied + self.create_task(asyncio.sleep(0)) + + # advance the clock by the given offset + self._offset += seconds + + # Once the clock is adjusted, new tasks may have just been + # scheduled for running in the next pass through the event loop and + # advance again for the newly ready tasks + return self.create_task(asyncio.sleep(0)) + + return ClockEventLoop + + @pytest.yield_fixture def clock_event_loop(request): """Create an instance of the default event loop for each test case.""" - loop = ClockEventLoop() + loop = _clock_event_loop_class()() asyncio.get_event_loop_policy().set_event_loop(loop) yield loop loop.close() diff --git a/tests/test_simple_35.py b/tests/test_simple_35.py index 05628e48..cfca3e6e 100644 --- a/tests/test_simple_35.py +++ b/tests/test_simple_35.py @@ -93,16 +93,14 @@ async def test_mark_asyncio_clock(): """ Test that coroutines marked with asyncio_clock are run with a ClockEventLoop """ - import pytest_asyncio - assert isinstance(asyncio.get_event_loop(), pytest_asyncio.plugin.ClockEventLoop) + assert hasattr(asyncio.get_event_loop(), 'advance_time') def test_clock_loop_loop_fixture(clock_event_loop): """ Test that the clock_event_loop fixture returns a proper instance of the loop """ - import pytest_asyncio - assert isinstance(asyncio.get_event_loop(), pytest_asyncio.plugin.ClockEventLoop) + assert hasattr(asyncio.get_event_loop(), 'advance_time') clock_event_loop.close() return 'ok' @@ -112,11 +110,11 @@ async def test_clock_loop_advance_time(clock_event_loop): """ Test the sliding time event loop fixture """ - async def short_nap(): - await asyncio.sleep(1) + # a timeout for operations using advance_time + NAP_TIME = 10 # create the task - task = clock_event_loop.create_task(short_nap()) + task = clock_event_loop.create_task(asyncio.sleep(NAP_TIME)) assert not task.done() # start the task @@ -124,5 +122,5 @@ async def short_nap(): assert not task.done() # process the timeout - await clock_event_loop.advance_time(1) + await clock_event_loop.advance_time(NAP_TIME) assert task.done() From 559e8838086067b9b21052603711467d472762fc Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Tue, 8 Jan 2019 22:53:25 -0800 Subject: [PATCH 13/15] further simplify advance_time method --- pytest_asyncio/plugin.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index fe2c387a..c058f7e1 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -237,14 +237,7 @@ def advance_time(self, seconds): ''' if seconds <= 0: # cannot go backwards in time, so return after one iteration of a loop - return asyncio.sleep(0) - - # Add a task associated with iterating the currently "ready" tasks and handles - # - # NOTE: This can actually take place after the offset changed, but - # it is here to highlight that the loop is for currently ready - # items before offset is applied - self.create_task(asyncio.sleep(0)) + return self.create_task(asyncio.sleep(0)) # advance the clock by the given offset self._offset += seconds From 1d3cd5189437ca52c863225b171d8b582eeff333 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Tue, 8 Jan 2019 23:55:03 -0800 Subject: [PATCH 14/15] rework advance_time to reduce complexity --- pytest_asyncio/plugin.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index c058f7e1..9eff17e0 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -235,16 +235,12 @@ def advance_time(self, seconds): that will complete after all tasks scheduled for after advancement of time are proceeding. ''' - if seconds <= 0: - # cannot go backwards in time, so return after one iteration of a loop - return self.create_task(asyncio.sleep(0)) - - # advance the clock by the given offset - self._offset += seconds + if seconds > 0: + # advance the clock by the given offset + self._offset += seconds # Once the clock is adjusted, new tasks may have just been - # scheduled for running in the next pass through the event loop and - # advance again for the newly ready tasks + # scheduled for running in the next pass through the event loop return self.create_task(asyncio.sleep(0)) return ClockEventLoop From ed8b05d7d0d64047c1e7c3efa5a71d805d450162 Mon Sep 17 00:00:00 2001 From: "Derek A. Thomas" Date: Tue, 8 Jan 2019 23:56:36 -0800 Subject: [PATCH 15/15] make the clock_event_loop test easier to read --- tests/test_simple_35.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_simple_35.py b/tests/test_simple_35.py index cfca3e6e..d148521a 100644 --- a/tests/test_simple_35.py +++ b/tests/test_simple_35.py @@ -110,11 +110,11 @@ async def test_clock_loop_advance_time(clock_event_loop): """ Test the sliding time event loop fixture """ - # a timeout for operations using advance_time - NAP_TIME = 10 + # A task is created that will sleep some number of seconds + SLEEP_TIME = 10 # create the task - task = clock_event_loop.create_task(asyncio.sleep(NAP_TIME)) + task = clock_event_loop.create_task(asyncio.sleep(SLEEP_TIME)) assert not task.done() # start the task @@ -122,5 +122,5 @@ async def test_clock_loop_advance_time(clock_event_loop): assert not task.done() # process the timeout - await clock_event_loop.advance_time(NAP_TIME) + await clock_event_loop.advance_time(SLEEP_TIME) assert task.done()