From 6101c80e4d2b6ffe672aa486f541a81c27111163 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Fri, 25 Oct 2024 14:04:55 -0400 Subject: [PATCH 1/3] add upload to attic cache --- nix_fast_build/__init__.py | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/nix_fast_build/__init__.py b/nix_fast_build/__init__.py index 08f1fbb..5790270 100644 --- a/nix_fast_build/__init__.py +++ b/nix_fast_build/__init__.py @@ -84,6 +84,8 @@ class Options: cachix_cache: str | None = None + attic_cache: str | None = None + @property def remote_url(self) -> None | str: if self.remote is None: @@ -97,6 +99,7 @@ class ResultType(enum.Enum): UPLOAD = enum.auto() DOWNLOAD = enum.auto() CACHIX = enum.auto() + ATTIC = enum.auto() @dataclass @@ -183,6 +186,11 @@ async def parse_args(args: list[str]) -> Options: help="Cachix cache to upload to", default=None, ) + parser.add_argument( + "--attic-cache", + help="Attic cache to upload to", + default=None, + ) parser.add_argument( "--no-nom", help="Don't use nix-output-monitor to print build output (default: false)", @@ -318,6 +326,7 @@ async def parse_args(args: list[str]) -> Options: eval_workers=a.eval_workers, copy_to=a.copy_to, cachix_cache=a.cachix_cache, + attic_cache=a.attic_cache, no_link=a.no_link, out_link=a.out_link, result_format=ResultFormat[a.result_format.upper()], @@ -633,6 +642,22 @@ async def upload_cachix( proc = await asyncio.create_subprocess_exec(*cmd) return await proc.wait() + async def upload_attic(self, opts: Options) -> int: + if opts.attic_cache is None: + return 0 + cmd = maybe_remote( + [ + *nix_shell("nixpkgs#attic-client", "attic"), + "push", + opts.attic_cache, + *list(self.outputs.values()), + ], + opts, + ) + logger.debug("run %s", shlex.join(cmd)) + proc = await asyncio.create_subprocess_exec(*cmd) + return await proc.wait() + async def download(self, exit_stack: AsyncExitStack, opts: Options) -> int: if not opts.remote_url or not opts.download: return 0 @@ -757,6 +782,7 @@ async def run_builds( build_queue: QueueWithContext[Job | StopTask], upload_queue: QueueWithContext[Build | StopTask], cachix_queue: QueueWithContext[Build | StopTask], + attic_queue: QueueWithContext[Build | StopTask], download_queue: QueueWithContext[Build | StopTask], results: list[Result], opts: Options, @@ -791,6 +817,7 @@ async def run_builds( upload_queue.put_nowait(build) download_queue.put_nowait(build) cachix_queue.put_nowait(build) + attic_queue.put_nowait(build) async def run_uploads( @@ -842,6 +869,29 @@ async def run_cachix_upload( ) +async def run_attic_upload( + attic_queue: QueueWithContext[Build | StopTask], + results: list[Result], + opts: Options, +) -> int: + while True: + async with attic_queue.get_context() as build: + if isinstance(build, StopTask): + logger.debug("finish attic upload task") + return 0 + start_time = timeit.default_timer() + rc = await build.upload_attic(opts) + results.append( + Result( + result_type=ResultType.ATTIC, + attr=build.attr, + success=rc == 0, + duration=start_time - timeit.default_timer(), + error=f"attic upload exited with {rc}" if rc != 0 else None, + ) + ) + + async def run_downloads( stack: AsyncExitStack, download_queue: QueueWithContext[Build | StopTask], @@ -915,6 +965,7 @@ async def run(stack: AsyncExitStack, opts: Options) -> int: results: list[Result] = [] build_queue: QueueWithContext[Job | StopTask] = QueueWithContext() cachix_queue: QueueWithContext[Build | StopTask] = QueueWithContext() + attic_queue: QueueWithContext[Build | StopTask] = QueueWithContext() upload_queue: QueueWithContext[Build | StopTask] = QueueWithContext() download_queue: QueueWithContext[Build | StopTask] = QueueWithContext() @@ -937,6 +988,7 @@ async def run(stack: AsyncExitStack, opts: Options) -> int: build_queue, upload_queue, cachix_queue, + attic_queue, download_queue, results, opts, @@ -961,6 +1013,16 @@ async def run(stack: AsyncExitStack, opts: Options) -> int: name=f"cachix-{i}", ) ) + tasks.append( + tg.create_task( + run_attic_upload( + attic_queue, + results, + opts, + ), + name=f"attic-{i}", + ) + ) tasks.append( tg.create_task( run_downloads(stack, download_queue, results, opts), @@ -993,6 +1055,11 @@ async def run(stack: AsyncExitStack, opts: Options) -> int: cachix_queue.put_nowait(StopTask()) await cachix_queue.join() + logger.debug("Uploads finished, waiting for attic uploads to finish...") + for _ in range(opts.max_jobs): + attic_queue.put_nowait(StopTask()) + await attic_queue.join() + logger.debug("Uploads finished, waiting for downloads to finish...") for _ in range(opts.max_jobs): download_queue.put_nowait(StopTask()) From 6b119b82d967b41ea3deb8a99ba4a99080d6a425 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Fri, 25 Oct 2024 14:40:06 -0400 Subject: [PATCH 2/3] only push 'out' --- nix_fast_build/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix_fast_build/__init__.py b/nix_fast_build/__init__.py index 5790270..8b6e78c 100644 --- a/nix_fast_build/__init__.py +++ b/nix_fast_build/__init__.py @@ -650,7 +650,7 @@ async def upload_attic(self, opts: Options) -> int: *nix_shell("nixpkgs#attic-client", "attic"), "push", opts.attic_cache, - *list(self.outputs.values()), + self.outputs["out"], ], opts, ) From f41bcd6843e09934d67146c8749913f52f1cd3f7 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Fri, 25 Oct 2024 15:13:00 -0400 Subject: [PATCH 3/3] add docs --- README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c108766..1cb4343 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,19 @@ set. These environment variables are currently not propagated to ssh when using the `--remote` flag, instead the user is expected that cachix credentials are configured on the remote machine. +## Attic support + +nix-fast-build can upload to attic like this: + +```console +$ nix-fast-build --attic-cache mic92 +``` + +nix-fast-build assumes that your current machine is either logged in to attic. +Authentication is not propagated to ssh when using the `--remote` flag, instead +the user is expected that attic credentials are configured on the remote +machine. + ## Machine-readable builds results nix-fast-build supports both its own json format and junit: @@ -175,7 +188,8 @@ nix-shell -p python3Packages.junit2html --run 'junit2html result.xml result.html ```console usage: nix-fast-build [-h] [-f FLAKE] [-j MAX_JOBS] [--option name value] [--remote-ssh-option name value] - [--cachix-cache CACHIX_CACHE] [--no-nom] + [--cachix-cache CACHIX_CACHE] + [--attic-cache ATTIC_CACHE] [--no-nom] [--systems SYSTEMS] [--retries RETRIES] [--no-link] [--out-link OUT_LINK] [--remote REMOTE] [--always-upload-source] [--no-download] [--skip-cached] @@ -197,6 +211,8 @@ options: ssh option when accessing remote --cachix-cache CACHIX_CACHE Cachix cache to upload to + --attic-cache ATTIC_CACHE + Attic cache to upload to --no-nom Don't use nix-output-monitor to print build output (default: false) --systems SYSTEMS Space-separated list of systems to build for (default: