From b30e2742510c6c8fa930badbad8b0fb5721afa7a Mon Sep 17 00:00:00 2001 From: toxic-recker <62395124+toxicrecker@users.noreply.github.com> Date: Sat, 7 Jan 2023 17:15:34 +0530 Subject: [PATCH 1/6] Added support for moving games between disks Used `[shutil.move()](https://docs.python.org/3/library/shutil.html#shutil.move)` instead of `os.rename()` to add support for moving games to different disks. --- legendary/cli.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 79cd7309..70f1c68c 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -6,6 +6,7 @@ import json import logging import os +import shutil import shlex import subprocess import time @@ -2507,20 +2508,28 @@ def move(self, args): old_base, game_folder = os.path.split(igame.install_path.replace('\\', '/')) new_path = os.path.join(args.new_path, game_folder) logger.info(f'Moving "{game_folder}" from "{old_base}" to "{args.new_path}"') - if not args.skip_move: + if not os.path.exists(args.new_path): + os.makedirs(args.new_path) + if os.stat(args.new_path).st_dev != os.stat(igame.install_path).st_dev: + choice = get_boolean_choice(f'Moving files to another drive can cause errors when running the game, ' + f'however, it is unlikely to happen. Do you still wish to continue? (default=no)', default=False) + if not choice: + logger.info(f'To move it without a risk move the folder manually to {new_path} ' + f'and run "legendary move {app_name} "{args.new_path}" --skip-move"') + print("Aborting...") + return + if os.path.exists(new_path): + logger.error(f'The target path already contains a folder called "{game_folder}", ' + f'please remove or rename it first.') + return try: - if not os.path.exists(args.new_path): - os.makedirs(args.new_path) - - os.rename(igame.install_path, new_path) + shutil.move(igame.install_path, new_path) except Exception as e: - if isinstance(e, OSError) and e.errno == 18: - logger.error(f'Moving to a different drive is not supported. Move the folder manually to ' - f'"{new_path}" and run "legendary move {app_name} "{args.new_path}" --skip-move"') - elif isinstance(e, FileExistsError): - logger.error(f'The target path already contains a folder called "{game_folder}", ' - f'please remove or rename it first.') + if isinstance(e, shutil.Error): + logger.error(f'Cannot move the folder into itself.') + elif isinstance(e, PermissionError): + logger.error(f'Cannot move the directory "{igame.install_path}", lacking write permission to it.') else: logger.error(f'Moving failed with unknown error {e!r}.') logger.info(f'Try moving the folder manually to "{new_path}" and running ' From 8a95e635cd56e72a51055cd24d38a1cf505be42c Mon Sep 17 00:00:00 2001 From: toxic-recker <62395124+toxicrecker@users.noreply.github.com> Date: Mon, 9 Jan 2023 21:42:16 +0530 Subject: [PATCH 2/6] Added `scan_dir` function This is a helper function for implementing a progress indication in the moving games between disks functionality --- legendary/utils/cli.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/legendary/utils/cli.py b/legendary/utils/cli.py index c0fe14a7..bad3a597 100644 --- a/legendary/utils/cli.py +++ b/legendary/utils/cli.py @@ -1,3 +1,5 @@ +import os + def get_boolean_choice(prompt, default=True): yn = 'Y/n' if default else 'y/N' @@ -89,3 +91,15 @@ def strtobool(val): else: raise ValueError("invalid truth value %r" % (val,)) +def scan_dir(src): + files = 0 + chunks = 0 + for entry in os.scandir(src): + if entry.is_dir(): + cnt, sz = scan_dir(entry.path) + files += cnt + chunks += sz + else: + files += 1 + chunks += entry.stat().st_size + return files, chunks From eec5b9ef69f4ee4e36ec40722dfde930c1030993 Mon Sep 17 00:00:00 2001 From: toxic-recker <62395124+toxicrecker@users.noreply.github.com> Date: Mon, 9 Jan 2023 21:43:45 +0530 Subject: [PATCH 3/6] Added progress indication for moving games Moving between disks can take a long time and before there was no progress indication so the user might think that the program was stuck but I have added it now --- legendary/cli.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 70f1c68c..41e422a5 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -22,7 +22,7 @@ from legendary.core import LegendaryCore from legendary.models.exceptions import InvalidCredentialsError from legendary.models.game import SaveGameStatus, VerifyResult, Game -from legendary.utils.cli import get_boolean_choice, get_int_choice, sdl_prompt, strtobool +from legendary.utils.cli import get_boolean_choice, get_int_choice, sdl_prompt, strtobool, scan_dir from legendary.lfs.crossover import * from legendary.utils.custom_parser import HiddenAliasSubparsersAction from legendary.utils.env import is_windows_mac_or_pyi @@ -2524,7 +2524,47 @@ def move(self, args): f'please remove or rename it first.') return try: - shutil.move(igame.install_path, new_path) + total_files, total_chunks = scan_dir(igame.install_path) + copied_files = 0 + last_copied_chunks = 0 + copied_chunks = 0 + last = start = time.perf_counter() + def copy_function(src, dst, *, follow_symlinks=True): + nonlocal total_files, copied_files + nonlocal total_chunks, last_copied_chunks, copied_chunks + nonlocal last, start + + shutil.copy2(src, dst, follow_symlinks=follow_symlinks) + size = os.path.getsize(dst) + last_copied_chunks += size + copied_chunks += size + copied_files += 1 + + now = time.perf_counter() + runtime = now - start + delta = now - last + if delta < 0.5 and copied_files < total_files: # to prevent spamming the console + return + + last = now + speed = last_copied_chunks / delta + last_copied_chunks = 0 + perc = copied_files / total_files * 100 + + average_speed = copied_chunks / runtime + estimate = (total_chunks - copied_chunks) / average_speed + minutes, seconds = int(estimate//60), int(estimate%60) + hours, minutes = int(minutes//60), int(minutes%60) + + rt_minutes, rt_seconds = int(runtime//60), int(runtime%60) + rt_hours, rt_minutes = int(rt_minutes//60), int(rt_minutes%60) + + logger.info(f'= Progress: {perc:.02f}% ({copied_files}/{total_files}), ') + logger.info(f' + Running for {rt_hours:02d}:{rt_minutes:02d}:{rt_seconds:02d}, ') + logger.info(f' + ETA: {hours:02d}:{minutes:02d}:{seconds:02d}') + logger.info(f' + Speed: {speed / 1024 / 1024:.02f} MiB/s') + + shutil.move(igame.install_path, new_path, copy_function=copy_function) except Exception as e: if isinstance(e, shutil.Error): logger.error(f'Cannot move the folder into itself.') From c96afd76f946212e7c67a376ec2b5c8064459350 Mon Sep 17 00:00:00 2001 From: toxic-recker <62395124+toxicrecker@users.noreply.github.com> Date: Mon, 9 Jan 2023 22:00:05 +0530 Subject: [PATCH 4/6] Removed warning --- legendary/cli.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 41e422a5..7f0fabc6 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -2511,14 +2511,6 @@ def move(self, args): if not args.skip_move: if not os.path.exists(args.new_path): os.makedirs(args.new_path) - if os.stat(args.new_path).st_dev != os.stat(igame.install_path).st_dev: - choice = get_boolean_choice(f'Moving files to another drive can cause errors when running the game, ' - f'however, it is unlikely to happen. Do you still wish to continue? (default=no)', default=False) - if not choice: - logger.info(f'To move it without a risk move the folder manually to {new_path} ' - f'and run "legendary move {app_name} "{args.new_path}" --skip-move"') - print("Aborting...") - return if os.path.exists(new_path): logger.error(f'The target path already contains a folder called "{game_folder}", ' f'please remove or rename it first.') From b2e9f6ae81e1393ec30cd257740f3a06fb7bb3b4 Mon Sep 17 00:00:00 2001 From: toxic-recker <62395124+toxicrecker@users.noreply.github.com> Date: Mon, 9 Jan 2023 23:55:34 +0530 Subject: [PATCH 5/6] Handled user interruption during the process of moving a game --- legendary/cli.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/legendary/cli.py b/legendary/cli.py index 7f0fabc6..4c9176a5 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -2515,6 +2515,10 @@ def move(self, args): logger.error(f'The target path already contains a folder called "{game_folder}", ' f'please remove or rename it first.') return + logger.info('This process could be cancelled and reverted by interrupting it with CTRL-C') + if not get_boolean_choice(f'Are you sure you wish to move "{igame.title}" from "{old_base}" to "{args.new_path}"?'): + print('Aborting...') + exit(0) try: total_files, total_chunks = scan_dir(igame.install_path) copied_files = 0 @@ -2567,6 +2571,11 @@ def copy_function(src, dst, *, follow_symlinks=True): logger.info(f'Try moving the folder manually to "{new_path}" and running ' f'"legendary move {app_name} "{args.new_path}" --skip-move"') return + except KeyboardInterrupt: + # TODO: Make it resumable + shutil.rmtree(new_path) + logger.info("The process has been cancelled.") + return else: logger.info(f'Not moving, just rewriting legendary metadata...') From ed1afbd367e78e3604dbf7a704db94ace0099898 Mon Sep 17 00:00:00 2001 From: toxic-recker <62395124+toxicrecker@users.noreply.github.com> Date: Wed, 1 Feb 2023 18:40:22 +0530 Subject: [PATCH 6/6] Added ability to cancel and resume moving games And cleaned up the code a bit --- legendary/cli.py | 115 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 4c9176a5..dcd701a1 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -2511,60 +2511,102 @@ def move(self, args): if not args.skip_move: if not os.path.exists(args.new_path): os.makedirs(args.new_path) - if os.path.exists(new_path): - logger.error(f'The target path already contains a folder called "{game_folder}", ' - f'please remove or rename it first.') + if shutil._destinsrc(igame.install_path, new_path): + logger.error(f'Cannot move the folder into itself.') + return + if (shutil._is_immutable(igame.install_path) or (not os.access(igame.install_path, os.W_OK) \ + and os.listdir(igame.install_path) and sys_platform == 'darwin')): + logger.error(f'Cannot move the directory "{igame.install_path}", lacking write permission to it.') return - logger.info('This process could be cancelled and reverted by interrupting it with CTRL-C') + existing_files = dict(__total__ = 0) + existing_chunks = 0 + if os.path.exists(new_path): + if not get_boolean_choice(f'"{game_folder}" is found in the target path, would you like to resume moving this game?'): + return + logger.info('Attempting to resume the process... DO NOT PANIC IF IT LOOKS STUCK') + manifest_data, _ = self.core.get_installed_manifest(igame.app_name) + if manifest_data is None: + logger.critical(f'Manifest appears to be missing! To repair, run "legendary repair ' + f'{args.app_name} --repair-and-update", this will however redownload all files ' + f'that do not match the latest manifest in their entirety.') + return + manifest = self.core.load_manifest(manifest_data) + files = manifest.file_manifest_list.elements + if config_tags := self.core.lgd.config.get(args.app_name, 'install_tags', fallback=None): + install_tags = set(i.strip() for i in config_tags.split(',')) + file_list = [ + (f.filename, f.sha_hash.hex()) + for f in files + if any(it in install_tags for it in f.install_tags) or not f.install_tags + ] + else: + file_list = [(f.filename, f.sha_hash.hex()) for f in files] + for result, path, _, bytes_read in validate_files(new_path, file_list): + if result == VerifyResult.HASH_MATCH: + path = path.replace('\\', '/').split('/') + dir, filename = ('/'.join(path[:-1]), path[-1]) # 'foo/bar/baz.txt' -> ('foo/bar', 'baz.txt') + if dir not in existing_files: + existing_files[dir] = [] + existing_files[dir].append(filename) + existing_files['__total__'] += 1 + existing_chunks += bytes_read + + def _ignore(dir, existing_files, game_folder): + dir = dir.replace('\\', '/').split(f'{game_folder}')[1:][0] + if dir.startswith('/'): dir = dir[1:] + return existing_files.get(dir, []) + ignore = lambda dir, _: _ignore(dir, existing_files, game_folder) + + logger.info('This process could be stopped and resumed by interrupting it with CTRL-C') if not get_boolean_choice(f'Are you sure you wish to move "{igame.title}" from "{old_base}" to "{args.new_path}"?'): print('Aborting...') exit(0) + try: - total_files, total_chunks = scan_dir(igame.install_path) - copied_files = 0 - last_copied_chunks = 0 - copied_chunks = 0 - last = start = time.perf_counter() - def copy_function(src, dst, *, follow_symlinks=True): - nonlocal total_files, copied_files - nonlocal total_chunks, last_copied_chunks, copied_chunks - nonlocal last, start - - shutil.copy2(src, dst, follow_symlinks=follow_symlinks) + data = dict( + **dict(zip(('total_files', 'total_chunks'), scan_dir(igame.install_path))), + copied_files = 0 + existing_files['__total__'], + last_copied_chunks = 0, + copied_chunks = 0 + existing_chunks, + **dict.fromkeys(('start', 'last'), time.perf_counter()) + ) + def _copy_function(src, dst, data): + shutil.copy2(src, dst, follow_symlinks=True) size = os.path.getsize(dst) - last_copied_chunks += size - copied_chunks += size - copied_files += 1 + data['last_copied_chunks'] += size + data['copied_chunks'] += size + data['copied_files'] += 1 now = time.perf_counter() - runtime = now - start - delta = now - last - if delta < 0.5 and copied_files < total_files: # to prevent spamming the console + runtime = now - data['start'] + delta = now - data['last'] + if delta < 1 and data['copied_files'] < data['total_files']: # to prevent spamming the console return - last = now - speed = last_copied_chunks / delta - last_copied_chunks = 0 - perc = copied_files / total_files * 100 + data['last'] = now + speed = data['last_copied_chunks'] / delta + data['last_copied_chunks'] = 0 + perc = data['copied_files'] / data['total_files'] * 100 - average_speed = copied_chunks / runtime - estimate = (total_chunks - copied_chunks) / average_speed + average_speed = data['copied_chunks'] / runtime + estimate = (data['total_chunks'] - data['copied_chunks']) / average_speed minutes, seconds = int(estimate//60), int(estimate%60) hours, minutes = int(minutes//60), int(minutes%60) rt_minutes, rt_seconds = int(runtime//60), int(runtime%60) rt_hours, rt_minutes = int(rt_minutes//60), int(rt_minutes%60) - logger.info(f'= Progress: {perc:.02f}% ({copied_files}/{total_files}), ') + logger.info(f'= Progress: {perc:.02f}% ({data["copied_files"]}/{data["total_files"]}), ') logger.info(f' + Running for {rt_hours:02d}:{rt_minutes:02d}:{rt_seconds:02d}, ') logger.info(f' + ETA: {hours:02d}:{minutes:02d}:{seconds:02d}') logger.info(f' + Speed: {speed / 1024 / 1024:.02f} MiB/s') - shutil.move(igame.install_path, new_path, copy_function=copy_function) + def copy_function(src, dst, *_, **__): + _copy_function(src, dst, data) + + shutil.copytree(igame.install_path, new_path, copy_function=copy_function, dirs_exist_ok=True, ignore=ignore) except Exception as e: - if isinstance(e, shutil.Error): - logger.error(f'Cannot move the folder into itself.') - elif isinstance(e, PermissionError): + if isinstance(e, PermissionError): logger.error(f'Cannot move the directory "{igame.install_path}", lacking write permission to it.') else: logger.error(f'Moving failed with unknown error {e!r}.') @@ -2572,10 +2614,13 @@ def copy_function(src, dst, *, follow_symlinks=True): f'"legendary move {app_name} "{args.new_path}" --skip-move"') return except KeyboardInterrupt: - # TODO: Make it resumable - shutil.rmtree(new_path) - logger.info("The process has been cancelled.") + logger.info('The process has been cancelled.') return + try: + shutil.rmtree(igame.install_path) + except KeyboardInterrupt: + logger.info('The process cannot be cancelled now. Please wait patiently for a few seconds.') + shutil.rmtree(igame.install_path) else: logger.info(f'Not moving, just rewriting legendary metadata...')