diff --git a/alas.py b/alas.py index 46a91e4f65..152eec68eb 100644 --- a/alas.py +++ b/alas.py @@ -1,5 +1,6 @@ import os import re +import shutil import threading import time from datetime import datetime, timedelta @@ -131,20 +132,41 @@ def run(self, command): ) exit(1) + def keep_last_errlog(self, folder_path, n: int = 30): + """ + + Keep last n folders in folder_path, delete others. + If n is negative or 0, do nothing.(Keep all errlog folders) + + Args: + folder_path (str): Path to folder.\n + n (int): Number of folders to keep. + """ + if n <= 0: + return + folders = [ + os.path.join(folder_path, f) + for f in os.listdir(folder_path) + if os.path.isdir(os.path.join(folder_path, f)) + ] + for folder in folders[:-n]: + shutil.rmtree(folder) + def save_error_log(self): """ - Save last 60 screenshots in ./log/error/ - Save logs to ./log/error//log.txt + Save last 60 screenshots in ./log/error// + Save logs to ./log/error///log.txt """ + import pathlib from module.base.utils import save_image from module.handler.sensitive_info import (handle_sensitive_image, handle_sensitive_logs) if self.config.Error_SaveError: - if not os.path.exists('./log/error'): - os.mkdir('./log/error') - folder = f'./log/error/{int(time.time() * 1000)}' + config_folder = pathlib.Path(f"./log/error/{self.config_name}") + folder = config_folder.joinpath(str(int(time.time() * 1000))) + folder.mkdir(parents=True, exist_ok=True) logger.warning(f'Saving error: {folder}') - os.mkdir(folder) + for data in self.device.screenshot_deque: image_time = datetime.strftime(data['time'], '%Y-%m-%d_%H-%M-%S-%f') image = handle_sensitive_image(data['image']) @@ -160,6 +182,7 @@ def save_error_log(self): lines = handle_sensitive_logs(lines) with open(f'{folder}/log.txt', 'w', encoding='utf-8') as f: f.writelines(lines) + self.keep_last_errlog(config_folder, self.config.Error_SaveErrorCount) def restart(self): from module.handler.login import LoginHandler diff --git a/config/template.json b/config/template.json index 21450d3031..1cfccc1112 100644 --- a/config/template.json +++ b/config/template.json @@ -17,6 +17,7 @@ "Error": { "HandleError": true, "SaveError": true, + "SaveErrorCount": 30, "OnePushConfig": "provider: null", "ScreenshotLength": 1 }, @@ -42,6 +43,11 @@ } }, "General": { + "Log": { + "LogKeepCount": 7, + "LogBackUpMethod": "delete", + "ZipMethod": "bz2" + }, "Retirement": { "RetireMode": "one_click_retire" }, diff --git a/gui.py b/gui.py index 97b5f3cf4f..953cc93cbc 100644 --- a/gui.py +++ b/gui.py @@ -72,7 +72,7 @@ def func(ev: threading.Event): should_exit = False while not should_exit: event = Event() - process = Process(target=func, args=(event,)) + process = Process(target=func, args=(event,), name="gui") process.start() while not should_exit: try: diff --git a/module/config/argument/args.json b/module/config/argument/args.json index 551c710580..8345746379 100644 --- a/module/config/argument/args.json +++ b/module/config/argument/args.json @@ -179,6 +179,10 @@ "type": "checkbox", "value": true }, + "SaveErrorCount": { + "type": "input", + "value": 30 + }, "OnePushConfig": { "type": "textarea", "value": "provider: null", @@ -296,6 +300,31 @@ } }, "General": { + "Log": { + "LogKeepCount": { + "type": "input", + "value": 7 + }, + "LogBackUpMethod": { + "type": "select", + "value": "delete", + "option": [ + "delete", + "zip", + "copy" + ] + }, + "ZipMethod": { + "type": "select", + "value": "bz2", + "option": [ + "bz2", + "gzip", + "xz", + "zip" + ] + } + }, "Retirement": { "RetireMode": { "type": "select", diff --git a/module/config/argument/argument.yaml b/module/config/argument/argument.yaml index cc20982a17..f03712bded 100644 --- a/module/config/argument/argument.yaml +++ b/module/config/argument/argument.yaml @@ -83,6 +83,7 @@ EmulatorInfo: Error: HandleError: true SaveError: true + SaveErrorCount: 30 OnePushConfig: type: textarea mode: yaml @@ -119,6 +120,14 @@ DropRecord: MeowfficerTalent: value: do_not option: [ do_not, save, upload, save_and_upload ] +Log: + LogKeepCount: 7 + LogBackUpMethod: + value: delete + option: [ delete, zip, copy ] + ZipMethod: + value: bz2 + option: [ bz2, gzip, xz, zip ] Retirement: RetireMode: value: one_click_retire diff --git a/module/config/argument/task.yaml b/module/config/argument/task.yaml index 74e40c0883..c3512ca718 100644 --- a/module/config/argument/task.yaml +++ b/module/config/argument/task.yaml @@ -15,6 +15,7 @@ Alas: - Optimization - DropRecord General: + - Log - Retirement - OneClickRetire - Enhance diff --git a/module/config/config_generated.py b/module/config/config_generated.py index 8ac69edaa6..835aad071a 100644 --- a/module/config/config_generated.py +++ b/module/config/config_generated.py @@ -34,6 +34,7 @@ class GeneratedConfig: # Group `Error` Error_HandleError = True Error_SaveError = True + Error_SaveErrorCount = 30 Error_OnePushConfig = 'provider: null' Error_ScreenshotLength = 1 @@ -54,6 +55,11 @@ class GeneratedConfig: DropRecord_MeowfficerBuy = 'do_not' # do_not, save DropRecord_MeowfficerTalent = 'do_not' # do_not, save, upload, save_and_upload + # Group `Log` + Log_LogKeepCount = 7 + Log_LogBackUpMethod = 'delete' # delete, zip, copy + Log_ZipMethod = 'bz2' # bz2, gzip, xz, zip + # Group `Retirement` Retirement_RetireMode = 'one_click_retire' # one_click_retire, enhance, old_retire diff --git a/module/config/i18n/en-US.json b/module/config/i18n/en-US.json index 927dc0c21b..c3b1997068 100644 --- a/module/config/i18n/en-US.json +++ b/module/config/i18n/en-US.json @@ -475,6 +475,10 @@ "name": "Record Exception", "help": "Records exception and log into directory for review or sharing" }, + "SaveErrorCount": { + "name": "Error storage limit", + "help": "Error logs that exceed will be deleted, unlimited if it is negative" + }, "OnePushConfig": { "name": "Error notify config", "help": "When Alas cannot handle exception, send a message through Onepush. Configuration document: \nhttps://github.com/LmeSzinc/AzurLaneAutoScript/wiki/Onepush-configuration-%5BEN%5D" @@ -573,6 +577,31 @@ "save_and_upload": "Save and upload" } }, + "Log": { + "_info": { + "name": "Log", + "help": "General settings, effective after the scheduler is restarted" + }, + "LogKeepCount": { + "name": "Count of log rotation", + "help": "Number of days to rotate logs" + }, + "LogBackUpMethod": { + "name": "Log backup method", + "help": "Back up logs or delete expired logs directly", + "delete": "delete", + "zip": "create zip in ./log/bak", + "copy": "copy to ./log/bak" + }, + "ZipMethod": { + "name": "Zip Method", + "help": "Log.ZipMethod.help", + "bz2": "bz2", + "gzip": "gzip", + "xz": "xz", + "zip": "zip" + } + }, "Retirement": { "_info": { "name": "Retirement Settings", diff --git a/module/config/i18n/ja-JP.json b/module/config/i18n/ja-JP.json index 52ceba496a..53940a43d6 100644 --- a/module/config/i18n/ja-JP.json +++ b/module/config/i18n/ja-JP.json @@ -475,6 +475,10 @@ "name": "Error.SaveError.name", "help": "Error.SaveError.help" }, + "SaveErrorCount": { + "name": "Error.SaveErrorCount.name", + "help": "Error.SaveErrorCount.help" + }, "OnePushConfig": { "name": "Error.OnePushConfig.name", "help": "Error.OnePushConfig.help" @@ -573,6 +577,31 @@ "save_and_upload": "save_and_upload" } }, + "Log": { + "_info": { + "name": "Log._info.name", + "help": "Log._info.help" + }, + "LogKeepCount": { + "name": "Log.LogKeepCount.name", + "help": "Log.LogKeepCount.help" + }, + "LogBackUpMethod": { + "name": "Log.LogBackUpMethod.name", + "help": "Log.LogBackUpMethod.help", + "delete": "delete", + "zip": "zip", + "copy": "copy" + }, + "ZipMethod": { + "name": "Log.ZipMethod.name", + "help": "Log.ZipMethod.help", + "bz2": "bz2", + "gzip": "gzip", + "xz": "xz", + "zip": "zip" + } + }, "Retirement": { "_info": { "name": "Retirement._info.name", diff --git a/module/config/i18n/zh-CN.json b/module/config/i18n/zh-CN.json index 24e75d5dde..6951fb17e4 100644 --- a/module/config/i18n/zh-CN.json +++ b/module/config/i18n/zh-CN.json @@ -475,6 +475,10 @@ "name": "出错时，保存 Log 和截图", "help": "" }, + "SaveErrorCount": { + "name": "Error Log 保存上限", + "help": "超出上限的错误日志将被删除，若为0或负值则没有限制" + }, "OnePushConfig": { "name": "错误推送设置", "help": "发生无法处理的异常后，使用 Onepush 推送一条错误信息。配置方法见文档：https://github.com/LmeSzinc/AzurLaneAutoScript/wiki/Onepush-configuration-%5BCN%5D" @@ -573,6 +577,31 @@ "save_and_upload": "保存并上传" } }, + "Log": { + "_info": { + "name": "日志", + "help": "调度器重启后生效" + }, + "LogKeepCount": { + "name": "Log保留数量", + "help": "Log过期时间(天)" + }, + "LogBackUpMethod": { + "name": "过期Log处理方式", + "help": "备份目录为./log/bak", + "delete": "删除", + "zip": "压缩备份", + "copy": "拷贝备份" + }, + "ZipMethod": { + "name": "压缩格式", + "help": "", + "bz2": "bz2", + "gzip": "gzip", + "xz": "xz", + "zip": "zip" + } + }, "Retirement": { "_info": { "name": "退役设置", diff --git a/module/config/i18n/zh-TW.json b/module/config/i18n/zh-TW.json index 00b9a450b6..753196e3a4 100644 --- a/module/config/i18n/zh-TW.json +++ b/module/config/i18n/zh-TW.json @@ -475,6 +475,10 @@ "name": "出錯時，保存 Log 和截圖", "help": "" }, + "SaveErrorCount": { + "name": "Error Log 保存上限", + "help": "超出上限的錯誤日志將被刪除, 若爲0或負數則沒有限制" + }, "OnePushConfig": { "name": "錯誤推送設定", "help": "發生無法處理的異常後，使用 Onepush 推送错误消息。設定參考文檔：https://github.com/LmeSzinc/AzurLaneAutoScript/wiki/Onepush-configuration-%5BCN%5D" @@ -573,6 +577,31 @@ "save_and_upload": "保存並上傳" } }, + "Log": { + "_info": { + "name": "Log", + "help": "調度器重啟後生效" + }, + "LogKeepCount": { + "name": "Log保留數量", + "help": "Log過期時間(天)" + }, + "LogBackUpMethod": { + "name": "Log處理方式", + "help": "備份目錄為./log/bak", + "delete": "刪除", + "zip": "壓縮備份", + "copy": "copy" + }, + "ZipMethod": { + "name": "壓縮格式", + "help": "", + "bz2": "bz2", + "gzip": "gzip", + "xz": "xz", + "zip": "zip" + } + }, "Retirement": { "_info": { "name": "退役設定", diff --git a/module/logger.py b/module/logger.py index 97dc148dec..c1b00323e4 100644 --- a/module/logger.py +++ b/module/logger.py @@ -1,11 +1,21 @@ import datetime +import io +import json import logging +import multiprocessing import os +import shutil import sys +import tarfile +import threading +import time +import zipfile +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path from typing import Callable, List from rich.console import Console, ConsoleOptions, ConsoleRenderable, NewLine -from rich.highlighter import RegexHighlighter, NullHighlighter +from rich.highlighter import NullHighlighter, RegexHighlighter from rich.logging import RichHandler from rich.rule import Rule from rich.style import Style @@ -86,6 +96,207 @@ def handle(self, record: logging.LogRecord) -> bool: super().handle(record) +class RichTimedRotatingHandler(TimedRotatingFileHandler): + ZIPMAP = { + "gzip": "gz", + "gz" : "gz", + "bz2" : "bz2", + "xz": "xz", + "zip": "zip", + } + def __init__(self, pname:str, *args, **kwargs) -> None: + count, bak_method, zip_method = self._read_file_logger_config(pname) + TimedRotatingFileHandler.__init__(self, backupCount=count,* args, **kwargs) + self.console = Console(file=io.StringIO(), no_color=True, highlight=False, width=119) + self.richd = RichHandler( + console=self.console, + show_path=False, + show_time=False, + show_level=False, + rich_tracebacks=True, + tracebacks_show_locals=True, + tracebacks_extra_lines=3, + highlighter=NullHighlighter(), + ) + # Keep the same format + self.richd.setFormatter( + logging.Formatter( + fmt="%(asctime)s.%(msecs)03d | %(levelname)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + # To handle the API of alas.save_error_log() + self.log_file = None + # For expire method + self.pname = pname + self.bak = bak_method.lower() + self.compression = zip_method.lower() + + # Override the initial rolloverAt and rich.console.file + self.rolloverAt = time.time() + self.doRollover() + + # Close unnecessary stream + self.stream.close() + self.stream = None + + def _read_file_logger_config(self, process_name): + cfg_name = "alas" if process_name == "gui" else process_name + config_file = Path("./config").joinpath(f"{cfg_name}.json") + if config_file.exists(): + try: + with config_file.open("r") as f: + config = json.load(f) + log_config = config.get("General", {}).get("Log", {}) + count = log_config.get("LogKeepCount", 7) + bak_method = log_config.get("LogBackUpMethod", "copy") + zip_method = log_config.get("ZipMethod", "bz2") + except Exception as e: + logging.exception(e) + count = 7 + bak_method = "copy" + zip_method = "bz2" + else: + count = 7 + bak_method = "zip" if process_name == "gui" else "copy" + zip_method = "bz2" + return count, bak_method, zip_method + + def getFilesToDelete(self) -> List[Path]: + """ + Determine the files to delete when rolling over.\n + Override the original method to use RichHandler and keep the same format + """ + dirName, baseName = os.path.split(self.baseFilename) + fileNames = os.listdir(dirName) + result = [] + suffix = "_" + baseName + plen = len(suffix) + for fileName in fileNames: + if fileName[-plen:] == suffix: + prefix = fileName[:-plen] + if self.extMatch.match(prefix): + result.append(Path(dirName).joinpath(fileName).resolve()) + if len(result) < self.backupCount: + result = [] + else: + result.sort() + result = result[: len(result) - self.backupCount] + return result + + def doRollover(self) -> None: + """ + Do a rollover.\n + Override the original method to use RichHandler + """ + if self.richd.console: + self.richd.console.file.close() + self.richd.console.file = None + + currentTime = int(time.time()) + dstNow = time.localtime(currentTime)[-1] + t = self.rolloverAt + if self.utc: + timeTuple = time.gmtime(t) + else: + timeTuple = time.localtime(t) + dstThen = timeTuple[-1] + if dstNow != dstThen: + if dstNow: + addend = 3600 + else: + addend = -3600 + timeTuple = time.localtime(t + addend) + + path = Path(self.baseFilename) + # 2021-08-01 + _ + alas.txt -> "2021-08-01_alas.txt" + newPath = path.with_name( + time.strftime(self.suffix, timeTuple) + "_" + path.name + ) + self.richd.console.file = open(newPath, "a", encoding="utf-8") + + if self.backupCount > 0: + files = self.getFilesToDelete() + if files: + threading.Thread(target=self.expire, args=(files,), daemon=True).start() + # self.expire(files) + + newRolloverAt = self.computeRollover(currentTime) + while newRolloverAt <= currentTime: + newRolloverAt = newRolloverAt + self.interval + # If DST changes and midnight or weekly rollover, adjust for this. + if (self.when == "MIDNIGHT" or self.when.startswith("W")) and not self.utc: + dstAtRollover = time.localtime(newRolloverAt)[-1] + if dstNow != dstAtRollover: + if ( + not dstNow + ): # DST kicks in before next rollover, so we need to deduct an hour + addend = -3600 + else: # DST bows out before next rollover, so we need to add an hour + addend = 3600 + newRolloverAt += addend + self.rolloverAt = newRolloverAt + + self.log_file = str(newPath.resolve()) + + def expire(self, files: List[Path]) -> None: + """ + Remove or backup the expired log files\n + + Template: + 2021-08-01_alas.txt...2021-08-07_alas.txt -> bak/2021-08-01~2021-08-07_alas.tar.bz2 \n + 2021-08-01_gui.txt -> bak/2021-08-01_gui.zip \n + 2021-08-01_gui.txt(copy) -> bak/2021-08-01_gui.txt(copy) \n + """ + basePath = Path(self.baseFilename) + bakPath = basePath.parent / "bak" + bakPath.mkdir(parents=True, exist_ok=True) + if self.bak == "delete": + for file in files: + file.unlink() + return + elif self.bak == "copy": + for file in files: + dst = bakPath.joinpath(file.name) + if not dst.exists(): + shutil.copy2(file, dst) + file.unlink() + return + try: + dates = [file.stem.split("_")[0] for file in files] + name = ( + min(dates) + "~" + max(dates) + "_" + basePath.name + if len(dates) > 1 + else files[0].name + ) + ext = self.ZIPMAP[self.compression] + if ext == "zip": + zipFile = bakPath.joinpath(name).with_suffix(".zip") + with zipfile.ZipFile(zipFile, "w", zipfile.ZIP_DEFLATED) as zipf: + for file in files: + zipf.write(file, arcname=file.name) + file.unlink() + else: + zipFile = bakPath.joinpath(name).with_suffix(".tar." + ext) + with tarfile.open(zipFile, "w:" + ext) as tar: + for file in files: + tar.add(file, arcname=file.name) + file.unlink() + except Exception as e: + logger.exception(e) + + def print(self, *objects: ConsoleRenderable, **kwargs) -> None: + Console.print(self.console, *objects, **kwargs) + + def emit(self, record: logging.LogRecord) -> None: + try: + if self.shouldRollover(record): + self.doRollover() + RichHandler.emit(self.richd, record) + except Exception: + RichHandler.handleError(self.richd, record) + + class HTMLConsole(Console): """ Force full feature console @@ -186,38 +397,51 @@ def _set_file_logger(name=pyw_name): def set_file_logger(name=pyw_name): - if '_' in name: - name = name.split('_', 1)[0] - log_file = f'./log/{datetime.date.today()}_{name}.txt' - try: - file = open(log_file, mode='a', encoding='utf-8') - except FileNotFoundError: - os.mkdir('./log') - file = open(log_file, mode='a', encoding='utf-8') - - file_console = Console( - file=file, - no_color=True, - highlight=False, - width=119, + if "_" in name: + name = name.split("_", 1)[0] + # Handler Windows : Windows have "SyncManager-N:N", "MainProcess", "Process-N", "gui" 4 Processes + # There have no process named "SyncManager", only "MainProcess" on Linux + if os.name == "nt": + # These process needn't to save log file on Windows + processes = ["SyncManager-", "MainProcess", "Process-"] + pname = multiprocessing.current_process().name.replace(":", "_") + # Each process should only call once when alas start. + if any(isinstance(hdlr, RichTimedRotatingHandler) for hdlr in logger.handlers): + return + else: + processes = [] + pname = name + for hdlr in logger.handlers: + if isinstance(hdlr, RichTimedRotatingHandler): + # Each process should only call once when alas start. + if hdlr.pname == name: + return + else: + logger.handlers = [h for h in logger.handlers if not isinstance( + h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] + + log_dir = Path("./log") + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") + if any(p in log_file.name for p in processes): + return + + hdlr = RichTimedRotatingHandler( + pname=name, + filename=str(log_file), + when="midnight", + interval=1, + encoding="utf-8", ) - hdlr = RichFileHandler( - console=file_console, - show_path=False, - show_time=False, - show_level=False, - rich_tracebacks=True, - tracebacks_show_locals=True, - tracebacks_extra_lines=3, - highlighter=NullHighlighter(), - ) - hdlr.setFormatter(file_formatter) - - logger.handlers = [h for h in logger.handlers if not isinstance( - h, (logging.FileHandler, RichFileHandler))] logger.addHandler(hdlr) - logger.log_file = log_file + logger.log_file = hdlr.log_file + try: + if log_file.exists(): + log_file.unlink() + except Exception: + pass + def set_func_logger(func): @@ -280,6 +504,8 @@ def print(*objects: ConsoleRenderable, **kwargs): hdlr._func(renderable) elif isinstance(hdlr, RichHandler): hdlr.console.print(*objects) + elif isinstance(hdlr, RichTimedRotatingHandler): + hdlr.print(*objects, **kwargs) def rule(title="", *, characters="─", style="rule.line", end="\n", align="center"):