diff --git a/tests/metadate.py b/tests/metadate.py new file mode 100644 index 0000000000..0bf990bf05 --- /dev/null +++ b/tests/metadate.py @@ -0,0 +1,57 @@ +import subprocess + +import h5py + + +def remove_metadata(input_file, output_file): + with h5py.File(input_file, "r") as src, h5py.File(output_file, "w") as dst: + + def copy_attrs(src_obj, dst_obj): + # 不复制属性（元数据） + pass + + def copy_group(src_group, dst_group): + copy_attrs(src_group, dst_group) + for name, item in src_group.items(): + if isinstance(item, h5py.Group): + new_group = dst_group.create_group(name) + copy_group(item, new_group) + elif isinstance(item, h5py.Dataset): + dst_group.create_dataset(name, data=item[...]) + + copy_group(src, dst) + + +def get_file_md5(file) -> str: + """ + Get the file md5 + + Args: + file (str): The path of the gzip input file. + """ + try: + result = subprocess.run(["md5sum", file], capture_output=True, text=True) + return result.stdout.split(" ")[0] + except Exception as e: + return "" + + +if __name__ == "__main__": + f1 = "11.hdf5.gz" + f2 = "22.hdf5.gz" + f3 = "33.hdf5" # 输出去除元数据后的 HDF5 文件路径 + f4 = "44.hdf5" # 输出去除元数据后的 HDF5 文件路径 + + md5_f1 = get_file_md5("11.hdf5.gz") + print(md5_f1) + md5_f2 = get_file_md5("22.hdf5.gz") + print(md5_f2) + + # 移除元数据 + remove_metadata(f1, f3) + remove_metadata(f2, f4) + + md5_f3 = get_file_md5(f3) + print(md5_f3) + md5_f4 = get_file_md5(f4) + print(md5_f4) diff --git a/tests/test_web/test_webapi.py b/tests/test_web/test_webapi.py index a654934056..91deab6237 100644 --- a/tests/test_web/test_webapi.py +++ b/tests/test_web/test_webapi.py @@ -37,6 +37,7 @@ upload, ) from tidy3d.web.core.environment import Env +from tidy3d.web.core.system_config import SystemConfig from tidy3d.web.core.types import TaskType TASK_NAME = "task_name_test" @@ -174,6 +175,42 @@ def mock_get_info(monkeypatch, set_api_key): ) +@pytest.fixture +def mock_get_system_config(monkeypatch, set_api_key): + """Mocks webapi.get_system_config.""" + + responses.add( + responses.GET, + f"{Env.current.web_api_endpoint}/tidy3d/system/py/config", + json={ + "data": { + "runStatuses": [ + "draft", + "queued", + "preprocess", + "queued_solver", + "running", + "postprocess", + "visualize", + "success", + ], + "endStatuses": [ + "success", + "error", + "errored", + "diverged", + "diverge", + "deleted", + "draft", + "abort", + "aborted", + ], + } + }, + status=200, + ) + + @pytest.fixture def mock_start(monkeypatch, set_api_key, mock_get_info): """Mocks webapi.start.""" @@ -225,10 +262,38 @@ def mock_get_run_info(task_id): run_count[0] += 1 return perc_done, 1 + def mock_get_system_info(): + return SystemConfig( + **{ + "runStatuses": [ + "draft", + "queued", + "preprocess", + "queued_solver", + "running", + "postprocess", + "visualize", + "success", + ], + "endStatuses": [ + "success", + "error", + "errored", + "diverged", + "diverge", + "deleted", + "draft", + "abort", + "aborted", + ], + } + ) + monkeypatch.setattr("tidy3d.web.api.connect_util.REFRESH_TIME", 0.00001) monkeypatch.setattr(f"{api_path}.RUN_REFRESH_TIME", 0.00001) monkeypatch.setattr(f"{api_path}.get_status", mock_get_status) monkeypatch.setattr(f"{api_path}.get_run_info", mock_get_run_info) + monkeypatch.setattr(f"{api_path}.get_system_config", mock_get_system_info) @pytest.fixture @@ -290,7 +355,13 @@ def mock_get_run_info(monkeypatch, set_api_key): @pytest.fixture def mock_webapi( - mock_upload, mock_metadata, mock_get_info, mock_start, mock_monitor, mock_download, mock_load + mock_upload, + mock_metadata, + mock_get_info, + mock_start, + mock_monitor, + mock_download, + mock_load, ): """Mocks all webapi operation.""" @@ -545,7 +616,7 @@ def mock_job_status(monkeypatch): @responses.activate -def test_batch(mock_webapi, mock_job_status, mock_load, tmp_path): +def test_batch(mock_webapi, mock_job_status, mock_load, mock_get_system_config, tmp_path): # monkeypatch.setattr("tidy3d.web.api.container.Batch.monitor", lambda self: time.sleep(0.1)) # monkeypatch.setattr("tidy3d.web.api.container.Job.status", property(lambda self: "success")) @@ -569,7 +640,7 @@ def test_batch(mock_webapi, mock_job_status, mock_load, tmp_path): @responses.activate -def test_async(mock_webapi, mock_job_status): +def test_async(mock_webapi, mock_get_system_config, mock_job_status): # monkeypatch.setattr("tidy3d.web.api.container.Job.status", property(lambda self: "success")) sims = {TASK_NAME: make_sim()} _ = run_async(sims, folder_name=PROJECT_NAME) diff --git a/tests/test_web/test_webapi_system.py b/tests/test_web/test_webapi_system.py new file mode 100644 index 0000000000..fe5df2d322 --- /dev/null +++ b/tests/test_web/test_webapi_system.py @@ -0,0 +1,46 @@ +# Tests webapi and things that depend on it + +import pytest +import responses +import tidy3d as td +from tidy3d.web.api.webapi import ( + get_system_config, +) +from tidy3d.web.core.environment import Env + +task_core_path = "tidy3d.web.core.task_core" +api_path = "tidy3d.web.api.webapi" + +Env.dev.active() + + +@pytest.fixture +def set_api_key(monkeypatch): + """Set the api key.""" + import tidy3d.web.core.http_util as http_module + + monkeypatch.setattr(http_module, "api_key", lambda: "apikey") + monkeypatch.setattr(http_module, "get_version", lambda: td.version.__version__) + + +@pytest.fixture +def mock_get_system_config(monkeypatch, set_api_key): + """Mocks webapi.get_info.""" + + responses.add( + responses.GET, + f"{Env.current.web_api_endpoint}/tidy3d/system/py/config", + json={ + "data": { + "runStatuses": [], + "endStatuses": [], + } + }, + status=200, + ) + + +@responses.activate +def test_system_config(mock_get_system_config): + info = get_system_config() + assert info is not None diff --git a/tidy3d/web/api/container.py b/tidy3d/web/api/container.py index 338d169734..41d9fc9c25 100644 --- a/tidy3d/web/api/container.py +++ b/tidy3d/web/api/container.py @@ -17,6 +17,7 @@ from ...exceptions import DataError from ...log import get_logging_console, log from ..api import webapi as web +from ..api.webapi import get_system_config from ..core.constants import TaskId, TaskName from ..core.task_info import RunInfo, TaskInfo from .tidy3d_stub import SimulationDataType, SimulationType @@ -692,7 +693,7 @@ def pbar_description(task_name: str, status: str) -> str: return description - run_statuses = [ + run_statuses = ( "draft", "queued", "preprocess", @@ -701,8 +702,28 @@ def pbar_description(task_name: str, status: str) -> str: "postprocess", "visualize", "success", - ] - end_statuses = ("success", "error", "errored", "diverged", "diverge", "deleted", "draft") + ) + + end_statuses = ( + "success", + "error", + "errored", + "diverged", + "diverge", + "deleted", + "draft", + "abort", + "aborted", + ) + + config = get_system_config() + if config is not None: + config_end_statuses = config.end_statuses + config_run_statuses = config.run_statuses + if config_end_statuses is not None: + end_statuses = config_end_statuses + if config_run_statuses is not None: + run_statuses = config_run_statuses if self.verbose: console = get_logging_console() diff --git a/tidy3d/web/api/webapi.py b/tidy3d/web/api/webapi.py index 45304cf085..fbd8d31311 100644 --- a/tidy3d/web/api/webapi.py +++ b/tidy3d/web/api/webapi.py @@ -15,6 +15,7 @@ from ..core.account import Account from ..core.constants import SIM_FILE_HDF5, TaskId from ..core.environment import Env +from ..core.system_config import SystemConfig from ..core.task_core import Folder, SimulationTask from ..core.task_info import ChargeType, TaskInfo from .connect_util import ( @@ -359,16 +360,17 @@ def monitor(task_id: TaskId, verbose: bool = True) -> None: ---- To load results when finished, may call :meth:`load`. """ + break_statuses = ("success", "error", "diverged", "deleted", "draft", "abort", "aborted") console = get_logging_console() if verbose else None - task_info = get_info(task_id) - task_name = task_info.taskName - task_type = task_info.taskType - - break_statuses = ("success", "error", "diverged", "deleted", "draft", "abort", "aborted") + config = get_system_config() + if config is not None: + config_end_statuses = config.end_statuses + if config_end_statuses is not None: + break_statuses = config_end_statuses def get_estimated_cost() -> float: """Get estimated cost, if None, is not ready.""" @@ -959,6 +961,21 @@ def account(verbose=True) -> Account: return account_info +@wait_for_connection +def get_system_config() -> SystemConfig: + """Return system config. + + Parameters + ---------- + + Returns + ------- + :class:`SystemConfig` + Object containing information about system configuration. + """ + return SystemConfig.get() + + @wait_for_connection def test() -> None: """ diff --git a/tidy3d/web/core/system_config.py b/tidy3d/web/core/system_config.py new file mode 100644 index 0000000000..96124eb1b6 --- /dev/null +++ b/tidy3d/web/core/system_config.py @@ -0,0 +1,45 @@ +"""Tidy3d system config.""" + +from __future__ import annotations + +from typing import Optional + +from pydantic.v1 import Extra, Field + +from .http_util import http +from .types import Tidy3DResource + + +class SystemConfig(Tidy3DResource, extra=Extra.allow): + """Tidy3D system config.""" + + end_statuses: Optional[tuple] = Field( + None, + title="End Statuses", + description="Tuple of status keys that signify that the task has completed.", + alias="endStatuses", + ) + run_statuses: Optional[tuple] = Field( + None, + title="Run Statuses", + description="Tuple of ordered status keys that signify that the task is in progress.", + alias="runStatuses", + ) + + @classmethod + def get(cls): + """Get user SystemConfig information. + + Parameters + ---------- + + Returns + ------- + systemConfig : SystemConfig + """ + resp = http.get("tidy3d/system/py/config") + if resp: + config = SystemConfig(**resp) + return config + else: + return None