New Workflow: cp_process_singlecells !

configs/analysis_configs/cytotable_convert.yaml

@@ -0,0 +1,11 @@
+cytotable_convert:
+  params:
+    dest_datatype: parquet
+    source_datatype: null
+    concat: True
+    join: True
+    infer_common_schema: True
+    drop_null: True
+    preset: cellprofiler_sqlite
+    log_level: ERROR
+

configs/configuration.yaml

@@ -1,17 +1,27 @@
 config_name: cytopipe_defaults
 
+env_manager: conda
+
 # computation configs
 analysis_configs:
   preprocessing:
     threads: 4
 
+# data configurations
+data_configs:
+  plate_data_format: sqlite
+
 # Analysis configuration file paths
 config_paths:
+  # general configs
+  general_configs: "configs/configuration.yaml"
+
   # CellProfiler Specific analysis configurations
   single_cell: "configs/analysis_configs/single_cell_configs.yaml"
   normalize: "configs/analysis_configs/normalize_configs.yaml"
   feature_select: "configs/analysis_configs/feature_select_configs.yaml"
   aggregate: "configs/analysis_configs/aggregate_configs.yaml"
+  cytotable_config: "configs/analysis_configs/cytotable_convert.yaml"
 
   # DeepProfiler Specific analysis configurations
   dp_data: "configs/analysis_configs/dp_data_configs.yaml"

configs/wf_configs/cp_process_singlecells.yaml

@@ -0,0 +1,50 @@
+name: cp_process_singlecells_configs
+cytotable_convert:
+  params:
+    dest_datatype: parquet
+    source_datatype: null
+    concat: True
+    join: True
+    infer_common_schema: True
+    drop_null: True
+    preset: cellprofiler_sqlite
+    log_level: ERROR
+normalize_configs:
+  params:
+    features: infer
+    image_features: False
+    meta_features: infer
+    samples: all
+    method: mad_robustize
+    compression_options:
+      method: gzip
+      mtime: 1
+    float_format: null
+    mad_robustize_epsilon: 1.0e-18
+    spherize_center: True
+    spherize_method: ZCA-cor
+    spherize_epsilon: 1.0e-6
+feature_select_configs:
+  params:
+    features: infer
+    image_features: False
+    samples: all
+    operation:
+      - variance_threshold
+      - drop_na_columns
+      - correlation_threshold
+      - drop_outliers
+      - blocklist
+    na_cutoff: 0.05
+    corr_threshold: 0.9
+    corr_method: pearson
+    freq_cut: 0.05
+    unique_cut: 0.1
+    compression_options:
+      method: gzip
+      mtime: 1
+    float_format: null
+    blocklist_file: null
+    outlier_cutoff: 15
+    noise_removal_perturb_groups: null
+    noise_removal_stdev_cutoff: null

cytosnake/common/errors.py

@@ -87,6 +87,10 @@ class ProjectExistsError(BaseFileExistsError):
     that the current directory has already been set up for cytosnake analysis"""
 
 
+class ExtensionError(BaseValueError):
+    """Raised when invalid extensions are captured"""
+
+
 # -----------------------
 # Error handling functions
 # -----------------------

cytosnake/guards/ext_guards.py

@@ -0,0 +1,50 @@
+"""
+module: ext_guards.py
+
+Checks if the correct extensions are provided
+"""
+
+import pathlib
+from cytosnake.guards.path_guards import is_valid_path
+from typing import TypeGuard
+
+
+def has_parquet_ext(file_name: str | pathlib.Path) -> TypeGuard[str]:
+    """Checks if the provided path is a parquet file.
+
+    Parameters
+    ----------
+    file_name : str | pathlib.Path
+        path to file
+
+    Returns
+    -------
+    TypeGuard[str]
+        return True if it is a parquet file, else False
+    """
+    return (
+        file_name.suffix in [".parquet", ".parq", ".pq"]
+        if is_valid_path(file_name)
+        else False
+    )
+
+
+def has_sqlite_ext(file_name: str | pathlib.Path) -> TypeGuard[str]:
+    """Checks if the provided path is a parquet file.
+
+    Parameters
+    ----------
+    file_name : str | pathlib.Path
+        path to file
+
+    Returns
+    -------
+    TypeGuard[str]
+        return True if it is a parquet file, else False
+    """
+
+    return (
+        file_name.suffix in [".sqlite", ".sqlite3"]
+        if is_valid_path(file_name)
+        else False
+    )

cytosnake/guards/path_guards.py

@@ -9,20 +9,19 @@
     - valid path strings
 """
 
-from pathlib import Path
+import pathlib
 from typing import TypeGuard
 
 
-def is_valid_path(val: object) -> TypeGuard[Path]:
+def is_valid_path(val: object) -> TypeGuard[pathlib.Path]:
     """checks if provided value is a valid path"""
 
-    # check if the val is valid type
-    # -- if string, convert to Path
-    accepted_types = (str, Path)
-    if not isinstance(val, accepted_types):
+    # type checking
+    if not isinstance(val, (str, pathlib.Path)):
         return False
+    # convert to pathlib.Path
     if isinstance(val, str):
-        val = Path(val)
+        val = pathlib.Path(val).resolve()
 
     # check if the path exists
-    return bool(val.exists())
+    return val.exists()

cytosnake/helpers/helper_funcs.py

@@ -9,11 +9,12 @@
 from typing import Optional
 from snakemake.io import expand
 from pathlib import Path
-from cytosnake.utils.config_utils import load_meta_path_configs
+from cytosnake.utils.config_utils import load_meta_path_configs, load_general_configs
 from cytosnake.guards.path_guards import is_valid_path
 
 # loading in config as global variables
 PATHS = load_meta_path_configs()
+CYTOSNAKE_CONFIGS = load_general_configs()
 
 
 # ------------------------------
@@ -151,7 +152,6 @@ def annotated_output() -> str:
 
 
 def normalized_output() -> str:
-
     """Generates output path for normalized dataset
 
     Returns
@@ -169,7 +169,6 @@ def normalized_output() -> str:
 
 
 def selected_features_output() -> str:
-
     """Generates output path for selected features dataset
 
     Returns
@@ -187,7 +186,6 @@ def selected_features_output() -> str:
 
 
 def consensus_output() -> str:
-
     """Generates output path for consensus  dataset
 
     Returns
@@ -204,6 +202,23 @@ def consensus_output() -> str:
     return str(results_path / f"{output_name}.{ext}")
 
 
+def parquet_output():
+    """Generates output path for parquet profiles
+
+    Returns
+    -------
+    str
+        path to generated parquet files
+
+    """
+    data_path = Path(PATHS["project_dir_path"]) / "data"
+    output_name = "{file_name}"
+    ext = "parquet"
+
+    # constructing file output string
+    return str(data_path / f"{output_name}.{ext}")
+
+
 # ------------------------------
 # Formatting I/O functions
 # ------------------------------

cytosnake/utils/config_utils.py

@@ -30,9 +30,7 @@ def load_configs(config_path: str | Path) -> dict:
     if not is_valid_path(config_path):
         raise FileNotFoundError("Invalid config path provided")
     if isinstance(config_path, str):
-        config_path = Path(config_path).absolute()
-    if not config_path.is_absolute():
-        config_path = config_path.absolute()
+        config_path = Path(config_path).resolve(strict=True)
 
     # loading in config_path
     with open(config_path, "r") as yaml_contents:
@@ -41,6 +39,19 @@ def load_configs(config_path: str | Path) -> dict:
     return loaded_configs
 
 
+def load_general_configs() -> dict:
+    """Loads cytosnake's general configurations
+
+    Returns:
+    -------
+    dict
+        dictionary containing the cytosnake general configs
+    """
+    config_dir_path = cp.get_config_dir_path() / "configuration.yaml"
+    print(config_dir_path)
+    return load_configs(config_dir_path)
+
+
 def load_meta_path_configs() -> dict:
     """Loads the metadata path from `.cytosnake/_paths.yaml` file
 

cytosnake/utils/cyto_paths.py

@@ -105,12 +105,9 @@ def get_project_root() -> Path:
     """
 
     # get current working directory
-    project_dir = Path().absolute()
-
-    # check if the `.cytosnake` folder exist
-    project_folder = project_dir / ".cytosnake"
-    if not project_folder.exists():
-        raise FileNotFoundError("Current directory is not a project folder")
+    project_dir = find_project_dir()
+    if project_dir is None:
+        raise FileNotFoundError("Unable to find project directory")
 
     return project_dir
 
@@ -139,20 +136,32 @@ def get_workflow_fpaths() -> dict:
     return file_search(workflow_path)
 
 
-def get_config_fpaths() -> dict:
-    """Obtains all file paths located in the `configs` folder as a dictionary.
+def get_config_dir_path() -> Path:
+    """Returns path to configuration folder
 
     Returns
     -------
-    dict
-        structured dictionary directory name and file paths as key value pairs
+    Path
+        Path to config directory
     """
+
     proj_root_path = get_project_root()
     config_path = proj_root_path / "configs"
     if not is_valid_path(config_path):
         raise FileNotFoundError("Unable to find config directory")
 
-    return file_search(config_path)
+    return config_path
+
+
+def get_config_fpaths() -> dict:
+    """Obtains all file paths located in the `configs` folder as a dictionary.
+
+    Returns
+    -------
+    dict
+        structured dictionary directory name and file paths as key value pairs
+    """
+    return file_search(get_config_dir_path())
 
 
 def get_project_dirpaths(args: Namespace) -> dict:

cytosnake/utils/file_utils.py

@@ -97,7 +97,7 @@ def find_project_dir(steps: Optional[int] = 10) -> Path | None:
             # check if the file is a directory and has the name `cytosnake`
             # -- if true, return the complete path
             if _file.is_dir() and _file.name == ".cytosnake":
-                return _file.parent.absolute()
+                return _file.parent.resolve(strict=True)
 
         start_point = start_point.parent
 

workflows/envs/cytotable.yaml

@@ -0,0 +1,10 @@
+name: cytotable
+channels:
+  - conda-forge
+  - anaconda
+  - defaults
+dependencies:
+  - python==3.8
+  - pip
+  - pip:
+    - git+https://github.com/cytomining/CytoTable.git@f336d26baf96b8b3d9ed6ca07f617895030bf038

workflows/rules/common.smk

@@ -26,10 +26,13 @@ METADATA_DIR = hf.get_metadata_dir()
 # OUTPUTS
 # -------
 # -- extended = list of the file names with a given wildcard
-AGGREGATE_DATA = hf.aggregate_output()
-CELL_COUNTS = hf.cell_count_output()
+CONVERTED_DATA = hf.parquet_output()
+CONVERTED_DATA_EXTENDED = expand(CONVERTED_DATA, file_name=plate_name)
 
+CELL_COUNTS = hf.cell_count_output()
 CELL_COUNTS_EXPANDED = expand(CELL_COUNTS, file_name=plate_name)
+
+AGGREGATE_DATA = hf.aggregate_output()
 AGGREGATE_DATA_EXPAND = expand(AGGREGATE_DATA, file_name=plate_name)
 
 ANNOTATED_DATA = hf.annotated_output()

workflows/rules/cytotable_convert.smk

@@ -0,0 +1,32 @@
+"""
+rule module: cytotable_convert.smk
+
+Utilizes CytoTable's convert workflow module:
+https://github.com/cytomining/CytoTable/blob/main/cytotable/convert.py
+
+Parameters:
+-----------
+
+
+Returns:
+--------
+    parquet files stored within the data/ folder
+
+"""
+
+
+configfile: "configs/configuration.yaml"
+
+
+rule convert:
+    input:
+        PLATE_DATA,
+    output:
+        CONVERTED_DATA,
+    conda:
+        "../envs/cytotable.yaml"
+    params:
+        data_configs=config["data_configs"]["plate_data_format"],
+        cytotable_config=config["cytotable_convert"],
+    script:
+        "../scripts/convert.py"