From 134007bd7c7f21388265c379cdbe70422b458be9 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Thu, 11 Jul 2024 12:09:42 +0200 Subject: [PATCH 1/7] Rewrite constructors for ceci v2 --- src/rail/yaw_rail/correlation.py | 4 ++-- src/rail/yaw_rail/summarize.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rail/yaw_rail/correlation.py b/src/rail/yaw_rail/correlation.py index 97eedd3..3722c04 100644 --- a/src/rail/yaw_rail/correlation.py +++ b/src/rail/yaw_rail/correlation.py @@ -88,8 +88,8 @@ class YawBaseCorrelate(YawRailStage): inputs: list[tuple[str, YawCacheHandle]] outputs: list[tuple[str, YawCorrFuncHandle]] - def __init__(self, args, comm=None): - super().__init__(args, comm=comm) + def __init__(self, args, **kwargs): + super().__init__(args, **kwargs) self.yaw_config = Configuration.create(**self.get_algo_config_dict()) @abstractmethod diff --git a/src/rail/yaw_rail/summarize.py b/src/rail/yaw_rail/summarize.py index 4bfff95..b32c3a7 100644 --- a/src/rail/yaw_rail/summarize.py +++ b/src/rail/yaw_rail/summarize.py @@ -143,8 +143,8 @@ class YawSummarize( ("yaw_cc", YawRedshiftDataHandle), ] - def __init__(self, args, comm=None): - super().__init__(args, comm=comm) + def __init__(self, args, **kwargs): + super().__init__(args, **kwargs) config = {p: self.config_options[p].value for p in config_yaw_resampling} self.yaw_config = ResamplingConfig.create(**config) From 97651cb111dca07ed785cbe82b7f92d0296b0361 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Thu, 11 Jul 2024 14:01:49 +0200 Subject: [PATCH 2/7] Temporarily switch to use the ceci2 branch of rail_base in testing to demonstrate things are working --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f8d65e5..33c6902 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dynamic = ["version"] dependencies = [ "numpy", "h5py", - "pz-rail-base", + "ceci2 @ https://github.com/LSSTDESC/rail_base", "qp-prob", "yet_another_wizz", ] From 9698ba1492821157cd069619ea6e37c83cccf5eb Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Thu, 11 Jul 2024 14:04:50 +0200 Subject: [PATCH 3/7] Fix error in pyproject.toml github spec --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 33c6902..786ceec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dynamic = ["version"] dependencies = [ "numpy", "h5py", - "ceci2 @ https://github.com/LSSTDESC/rail_base", + "ceci2 @ git+https://github.com/LSSTDESC/rail_base", "qp-prob", "yet_another_wizz", ] From b5b1afc139da568b1f1d7308bae431e814e350b1 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Thu, 11 Jul 2024 14:41:22 +0200 Subject: [PATCH 4/7] Fix error in pyproject.toml github spec --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 786ceec..8a83529 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dynamic = ["version"] dependencies = [ "numpy", "h5py", - "ceci2 @ git+https://github.com/LSSTDESC/rail_base", + "pz_rail_base @ git+https://github.com/LSSTDESC/rail_base@ceci2", "qp-prob", "yet_another_wizz", ] From 2c54f284ca162b2c42feed389b26dbbe2f996aac Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Wed, 17 Jul 2024 09:35:34 +0100 Subject: [PATCH 5/7] Move to pypi pz-rail-base --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8a83529..4034b57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dynamic = ["version"] dependencies = [ "numpy", "h5py", - "pz_rail_base @ git+https://github.com/LSSTDESC/rail_base@ceci2", + "pz-rail-base>=1.0.3", "qp-prob", "yet_another_wizz", ] From a606efab2b770dfdf72dcec4f838e0588bf34f47 Mon Sep 17 00:00:00 2001 From: Jan Luca van den Busch Date: Wed, 17 Jul 2024 08:42:21 +0000 Subject: [PATCH 6/7] ported over Joe's changes after pull changes from main --- pyproject.toml | 2 +- src/rail/estimation/algos/cc_yaw.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3330123..ff71c08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dynamic = ["version"] dependencies = [ "numpy", "h5py", - "pz_rail_base @ git+https://github.com/LSSTDESC/rail_base@ceci2", + "pz_rail_base>=1.0.3", "qp-prob", # fix yet_another_wizz version; if there is a new release, issue a new # patch or minor version release, depending on the changes requred diff --git a/src/rail/estimation/algos/cc_yaw.py b/src/rail/estimation/algos/cc_yaw.py index 3a28689..ef69f5a 100644 --- a/src/rail/estimation/algos/cc_yaw.py +++ b/src/rail/estimation/algos/cc_yaw.py @@ -226,8 +226,8 @@ class YawAutoCorrelate( ("output", YawCorrFuncHandle), ] - def __init__(self, args, comm=None): - super().__init__(args, comm) + def __init__(self, args, **kwargs): + super().__init__(args, **kwargs) warn_thread_num_deprecation(self.get_algo_config_dict()) def correlate(self, sample: YawCacheHandle | YawCache) -> YawCorrFuncHandle: @@ -294,8 +294,8 @@ class YawCrossCorrelate( ("output", YawCorrFuncHandle), ] - def __init__(self, args, comm=None): - super().__init__(args, comm) + def __init__(self, args, **kwargs): + super().__init__(args, **kwargs) warn_thread_num_deprecation(self.get_algo_config_dict()) def correlate( @@ -387,8 +387,8 @@ class YawSummarize( ("output", ModelHandle), ] - def __init__(self, args, comm=None): - super().__init__(args, comm=comm) + def __init__(self, args, **kwargs): + super().__init__(args, **kwargs) config = {p: self.config_options[p].value for p in stage_config.yaw_resampling} self.yaw_config = ResamplingConfig.create(**config) From 3ac6f56814d885576a4fbd0a9e87aae15b24e57c Mon Sep 17 00:00:00 2001 From: Jan Luca van den Busch Date: Wed, 17 Jul 2024 08:46:36 +0000 Subject: [PATCH 7/7] regenerated pipeline with ceci v2 --- .../pipelines/estimation/yaw_pipeline.yml | 63 ++++++---- .../estimation/yaw_pipeline_config.yml | 117 +++++++----------- 2 files changed, 84 insertions(+), 96 deletions(-) diff --git a/src/rail/pipelines/estimation/yaw_pipeline.yml b/src/rail/pipelines/estimation/yaw_pipeline.yml index a424818..1179e7c 100644 --- a/src/rail/pipelines/estimation/yaw_pipeline.yml +++ b/src/rail/pipelines/estimation/yaw_pipeline.yml @@ -1,36 +1,57 @@ +output_dir: data +log_dir: logs +resume: false config: yaw_pipeline_config.yml +modules: rail inputs: - auto_corr_unk: none data_ref: data/input_data.parquet - data_unk: data/input_data.parquet - patch_source_ref: none rand_ref: data/input_rand.parquet + data_unk: data/input_data.parquet rand_unk: none -log_dir: logs -modules: rail -output_dir: data -resume: false -site: - max_threads: 2 - name: local + patch_source_ref: none + auto_corr_unk: none stages: -- classname: YawCacheCreate - module_name: rail.estimation.algos.cc_yaw - name: cache_ref +- name: cache_ref + classname: YawCacheCreate nprocess: 1 -- classname: YawCacheCreate module_name: rail.estimation.algos.cc_yaw - name: cache_unk + aliases: + data: data_ref + rand: rand_ref + patch_source: patch_source_ref + output: output_cache_ref +- name: cache_unk + classname: YawCacheCreate nprocess: 1 -- classname: YawAutoCorrelate module_name: rail.estimation.algos.cc_yaw - name: auto_corr + aliases: + data: data_unk + rand: rand_unk + patch_source: output_cache_ref + output: output_cache_unk +- name: auto_corr + classname: YawAutoCorrelate nprocess: 1 -- classname: YawCrossCorrelate module_name: rail.estimation.algos.cc_yaw - name: cross_corr + aliases: + output: output_auto_corr + sample: output_cache_ref +- name: cross_corr + classname: YawCrossCorrelate nprocess: 1 -- classname: YawSummarize module_name: rail.estimation.algos.cc_yaw - name: summarize + aliases: + output: output_cross_corr + reference: output_cache_ref + unknown: output_cache_unk +- name: summarize + classname: YawSummarize nprocess: 1 + module_name: rail.estimation.algos.cc_yaw + aliases: + output: output_summarize + cross_corr: output_cross_corr + auto_corr_ref: output_auto_corr +site: + max_threads: 2 + name: local diff --git a/src/rail/pipelines/estimation/yaw_pipeline_config.yml b/src/rail/pipelines/estimation/yaw_pipeline_config.yml index 2cb515b..39ab837 100644 --- a/src/rail/pipelines/estimation/yaw_pipeline_config.yml +++ b/src/rail/pipelines/estimation/yaw_pipeline_config.yml @@ -1,100 +1,67 @@ -auto_corr: - aliases: - output: output_auto_corr - sample: output_cache_ref - config: null - method: linear - name: auto_corr - output_mode: default - rbin_num: 50 - rmax: 1000.0 - rmin: 100.0 - rweight: null - sample: None - thread_num: null - verbose: debug - zbin_num: 8 - zbins: null - zmax: 3.0 - zmin: 0.0 cache_ref: - aliases: - data: data_ref - output: output_cache_ref - patch_source: patch_source_ref - rand: rand_ref - config: null - data: None - dec_name: dec - n_patches: 5 - name: cache_ref output_mode: default - overwrite: true - patch_file: null - patch_name: null - patch_source: None path: data/test_ref + overwrite: true ra_name: ra - rand: None + dec_name: dec redshift_name: z - verbose: debug weight_name: null -cache_unk: - aliases: - data: data_unk - output: output_cache_unk - patch_source: output_cache_ref - rand: rand_unk - config: null - data: None - dec_name: dec - n_patches: null - name: cache_unk - output_mode: default - overwrite: true patch_file: null patch_name: null - patch_source: None + n_patches: 5 + verbose: debug + name: cache_ref + config: null +cache_unk: + output_mode: default path: data/test_unk + overwrite: true ra_name: ra - rand: None + dec_name: dec redshift_name: null - verbose: debug weight_name: null -cross_corr: - aliases: - output: output_cross_corr - reference: output_cache_ref - unknown: output_cache_unk + patch_file: null + patch_name: null + n_patches: null + verbose: debug + name: cache_unk config: null - method: linear - name: cross_corr +auto_corr: output_mode: default - rbin_num: 50 - reference: None - rmax: 1000.0 rmin: 100.0 + rmax: 1000.0 rweight: null + rbin_num: 50 + zmin: 0.0 + zmax: 3.0 + zbin_num: 8 + method: linear + zbins: null thread_num: null - unknown: None verbose: debug + name: auto_corr + config: null +cross_corr: + output_mode: default + rmin: 100.0 + rmax: 1000.0 + rweight: null + rbin_num: 50 + zmin: 0.0 + zmax: 3.0 zbin_num: 8 + method: linear zbins: null - zmax: 3.0 - zmin: 0.0 -summarize: - aliases: - auto_corr_ref: output_auto_corr - cross_corr: output_cross_corr - output: output_summarize - auto_corr_ref: None - auto_corr_unk: None + thread_num: null + verbose: debug + name: cross_corr config: null - cross_corr: None - cross_est: null - crosspatch: true - name: summarize +summarize: output_mode: default + cross_est: null ref_est: null unk_est: null + crosspatch: true verbose: debug + name: summarize + config: null