diff --git a/example/cfis_image_sims/config_exp_Gie_symlink.ini b/example/cfis_image_sims/config_exp_Gie_symlink.ini new file mode 100644 index 000000000..a6a3aed92 --- /dev/null +++ b/example/cfis_image_sims/config_exp_Gie_symlink.ini @@ -0,0 +1,90 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_MODULE = last:find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_DIR/input_exp/,$SP_DIR/input_exp/,$SP_DIR/input_exp/ + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = simu_image-0000000,simu_weight-0000000,simu_flag-0000000 + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits,.fits + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{7} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-,weight-,flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink diff --git a/example/cfis_image_sims/config_exp_Ma_onthefly.ini b/example/cfis_image_sims/config_exp_Ma_onthefly.ini new file mode 100644 index 000000000..df0307a19 --- /dev/null +++ b/example/cfis_image_sims/config_exp_Ma_onthefly.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for masking of exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Ma + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask exposures +[MASK_RUNNER] + +# Parent module +INPUT_DIR = last:split_exp_runner + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask_simu + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = False + +# File name suffix for the output flag files (optional) +PREFIX = pipeline diff --git a/example/cfis_image_sims/config_exp_Sp.ini b/example/cfis_image_sims/config_exp_Sp.ini new file mode 100644 index 000000000..ca010b643 --- /dev/null +++ b/example/cfis_image_sims/config_exp_Sp.ini @@ -0,0 +1,74 @@ +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = last:get_images_runner + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/example/cfis_image_sims/config_exp_psfex.ini b/example/cfis_image_sims/config_exp_psfex.ini new file mode 100644 index 000000000..f44bb38d7 --- /dev/null +++ b/example/cfis_image_sims/config_exp_psfex.ini @@ -0,0 +1,54 @@ +# ShapePipe configuration file for fake PSF postage stamps (image simulations) + + +## Default ShapePipe options +[DEFAULT] + +VERBOSE = True + +RUN_NAME = run_sp_tile_fpsf + +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +MODULE = fake_psf_runner + +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +LOG_NAME = log_sp + +RUN_LOG_NAME = log_run_sp + +INPUT_DIR = $SP_RUN/output + +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +SMP_BATCH_SIZE = 1 + +TIMEOUT = 96:00:00 + + +## Module options +[FAKE_PSF_RUNNER] + +INPUT_MODULE = sextractor_runner + +FILE_PATTERN = sexcat + +FILE_EXT = .fits + +NUMBERING_SCHEME = -000-000 + +# Path to the pickled PSF dictionary +PSF_DICT_PATH = /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle diff --git a/example/cfis_image_sims/config_onthefly.mask_simu b/example/cfis_image_sims/config_onthefly.mask_simu new file mode 100644 index 000000000..1a63cc2e5 --- /dev/null +++ b/example/cfis_image_sims/config_onthefly.mask_simu @@ -0,0 +1,86 @@ +# Mask module configuration file for single-exposure images + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + + +## Border mask +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 50 +BORDER_FLAG_VALUE = 4 + + +## Halo mask +[HALO_PARAMETERS] + +HALO_MAKE = False + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction spike mask +[SPIKE_PARAMETERS] + +SPIKE_MAKE = False + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier mask +[MESSIER_PARAMETERS] + +MESSIER_MAKE = False + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = False + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +TEMP_DIRECTORY = .temp + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis_image_sims/config_tile_Fe.ini b/example/cfis_image_sims/config_tile_Fe.ini new file mode 100644 index 000000000..9c3cb8cd5 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Fe.ini @@ -0,0 +1,68 @@ +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[FIND_EXPOSURES_RUNNER] + +INPUT_MODULE = last:get_images_runner + +FILE_PATTERN = CFIS_simu_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +COLNUM = 2 + +EXP_PREFIX = simu_image- diff --git a/example/cfis_image_sims/config_tile_Git_symlink.ini b/example/cfis_image_sims/config_tile_Git_symlink.ini new file mode 100644 index 000000000..87c4a8816 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Git_symlink.ini @@ -0,0 +1,87 @@ +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_DIR/input_tiles, $SP_DIR/input_tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS_simu_image-000-000, CFIS_simu_weight-000-000 + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}-\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_simu_image-, CFIS_simu_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# Copy command options, optional +RETRIEVE_OPTIONS = -L diff --git a/example/cfis_image_sims/config_tile_Ma_onthefly.ini b/example/cfis_image_sims/config_tile_Ma_onthefly.ini new file mode 100644 index 000000000..0f49eccea --- /dev/null +++ b/example/cfis_image_sims/config_tile_Ma_onthefly.ini @@ -0,0 +1,82 @@ +# ShapePipe configuration file for masking of tiles + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Ma + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask tiles +[MASK_RUNNER] + +# Input directory, containing input files, single string or list of names +INPUT_DIR = run_sp_Git:get_images_runner, last:uncompress_fits_runner + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .fits + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask_simu + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = False + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = False + +# File name suffix for the output flag files (optional) +PREFIX = pipeline diff --git a/example/cfis_image_sims/config_tile_Mc_psfex.ini b/example/cfis_image_sims/config_tile_Mc_psfex.ini new file mode 100644 index 000000000..728ae89e7 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Mc_psfex.ini @@ -0,0 +1,73 @@ +# ShapePipe post-run configuration file: create final catalogs for psfex + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mc_psfex + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:fake_psf_runner, last:ngmix_runner + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/cfis_image_sims/config_tile_Mh_exp.ini b/example/cfis_image_sims/config_tile_Mh_exp.ini new file mode 100644 index 000000000..94270a7e6 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Mh_exp.ini @@ -0,0 +1,77 @@ +# ShapePipe configuration file for merging per-exposure WCS headers +# at the tile level. Input is the exp_numbers file produced by +# find_exposures_runner; EXP_BASE_DIR tells the runner where to find +# the per-exposure split_exp_runner header .npy files. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mh_exp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_HEADERS_RUNNER] + +# Input: exp_numbers txt file from find_exposures_runner +INPUT_DIR = last:find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# Tile numbering scheme (RA-Dec, e.g. -301-279) +NUMBERING_SCHEME = -000-000 + +# Root directory containing all per-exposure work directories. +# The runner will walk this tree to collect headers-.npy files. +EXP_BASE_DIR = $SP_EXP + +# If True, log a warning and continue when split_exp_runner output is missing +# for some exposures (e.g. those that failed job 16). If False (default), +# raise an error and stop. +WARN_MISSING_EXP = False diff --git a/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini b/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini new file mode 100644 index 000000000..ae08976a2 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini @@ -0,0 +1,79 @@ +# ShapePipe configuration file for tiles: ngmix + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Ng + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner,run_sp_tile_fpsf:fake_psf_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner + +FILE_PATTERN = sexcat, image_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# No background subtraction for image sims (background not simulated) +BKG_SUB = False + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +SAVE_BATCH = 1000 + +ID_OBJ_MIN = -1 +ID_OBJ_MAX = -1 diff --git a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini new file mode 100644 index 000000000..025097f81 --- /dev/null +++ b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini @@ -0,0 +1,127 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# Fake PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_ViVi + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names + +MODULE = vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:uncompress_fits_runner + +FILE_PATTERN = sexcat, CFIS_simu_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner +ME_IMAGE_PATTERN = flag, image, weight diff --git a/example/cfis_image_sims/config_tile_Sx_nomask.ini b/example/cfis_image_sims/config_tile_Sx_nomask.ini new file mode 100644 index 000000000..810614c5d --- /dev/null +++ b/example/cfis_image_sims/config_tile_Sx_nomask.ini @@ -0,0 +1,114 @@ +# ShapePipe configuration file for tile detection + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner + +FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight, log_exp_headers + +FILE_EXT = .fits, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = False + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/example/cfis_image_sims/config_tile_onthefly.mask_simu b/example/cfis_image_sims/config_tile_onthefly.mask_simu new file mode 100644 index 000000000..f42d05d39 --- /dev/null +++ b/example/cfis_image_sims/config_tile_onthefly.mask_simu @@ -0,0 +1,90 @@ +# Mask module config file for tiles + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + +## Border parameters +[BORDER_PARAMETERS] + +BORDER_MAKE = False + +BORDER_WIDTH = 1 +BORDER_FLAG_VALUE = 4 + + +## Halo parameters +[HALO_PARAMETERS] + +HALO_MAKE = False + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction pike parameters +[SPIKE_PARAMETERS] + +SPIKE_MAKE = False + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier parameters +[MESSIER_PARAMETERS] + +MESSIER_MAKE = False + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_PIXEL_SCALE = 0.187 +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = False + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + +## External flag +[EXTERNAL_FLAG] + +EF_MAKE = False + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False + +TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis_image_sims/default.conv b/example/cfis_image_sims/default.conv new file mode 120000 index 000000000..bd71df850 --- /dev/null +++ b/example/cfis_image_sims/default.conv @@ -0,0 +1 @@ +../cfis/default.conv \ No newline at end of file diff --git a/example/cfis_image_sims/default.param b/example/cfis_image_sims/default.param new file mode 120000 index 000000000..49e000314 --- /dev/null +++ b/example/cfis_image_sims/default.param @@ -0,0 +1 @@ +../cfis/default.param \ No newline at end of file diff --git a/example/cfis_image_sims/default_noimaflags.param b/example/cfis_image_sims/default_noimaflags.param new file mode 120000 index 000000000..75451801e --- /dev/null +++ b/example/cfis_image_sims/default_noimaflags.param @@ -0,0 +1 @@ +../cfis/default_noimaflags.param \ No newline at end of file diff --git a/example/cfis_image_sims/default_tile.sex b/example/cfis_image_sims/default_tile.sex new file mode 120000 index 000000000..8770da87e --- /dev/null +++ b/example/cfis_image_sims/default_tile.sex @@ -0,0 +1 @@ +../cfis/default_tile.sex \ No newline at end of file diff --git a/example/cfis_image_sims/final_cat.param b/example/cfis_image_sims/final_cat.param new file mode 100644 index 000000000..0357c4850 --- /dev/null +++ b/example/cfis_image_sims/final_cat.param @@ -0,0 +1,107 @@ +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +# Can maybe be removed. +TILE_ID + +# SExtractor number. TODO: Create unique UNIONS number +# from TILD_ID + NUMBER +NUMBER + +# flags +FLAGS +#IMAFLAGS_ISO +NGMIX_MCAL_FLAGS + +# PSF ellipticity +NGMIX_ELL_PSFo_NOSHEAR + +# spread class +#SPREAD_CLASS + +# spread model flag and error +#SPREAD_MODEL +#SPREADERR_MODEL + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_ELL_1M +NGMIX_ELL_1P +NGMIX_ELL_2M +NGMIX_ELL_2P +NGMIX_ELL_NOSHEAR +#NGMIX_ELL_ERR_1M +#NGMIX_ELL_ERR_1P +#NGMIX_ELL_ERR_2M +#NGMIX_ELL_ERR_2P +NGMIX_ELL_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR +NGMIX_Tpsf_1M +NGMIX_Tpsf_1P +NGMIX_Tpsf_2M +NGMIX_Tpsf_2P +NGMIX_Tpsf_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSFo_NOSHEAR + +# PSF size measured on reconvolved image +# NGMIX_Tpsf_NOSHEAR + +# ngmix moment failure flag +NGMIX_MOM_FAIL diff --git a/example/cfis_image_sims/input_dirs.conf b/example/cfis_image_sims/input_dirs.conf new file mode 100644 index 000000000..cd3661983 --- /dev/null +++ b/example/cfis_image_sims/input_dirs.conf @@ -0,0 +1,12 @@ +# Configuration template for init_run_v2.0.sh +# Specifies input directories for image simulations +# Copy and edit this file, then pass with: -c /path/to/config + +# Base directory for input data +input_dir_base="/n09data/hervas/skills_out" + +# Tile images subdirectory +input_dir_tiles="${input_dir_base}/${subdir}/images/SP_tiles" + +# Exposure images subdirectory +input_dir_exp="${input_dir_base}/${subdir}/images/SP_exp" diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index a0b10e8c9..75e3a67c0 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -61,7 +61,7 @@ def params_default(): Set default parameter values. """ - _params = { + _params = { "input_root_dir": ".", "merged_cat_path": "final_cat.hdf5", "param_path": None, @@ -71,29 +71,33 @@ def params_default(): "output_summary": "n_tiles_final.txt", "ID": None, "single_op": None, + "image_sims": False, } - _short_options = { - "input_root_dir": "-i", + _short_options = { + "input_root_dir": "-i", "merged_cat_path": "-m", "param_path": "-p", "patch": "-P", "list_only": "-l", "output_summary": "-o", "single_op": "-s", - } - _types = { - "hdu_num": "int", + "image_sims": "-I", + } + _types = { + "hdu_num": "int", "list_only": "bool", - } - _help_strings = { + "image_sims": "bool", + } + _help_strings = { "input_root_dir": "input root_dir for tile catalogues, default={}", "merged_cat_path": "merged catalogue path (hdf5 file), default={}", "param_path": "parameter file path, if not given use all columns, default={}", - "patch": "patch number, default={}", + "patch": "patch number (data) or grid subdir (image_sims), default={}", "list_only": "print list of patches and IDs only, default={}", "output_summary": "output file for numbre of tiles, default={}", "ID": "ID for single-ID operation, default={}", "single_op": "single ID operation, allowed are 'check', 'add', 'remove'; default={}", + "image_sims": "image simulations mode (different dir layout and run prefix), default={}", } return _params, _short_options, _types, _help_strings @@ -231,17 +235,34 @@ def check_ID(merged_cat_path, ID, verbose=False): def print_list(params): + verbose = params.get("verbose", False) n_tiles = 0 + + if not os.path.exists(params["merged_cat_path"]): + print(f"File {params['merged_cat_path']} not found") + return + with h5py.File(params["merged_cat_path"], "r") as hdf5_file: - for patch in hdf5_file["patches"]: - #print(patch) - for id in hdf5_file[f"patches/{patch}"]: - n_tiles += 1 + if "patches" not in hdf5_file: + print("Warning: no 'patches' group in output file (0 tiles added?)") + else: + for patch in hdf5_file["patches"]: + for id in hdf5_file[f"patches/{patch}"]: + n_tiles += 1 + if verbose: + print(f" {patch}/{id}") + + if verbose: + print(f"Total: {n_tiles} tiles") with open(params["output_summary"], "w") as f_out: print(n_tiles, file=f_out) + # Write n_tiles to HDF5 file header + with h5py.File(params["merged_cat_path"], "a") as hdf5_file: + hdf5_file.attrs["n_tiles"] = n_tiles + def get_patch_group(hdf5_file, patch, verbose=False): """Get Patch group. @@ -339,20 +360,50 @@ def copy_data(param_list, extracted_data, dtype): return structured_data +def collect_tile_ids_image_sims(patch_path): + """Collect tile IDs from image-sims layout: tiles/// + + Parameters + ---------- + patch_path : str + path to the grid subdir (e.g. .../1p2z_grid_1) + + Returns + ------- + list of (tile_id, tile_path) tuples + """ + id_pattern = re.compile(r"^\d+\.\d+$") + tiles_root = os.path.join(patch_path, "tiles") + result = [] + if not os.path.isdir(tiles_root): + return result + for prefix in os.listdir(tiles_root): + prefix_path = os.path.join(tiles_root, prefix) + if not os.path.isdir(prefix_path): + continue + for tile_id in os.listdir(prefix_path): + if id_pattern.match(tile_id): + result.append((tile_id, os.path.join(prefix_path, tile_id))) + return result + + def process(params): - - patch = rf"P{params['patch']}" - patch_pattern = re.compile(patch) - - # Define the nested path pattern for locating `.fits` files - file_pattern = "output/run_sp_Mc_*/make_cat_runner/output/*.fits" + + if params["image_sims"]: + patch_name = params["patch"] + run_prefix = "run_sp_tile_Mc_*" + else: + patch_name = rf"P{params['patch']}" + run_prefix = "run_sp_Mc_*" + + patch_pattern = re.compile(patch_name) # Regex pattern for tile IDs id_pattern = re.compile(r"^\d+\.\d+$") n_added = 0 IDs_added = [] - + # Open the HDF5 file (create it if it doesn't exist) if params["verbose"]: print(f"Initializing file {params['merged_cat_path']}") @@ -364,21 +415,30 @@ def process(params): # Skip non-matching entries if not patch_pattern.fullmatch(patch): continue - + # Full path to patch patch_path = os.path.join(params["input_root_dir"], patch) if not os.path.isdir(patch_path): if params["verbose"]: print(f"Path {patch_path} not found, skipping") continue - + # Get hdf5 group for this patch patch_group = get_patch_group(hdf5_file, patch, params["verbose"]) - # Get paths to all tile IDs and loop - tile_runs_path = os.path.join(patch_path, "tile_runs") - subdirs = os.listdir(tile_runs_path) - for id in tqdm.tqdm(subdirs, total=len(subdirs)): + # Collect (tile_id, tile_path) pairs depending on layout + if params["image_sims"]: + tile_items = collect_tile_ids_image_sims(patch_path) + else: + tile_runs_path = os.path.join(patch_path, "tile_runs") + tile_items = [ + (tid, os.path.join(tile_runs_path, tid)) + for tid in os.listdir(tile_runs_path) + if id_pattern.match(tid) + and os.path.isdir(os.path.join(tile_runs_path, tid)) + ] + + for id, id_path in tqdm.tqdm(tile_items, total=len(tile_items)): # Skip if the patch/ID data already exists if id in patch_group: @@ -386,44 +446,40 @@ def process(params): print(f"Skipping {id} (already processed)") continue - if id_pattern.match(id) and os.path.isdir(os.path.join(tile_runs_path, id)): - id_path = os.path.join(tile_runs_path, id) - - base_pattern = os.path.join(id_path, "output", "run_sp_Mc_*") - all_matches = [d for d in glob.glob(base_pattern) if os.path.isdir(d)] - if not all_matches: - if params["verbose"]: - print(f"Final cat for {id} not found, continuing") - continue - newest_dir = max(all_matches, key=os.path.getmtime) - - id_dash = re.sub("\.", "-", id) - fits_file = f"{newest_dir}/make_cat_runner/output/final_cat-{id_dash}.fits" - - # Exclude unsuccessful run without output FITS file - if not os.path.exists(fits_file): - if params["verbose"]: - print(f"Run without output file found for {id}, skipping") - continue - - if True: - extracted_data, dtype = read_data(fits_file, params) - - structured_data = copy_data(params["param_list"], extracted_data, dtype) - - # Create a new dataset - try: - patch_group.create_dataset( - str(id), - data=structured_data, - dtype=dtype, - ) - except: - print(f"Error for {id}: Could not create dataset in group {patch}") - raise - - n_added += 1 - IDs_added.append(id) + base_pattern = os.path.join(id_path, "output", run_prefix) + all_matches = [d for d in glob.glob(base_pattern) if os.path.isdir(d)] + if not all_matches: + if params["verbose"]: + print(f"Final cat for {id} not found, continuing") + continue + newest_dir = max(all_matches, key=os.path.getmtime) + + id_dash = re.sub(r"\.", "-", id) + fits_file = f"{newest_dir}/make_cat_runner/output/final_cat-{id_dash}.fits" + + # Exclude unsuccessful run without output FITS file + if not os.path.exists(fits_file): + if params["verbose"]: + print(f"Run without output file found for {id}, skipping") + continue + + extracted_data, dtype = read_data(fits_file, params) + + structured_data = copy_data(params["param_list"], extracted_data, dtype) + + # Create a new dataset + try: + patch_group.create_dataset( + str(id), + data=structured_data, + dtype=dtype, + ) + except: + print(f"Error for {id}: Could not create dataset in group {patch}") + raise + + n_added += 1 + IDs_added.append(id) if params["verbose"]: print(f"{n_added} tiles added ({' '.join(IDs_added)})") diff --git a/src/shapepipe/modules/fake_psf_package/__init__.py b/src/shapepipe/modules/fake_psf_package/__init__.py new file mode 100644 index 000000000..c77dd5658 --- /dev/null +++ b/src/shapepipe/modules/fake_psf_package/__init__.py @@ -0,0 +1,28 @@ +"""FAKE PSF PACKAGE. + +This package creates fake PSF postage-stamp dictionaries for image simulations. + +:Author: Martin Kilbinger + +:Parent module: ``sextractor_runner`` + +:Input: Tile SExtractor catalogue (multi-epoch FITS format) + +:Output: Per-galaxy PSF sqlite dictionary (``galaxy_psf-XXX-XXX.sqlite``) + +Description +=========== + +For each galaxy in the tile catalogue the module looks up the pre-computed +PSF stamp for every contributing exposure-CCD combination from a pickled +PSF dictionary and writes the result to a SqliteDict file in the same format +produced by ``psfex_interp_runner``. + +Module-specific config file entries +==================================== + +PSF_DICT_PATH : str + Path to the pickled PSF dictionary (``Full_psf_dict.pickle``). +""" + +__all__ = ["fake_psf.py"] diff --git a/src/shapepipe/modules/fake_psf_package/fake_psf.py b/src/shapepipe/modules/fake_psf_package/fake_psf.py new file mode 100644 index 000000000..4c2ac4bc2 --- /dev/null +++ b/src/shapepipe/modules/fake_psf_package/fake_psf.py @@ -0,0 +1,89 @@ +"""FAKE PSF. + +Create fake PSF postage-stamp dictionaries for image simulations. + +:Author: Martin Kilbinger + +""" + +import pickle + +import numpy as np +from astropy.io import fits +from sqlitedict import SqliteDict + + +class FakePsf: + """Fake PSF. + + Parameters + ---------- + sexcat_path : str + Path to the tile SExtractor catalogue (multi-epoch FITS format). + psf_dict_path : str + Path to the pickled PSF dictionary. + output_path : str + Path for the output SqliteDict file. + w_log : logging.Logger + Pipeline logger. + """ + + def __init__(self, sexcat_path, psf_dict_path, output_path, w_log): + self._sexcat_path = sexcat_path + self._psf_dict_path = psf_dict_path + self._output_path = output_path + self._w_log = w_log + + def process(self): + """Run fake PSF creation.""" + self._w_log.info(f"Reading sexcat: {self._sexcat_path}") + sex = fits.open(self._sexcat_path, ignore_missing_simple=True) + + self._w_log.info(f"Loading PSF dictionary: {self._psf_dict_path}") + with open(self._psf_dict_path, "rb") as f: + psf_dict = pickle.load(f) + + n_gal = len(sex[3].data.field("NUMBER")) + n_exp = len(sex) - 3 + + self._w_log.info(f"Processing {n_gal} galaxies over {n_exp} epochs") + + # Build (n_gal, n_exp) array of "expname-ccdnum" strings + string_array = np.empty((n_gal, n_exp), dtype="object") + for exp_ind in range(3, len(sex)): + ccd_n = sex[exp_ind].data.field("CCD_N") + exp_name = sex[exp_ind].data.field("EXP_NAME") + string_array[:, exp_ind - 3] = [ + f"{int(exp)}-{int(ccd)}" + for exp, ccd in zip(exp_name, ccd_n) + ] + + # Mask invalid entries (exp or ccd == -99 produces strings with "--") + mask = np.array( + [["--" in item for item in row] for row in string_array] + ) + masked = np.ma.masked_array(string_array, mask) + + # Build per-galaxy PSF dictionaries + output_file = SqliteDict(self._output_path) + missing = 0 + for idx, gal_row in enumerate(masked): + galaxy_number = idx + 1 # 1-based, matches NUMBER field + gal_dict = {} + for exp_ccd in gal_row.compressed(): + if exp_ccd not in psf_dict: + missing += 1 + self._w_log.warning( + f"Galaxy {galaxy_number}: key '{exp_ccd}' not in PSF dict" + ) + continue + gal_dict[exp_ccd] = psf_dict[exp_ccd] + output_file[str(galaxy_number)] = gal_dict + + output_file.commit() + output_file.close() + sex.close() + + if missing: + self._w_log.warning(f"{missing} missing PSF entries across all galaxies") + self._w_log.info(f"Written: {self._output_path}") diff --git a/src/shapepipe/modules/fake_psf_runner.py b/src/shapepipe/modules/fake_psf_runner.py new file mode 100644 index 000000000..d3dcbbc0e --- /dev/null +++ b/src/shapepipe/modules/fake_psf_runner.py @@ -0,0 +1,36 @@ +"""FAKE PSF RUNNER. + +Module runner for ``fake_psf``. + +:Author: Martin Kilbinger + +""" + +from shapepipe.modules.fake_psf_package import fake_psf +from shapepipe.modules.module_decorator import module_runner + + +@module_runner( + version="1.0", + file_pattern=["sexcat"], + file_ext=".fits", + depends=["numpy", "astropy", "sqlitedict"], + numbering_scheme="-000-000", +) +def fake_psf_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Fake PSF Runner.""" + sexcat_path = input_file_list[0] + psf_dict_path = config.getexpanded(module_config_sec, "PSF_DICT_PATH") + output_path = f'{run_dirs["output"]}/galaxy_psf{file_number_string}.sqlite' + + inst = fake_psf.FakePsf(sexcat_path, psf_dict_path, output_path, w_log) + inst.process() + + return None, None diff --git a/src/shapepipe/modules/merge_headers_runner.py b/src/shapepipe/modules/merge_headers_runner.py index 9bee35159..6c3cfa38e 100644 --- a/src/shapepipe/modules/merge_headers_runner.py +++ b/src/shapepipe/modules/merge_headers_runner.py @@ -43,6 +43,9 @@ def merge_headers_runner( f"Tile-level merge: collecting headers from {exp_base_dir} " f"using {exp_numbers_file}" ) + warn_missing = config.getboolean( + module_config_sec, "WARN_MISSING_EXP", fallback=False + ) headers_file_list = get_exp_output_files( exp_base_dir, exp_numbers_file, @@ -50,6 +53,7 @@ def merge_headers_runner( "headers", ".npy", w_log=w_log, + warn_only=warn_missing, ) # Extract tile number from the exp_numbers filename, e.g. # "exp_numbers-284.272-1.000.txt" -> "-284.272-1.000" diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 371264e23..de6c87e32 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -213,11 +213,10 @@ def __init__( flag_vignet_path, f_wcs_path, bkg_rms_vignet_path=None, - ): self.f_wcs_file = SqliteDict(f_wcs_path) self.gal_vign_cat = SqliteDict(gal_vignet_path) - self.bkg_vign_cat = SqliteDict(bkg_vignet_path) + self.bkg_vign_cat = SqliteDict(bkg_vignet_path) if bkg_vignet_path is not None else None self.psf_vign_cat = SqliteDict(psf_vignet_path) self.weight_vign_cat = SqliteDict(weight_vignet_path) self.flag_vign_cat = SqliteDict(flag_vignet_path) @@ -230,7 +229,8 @@ def __init__( def close(self): self.f_wcs_file.close() self.gal_vign_cat.close() - self.bkg_vign_cat.close() + if self.bkg_vign_cat is not None: + self.bkg_vign_cat.close() self.flag_vign_cat.close() self.weight_vign_cat.close() self.psf_vign_cat.close() @@ -293,24 +293,45 @@ def __init__( save_batch=-1, id_obj_min=-1, id_obj_max=-1, + bkg_sub=True, centroid_source="hsm", ): - if len(input_file_list) not in {6, 7}: + # Base count = catalogue + vignets, excluding the f_wcs headers (passed + # separately). One fewer when the background vignet is absent + # (``bkg_sub=False``, image sims). An extra trailing slot may carry the + # optional background-rms vignet (#779), so both counts are valid. + n_base = 6 if bkg_sub else 5 + if len(input_file_list) not in {n_base, n_base + 1}: raise IndexError( f"Input file list has length {len(input_file_list)}," - + " required is 6 or 7" + + f" required is {n_base} or {n_base + 1}" ) self._tile_cat_path = input_file_list[0] + if bkg_sub: + bkg_path, psf_path, weight_path, flag_path = ( + input_file_list[2], input_file_list[3], + input_file_list[4], input_file_list[5], + ) + else: + bkg_path, psf_path, weight_path, flag_path = ( + None, input_file_list[2], + input_file_list[3], input_file_list[4], + ) + bkg_rms_vignet_path = ( + input_file_list[n_base] + if len(input_file_list) == n_base + 1 + else None + ) self._vignet_cat = Vignet( input_file_list[1], - input_file_list[2], - input_file_list[3], - input_file_list[4], - input_file_list[5], + bkg_path, + psf_path, + weight_path, + flag_path, f_wcs_path, - input_file_list[6] if len(input_file_list) == 7 else None, + bkg_rms_vignet_path, ) self._output_dir = output_dir @@ -324,6 +345,7 @@ def __init__( self._save_batch = save_batch self._id_obj_min = id_obj_min self._id_obj_max = id_obj_max + self._bkg_sub = bkg_sub self._centroid_source = centroid_source self._w_log = w_log @@ -629,6 +651,61 @@ def check_key(self, expccd_name_tmp, vign_cat, vignet_path): + f" file '{vignet_path}'" ) + def log_mean_ellipticity(self): + """Log mean ellipticity from NOSHEAR HDU to the run log. + + Reports , with standard errors for all objects and for + objects passing the default metacal cuts (flags==0, mcal_flags==0, + 10 < SNR < 500, T/Tpsf > 0.5). + """ + output_path = self.get_output_path(self._output_dir) + try: + with fits.open(output_path) as hdul: + d = hdul['NOSHEAR'].data + g1 = d['g1'].astype(float) + g2 = d['g2'].astype(float) + flags = d['flags'] + mcal_flags = d['mcal_flags'] + s2n = d['s2n'].astype(float) + T = d['T'].astype(float) + Tpsf = d['Tpsf'].astype(float) + except Exception as e: + self._w_log.warning(f"Could not compute mean ellipticity: {e}") + return + + n_total = len(g1) + if n_total == 0: + self._w_log.info("Mean ellipticity: no objects in output catalogue") + return + + def _log_stats(g1_sel, g2_sel, label): + n = len(g1_sel) + if n == 0: + self._w_log.info(f" {label}: 0 objects") + return + mean_g1 = g1_sel.mean() + mean_g2 = g2_sel.mean() + err_g1 = g1_sel.std() / np.sqrt(n) + err_g2 = g2_sel.std() / np.sqrt(n) + self._w_log.info( + f" {label} (N={n}):" + f" = {mean_g1:+.4e} +/- {err_g1:.4e}," + f" = {mean_g2:+.4e} +/- {err_g2:.4e}" + ) + + self._w_log.info(f"Mean ellipticity (NOSHEAR, N_total={n_total}):") + _log_stats(g1, g2, "no cuts") + + with np.errstate(invalid='ignore'): + mask = ( + (flags == 0) + & (mcal_flags == 0) + & (s2n >= 10.0) + & (s2n <= 500.0) + & (T / Tpsf >= 0.5) + ) + _log_stats(g1[mask], g2[mask], "SNR in [10, 500], T/Tpsf > 0.5") + def process(self): """Process. @@ -678,7 +755,7 @@ def process(self): n_empty_cat += 1 continue - stamp = prepare_postage_stamps(vignet_cat, obj_id, i_tile, tile_cat) + stamp = prepare_postage_stamps(vignet_cat, obj_id, i_tile, tile_cat, self._bkg_sub) if len(stamp.gals) == 0: n_no_epoch += 1 @@ -769,9 +846,12 @@ def process(self): # Save results self.save_results(res_dict) -def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat): + # Log mean ellipticity statistics + self.log_mean_ellipticity() + +def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): # define per-object lists of individual exposures to go into ngmix - stamp = Postage_stamp() + stamp = Postage_stamp(bkg_sub=bkg_sub) #identify exposure and ccd number from psf catalog psf_expccd_names = list(vignet.psf_vign_cat[str(obj_id)].keys()) for expccd_name in psf_expccd_names: diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 31028c7f8..6367978e8 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -49,9 +49,18 @@ def ngmix_runner( # Pixel scale pixel_scale = config.getfloat(module_config_sec, "PIXEL_SCALE") + # Background subtraction: disable for image sims, where there is no + # background image to subtract. The background vignet occupies one input + # slot, so the f_wcs headers -- and the optional background-rms vignet -- + # shift by one when it is absent. + bkg_sub = config.getboolean(module_config_sec, "BKG_SUB", fallback=True) + wcs_idx = 6 if bkg_sub else 5 + # Path to merged single-exposure single-HDU headers - f_wcs_path = input_file_list[6] + f_wcs_path = input_file_list[wcs_idx] + # Optional background-rms vignet -> ngmix inverse-variance weight map (#779); + # all-or-nothing. Appended as the final positional input after the wcs slice. if config.has_option(module_config_sec, "BKG_RMS_VIGNET_PATH"): bkg_rms_vignet_path = config.getexpanded( module_config_sec, @@ -61,9 +70,9 @@ def ngmix_runner( raise FileNotFoundError( f"Background RMS vignet file not found: {bkg_rms_vignet_path}" ) - input_file_list = input_file_list[:6] + [bkg_rms_vignet_path] + input_file_list = input_file_list[:wcs_idx] + [bkg_rms_vignet_path] else: - input_file_list = input_file_list[:6] + input_file_list = input_file_list[:wcs_idx] # Batch save option if config.has_option(module_config_sec, "SAVE_BATCH"): @@ -79,10 +88,10 @@ def ngmix_runner( id_obj_min = config.getint(module_config_sec, "ID_OBJ_MIN") id_obj_max = config.getint(module_config_sec, "ID_OBJ_MAX") - # Centroid source for the galaxy Jacobian origin: "wcs" (default — the + # Centroid source for the galaxy Jacobian origin: "wcs" (default -- the # catalog sky position projected through the WCS, trusting the astrometry) # or "hsm" (legacy HSM adaptive-moment centroid, being phased out: noisy - # for stars and flagged as incorrect by Fabian — see #767). + # for stars and flagged as incorrect by Fabian -- see #767). if config.has_option(module_config_sec, "CENTROID_SOURCE"): centroid_source = config.get(module_config_sec, "CENTROID_SOURCE") else: @@ -100,6 +109,7 @@ def ngmix_runner( save_batch=save_batch, id_obj_min=id_obj_min, id_obj_max=id_obj_max, + bkg_sub=bkg_sub, centroid_source=centroid_source, ) diff --git a/src/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py index f124612b7..89e3be6df 100644 --- a/src/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/src/shapepipe/modules/sextractor_package/sextractor_script.py @@ -98,9 +98,13 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): history.append(idx) exp_list = [] - pattern = r"([0-9]*)p\.(.*)" + pattern = r"([0-9]+)p?\.(.*)" for hist in history: m = re.search(pattern, hist) + if m is None: + raise ValueError( + f"Could not parse exposure ID from HISTORY entry: '{hist}'" + ) exp_list.append(m.group(1)) obj_id = np.copy(cat.get_data()["NUMBER"]) diff --git a/src/shapepipe/pipeline/exp_utils.py b/src/shapepipe/pipeline/exp_utils.py index 8f0df784a..b934f0bb3 100644 --- a/src/shapepipe/pipeline/exp_utils.py +++ b/src/shapepipe/pipeline/exp_utils.py @@ -22,6 +22,7 @@ def get_exp_output_files( file_pattern, file_ext, w_log=None, + warn_only=False, ): """Collect output files from a per-exposure runner for all tile exposures. @@ -31,9 +32,9 @@ def get_exp_output_files( ///output/run_sp_*//output/ - where ``exp_prefix = exp_id[:2]`` and ``exp_base = exp_id[:-1]`` (the - trailing letter, typically ``p``, is stripped because directory names - do not carry it). + where ``exp_prefix = exp_id[:2]`` and ``exp_base`` is ``exp_id`` with the + trailing letter stripped if present (e.g. ``2113864p`` → ``2113864``), or + the full ``exp_id`` for numeric-only IDs (image simulations). Parameters ---------- @@ -52,6 +53,10 @@ def get_exp_output_files( File extension including the leading dot, e.g. ``.npy`` w_log : logging.Logger, optional Pipeline logger; ``None`` silences all logging + warn_only : bool, optional + If ``True``, log a warning for missing exposures and continue with + the files that were found instead of raising ``FileNotFoundError``. + Default is ``False``. Returns ------- @@ -88,9 +93,10 @@ def get_exp_output_files( for exp_id in exp_ids: # Directory structure mirrors run_job_canfar_v2.0.sh: # exp_prefix = first 2 chars of exp_id (e.g. "21") - # exp_base = exp_id without trailing letter (e.g. "2113864") + # exp_base = exp_id without trailing letter if present (e.g. "2113864"), + # or full exp_id for numeric-only ids (image sims) exp_prefix = exp_id[:2] - exp_base = exp_id[:-1] + exp_base = exp_id[:-1] if exp_id[-1].isalpha() else exp_id pattern = os.path.join( exp_base_dir, @@ -115,10 +121,12 @@ def get_exp_output_files( w_log.warning(f" {exp_id}: no match for {pattern}") if missing: - raise FileNotFoundError( - f"No {runner_name} output found for " - f"{len(missing)} exposure(s): {missing}" - ) + msg = f"No {runner_name} output found for {len(missing)} exposure(s): {missing}" + if warn_only: + if w_log: + w_log.warning(msg) + else: + raise FileNotFoundError(msg) if w_log: w_log.info(f"Found {len(file_list)} exposure output files") @@ -183,7 +191,7 @@ def get_exp_output_dirs( for exp_id in exp_ids: exp_prefix = exp_id[:2] - exp_base = exp_id[:-1] + exp_base = exp_id[:-1] if exp_id[-1].isalpha() else exp_id pattern = os.path.join( exp_base_dir, diff --git a/src/shapepipe/testing/simulate.py b/src/shapepipe/testing/simulate.py index 8ab5a06f0..ee62f43c6 100644 --- a/src/shapepipe/testing/simulate.py +++ b/src/shapepipe/testing/simulate.py @@ -103,3 +103,98 @@ def make_data( jacob_lists.append(wcs.jacobian()) return gals, psfs, psfs_sigmas, weights, flags, jacob_lists + + +def make_data_pujol( + rng, + shear_list, + noise=1e-5, + n_epochs=1, + share_shift=False, + gal_hlr=0.3, + gal_flux=1000.0, + psf_fwhm=0.55, + pixel_scale=0.1857, + img_size=201, +): + """Simulate galaxies at multiple shear values sharing the same noise per epoch. + + Implements the Pujol (2018) denoising approach: all shear variants receive + the identical noise and sub-pixel shift each epoch, so noise contributions + to the shear response R and to the m/c estimator cancel exactly. + + Parameters + ---------- + rng : numpy.random.RandomState + shear_list : list of tuple of float + List of (g1, g2) values to simulate simultaneously. + noise : float, optional + Per-pixel noise sigma. + n_epochs : int, optional + share_shift : bool, optional + If True all epochs share the same sub-pixel shift; the shift is + always shared across shear variants regardless of this flag. + gal_hlr, gal_flux, psf_fwhm, pixel_scale, img_size : float / int, optional + Same meaning as in make_data. + + Returns + ------- + list of tuple + One ``(gals, psfs, psfs_sigmas, weights, flags, jacob_lists)`` per + entry in ``shear_list``, all sharing identical per-epoch noise. + """ + psf_noise_sigma = 1.0e-6 + scale = pixel_scale + wcs = galsim.PixelScale(scale) + n_shears = len(shear_list) + + # output accumulators + all_gals = [[] for _ in range(n_shears)] + all_psfs = [[] for _ in range(n_shears)] + all_psfs_sigmas = [[] for _ in range(n_shears)] + all_weights = [[] for _ in range(n_shears)] + all_flags = [[] for _ in range(n_shears)] + all_jacobs = [[] for _ in range(n_shears)] + + if share_shift: + dy, dx = rng.uniform(low=-scale / 2, high=scale / 2, size=2) + + for epoch in range(n_epochs): + if not share_shift: + dy, dx = rng.uniform(low=-scale / 2, high=scale / 2, size=2) + + # One noise draw shared by all shear variants this epoch + noise_img = rng.normal(scale=noise, size=(img_size, img_size)) + psf_noise_img = rng.normal(scale=psf_noise_sigma, size=(img_size, img_size)) + + psf = galsim.Moffat(beta=2.5, fwhm=psf_fwhm) + psf_im_ = psf.drawImage(nx=img_size, ny=img_size, wcs=wcs) + psf_sigma = galsim.hsm.FindAdaptiveMom(psf_im_).moments_sigma + psf_im = psf_im_.array.astype(np.float64) + psf_noise_img + + jacob = wcs.jacobian() + weight = np.full((img_size, img_size), 1.0 / noise ** 2) + flag = np.zeros((img_size, img_size)) + + for j, shear in enumerate(shear_list): + obj = galsim.Convolve( + psf, + galsim.Exponential(half_light_radius=gal_hlr, flux=gal_flux).shear( + g1=shear[0], g2=shear[1] + ), + ).shift(dx, dy) + im = obj.drawImage(nx=img_size, ny=img_size, wcs=wcs).array.astype(np.float64) + im += noise_img # same noise for every shear variant + + all_gals[j].append(im) + all_psfs[j].append(psf_im) + all_psfs_sigmas[j].append(psf_sigma) + all_weights[j].append(weight.copy()) + all_flags[j].append(flag.copy()) + all_jacobs[j].append(jacob) + + return [ + (all_gals[j], all_psfs[j], all_psfs_sigmas[j], + all_weights[j], all_flags[j], all_jacobs[j]) + for j in range(n_shears) + ]