Image simulations workflow

example/cfis_image_sims/config_exp_Gie_symlink.ini

@@ -0,0 +1,90 @@
+# ShapePipe configuration file for: get images
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = False
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_exp_Gie
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = get_images_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+# Get exposures
+[GET_IMAGES_RUNNER]
+
+INPUT_MODULE = last:find_exposures_runner
+
+FILE_PATTERN = exp_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+
+# Paths
+
+# Output path (optional, default is [FILE]:OUTPUT_DIR
+# OUTPUT_PATH = input_images
+
+# Input path where original images are stored. Can be local path or vos url.
+# Single string or list of strings
+INPUT_PATH = $SP_DIR/input_exp/,$SP_DIR/input_exp/,$SP_DIR/input_exp/
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = simu_image-0000000,simu_weight-0000000,simu_flag-0000000
+
+# Input file extensions
+INPUT_FILE_EXT = .fits, .fits,.fits
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{7}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = image-,weight-,flag-
+
+# Method to retrieve images, one in 'vos', 'symlink'
+RETRIEVE = symlink

example/cfis_image_sims/config_exp_Ma_onthefly.ini

@@ -0,0 +1,76 @@
+# ShapePipe configuration file for masking of exposures
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_exp_Ma
+
+# Add date and time to RUN_NAME, optional, default: False
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = mask_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+### Mask exposures
+[MASK_RUNNER]
+
+# Parent module
+INPUT_DIR = last:split_exp_runner
+
+# Update numbering convention, accounting for HDU number of
+# single-exposure single-HDU files
+NUMBERING_SCHEME = -0000000-0
+
+# Path of mask config file
+MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask_simu
+
+# External mask file flag, use if True, otherwise ignore
+USE_EXT_FLAG = True
+
+# External star catalogue flag, use external cat if True,
+# obtain from online catalogue if False
+USE_EXT_STAR = False
+
+# File name suffix for the output flag files (optional)
+PREFIX = pipeline

example/cfis_image_sims/config_exp_Sp.ini

@@ -0,0 +1,74 @@
+# ShapePipe configuration file for single-exposures,
+# split images
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_exp_Sp
+
+# Add date and time to RUN_NAME, optional, default: True
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = split_exp_runner
+
+# Run mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+[SPLIT_EXP_RUNNER]
+
+INPUT_DIR = last:get_images_runner
+
+FILE_PATTERN = image, weight, flag
+
+# Matches compressed single-exposure files
+FILE_EXT = .fits, .fits, .fits
+
+NUMBERING_SCHEME = -0000000
+
+# OUTPUT_SUFFIX, actually file name prefixes.
+# Expected keyword "flag" will lead to a behavior where the data are saved as int.
+# The code also expects the image data to use the "image" suffix
+# (default value in the pipeline).
+OUTPUT_SUFFIX = image, weight, flag
+
+# Number of HDUs/CCDs of mosaic
+N_HDU = 40

example/cfis_image_sims/config_exp_psfex.ini

@@ -0,0 +1,54 @@
+# ShapePipe configuration file for fake PSF postage stamps (image simulations)
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+VERBOSE = True
+
+RUN_NAME = run_sp_tile_fpsf
+
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+MODULE = fake_psf_runner
+
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+LOG_NAME = log_sp
+
+RUN_LOG_NAME = log_run_sp
+
+INPUT_DIR = $SP_RUN/output
+
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+SMP_BATCH_SIZE = 1
+
+TIMEOUT = 96:00:00
+
+
+## Module options
+[FAKE_PSF_RUNNER]
+
+INPUT_MODULE = sextractor_runner
+
+FILE_PATTERN = sexcat
+
+FILE_EXT = .fits
+
+NUMBERING_SCHEME = -000-000
+
+# Path to the pickled PSF dictionary
+PSF_DICT_PATH = /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle

example/cfis_image_sims/config_onthefly.mask_simu

@@ -0,0 +1,86 @@
+# Mask module configuration file for single-exposure images
+
+## Paths to executables
+[PROGRAM_PATH]
+
+WW_PATH = weightwatcher
+WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww
+
+# Indicate cds client executable if no external star catalogue is available
+# (e.g. no internet access on run nodes)
+CDSCLIENT_PATH = findgsc2.2
+
+
+## Border mask
+[BORDER_PARAMETERS]
+
+BORDER_MAKE = True
+
+BORDER_WIDTH = 50
+BORDER_FLAG_VALUE = 4
+
+
+## Halo mask
+[HALO_PARAMETERS]
+
+HALO_MAKE = False
+
+HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg
+HALO_MAG_LIM = 13.
+HALO_SCALE_FACTOR = 0.05
+HALO_MAG_PIVOT = 13.8
+HALO_FLAG_VALUE = 2
+HALO_REG_FILE = halo.reg
+
+
+## Diffraction spike mask
+[SPIKE_PARAMETERS]
+
+SPIKE_MAKE = False
+
+SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg
+SPIKE_MAG_LIM = 18.
+SPIKE_SCALE_FACTOR = 0.3
+SPIKE_MAG_PIVOT = 13.8
+SPIKE_FLAG_VALUE = 128
+SPIKE_REG_FILE = spike.reg
+
+
+## Messier mask
+[MESSIER_PARAMETERS]
+
+MESSIER_MAKE = False
+
+MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits
+MESSIER_SIZE_PLUS = 0.
+MESSIER_FLAG_VALUE = 16
+
+
+## NGC mask
+[NGC_PARAMETERS]
+
+NGC_MAKE = False
+
+NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits
+NGC_SIZE_PLUS = 0.
+NGC_FLAG_VALUE = 32
+
+
+
+## Missing data parameters
+[MD_PARAMETERS]
+
+MD_MAKE = False
+
+MD_THRESH_FLAG = 0.3
+MD_THRESH_REMOVE = 0.75
+MD_REMOVE = False
+
+
+## Other parameters
+[OTHER]
+
+TEMP_DIRECTORY = .temp
+
+KEEP_REG_FILE = False
+KEEP_INDIVIDUAL_MASK = False