GMTB Workflow Documentation
met_qpf_verf_all.ksh
Go to the documentation of this file.
1 #!/bin/ksh -l
2 
3 ##########################################################################
4 #
5 # Script Name: met_qpf_verf_all.ksh
6 #
7 # Author: J.Wolff & M.Harrold
8 # NCAR/RAL & DTC
9 #
10 # Released: 6/7/2016
11 #
12 # Description:
13 # This script runs the MET/Grid-Stat tool to verify gridded
14 # precipitation forecasts against gridded precipitation analyses.
15 # The precipitation fields must first be placed on a common grid prior
16 # to running this script.
17 #
18 # START_TIME = The cycle time to use for the initial time.
19 # FCST_TIME = The three-digit forecasts that is to be verified.
20 # ACCUM_TIME = The two-digit accumulation time: 03 or 24.
21 # MODEL_BUCKET = The accumulation time in the model (bucket): 6.
22 # OBS_BUCKET = The accumulation time in the obs (bucket): 6 (ccpa) 1 (cmorph).
23 # MET_EXE_ROOT = The full path of the MET executables.
24 # MET_CONFIG = The full path of the MET configuration files.
25 # UTIL_EXEC = The full path of the UPP executables.
26 # MOAD_DATAROOT = Top-level data directory of WRF output.
27 # RAW_OBS = Directory containing observations to be used.
28 # OBTYPE = Gridded observation source data.
29 # MODEL = The model being evaluated.
30 # OP_SWITCH = Op-like data (TRUE) to determine if different processing for after 240 h.
31 #
32 ##########################################################################
33 
34 # Name of this script
35 SCRIPT=met_qpf_verf_all.ksh
36 
37 # Set path for manual testing of script
38 #export CONSTANT=/scratch4/BMC/gmtb/gmtb-tierIII/bin/constants.ksh
39 
40 # Make sure ${CONSTANT} exists
41 if [ ! -x "${CONSTANT}" ]; then
42  ${ECHO} "ERROR: ${CONSTANT} does not exist or is not executable"
43  exit 1
44 fi
45 
46 # Read constants into the current shell
47 . ${CONSTANT}
48 
49 # Vars used for manual testing of the script
50 #export START_TIME=2016011500
51 #export FCST_TIME=036
52 #export ACCUM_TIME=24
53 #export MODEL_BUCKET=6
54 #export OBS_BUCKET=1
55 #export RES_LIST="0p25"
56 #export GRID_VX_LIST="G3"
57 #export MET_EXE_ROOT=/scratch4/BMC/dtc/MET/met-5.2_beta3/bin
58 #export MET_CONFIG=/scratch4/BMC/gmtb/gmtb-tierIII/parm/met_config
59 #export UTIL_EXEC=/scratch4/BMC/gmtb/gmtb-tierIII/util
60 #export MOAD_DATAROOT=/scratch4/BMC/gmtb/gmtb-tierIII/vx_out/refcst/2016011500
61 #export RAW_OBS=/scratch3/BMC/dtc-hwrf/GMTB/vx_data/cmorph
62 #export ROTDIR=/scratch4/BMC/gmtb/gmtb-tierIII/refcst/2016011500
63 #export OBTYPE=cmorph
64 #export MODEL=refcst
65 #export OP_SWITCH=TRUE
66 
67 # Print run parameters
68 ${ECHO}
69 ${ECHO} "${SCRIPT} started at `${DATE}`"
70 ${ECHO}
71 ${ECHO} " START_TIME = ${START_TIME}"
72 ${ECHO} " FCST_TIME = ${FCST_TIME}"
73 ${ECHO} " ACCUM_TIME = ${ACCUM_TIME}"
74 ${ECHO} " MODEL_BUCKET = ${MODEL_BUCKET}"
75 ${ECHO} " OBS_BUCKET = ${OBS_BUCKET}"
76 ${ECHO} " MET_EXE_ROOT = ${MET_EXE_ROOT}"
77 ${ECHO} " MET_CONFIG = ${MET_CONFIG}"
78 ${ECHO} " UTIL_EXEC = ${UTIL_EXEC}"
79 ${ECHO} " MOAD_DATAROOT = ${MOAD_DATAROOT}"
80 ${ECHO} " ROTDIR = ${ROTDIR}"
81 ${ECHO} " RAW_OBS = ${RAW_OBS}"
82 ${ECHO} " OBTYPE = ${OBTYPE}"
83 ${ECHO} " MODEL = ${MODEL}"
84 
85 # Make sure $MOAD_DATAROOT exists
86 if [ ! -d "${MOAD_DATAROOT}" ]; then
87  ${ECHO} "MOAD_DATAROOT, ${MOAD_DATAROOT} does not exist; will create it now!"
88  ${MKDIR} -p ${MOAD_DATAROOT}
89 fi
90 
91 # Make sure ROTDIR exists
92 if [ ! -d "${ROTDIR}" ]; then
93  ${ECHO} "ERROR: ROTDIR, ${ROTDIR} does not exist"
94  exit 1
95 fi
96 
97 # Make sure RAW_OBS directory exists
98 if [ ! -d ${RAW_OBS} ]; then
99  ${ECHO} "ERROR: RAW_OBS, ${RAW_OBS}, does not exist!"
100  exit 1
101 fi
102 
103 export MODEL
104 export VERSION
105 export OBTYPE
106 export FCST_TIME
107 export OP_SWITCH
108 ${ECHO} "MODEL=${MODEL}"
109 ${ECHO} "VERSION=${VERSION}"
110 ${ECHO} "OBTYPE=${OBTYPE}"
111 ${ECHO} "FCST_TIME=${FCST_TIME}"
112 ${ECHO} "OP_SWITCH=${OP_SWITCH}"
113 
114 # Go to working directory
115 workdir=${MOAD_DATAROOT}/metprd
116 ${MKDIR} -p ${workdir}
117 pcp_combine_dir_obs=${MOAD_DATAROOT}/metprd/pcp_combine/${OBTYPE}
118 ${MKDIR} -p ${pcp_combine_dir_obs}
119 pcp_combine_dir_model=${MOAD_DATAROOT}/metprd/pcp_combine/${MODEL}
120 ${MKDIR} -p ${pcp_combine_dir_model}
121 cd ${workdir}
122 
123 # Loop through the domain list
124 for RES in ${RES_LIST}; do
125 
126  export RES
127  ${ECHO} "RES=${RES}"
128 
129  for GRID_VX in ${GRID_VX_LIST}; do
130  GRID_VX_NUM=`${ECHO} ${GRID_VX} | cut -c2-`
131  typeset -Z3 GRID_VX_NUM
132  GRID_VX_FORMAT="G${GRID_VX_NUM}"
133  export GRID_VX
134  export GRID_VX_FORMAT
135 
136  ${ECHO} "FCST_TIME=${FCST_TIME}"
137 
138  # Specify mask directory structure
139  MASKS=${MET_CONFIG}/masks
140  export MASKS
141 
142  # Specify the MET Grid-Stat configuration file to be used
143  GS_CONFIG="${MET_CONFIG}/GridStatConfig_APCP${ACCUM_TIME}_REGRID_${OBTYPE}"
144 
145  # Make sure the Point-Stat configuration files exists
146  if [ ! -e ${GS_CONFIG} ]; then
147  ${ECHO} "ERROR: ${GS_CONFIG} does not exist!"
148  exit 1
149  fi
150 
151  # Compute the verification date
152  YYYYMMDD=`${ECHO} ${START_TIME} | ${CUT} -c1-8`
153  HH=`${ECHO} ${START_TIME} | ${CUT} -c9-10`
154  VDATE=`${UTIL_EXEC}/ndate +${FCST_TIME} ${START_TIME}`
155  VYYYYMMDD=`${ECHO} ${VDATE} | ${CUT} -c1-8`
156  VHH=`${ECHO} ${VDATE} | ${CUT} -c9-10`
157  ${ECHO} 'valid time for ' ${FCST_TIME} 'h forecast = ' ${VDATE}
158 
159  PVDATE=`${UTIL_EXEC}/ndate -24 ${VDATE}`
160  PVYYYYMMDD=`${ECHO} ${PVDATE} | ${CUT} -c1-8`
161 
162  # Run pcp_combine on model to make appropriate accumulation times
163  FCST_GRIB_FILE_DIR=${ROTDIR}
164  if [ ! -e ${FCST_GRIB_FILE_DIR} ]; then
165  ${ECHO} "ERROR: ${FCST_GRIB_FILE_DIR} does not exist!"
166  exit 1
167  fi
168 
169  # 6-h accumulations from the model: Simply pass through because already in 6-hrly buckets (just create .nc file)
170  # /scratch4/BMC/dtc/MET/met-5.1/bin/pcp_combine -sum 20160122_000000 6 20160122_060000 06 -pcpdir /scratch4/BMC/gmtb/mid_tier/OUTPUT/prtutornems_0p25/DOMAINS/2016012200/mdlprd -name "APCP_06" /scratch4/BMC/gmtb/mid_tier/OUTPUT/prtutornems_0p25/DOMAINS/2016012200/metprd/pcp_combine/test_pgrbq006.gfs.2016012200_06h.nc
171 
172  # 24-h accumulations from the model: Add (4) 6-hr files (valid 12-12 UTC)
173  # /scratch4/BMC/dtc/MET/met-5.1/bin/pcp_combine -sum 20160122_000000 6 20160123_120000 24 -pcpdir /scratch4/BMC/gmtb/mid_tier/OUTPUT/prtutornems_0p25/DOMAINS/2016012200/mdlprd -name "APCP_24" /scratch4/BMC/gmtb/mid_tier/OUTPUT/prtutornems_0p25/DOMAINS/2016012200/metprd/pcp_combine/test_pgrbq036.gfs.2016012200_24h.nc
174 
175  FCST_FILE=${pcp_combine_dir_model}/pgrbq${FCST_TIME}.gfs.${START_TIME}_${ACCUM_TIME}h.nc
176 
177  if [[ ! -e ${FCST_FILE} ]]; then
178  if [[ ${OP_SWITCH} == "FALSE" ]]; then
179  PCP_COMBINE_ARGS="-sum ${YYYYMMDD}_${HH}0000 ${MODEL_BUCKET} ${VYYYYMMDD}_${VHH}0000 ${ACCUM_TIME} -pcpdir ${FCST_GRIB_FILE_DIR} -pcprx 'pgrbq.*grib2$' -name "APCP_${ACCUM_TIME}" ${FCST_FILE}"
180 
181  # Run the PCP-Combine command
182  ${ECHO} "CALLING: ${MET_EXE_ROOT}/pcp_combine ${PCP_COMBINE_ARGS}"
183 
184  ${MET_EXE_ROOT}/pcp_combine -sum ${YYYYMMDD}_${HH}0000 ${MODEL_BUCKET} ${VYYYYMMDD}_${VHH}0000 ${ACCUM_TIME} -pcpdir ${FCST_GRIB_FILE_DIR} -pcprx 'pgrbq.*grib2$' -name "APCP_${ACCUM_TIME}" ${FCST_FILE}
185  error=$?
186  if [ ${error} -ne 0 ]; then
187  ${ECHO} "${MET_EXE_ROOT}/pcp_combine crashed! Exit status=${error}"
188  exit ${error}
189  fi
190 
191  elif [[ ${OP_SWITCH} == "TRUE" ]]; then
192  if [[ ${HH} == "00" && ${FCST_TIME} -le 252 ]]; then
193  MODEL_BUCKET=${MODEL_BUCKET}
194  PCP_COMBINE_ARGS="-sum ${YYYYMMDD}_${HH}0000 ${MODEL_BUCKET} ${VYYYYMMDD}_${VHH}0000 ${ACCUM_TIME} -pcpdir ${FCST_GRIB_FILE_DIR} -pcprx 'pgrbq.*grib2$' -name "APCP_${ACCUM_TIME}" ${FCST_FILE}"
195 
196  # Run the PCP-Combine command
197  ${ECHO} "CALLING: ${MET_EXE_ROOT}/pcp_combine ${PCP_COMBINE_ARGS}"
198 
199  ${MET_EXE_ROOT}/pcp_combine -sum ${YYYYMMDD}_${HH}0000 ${MODEL_BUCKET} ${VYYYYMMDD}_${VHH}0000 ${ACCUM_TIME} -pcpdir ${FCST_GRIB_FILE_DIR} -pcprx 'pgrbq.*grib2$' -name "APCP_${ACCUM_TIME}" ${FCST_FILE}
200  error=$?
201  if [ ${error} -ne 0 ]; then
202  ${ECHO} "${MET_EXE_ROOT}/pcp_combine crashed! Exit status=${error}"
203  exit ${error}
204  fi
205 
206  elif [[ ${HH} == "00" && ${FCST_TIME} -gt 252 ]]; then
207  MODEL_BUCKET=12
208  PCP_COMBINE_ARGS="-sum ${YYYYMMDD}_${HH}0000 ${MODEL_BUCKET} ${VYYYYMMDD}_${VHH}0000 ${ACCUM_TIME} -pcpdir ${FCST_GRIB_FILE_DIR} -pcprx 'pgrbq.*grib2$' -name "APCP_${ACCUM_TIME}" ${FCST_FILE}"
209 
210  # Run the PCP-Combine command
211  ${ECHO} "CALLING: ${MET_EXE_ROOT}/pcp_combine ${PCP_COMBINE_ARGS}"
212 
213  ${MET_EXE_ROOT}/pcp_combine -sum ${YYYYMMDD}_${HH}0000 ${MODEL_BUCKET} ${VYYYYMMDD}_${VHH}0000 ${ACCUM_TIME} -pcpdir ${FCST_GRIB_FILE_DIR} -pcprx 'pgrbq.*grib2$' -name "APCP_${ACCUM_TIME}" ${FCST_FILE}
214  error=$?
215  if [ ${error} -ne 0 ]; then
216  ${ECHO} "${MET_EXE_ROOT}/pcp_combine crashed! Exit status=${error}"
217  exit ${error}
218  fi
219 
220  elif [[ ${HH} == "00" && ${FCST_TIME} -eq 252 ]]; then
221  export FIELD='name="CMORPH";level="(0,*,*)";'
222  F1=${FCST_GRIB_FILE_DIR}/pgbq234.gfs.${START_TIME}.grib2
223  F2=${FCST_GRIB_FILE_DIR}/pgbq240.gfs.${START_TIME}.grib2
224  F3=${FCST_GRIB_FILE_DIR}/pgbq252.gfs.${START_TIME}.grib2
225  PCP_COMBINE_ARGS="-add ${F1} '${FIELD}' ${F2} '${FIELD}' ${F3} '${FIELD}' -name "APCP_${ACCUM_TIME}" ${FCST_FILE}"
226 
227  # Run the PCP-Combine command
228  ${ECHO} "CALLING: ${MET_EXE_ROOT}/pcp_combine ${PCP_COMBINE_ARGS}"
229 
230  ${MET_EXE_ROOT}/pcp_combine -add ${F1} '${FIELD}' ${F2} '${FIELD}' ${F3} '${FIELD}' -name "APCP_${ACCUM_TIME}" ${FCST_FILE}
231  error=$?
232  if [ ${error} -ne 0 ]; then
233  ${ECHO} "${MET_EXE_ROOT}/pcp_combine crashed! Exit status=${error}"
234  exit ${error}
235  fi
236  else
237  ${ECHO} "Bad init and/or FCST_TIME specified OR not supported!"
238  exit 1
239  fi
240  else
241  ${ECHO} "Bad OP_SWITCH specified!"
242  exit 1
243  fi
244  fi # End forecase file processing, if necessary.
245 
246  # 6-h accumulations from the CCPA observations: One 6-hrly files
247  # /scratch4/BMC/dtc/MET/met-5.1/bin/pcp_combine -sum 00000000_000000 6 20160122_060000 06 -pcpdir /scratch4/BMC/gmtb/mid_tier/vx_data/gridded/raw/ccpa/20160122 -pcpdir /scratch4/BMC/gmtb/mid_tier/vx_data/gridded/raw/ccpa/20160121 -name "APCP_06" /scratch4/BMC/gmtb/mid_tier//OUTPUT/prtutornems_0p25/DOMAINS/2016012200/metprd/pcp_combine/ccpa_20160122_060000_06h.nc
248 
249  # 24-h accumulations from the CCPA observations: Add (8) 3-hrly files (valid 12-12 UTC)
250  # /scratch4/BMC/dtc/MET/met-5.1/bin/pcp_combine -sum 00000000_000000 6 20160123_120000 24 -pcpdir /scratch4/BMC/gmtb/mid_tier/vx_data/gridded/raw/ccpa/20160123 -pcpdir /scratch4/BMC/gmtb/mid_tier/vx_data/gridded/raw/ccpa/20160122 -name "APCP_24" /scratch4/BMC/gmtb/mid_tier//OUTPUT/prtutornems_0p25/DOMAINS/2016012200/metprd/pcp_combine/ccpa_20160123_0120000_24h.nc
251 
252  # 24-h accumulations from the CMORPH observations: Add (24) 1-hrly files (valid 12-12 UTC)
253  # /scratch4/BMC/dtc/MET/met-5.1/bin/pcp_combine -sum 00000000_000000 1 20160123_120000 24 -pcpdir /scratch4/BMC/gmtb/mid_tier/vx_data/gridded/proc/cmorph/20160123 -pcpdir /scratch4/BMC/gmtb/mid_tier/vx_data/gridded/proc/cmorph/20160122 -name "APCP_24" /scratch4/BMC/gmtb/mid_tier//OUTPUT/prtutornems_0p25/DOMAINS/2016012200/metprd/pcp_combine/cmorph_20160123_0120000_24h.nc
254 
255  if [[ ${OBTYPE} == "ccpa" ]]; then
256 
257  # Run pcp_combine on CCPA/CMORPH observations to make appropriate accumulation times
258  RAW_OBS_DIR=${RAW_OBS}/${VYYYYMMDD}
259  if [ ! -e ${RAW_OBS_DIR} ]; then
260  ${ECHO} "ERROR: ${RAW_OBS_DIR} does not exist!"
261  exit 1
262  fi
263  PREV_RAW_OBS_DIR=${RAW_OBS}/${PVYYYYMMDD}
264  if [ ! -e ${PREV_RAW_OBS_DIR} ]; then
265  ${ECHO} "ERROR: ${PREV_RAW_OBS_DIR} does not exist!"
266  exit 1
267  fi
268 
269  OBS_FILE=${pcp_combine_dir_obs}/${OBTYPE}_${VYYYYMMDD}_${VHH}0000_${ACCUM_TIME}h.nc
270  PCP_COMBINE_ARGS="-sum 00000000_000000 ${OBS_BUCKET} ${VYYYYMMDD}_${VHH}0000 ${ACCUM_TIME} -pcpdir ${RAW_OBS_DIR} -pcpdir ${PREV_RAW_OBS_DIR} -name "APCP_${ACCUM_TIME}" ${OBS_FILE}"
271 
272  # Run the PCP-Combine command
273  ${ECHO} "CALLING: ${MET_EXE_ROOT}/pcp_combine ${PCP_COMBINE_ARGS}"
274 
275  ${MET_EXE_ROOT}/pcp_combine ${PCP_COMBINE_ARGS}
276  error=$?
277  if [ ${error} -ne 0 ]; then
278  ${ECHO} "${MET_EXE_ROOT}/pcp_combine crashed! Exit status=${error}"
279  exit ${error}
280  fi
281 
282  elif [[ ${OBTYPE} == "cmorph" ]]; then
283  OBS_FILE=${CMORPH_24H_BUCKET}/${VYYYYMMDD}/CMORPH_${VYYYYMMDD}_${VHH}0000_${ACCUM_TIME}h.nc
284  if [[ ! -e ${OBS_FILE} ]]; then
285  # Run pcp_combine on CCPA/CMORPH observations to make appropriate accumulation times
286  RAW_OBS_DIR=${RAW_OBS}/${VYYYYMMDD}
287  if [ ! -e ${RAW_OBS_DIR} ]; then
288  ${ECHO} "ERROR: ${RAW_OBS_DIR} does not exist!"
289  exit 1
290  fi
291  PREV_RAW_OBS_DIR=${RAW_OBS}/${PVYYYYMMDD}
292  if [ ! -e ${PREV_RAW_OBS_DIR} ]; then
293  ${ECHO} "ERROR: ${PREV_RAW_OBS_DIR} does not exist!"
294  exit 1
295  fi
296  OBS_FILE=${pcp_combine_dir_obs}/${OBTYPE}_${VYYYYMMDD}_${VHH}0000_${ACCUM_TIME}h.nc
297  export FIELD='name="CMORPH";level="(0,*,*)";'
298  PCP_COMBINE_ARGS="-sum 00000000_000000 ${OBS_BUCKET} ${VYYYYMMDD}_${VHH}0000 ${ACCUM_TIME} -pcpdir ${RAW_OBS_DIR} -pcpdir ${PREV_RAW_OBS_DIR} -field '${FIELD}' -name "APCP_${ACCUM_TIME}" ${OBS_FILE}"
299 
300  # Run the PCP-Combine command
301  ${ECHO} "CALLING: ${MET_EXE_ROOT}/pcp_combine ${PCP_COMBINE_ARGS}"
302 
303  ${MET_EXE_ROOT}/pcp_combine ${PCP_COMBINE_ARGS}
304  error=$?
305  if [ ${error} -ne 0 ]; then
306  ${ECHO} "${MET_EXE_ROOT}/pcp_combine crashed! Exit status=${error}"
307  exit ${error}
308  fi
309  fi
310  else
311  ${ECHO} "ERROR: Bad OBTYPE specified!"
312  exit 1
313  fi
314 
315 #######################################################################
316 #
317 # Run Grid-Stat
318 #
319 #######################################################################
320 
321  # Make sure the Grid-Stat configuration file exists
322  if [ ! -e ${GS_CONFIG} ]; then
323  ${ECHO} "ERROR: ${GS_CONFIG} does not exist!"
324  exit 1
325  fi
326 
327  # Make sure the forecast file run through pcp_combine exists
328  if [ ! -e ${FCST_FILE} ]; then
329  ${ECHO} "ERROR: Could not find observation file: ${OBS_FILE}"
330  exit 1
331  fi
332 
333  # Make sure the observation file run through pcp_combine exists
334  if [ ! -e ${OBS_FILE} ]; then
335  ${ECHO} "ERROR: Could not find observation file: ${OBS_FILE}"
336  exit 1
337  fi
338 
339  ${ECHO} "CALLING: ${MET_EXE_ROOT}/grid_stat ${FCST_FILE} ${OBS_FILE} ${GS_CONFIG} -outdir . -v 2"
340 
341  ${MET_EXE_ROOT}/grid_stat \
342  ${FCST_FILE} \
343  ${OBS_FILE} \
344  ${GS_CONFIG} \
345  -outdir . \
346  -v 2
347 
348  error=$?
349  if [ ${error} -ne 0 ]; then
350  ${ECHO} "ERROR: For ${MODEL}, ${MET_EXE_ROOT}/grid_stat crashed Exit status: ${error}"
351  exit ${error}
352  fi
353 
354  done
355 done
356 
357 ##########################################################################
358 
359 ${ECHO} "${SCRIPT} completed at `${DATE}`"
360 
361 exit 0