#!/usr/bin/env python3 # -*- coding: utf-8 -*- #======================================================================================== # name: clm.py # # category: Python module script # # description: # This module script writes tc-shell scripts which construct monthly and/or annual mean # climatology files from all or specified variables in MIROC output directories # # input: individual year output = yYYYY (e.g., y0300, y0301,...) # # call: ngt commands (ngtavr, ngtcf) and CDO commands (settaxis) # # use: import clmall; see the sample main script, ClmAllMain.py # # note: works on rocky@AORI # # history # 2010.06.03: originally written # 2010.08.08: class structure introduced # 2012.02.05: code clean-up # 2012.08.14: code clean-up # 2012.08.15: fixed time axis problem with a CDO command and reversed # latitude from the South to the North for netCDF # 2013.06.24: code clean-up # 2014.08.05: code clean-up # 2015.06.16: ignore ERROUT, SYSOUT, Restart, RSTA, RSTO, ftrace files # 2015.06.25: ignore SYSIN file # 2016.07.12: add an option to compute annual mean climatology # 2017.06.03: replace mydigit by str().zfill() # 2019.12.17: modularized with class # # author # m.yoshimori (masakazu AT aori.u-tokyo.ac.jp) #======================================================================================== import os class CLM(): def __init__(self, exp, subdir): self.inp = exp.inp self.year_start = exp.year_start self.year_end = exp.year_end self.subdir = subdir self.out = exp.out + '/' + self.subdir def annall(self, f, out_nc, out_gt, opt_lev41): nstep = self.year_end - self.year_start + 1 # number of iterations yyyys = str(self.year_start).zfill(4) yyyye = str(self.year_end).zfill(4) f.write('\n') f.write('# Create output directory if not exist\n') f.write('mkdir -p '+self.out+'\n') f.write('\n') # list files and dircotries from the first-year directory of the analysis din = self.inp + '/y' + str(yyyys) + '/' + self.subdir file_list = os.listdir(din) # search files in specified directory for var in file_list: # ignore files that contain the following words if ('SYSIN' in var) or ('SYSOUT' in var) or ('ERROUT' in var) or \ ('Restart' in var) or ('RSTA' in var) or ('RSTO' in var) or \ ('ftrace' in var) or ('.sh' in var) or ('md5' in var) or ('.gz' in var): continue # process other files x = din + '/' + var if os.path.isfile(x): # check if x is a file or not; use only files fout = self.out +'/av_'+var+'_y'+str(yyyys)+'-y'+str(yyyye)+'_ann' f.write('# Average multi-year files\n') f.write('ngtavr -o '+fout+' ') for i in range(nstep): year = self.year_start + i yyyy = str(year).zfill(4) dir = self.inp + '/y' + str(yyyy) + '/' + self.subdir f.write(dir+'/'+var+' ') f.write('\n') f.write('\n') if out_nc: # netCDF output f.write('# Convert the created file from Gtool to netCDF\n') f.write('ngtcf '+fout+' tmp1.nc\n') f.write('cdo settaxis,1850-07-01,12:00,1year tmp1.nc tmp2.nc\n') f.write('/bin/mv tmp2.nc '+fout+'.nc\n') if opt_lev41: f.write('ncrename -d HETA40.M,lev -v HETA40.M,lev '+fout+'.nc\n') f.write('/bin/rm tmp?.nc\n') f.write('\n') if not out_gt: # no gtool output f.write('# Delete the intermediate Gtool file\n') f.write('/bin/rm '+fout+'\n') f.write('\n') def monall(self, f, out_ann, out_nc, out_gt, opt_lev41): nstep = self.year_end - self.year_start + 1 # number of iterations yyyys = str(self.year_start).zfill(4) yyyye = str(self.year_end).zfill(4) nmonth = 12 + 1 f.write('\n') f.write('# Create output directory if not exist\n') f.write('mkdir -p '+self.out+'\n') f.write('\n') # list files and dircotries from the first-year directory of the analysis din = self.inp + '/y' + str(yyyys) + '/' + self.subdir file_list = os.listdir(din) # search files in specified directory for var in file_list: # ignore files that contain the following words if ('SYSIN' in var) or ('SYSOUT' in var) or ('ERROUT' in var) or \ ('Restart' in var) or ('RSTA' in var) or ('RSTO' in var) or \ ('ftrace' in var) or ('.sh' in var) or ('md5' in var) or ('.gz' in var): continue # process other files x = din + '/' + var if os.path.isfile(x): # check if x is a file or not; use only files fout = self.out +'/av_'+var+'_y'+str(yyyys)+'-y'+str(yyyye)+'_mon' # monthly mean climatology for imonth in range(1,nmonth): if imonth == 1: f.write('# Average multi-year files for month = '+str(imonth)+'\n') f.write('ngtavr -t '+str(imonth)+' -o '+fout+' ') else: f.write('# Average multi-year files for month = '+str(imonth)+', and append\n') f.write('ngtavr -t '+str(imonth)+' -a -o '+fout+' ') for i in range(nstep): year = self.year_start + i yyyy = str(year).zfill(4) dir = self.inp + '/y' + str(yyyy) + '/' + self.subdir f.write(dir+'/'+var+' ') f.write('\n') f.write('\n') if out_nc: # netCDF output f.write('# Convert the created file from Gtool to netCDF\n') f.write('ngtcf '+fout+' tmp1.nc\n') f.write('cdo settaxis,1850-01-16,12:00,1mon tmp1.nc tmp2.nc\n') f.write('/bin/mv tmp2.nc '+fout+'.nc\n') if opt_lev41: f.write('ncrename -d HETA40.M,lev -v HETA40.M,lev '+fout+'.nc\n') f.write('/bin/rm tmp?.nc\n') f.write('\n') if out_ann: # output annual mean climatology fout2 = self.out +'/av_'+var+'_y'+str(yyyys)+'-y'+str(yyyye)+'_ann' f.write('ngtavr -t 1:12 -o '+fout2+' '+fout+'\n') f.write('\n') if out_nc: f.write('# Convert the created file from Gtool to netCDF\n') f.write('ngtcf '+fout2+' tmp1.nc\n') f.write('cdo settaxis,1850-07-01,12:00,1year tmp1.nc tmp2.nc\n') f.write('/bin/mv tmp2.nc '+fout2+'.nc\n') if opt_lev41: f.write('ncrename -d HETA40.M,lev -v HETA40.M,lev '+fout2+'.nc\n') f.write('/bin/rm tmp?.nc\n') f.write('\n') if not out_gt: # no gtool output f.write('# Delete the intermediate Gtool file\n') f.write('/bin/rm '+fout2+'\n') f.write('\n') if not out_gt: # no gtool output f.write('# Delete the intermediate Gtool file\n') f.write('/bin/rm '+fout+'\n') f.write('\n') def dayall(self, f, out_nc, out_gt, opt_lev41, calendar): nstep = self.year_end - self.year_start + 1 # number of iterations yyyys = str(self.year_start).zfill(4) yyyye = str(self.year_end).zfill(4) # calendar type if calendar == "ideal": nday = 360 + 1 elif calendar == "noleap": nday = 365 + 1 else: print('Specified calendar type is not supported.') f.write('\n') f.write('# Create output directory if not exist\n') f.write('mkdir -p '+self.out+'\n') f.write('\n') # list files and dircotries from the first-year directory of the analysis din = self.inp + '/y' + str(yyyys) + '/' + self.subdir file_list = os.listdir(din) # search files in specified directory for var in file_list: # ignore files that contain the following words if ('SYSIN' in var) or ('SYSOUT' in var) or ('ERROUT' in var) or \ ('Restart' in var) or ('RSTA' in var) or ('RSTO' in var) or \ ('ftrace' in var) or ('.sh' in var) or ('md5' in var) or ('.gz' in var): continue # process other files x = din + '/' + var if os.path.isfile(x): # check if x is a file or not; use only files fout = self.out +'/av_'+var+'_y'+str(yyyys)+'-y'+str(yyyye)+'_day' # daily mean climatology for iday in range(1,nday): if iday == 1: f.write('# Average multi-year files for day = '+str(iday)+'\n') f.write('ngtavr -t '+str(iday)+' -o '+fout+' ') else: f.write('# Average multi-year files for day = '+str(iday)+', and append\n') f.write('ngtavr -t '+str(iday)+' -a -o '+fout+' ') for i in range(nstep): year = self.year_start + i yyyy = str(year).zfill(4) dir = self.inp + '/y' + str(yyyy) + '/' + self.subdir f.write(dir+'/'+var+' ') f.write('\n') f.write('\n') if out_nc: # netCDF output f.write('# Convert the created file from Gtool to netCDF\n') f.write('ngtcf '+fout+' tmp1.nc\n') f.write('cdo settaxis,1850-01-01,12:00,1day tmp1.nc tmp2.nc\n') f.write('/bin/mv tmp2.nc '+fout+'.nc\n') if opt_lev41: f.write('ncrename -d HETA40.M,lev -v HETA40.M,lev '+fout+'.nc\n') f.write('/bin/rm tmp?.nc\n') f.write('\n') if not out_gt: # no gtool output f.write('# Delete the intermediate Gtool file\n') f.write('/bin/rm '+fout+'\n') f.write('\n') def ann(self, f, out_nc, out_gt, opt_lev41, var): nstep = self.year_end - self.year_start + 1 # number of iterations yyyys = str(self.year_start).zfill(4) yyyye = str(self.year_end).zfill(4) f.write('\n') f.write('# Create output directory if not exist\n') f.write('mkdir -p '+self.out+'\n') f.write('\n') # list files and dircotries from the first-year directory of the analysis din = self.inp + '/y' + str(yyyys) + '/' + self.subdir file_list = os.listdir(din) fout = self.out +'/av_'+var+'_y'+str(yyyys)+'-y'+str(yyyye)+'_ann' f.write('# Average multi-year files\n') f.write('ngtavr -o '+fout+' ') for i in range(nstep): year = self.year_start + i yyyy = str(year).zfill(4) dir = self.inp + '/y' + str(yyyy) + '/' + self.subdir f.write(dir+'/'+var+' ') f.write('\n') f.write('\n') if out_nc: # netCDF output f.write('# Convert the created file from Gtool to netCDF\n') f.write('ngtcf '+fout+' tmp1.nc\n') f.write('cdo settaxis,1850-07-01,12:00,1year tmp1.nc tmp2.nc\n') f.write('/bin/mv tmp2.nc '+fout+'.nc\n') if opt_lev41: f.write('ncrename -d HETA40.M,lev -v HETA40.M,lev '+fout+'.nc\n') f.write('/bin/rm tmp?.nc\n') f.write('\n') if not out_gt: # no gtool output f.write('# Delete the intermediate Gtool file\n') f.write('/bin/rm '+fout+'\n') f.write('\n') def mon(self, f, out_ann, out_nc, out_gt, opt_lev41, var): nstep = self.year_end - self.year_start + 1 # number of iterations yyyys = str(self.year_start).zfill(4) yyyye = str(self.year_end).zfill(4) nmonth = 12 + 1 f.write('\n') f.write('# Create output directory if not exist\n') f.write('mkdir -p '+self.out+'\n') f.write('\n') # list files and dircotries from the first-year directory of the analysis din = self.inp + '/y' + str(yyyys) + '/' + self.subdir file_list = os.listdir(din) fout = self.out +'/av_'+var+'_y'+str(yyyys)+'-y'+str(yyyye)+'_mon' # monthly mean climatology for imonth in range(1,nmonth): if imonth == 1: f.write('# Average multi-year files for month = '+str(imonth)+'\n') f.write('ngtavr -t '+str(imonth)+' -o '+fout+' ') else: f.write('# Average multi-year files for month = '+str(imonth)+', and append\n') f.write('ngtavr -t '+str(imonth)+' -a -o '+fout+' ') for i in range(nstep): year = self.year_start + i yyyy = str(year).zfill(4) dir = self.inp + '/y' + str(yyyy) + '/' + self.subdir f.write(dir+'/'+var+' ') f.write('\n') f.write('\n') if out_nc: # netCDF output f.write('# Convert the created file from Gtool to netCDF\n') f.write('ngtcf '+fout+' tmp1.nc\n') f.write('cdo settaxis,1850-01-16,12:00,1mon tmp1.nc tmp2.nc\n') f.write('/bin/mv tmp2.nc '+fout+'.nc\n') if opt_lev41: f.write('ncrename -d HETA40.M,lev -v HETA40.M,lev '+fout+'.nc\n') f.write('/bin/rm tmp?.nc\n') f.write('\n') if out_ann: # output annual mean climatology fout2 = self.out +'/av_'+var+'_y'+str(yyyys)+'-y'+str(yyyye)+'_ann' f.write('ngtavr -t 1:12 -o '+fout2+' '+fout+'\n') f.write('\n') if out_nc: f.write('# Convert the created file from Gtool to netCDF\n') f.write('ngtcf '+fout2+' tmp1.nc\n') f.write('cdo settaxis,1850-07-01,12:00,1year tmp1.nc tmp2.nc\n') f.write('/bin/mv tmp2.nc '+fout2+'.nc\n') if opt_lev41: f.write('ncrename -d HETA40.M,lev -v HETA40.M,lev '+fout2+'.nc\n') f.write('/bin/rm tmp?.nc\n') f.write('\n') if not out_gt: # no gtool output f.write('# Delete the intermediate Gtool file\n') f.write('/bin/rm '+fout2+'\n') f.write('\n') if not out_gt: # no gtool output f.write('# Delete the intermediate Gtool file\n') f.write('/bin/rm '+fout+'\n') f.write('\n') def day(self, f, out_nc, out_gt, opt_lev41, calendar, var): nstep = self.year_end - self.year_start + 1 # number of iterations yyyys = str(self.year_start).zfill(4) yyyye = str(self.year_end).zfill(4) # calendar type if calendar == "ideal": nday = 360 + 1 elif calendar == "noleap": nday = 365 + 1 else: print('Specified calendar type is not supported.') f.write('\n') f.write('# Create output directory if not exist\n') f.write('mkdir -p '+self.out+'\n') f.write('\n') # list files and dircotries from the first-year directory of the analysis din = self.inp + '/y' + str(yyyys) + '/' + self.subdir file_list = os.listdir(din) fout = self.out +'/av_'+var+'_y'+str(yyyys)+'-y'+str(yyyye)+'_day' # daily mean climatology for iday in range(1,nday): if iday == 1: f.write('# Average multi-year files for day = '+str(iday)+'\n') f.write('ngtavr -t '+str(iday)+' -o '+fout+' ') else: f.write('# Average multi-year files for day = '+str(iday)+', and append\n') f.write('ngtavr -t '+str(iday)+' -a -o '+fout+' ') for i in range(nstep): year = self.year_start + i yyyy = str(year).zfill(4) dir = self.inp + '/y' + str(yyyy) + '/' + self.subdir f.write(dir+'/'+var+' ') f.write('\n') f.write('\n') if out_nc: # netCDF output f.write('# Convert the created file from Gtool to netCDF\n') f.write('ngtcf '+fout+' tmp1.nc\n') f.write('cdo settaxis,1850-01-01,12:00,1day tmp1.nc tmp2.nc\n') f.write('/bin/mv tmp2.nc '+fout+'.nc\n') if opt_lev41: f.write('ncrename -d HETA40.M,lev -v HETA40.M,lev '+fout+'.nc\n') f.write('/bin/rm tmp?.nc\n') f.write('\n') if not out_gt: # no gtool output f.write('# Delete the intermediate Gtool file\n') f.write('/bin/rm '+fout+'\n') f.write('\n')