PRO sylvania_converttoac1 ;rewrite toac1 files to toa5 format - looks to be the same except for ; header, and only in early 2012 ;<= Aug 15, 2012 is TOACI1, then TOA5 indir = '/sap/projects/cheas/sylvania/incoming/2012/TOACI1/' outdir = '/sap/projects/cheas/sylvania/incoming/2012/' mfiles = file_search(indir+'*met_data_*.dat',count=nmc) IF nmc GT 0 THEN BEGIN print,'Found ',nmc,' met files' FOR i = 0,nmc-1 DO BEGIN IF file_test(mfiles[i],/read) THEN BEGIN print,'Processing ',mfiles[i] outfname = outdir+(reverse(strsplit(mfiles[i],'/',/extract)))[0] openr,fl_in,mfiles[i],/get_lun openw,fl_out,outfname,/get_lun printf,fl_out,'"TOA5","CR3000_Sylvania","CR3000","6706","CR3000.Std.22","CPU:SylvaniaProgram_2012_CF.CR3","62550","met_data"' printf,fl_out,'"TIMESTAMP","RECORD","t_hmp_Avg","rh_hmp_Avg","e_hmp_Avg","batt_volt_Avg","panel_temp_Avg","SR01Up_Avg","SR01Dn_Avg","IR01Up_Avg","IR01Dn_Avg","TC_Avg","water_content_1_Avg","elec_cond_1_Avg","s_temp_1_Avg","water_content_2_Avg","elec_cond_2_Avg","s_temp_2_Avg","water_content_3_Avg","elec_cond_3_Avg","s_temp_3_Avg","water_content_4_Avg","elec_cond_4_Avg","s_temp_4_Avg","PAR_Den_Avg"' printf,fl_out,'"TS","RN","Deg C","%","kPa","V","Deg C","W/m2","W/m2","W/m2","W/m2","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","micromol/s/m2"' printf,fl_out,'"","","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg"' s = strarr(2) readf,fl_in,s WHILE ~eof(fl_in) DO BEGIN s = '' readf,fl_in,s printf,fl_out,s ENDWHILE free_lun,fl_in free_lun,fl_out ENDIF ELSE BEGIN print,'Cannot open ',mfiles[i] ENDELSE ENDFOR ENDIF ELSE BEGIN print,'No met_data files found' ENDELSE tfiles = file_search(indir+'*ts_data_*.dat',count=ntc) IF ntc GT 0 THEN BEGIN print,'Found ',ntc,' flux files' FOR i = 0,ntc-1 DO BEGIN IF file_test(tfiles[i],/read) THEN BEGIN print,'Processing ',tfiles[i] outfname = outdir+(reverse(strsplit(tfiles[i],'/',/extract)))[0] openr,fl_in,tfiles[i],/get_lun openw,fl_out,outfname,/get_lun printf,fl_out,'"TOA5","CR3000_Sylvania","CR3000","6706","CR3000.Std.22","CPU:SylvaniaProgram_2012_CF.CR3","62550","ts_data"' printf,fl_out,'"TIMESTAMP","RECORD","UxCSAT","UyCSAT","UzCSAT","TsCSAT","diag_CSAT","CO2","H2O","diag_irga","Ta_irga","press_irga","co2_signal","h2o_signal"' printf,fl_out,'"TS","RN","m/s","m/s","m/s","DegC","unitless","mg/m3","g/m3","unitless","DegC","kPa","fraction",""' printf,fl_out,'"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp"' s = strarr(2) readf,fl_in,s WHILE ~eof(fl_in) DO BEGIN s = '' readf,fl_in,s printf,fl_out,s ENDWHILE free_lun,fl_in free_lun,fl_out ENDIF ELSE BEGIN print,'Cannot open ',tfiles[i] ENDELSE ENDFOR ENDIF ELSE BEGIN print,'No met_data files found' ENDELSE END ;2019- flux file ;"TOA5","SylvaniaFlux","CR1000X","9701","CR1000X.Std.03.03","CPU:sylvania_flux.CR1X","61699","TSdata" ;"TIMESTAMP","RECORD","co2dry","h2odry","co2","h2o","ssli7200","pressure_total","Tin","Tout","cell_temp","li7200diag","Ux","Uy","Uz","Ts","csat_diag" ;"TS","RN","micromol/mol","mmol/mol","mmol/m3","mmol/m3","ave","kPa","degC","degC","degC","unitless","m/s","m/s","m/s","degC","unitless" ;"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp" ;"2019-07-25 15:30:01.6",1682189,378.7432,17.22542,14.19471,645.5823,99.56512,94.3497,24.63009,24.52737,24.54792,8191,3.4755,0.4505,0.258,24.68369,1 ;2019- surf_data file ;"TOA5","Sylvania","CR3000","3256","CR3000.Std.32.03","CPU:SylvaniaSurf_v1_0_0.CR3","4339","surf_data" ;"TIMESTAMP","RECORD","batt_volt_Avg","panel_temp_Avg","water_content_1_Avg","elec_cond_1_Avg","s_temp_1_Avg","water_content_2_Avg","elec_cond_2_Avg","s_temp_2_Avg","water_content_3_Avg","elec_cond_3_Avg","s_temp_3_Avg","water_content_4_Avg","elec_cond_4_Avg","s_temp_4_Avg","GMP_lower_Avg","GMP_upper_Avg" ;"TS","RN","V","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","ppm","ppm""","","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg" ;"2019-08-09 ;15:21:00",2162,13.44053,17.47415,0.1658,0.01355,15.44889,0.1737,0.01045,15.08444,"NAN",0,0,"NAN",0,0,201.4074,419.7627 ;do this by column name ;met file ;"TOA5","CR3000_Sylvania","CR3000","3256","CR3000.Std.24","CPU:SylvaniaProgram_2014-June-Renato.CR3","3453","met_data" ;"TIMESTAMP","RECORD","t_hmp_Avg","rh_hmp_Avg","e_hmp_Avg","batt_volt_Avg","panel_temp_Avg","SR01Up_Avg","SR01Dn_Avg","IR01Up_Avg","IR01Dn_Avg","TC_Avg","water_content_1_Avg","elec_cond_1_Avg","s_temp_1_Avg","water_content_2_Avg","elec_cond_2_Avg","s_temp_2_Avg","water_content_3_Avg","elec_cond_3_Avg","s_temp_3_Avg","water_content_4_Avg","elec_cond_4_Avg","s_temp_4_Avg","PAR_Den_Avg","Wind_Dir_min_Min","Wind_Dir_mean_Avg","Wind_Dir_max_Max","Wind_Speed_min_Min","Wind_Speed_mean_Avg","Wind_Speed_max_Max","Air_temp_Avg","RH_Avg","Pressure_Avg","Rain_accum_Max","Rain_dur_Max","Rain_intens_Avg","Hail_accum_Max","Hail_dur_Max","Hail_intens_Avg","Rain_pk_intens_Max","Hail_pk_intens_Max","GMP_upper_Avg","GMP_lower_Avg" ;"TS","RN","Deg C","%","kPa","V","Deg C","W/m2","W/m2","W/m2","W/m2","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","m3/m3","dS/m","Deg C","micromol/s/m2","deg","deg","deg","m/s","m/s","m/s","C","%","hPa","mm","s","mm/h","hits/cm2","s","hits/cm2/h","mm/h","hits/cm2/h","ppm","ppm""","","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Min","Avg","Max","Min","Avg","Max","Avg","Avg","Avg","Max","Max","Avg","Max","Max","Avg","Max","Max","Avg","Avg" ;"2014-07-03 00:01:00",12217,-39.99015,0.002008832,3.803532E-007,12.30687,11.31808,-3.826613,0.9663164,-67.0394,-3.975892,11.81481,0.3371,0.02313333,12.80353,0.3137,0.01503333,13.0621,"NAN",0,0,0.2229,0.01026667,13.7723,-0.05516443,233,246.3333,254,2.5,2.966666,3.5,12,66.96667,950.3,34.07,15910,0,0,20,0,0,0,410.5366,382.6202 ;;some columns added later (after Par_den) after June 18, 2014 ;new surf file ;flux file ;"TOA5","CR3000_Sylvania","CR3000","3256","CR3000.Std.24","CPU:SylvaniaProgram_2014-June-Renato.CR3","3453","ts_data" ;"TIMESTAMP","RECORD","UxCSAT","UyCSAT","UzCSAT","TsCSAT","diag_CSAT","LI7500_CO2","LI7500_H2O","press_LI7500","diag_LI7500" ;"TS","RN","m/s","m/s","m/s","Deg C","unitless","mmol/m^3","mmol/m^3","kPa","unitless" ;"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp" ;"2014-07-12 00:00:00",15104641,3.39625,-0.9587501,-1.278,21.26199,45,14.21547,743.7086,94.97427,248 ;flux file for EC-150 (up through May 1, 2013) ;"TOA5","CR3000_Sylvania","CR3000","6706","CR3000.Std.22","CPU:SylvaniaProgram_2012_CF.CR3","62550","ts_data" ;"TIMESTAMP","RECORD","UxCSAT","UyCSAT","UzCSAT","TsCSAT","diag_CSAT","CO2","H2O","diag_irga","Ta_irga","press_irga","co2_signal","h2o_signal" ;"TS","RN","m/s","m/s","m/s","DegC","unitless","mg/m3","g/m3","unitless","DegC","kPa","fraction","" ;"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp" ;"2012-09-23 00:00:00.0",45794999,1.294172,-3.189075,0.1458382,3.132324,0,692.089,4.199237,0,3.035248,96.0424,0.9570955,0.944715 ;TOACI1 files (up through August 15, 2012; then TOA5) ;"TOACI1","CR3000_sylvania","ts_data" ;"TMSTAMP","RECNBR","UxCSAT","UyCSAT","UzCSAT","TsCSAT","diag_CSAT","CO2","H2O","diag_irga","Ta_irga","press_irga","co2_signal","h2o_signal" ;"2012-05-02 13:11:15",5889267,4.587192,-3.605916,-0.7276728,15.97058,0,688.0979,10.20626,0,15.97519,94.90426,0.9812593,0.9801804 ;"TOACI1","CR3000_sylvania","met_data" ;"TMSTAMP","RECNBR","t_hmp_Avg","rh_hmp_Avg","e_hmp_Avg","batt_volt_Avg","panel_temp_Avg","SR01Up_Avg","SR01Dn_Avg","IR01Up_Avg","IR01Dn_Avg","TC_Avg","water_content_1_Avg","elec_cond_1_Avg","s_temp_1_Avg","water_content_2_Avg","elec_cond_2_Avg","s_temp_2_Avg","water_content_3_Avg","elec_cond_3_Avg","s_temp_3_Avg","water_content_4_Avg","elec_cond_4_Avg","s_temp_4_Avg","PAR_Den_Avg" ;"2012-05-02 16:32:00",10016,21.11232,53.05092,1.327916,12.5501,24.13233,878.6699,98.43698,-56.79611,0.8720093,24.35125,0.2827,0.0239,18.60967,0.2949,0.01545,14.49397,0.377,0.02091666,20.41723,0.3771,0.0137,13.71213,1750.917 ;2019- met file ;"TOA5","SylvaniaFlux","CR1000X","9701","CR1000X.Std.03.03","CPU:sylvania_flux.CR1X","61699","metdata" ;"TIMESTAMP","RECORD","sw_in_Avg","sw_out_Avg","lw_in_Avg","lw_out_Avg","kz_T_Avg","airT_Avg","RH_Avg","Thmp_Avg","PARden_top_Avg" ;"TS","RN","W/m2","W/m2","W/m2","W/m2","degC","degC","%","degC","micromols/s/m^2" ;"","","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg","Avg" ;"2019-07-24 05:31:00",765,-5.265282,-1.401461,-72.47894,-8.526684,18.2605,17.58786,81.1,17.42452,0 ;biomet file ;TIMESTAMP_1,Ta_1_1_1,RH_1_1_1,LWout_1_1_1,LWin_1_1_1,SWout_1_1_1,SWin_1_1_1,PPFD_1_1_1,Ts_1_1_1,SWC_1_1_1,MWS_1_1_1,WD_1_1_1,Pa_1_1_1 ;yyyy-mm-dd HHMM,C,%,W+1m-2,W+1m-2,W+1m-2,W+1m-2,umol+1m-2s-1,C,m+3m-3,m+1s-1,degrees,hPa ;2014-06-29 0000,22.00,87.50,-23.82,-0.60,-1.43,0.31,0.00,18.71,0.33,3.20,147.00,942.03 ;Ameriflux file ;Biomet output file ;NEED BIOMET CNR_T or from MET file PRO sylvania_processbiomet,yr,day ystr = string(yr,format='(i4.4)') dystr = jd_to_Dy(day,y=yr) doystr = string(day,format='(i3.3)') mostr = strmid(dystr,4,2) daystr = strmid(dystr,6,2) dr = ['/air/incoming/sylvania/'+ystr+'/'+ystr+mostr+daystr+'/','/air/incoming/sylvania/'+ystr+'/','/air/incoming/sylvania/current/'] drout = '/air/incoming/sylvania/'+ystr+'/' fout = drout + doystr + 'biomet.txt' mfiles = file_search(dr + 'sylvaniaflux_metdata_'+ystr+'_'+mostr+'_'+daystr+'*.dat',count=mcountp) IF mcountp GT 0 THEN BEGIN openw,flout,fout,/get_lun printf,flout,'TIMESTAMP_1,Ta_1_1_1,RH_1_1_1,LWout_1_1_1,LWin_1_1_1,SWout_1_1_1,SWin_1_1_1,PPFD_1_1_1,Ts_1_1_1,SWC_1_1_1,MWS_1_1_1,WD_1_1_1,Pa_1_1_1',format='(a0)' printf,flout,'yyyy-mm-dd HHMM,C,%,W+1m-2,W+1m-2,W+1m-2,W+1m-2,umol+1m-2s-1,C,m+3m-3,m+1s-1,degrees,hPa',format='(a0)' FOR i = 0,mcountp-1 DO BEGIN openr,fl,mfiles[i],/get_lun FOR j = 0,3 DO BEGIN dum = '' readf,fl,dum ENDFOR ;"TIMESTAMP","RECORD","sw_in_Avg","sw_out_Avg","lw_in_Avg","lw_out_Avg","kz_T_Avg","airT_Avg","RH_Avg","Thmp_Avg","PARden_top_Avg" ;"TS","RN","W/m2","W/m2","W/m2","W/m2","degC","degC","%","degC","micromols/s/m^2" ;TIMESTAMP_1,Ta_1_1_1,RH_1_1_1,LWout_1_1_1,LWin_1_1_1,SWout_1_1_1,SWin_1_1_1,PPFD_1_1_1,Ts_1_1_1,SWC_1_1_1,MWS_1_1_1,WD_1_1_1,Pa_1_1_1 ;5 -= sw_in WHILE ~eof(fl) DO BEGIN s = '' readf,fl,s s = strsplit(s,',": ',/extract) sout = make_Array(12,/float,value=-9999.0) IF yr GT 2023 OR (yr EQ 2023 AND day GT 318) THEN sout[0] = s[12] ELSE sout[0] = s[10] ;ta sout[1] = s[11] ;rh lw_out = float(s[8]) lw_in = float(s[7]) kz_t = float(s[9]) lw_out += 5.67e-8 * (kz_t+273.15)^4 lw_in += 5.67e-8 * (kz_t+273.15)^4 sout[2] = lw_out sout[3] = lw_in sout[4] = s[6] sout[5] = s[5] sout[6] = s[13] ;par printf,flout,s[0],s[1]+s[2],sout,format='(a10," ",a4,",",11(f0.1,","),f0.1)' ENDWHILE free_lun,fl ENDFOR free_lun,flout ENDIF END PRO sylvania_processflux,yr,day,noupload=noupload,debug=debug ;pattern of lcreek_flux ;except need separate .eddypro .metadata file for pre-May 2013 ;(LI-7200) and post-May 2013 (LI-7500) ;only one file per day - nice! ystr = string(yr,format='(i4.4)') dystr = jd_to_Dy(day,y=yr) doystr = string(day,format='(i3.3)') mostr = strmid(dystr,4,2) daystr = strmid(dystr,6,2) print,'Sylvania flux for '+dystr dr = ['/air/incoming/sylvania/'+ystr+'/'+ystr+mostr+daystr+'/','/air/incoming/sylvania/'+ystr+'/','/air/incoming/sylvania/current/'] drout = '/air/incoming/sylvania/output/'+ystr+'/' create_dir,drout ; sfile = drout + 'Sylvania_'+dystr+'_rawdata.sav' tarfile = drout + 'US-Syv_'+dystr+'_L0rawdata.tar.gz' IF yr GE 2019 THEN BEGIN tfiles = file_search(dr + 'sylvaniaflux_TSdata_'+ystr+'_'+mostr+'_'+daystr+'*.dat',count=tcountp) ENDIF ELSE BEGIN tfiles = file_search(dr + '*ts_data_'+ystr+'_'+mostr+'_'+daystr+'*.dat',count=tcountp) ENDELSE bfiles = file_search(dr + doystr + 'biomet.txt',count=bcountp) ; IF bcountp EQ 0 THEN sylvania_processbiomet,yr,day ; bfiles = file_search(dr + doystr + 'biomet.txt',count=bcountp) IF tcountp GT 0 THEN BEGIN tfilename = strarr(n_elements(tfiles)) FOR i = 0,n_elements(tfilename)-1 DO tfilename[i] = (reverse(strsplit(tfiles[i],'/',/extract)))[0] stfiles = sort(tfilename) tfilename = tfilename[stfiles] tfiles = tfiles[stfiles] utfiles = uniq(tfilename) tfiles = tfiles[utfiles] tcountp = n_elements(tfiles) print,' Calling EddyPro' oldfile = file_search('/air/incoming/sylvania/output/eddypro/input/*',count=delold) IF delold GT 0 THEN file_delete,oldfile,/allow_nonexistent oldfile = file_search('/air/incoming/sylvania/output/eddypro/output/*',count=delold) IF delold GT 0 THEN file_delete,oldfile,/allow_nonexistent,/recursive FOR i = 0,tcountp-1 DO file_link,tfiles[i],'/air/incoming/sylvania/output/eddypro/input/' IF bcountp GT 0 THEN file_link,bfiles[0],'/air/incoming/sylvania/output/eddypro/input/biomet.data' IF (yr GE 2019) THEN BEGIN ;LI-7200RS spawn,'cd /air/incoming/sylvania/output/eddypro/ini; /air/code/eddypro-engine/bin/eddypro_rp -s linux -c console sylvania3.eddypro' ENDIF ELSE BEGIN ;before May 4, 2013 - EC-150, after is LI-7500- use sylvania_ec150.eddypro IF (yr LT 2013) OR (yr EQ 2013 AND day LT 124) THEN BEGIN spawn,'cd /air/incoming/sylvania/output/eddypro/ini; /air/code/eddypro-engine/bin/eddypro_rp -s linux -c console sylvania_ec150.eddypro' ENDIF ELSE BEGIN IF (yr GT 2015) OR (yr EQ 2015 AND day GE 293) THEN BEGIN spawn,'cd /air/incoming/sylvania/output/eddypro/ini; /air/code/eddypro-engine/bin/eddypro_rp -s linux -c console sylvania2.eddypro' ENDIF ELSE BEGIN spawn,'cd /air/incoming/sylvania/output/eddypro/ini; /air/code/eddypro-engine/bin/eddypro_rp -s linux -c console sylvania.eddypro' ENDELSE ENDELSE ENDELSE ;IGNORE FOR NOW after June 18, 2014 - more columns past PAR in met file (averages) ;Upload ts_data and met_data to Ameriflux IF ~keyword_set(noupload) THEN BEGIN print,'Upload raw data to Ameriflux' spawn,'tar cvzf '+tarfile+' '+tfiles IF file_test(tarfile,/read) THEN BEGIN print,' Calling scp -i /home/adesai/ameriflux_keys/us-syv '+tarfile+' fluxnet@dtn01.nersc.gov:' spawn,'scp -v -i /home/adesai/ameriflux_keys/us-syv '+tarfile+' fluxnet@dtn01.nersc.gov:' ENDIF ; FOR i = 0,tcountp-1 DO spawn,'scp -v -i /home/adesai/ameriflux_keys/us-syv '+tfiles[i]+' fluxnet@dtn01.nersc.gov:' IF bcountp GT 0 THEN spawn,'scp -v -i /home/adesai/ameriflux_keys/us-syv '+bfiles[0]+' fluxnet@dtn01.nersc.gov:' ENDIF outfiles = file_search('/air/incoming/sylvania/output/eddypro/output/*.csv',count=ocount) IF yr GE 2019 THEN BEGIN fdir = '/air/incoming/sylvania/output/'+ystr+'/'+ystr+doystr IF ocount GT 0 THEN BEGIN FOR i = 0,n_elementS(outfiles)-1 DO BEGIN extracted = (reverse(strsplit(outfiles[i],'/_',/extract))) IF extracted[2] EQ 'eddy' THEN file_move,outfiles[i],fdir+extracted[3]+'.csv',/overwrite IF extracted[2] EQ 'ameriflux' THEN file_move,outfiles[i],fdir+extracted[2]+'.csv',/overwrite IF extracted[2] EQ 'biomet' AND extracted[3] EQ 'sylvania' THEN file_move,outfiles[i],fdir+extracted[2]+'.csv',/overwrite IF extracted[2] EQ 'essentials' THEN file_move,outfiles[i],fdir+extracted[2]+'.csv',/overwrite IF extracted[2] EQ 'full' THEN file_move,outfiles[i],fdir+extracted[2]+'.csv',/overwrite IF extracted[2] EQ 'metadata' THEN file_move,outfiles[i],fdir+extracted[2]+'.csv',/overwrite ENDFOR ENDIF ELSE print,'No eddypro output found!' ENDIF ELSE BEGIN IF ocount GT 0 THEN FOR i = 0,n_elementS(outfiles)-1 DO file_move,outfiles[i],'/air/incoming/sylvania/output/'+ystr+'/'+ystr+doystr+(reverse(strsplit(outfiles[i],'/_',/extract)))[1]+'.csv',/overwrite ELSE print,'No eddypro output found!' ENDELSE ENDIF END PRO sylvania_processfluxyear,yr,startday=startday,endday=endday,noupload=noupload,noread=noread,force=force,skipeddy=skipeddy,email=email,procfile=procfile,debug=debug ;Ameriflux ;Sitename: ;Location: Latitude: 46.2420170 - Longitude: -89.3476500 - Elevation (masl): 500. ;Principal investigator: ;Ecosystem type: ;File creation date: 2015-02-05T12:28 ;Datapolicy: -- The AmeriFlux data provided on this site are freely available and were furnished by individual AmeriFlux scientists who encourage their use. ;Please kindly inform in writing (or e-mail) the appropriate AmeriFlux scientist(s) of how you intend to use the data and of any publication plans. ;It is also important to contact the AmeriFlux investigator to assure you are downloading the latest revision of the data and to prevent potential misuse or misinterpretation of the data. ;Please acknowledge the data source as a citation or in the acknowledgments if no citation is available. ;If the AmeriFlux Principal Investigators (PIs) feel that they should be acknowledged or offered participation as authors they will let you know. ;And we assume that an agreement on such matters will be reached before publishing and/or use of the data for publication. ;If your work directly competes with the PIs analysis they may ask that they have the opportunity to submit a manuscript before you submit one that uses unpublished data. ;In addition when publishing please acknowledge the agency that supported the research. -- ;File Origin (4 lines) - To be compiled by AmeriFlux data management group. ;File Origin (4 lines) - To be compiled by AmeriFlux data management group. ;File Origin (4 lines) - To be compiled by AmeriFlux data management group. ;File Origin (4 lines) - To be compiled by AmeriFlux data management group. ;YEAR,GAP,DTIME,DOY,HRMIN,UST,TA,WD,WS,NEE,FC,SFC,H,SH,LE,SLE,FG,TS1,TSdepth1,TS2,TSdepth2,PREC,RH,PRESS,CO2,VPD,SWC1,SWC2,Rn,PAR,Rg,Rgdif,PARout,RgOut,Rgl,RglOut,H2O,RE,GPP,CO2top,CO2height,APAR,PARdif,APARpct,ZL ;YEAR,GAP,DTIME,DOY,HRMIN,m/s,deg C,deg,m/s,umol/m2/s,umol/m2/s,umol/m2/s,W/m2,W/m2,W/m2,W/m2,W/m2,deg C,cm,deg C,cm,mm,%,kPa,umol/mol,kPa,%,%,W/m2,umol/m2/s,W/m2,W/m2,umol/m2/s,W/m2,W/m2,W/m2,mmol/mol,umol/m2/s,umol/m2/s,umol/mol,m,umol/m2/s,umol/m2/s,%,unitless ; 2014,0,180.02082999999999,180,030,0.70930695482376172,20.899658060867409,215.13916198054542,3.7508821982648750,-6999.0000000000000,13.927327473250481,-9999.0000000000000,31.587104159085371,-9999.0000000000000,-288.80444632967055,-9999.0000000000000,-6999.0000000000000,-6999.0000000000000,-6999.0000000000000,-6999.0000000000000,-6999.0000000000000,-6999.0000000000000,97.488617898553386,94.296420978469371,410.97027085414931,6.18504818612050297E-002,-6999.,-6999.,-6999.,-6999.,-6999.,-6999.,-6999.,-6999.,-6999.,-6999.,25.664277761335022,-6999.,-6999.,-6999.,-6999.,-6999.,-6999.,-6999.,-2.27908248037736599E-002 ;biomet ;date,time,DOY,Ta_1_1_1,Pa_1_1_1,RH_1_1_1,LWin_1_1_1,LWout_1_1_1,SWin_1_1_1,SWout_1_1_1,PPFD_1_1_1,MWS_1_1_1,WD_1_1_1,SWC_1_1_1,Ts_1_1_1 ;[yyyy-mm-dd],[HH:MM],[ddd.ddd],[K],[Pa],[%],[W+1m-2],[W+1m-2],[W+1m-2],[W+1m-2],[µmol+1m-2s-1],[m+1s-1],[deg],[m+3m-3],[K] ;2014-06-29,00:30,180.020,294.00633333333332,94258.000000000000,84.827333333333314,-0.69433333333333325,-5.4279999999999999,-0.19466666666666671,-2.1456666666666662,4.00000000000000008E-003,6.2866666666666671,202.69999999999999,0.33000000000000002,291.86899999999997 ;date,time,DOY,Ta_1_1_1,RH_1_1_1,LWin_1_1_1,LWout_1_1_1,SWin_1_1_1,SWout_1_1_1,PPFD_1_1_1,SWC_1_1_1,Ts_1_1_1 ;[yyyy-mm-dd],[HH:MM],[ddd.ddd],[K],[%],[W+1m-2],[W+1m-2],[W+1m-2],[W+1m-2],[µmol+1m-2s-1],[m+3m-3],[K] ;2014-01-01,00:30,1.020,247.47433333333331,70.948000000000008,-6.1706666666666665,-9999.0,0.24233333333333335,-1.3749999999999998,-2.00000000000000004E-003,0.28999999999999992,273.51133333333348 ;ghg ;timestamp,gas_concentrations,,corrected_fluxes_and_quality_flags,,,,,,,,,,storage_terms,,,turbulence,,,footprint,, ;ISOdate,CO2_1_1_1,H2O_1_1_1,Tau_1_1_1,QcTau_1_1_1,H_1_1_1,QcH_1_1_1,LE_1_1_1, QcLE_1_1_1,Fc_1_1_1, QcFc_1_1_1,E_1_1_1, QcE_1_1_1,Sa_1_1_1,Sw_1_1_1,Sc_1_1_1,ustar_1_1_1,MO_length_1_1_1,ZL_1_1_1,Fetchmax_1_1_1,Fetch70_1_1_1,Fetch90_1_1_1 ;yyyymmddHHMM,[µmol+1mol_a-1],[mmol+1mol_a-1],[kg+1m-1s-2],[#],[W+1m-2],[#],[W+1m-2],[#],[µmol+1s-1m-2],[#],[mmol+1s-1m-2],[#],[W+1m-2],[W+1m-2],[µmol+1s-1m-2],[m+1s-1],[m],[#],[m],[m],[m] ;201205230030,-9999.,9.1818463831340118,0.86914960235649719,0,-42.367656748462025,0,-1.9845085633088557,2,-9999.,-9999.,-4.46398778204633792E-002,2,-9999.,-9999.,-9999.,0.86859185988602383,1303.1130665904577,1.65756913607777133E-002,119.16372608412219,217.86929660292674,326.40790729501100 ;after 2019 new ghg ;TIMESTAMP,CO2_1_1_1,H2O_1_1_1,TAU_1_1_1,TAU_SSITC_TEST_1_1_1,H_1_1_1,H_SSITC_TEST_1_1_1,LE_1_1_1,LE_SSITC_TEST_1_1_1,FC_1_1_1,FC_SSITC_TEST_1_1_1,WD_0_0_1,WS_0_0_1,WS_MAX_0_0_1,U_SIGMA_0_0_1,V_SIGMA_0_0_1,W_SIGMA_0_0_1,USTAR_0_0_1,MO_LENGTH_0_0_1,ZL_0_0_1,FETCH_MAX_0_0_1,FETCH_70_0_0_1,FETCH_80_0_0_1,FETCH_90_0_0_1,PA_0_0_1,RH_0_0_1,TA_0_0_1,VPD_0_0_1,T_SONIC_0_0_1,T_SONIC_SIGMA_0_0_1 yrstr = string(yr,format='(i4.4)') yrdr = yrstr+'/' IF n_elements(yr) EQ 0 THEN yr=2014 IF n_elements(startday) EQ 0 THEN startday = 1L diy = long(days_in_year(yr)) IF n_elements(endday) EQ 0 THEN endday = diy indir = '/air/incoming/sylvania/output/'+yrdr ;READ OLD FILE outfile = '/air/incoming/sylvania/flux/sylvaniaflux_'+yrstr+'.xdf' outfile_ascii = '/air/incoming/sylvania/flux/sylvaniaflux_'+yrstr+'.txt' outmfile = '/air/incoming/sylvania/flux/sylvaniamet_'+yrstr+'.xdf' outmfile_ascii = '/air/incoming/sylvania/flux/sylvaniamet_'+yrstr+'.txt' IF file_test(outfile,/read) AND ~keyword_set(noread) THEN flux = read_xdf(outfile,header=fheader) ELSE BEGIN fheader = strsplit('YEAR,GAP,DTIME,DOY,HRMIN,UST,TA,WD,WS,NEE,FC,SFC,H,SH,LE,SLE,FG,TS1,TSdepth1,TS2,TSdepth2,PREC,RH,PRESS,CO2,VPD,SWC1,SWC2,Rn,PAR,Rg,Rgdif,PARout,RgOut,Rgl,RglOut,H2O,RE,GPP,CO2top,CO2height,APAR,PARdif,APARpct,ZL,NEEfilled,Fluxu*,FlagH,FlagLE,FlagC,FlagE,FetchMax,Fetch90',',',/extract) diy = days_in_year(yr) flux = make_array(n_elements(fheader),diy*48,/double,value=nan()) flux[0,*] = yr flux[1,*] = 1 flux[3,*] = expand_arr(lindgen(diy)+1,48);long(flux[2,*]) flux[2,*] = flux[3,*]+replicate_arr((lindgen(48)+1l)/48.D,diy) ;(findgen(diy*48)/48.0)+1.0+(1.0/48.) hr = ((findgen(diy*48) MOD 48)/2)+0.5 flux[4,*] = long((long(hr)*100)+(60*(hr-long(hr)))) flux[46:50,*] = 0 ENDELSE IF file_test(outmfile,/read) AND ~keyword_set(noread) THEN met = read_xdf(outmfile,header=mheader) ELSE BEGIN mheader = ['Year','Flag','Dtime','DOY','HRMIN','SoilQ_10','SoilQ_20','SoilT_10','SoilT_20','AirT','RH','P','Rain_max','Rain_int','CO2_2.5m','CO2_21.5m','T_HMP','RH_HMP','ea','SW_In','SW_out','IR_in','IR_out','T_CNR4','PAR','Rnet','CO2_10m','Airt_low','Airt_mid','RH_low','RH_mid','PAR_low','PAR_mid','G','SoilQ_10_2','SoilQ_20_2','SoilT_10_2','SoilT_20_2','WS_520','WD_520'] ;0-39 mheader = [mheader,'SoilT_5_2','SoilT_10_2','SoilT_20_2','SoilT_30_2','SoilT_40_2','SoilT_50_2','SoilT_60_2','SoilT_75_2','SoilT_100_2'] ;40-48 mheader = [mheader,'SoilQ_5_2','SoilQ_10_2','SoilQ_20_2','SoilQ_30_2','SoilQ_40_2','SoilQ_50_2','SoilQ_60_2','SoilQ_75_2','SoilQ_100_2'] ;49-57 diy = days_in_year(yr) met = make_array(n_elements(mheader),diy*48,/double,value=nan()) met[0,*] = yr met[1,*] = 0 met[3,*] = expand_arr(lindgen(diy)+1,48) ;long(met[2,*]) met[2,*] = met[3,*]+replicate_arr((lindgen(48)+1l)/48.D,diy) ;(findgen(diy*48)/48.0)+1.0+(1.0/48.) hr = ((findgen(diy*48) MOD 48)/2)+0.5 met[4,*] = long((long(hr)*100)+(60*(hr-long(hr)))) ENDELSE met[2,*] = met[3,*]+replicate_arr((lindgen(48)+1l)/48.D,diy) bval = where(met LE -6999. OR met EQ -999.,nbv) IF nbv GT 0 THEN met[bval] = nan() bval = where(flux LE -6999. OR flux EQ -999.,nbv) IF nbv GT 0 THEN flux[bval] = nan() FOR doy = startday,endday DO BEGIN doystr = string(doy,format='(i3.3)') dystr = jd_to_Dy(doy,y=yr) mostr = strmid(dystr,4,2) daystr = strmid(dystr,6,2) lstart = (doy-1l)*48l lend = (doy*48l)-1l print,'Sylvania flux for '+dystr print,'Flux processing yr:'+yrstr+' doy: '+doystr ;call eddypro if needed fluxfile = indir+yrstr+doystr+'ameriflux.csv' IF ~keyword_set(skipeddy) AND (~file_test(fluxfile,/read) OR keyword_set(force)) THEN sylvania_processflux,yr,doy,noupload=noupload ;read file and copy into fields IF file_test(fluxfile,/read) THEN BEGIN print,' Reading flux output from '+fluxfile af = read_csv(fluxfile,n_table_header=19) locs = ((double(af.field04)-1)*48)+(round((double(af.field03)-double(af.field04))*48)-1) IF (yr EQ 2012) AND (doy LT 224) THEN locs-=4 ;2012 TIME ZONE OFFSET ;2017 CORRECTION!!! 11/08 IF ((yr EQ 2017) AND (doy GT 311)) OR (yr GT 2017) THEN locs-=12 flux[5,locs] = af.field06 flux[6,locs] = af.field07 flux[7,locs] = af.field08 flux[8,locs] = af.field09 flux[9,locs] = af.field10 flux[10,locs] = af.field11 flux[11,locs] = af.field12 flux[12,locs] = af.field13 flux[13,locs] = af.field14 flux[14,locs] = af.field15 flux[15,locs] = af.field16 flux[16,locs] = af.field17 flux[17,locs] = af.field18 flux[18,locs] = af.field19 flux[19,locs] = af.field20 flux[20,locs] = af.field21 flux[21,locs] = af.field22 flux[22,locs] = af.field23 flux[23,locs] = af.field24 flux[24,locs] = af.field25 flux[25,locs] = af.field26 flux[26,locs] = af.field27 flux[27,locs] = af.field28 flux[28,locs] = af.field29 flux[29,locs] = af.field30 flux[30,locs] = af.field31 flux[31,locs] = af.field32 flux[32,locs] = af.field33 flux[33,locs] = af.field34 flux[34,locs] = af.field35 flux[35,locs] = af.field36 flux[36,locs] = af.field37 flux[37,locs] = af.field38 flux[38,locs] = af.field39 flux[39,locs] = af.field40 flux[40,locs] = af.field41 flux[41,locs] = af.field42 flux[42,locs] = af.field43 flux[43,locs] = af.field44 flux[44,locs] = af.field45 ;copy to met met[9,locs] = merge_array(met[9,locs],flux[6,locs]) ;ta met[10,locs] = merge_array(met[10,locs],flux[22,locs]) ;rh met[11,locs] = merge_array(met[11,locs],flux[23,locs]) ;press kpa ;get quality flags and footprint from GHG file ghgfile = indir+yrstr+doystr+'ghg-europe.csv' IF file_test(ghgfile,/read) THEN BEGIN print,' Reading quality flags and footprint from '+ghgfile gh = read_csv(ghgfile,n_table_header=2) flux[46,locs] = gh.field05 ;mflag flux[47,locs] = gh.field07 ;hflag flux[48,locs] = gh.field09 ;qflag flux[49,locs] = gh.field11 ;cflag flux[50,locs] = gh.field13 ;eflag IF yr LT 2019 THEN flux[51,locs] = gh.field20 ELSE flux[51,locs] = gh.field21 ;fetchmax IF yr LT 2019 THEN flux[52,locs] = gh.field22 ELSE flux[52,locs] = gh.field24 ;fetch90 ENDIF ELSE print,' No GHG file found' ENDIF ELSE BEGIN print,' No flux file found' ENDELSE ;biomet uses 1-min input file and averages up ;/air/incoming/sylvania/YYYY/XXXbiomet.csv ;only need: Ts,tsdepth1, SWC,Rn, PAR, Rg, Rgout, Rgl, Rglout, Tair, Rh ;too 2019 bmetfile = '/air/incoming/sylvania/'+yrstr+'/'+doystr+'biomet.txt' IF ~file_test(bmetfile,/read) THEN bmetfile = '/air/incoming/sylvania/current/'+doystr+'biomet.txt' IF file_test(bmetfile,/read) THEN BEGIN IF file_lines(bmetfile) GT 2 THEN BEGIN print,' Reading biomet from '+bmetfile btime = (read_ascii(bmetfile,delim=' ',data_Start=2)).(0) bdata = (read_ascii(bmetfile,delim=',',data_Start=2)).(0) btime = btime[1,*] btimebin = fix((fix(btime/100) + ((btime mod 100)/60.0))*2) b_lw_out = bin_avg(btimebin,screen_arr(bdata[3,*],-1500,1000),min=0,max=47,binsize=1) b_lw_in = bin_avg(btimebin,screen_arr(bdata[4,*],-1500,1000),min=0,max=47,binsize=1) b_sw_out = bin_avg(btimebin,screen_arr(bdata[5,*],-200,1500),min=0,max=47,binsize=1) b_sw_in = bin_avg(btimebin,screen_arr(bdata[6,*],-200,1500),min=0,max=47,binsize=1) b_rnet = (b_sw_in+b_lw_in)-(b_sw_out+b_lw_out) b_tair = bin_avg(btimebin,screen_arr(bdata[1,*],-50,50),min=0,max=47,binsize=1) b_rh = bin_avg(btimebin,screen_arr(bdata[2,*],0,100),min=0,max=47,binsize=1) ; b_ts = bin_avg(btimebin,screen_arr(bdata[8,*],-40,40),min=0,max=47,binsize=1) ; b_swc = bin_avg(btimebin,screen_arr(bdata[9,*],0,1.0),min=0,max=47,binsize=1) b_par = bin_avg(btimebin,screen_arr(bdata[7,*],-100,2500),min=0,max=47,binsize=1) blocs = findgen(48)+(doy-1l)*48l IF (yr EQ 2012) AND (doy LT 224) THEN blocs-=4 ;2012 TIME ZONE OFFSET ;2017 correction!!! IF ((yr EQ 2017) AND (doy GT 311)) OR (yr GT 2017) THEN blocs-=12 ; flux[17,blocs] = b_ts ; flux[18,blocs] = 0.1 ; flux[26,blocs] = b_swc flux[28,blocs] = merge_array(b_rnet,flux[28,blocs]) flux[29,blocs] = merge_array(b_par,flux[29,blocs]) flux[30,blocs] = merge_array(b_sw_in,flux[30,blocs]) flux[33,blocs] = merge_array(b_sw_out,flux[33,blocs]) flux[34,blocs] = merge_array(b_lw_in,flux[34,blocs]) flux[35,blocs] = merge_array(b_lw_out,flux[35,blocs]) flux[6,blocs] = merge_array(b_tair,flux[6,blocs]) flux[22,blocs] = merge_array(b_rh,flux[22,blocs]) ; met[5,blocs] = b_swc ; met[7,blocs] = b_ts met[9,blocs] = merge_array(b_tair,met[9,blocs]) met[10,blocs] = merge_array(b_rh,met[10,blocs]) met[19,blocs] = merge_array(b_sw_in,met[19,blocs]) met[20,blocs] = merge_array(b_sw_out,met[20,blocs]) met[21,blocs] = merge_array(b_lw_in,met[21,blocs]) met[22,blocs] = merge_array(b_lw_out,met[22,blocs]) met[24,blocs] = merge_array(b_par,met[24,blocs]) met[25,blocs] = merge_array(b_rnet,met[25,blocs]) ENDIF ELSE print,' Biomet file found but empty '+bmetfile ENDIF ELSE BEGIN print,' No biomet file found' ENDELSE ;surf sfiles = file_search(['/air/incoming/sylvania/'+yrstr+'/'+yrstr+mostr+daystr+'/','/air/incoming/sylvania/'+yrstr+'/','/air/incoming/sylvania/current/']+'*surf*data_'+yrstr+'_'+mostr+'_'+daystr+'*.dat',count=scountp) IF scountp GT 0 THEN BEGIN sfilename = strarr(n_elements(sfiles)) FOR i = 0,n_elements(sfilename)-1 DO sfilename[i] = (reverse(strsplit(sfiles[i],'/',/extract)))[0] ssfiles = sort(sfilename) sfilename = sfilename[ssfiles] sfiles = sfiles[ssfiles] usfiles = uniq(sfilename) sfiles = sfiles[usfiles] scountp = n_elements(sfiles) s_q10 = make_array(1440,/float,value=nan()) s_q20 = s_q10 & s_t10 = s_q10 & s_t20 = s_q10 & s_co2low = s_q10 & s_co2high = s_q10 & s_co2mid = s_q10 s_mn = s_q10 & s_hr = s_q10 & s_airt1 = s_q10 & s_airt2 = s_q10 & s_rh1 = s_q10 & s_rh2 = s_q10 & s_par1 = s_q10 & s_par2 = s_q10 s_shf = s_q10 & s_q10_2 = s_q10 & s_q20_2 = s_q10 & s_t10_2 = s_q10 & s_t20_2 = s_q10 FOR k = 0,scountp-1 DO BEGIN IF file_test(sfiles[k],/read) THEN BEGIN print,' Reading surf_data from '+sfiles[k] sdata = toa5_read(sfiles[k]) snames = strlowcase(tag_names(sdata)) slocs = long(sdata.hour) * 60L +long(sdata.minute) ; s_hr[slocs] = sdata.hour ; s_mn[slocs] = sdata.minute s_q10[slocs] = sdata.water_content_1_avg s_q20[slocs] = sdata.water_content_2_avg s_t10[slocs] = sdata.s_temp_1_avg s_t20[slocs] = sdata.s_temp_2_avg s_co2low[slocs] = sdata.gmp_lower_avg s_co2high[slocs] = sdata.gmp_upper_avg IF in(snames,'gmp_mid_avg') THEN s_co2mid[slocs] = sdata.gmp_mid_avg IF in(snames,'airt1_avg') THEN s_airt1[slocs] = sdata.airt1_avg IF in(snames,'airt2_avg') THEN s_airt2[slocs] = sdata.airt2_avg IF in(snames,'rh1_avg') THEN s_rh1[slocs] = sdata.rh1_avg IF in(snames,'rh2_avg') THEN s_rh2[slocs] = sdata.rh2_avg IF in(snames,'par1_avg') THEN s_par1[slocs] = sdata.par1_avg IF in(snames,'par2_avg') THEN s_par2[slocs] = sdata.par2_avg IF in(snames,'shf_avg') THEN s_shf[slocs] = sdata.shf_avg IF in(snames,'water_content_3_avg') THEN s_q10_2[slocs] = sdata.water_content_3_avg IF in(snames,'water_content_4_avg') THEN s_q20_2[slocs] = sdata.water_content_4_avg IF in(snames,'s_temp_3_avg') THEN s_t10_2[slocs] = sdata.s_temp_3_Avg IF in(snames,'s_temp_4_avg') THEN s_t20_2[slocs] = sdata.s_temp_4_Avg ENDIF ENDFOR mstart = (doy-1l)*48l mstart-=12 mend = mstart+47l metvstart = 0 IF mstart LT 0 THEN BEGIN mstart = 0 metvstart = 12 ENDIF met[5,mstart:mend] = (average_arr(s_q10,30,/nan))[metvstart:47] met[6,mstart:mend] = (average_arr(s_q20,30,/nan))[metvstart:47] met[7,mstart:mend] = (average_arr(s_t10,30,/nan))[metvstart:47] met[8,mstart:mend] = (average_arr(s_t20,30,/nan))[metvstart:47] met[14,mstart:mend] = (average_arr(screen_arr(s_co2low,350,1000),30,/nan))[metvstart:47] met[15,mstart:mend] = (average_arr(screen_arr(s_co2high,350,1000),30,/nan))[metvstart:47] IF n_elements(mheader) GE 34 THEN BEGIN met[26,mstart:mend] = (average_arr(screen_arr(s_co2mid,350,1000),30,/nan))[metvstart:47] met[27,mstart:mend] = (average_arr(s_airt1,30,/nan))[metvstart:47] met[28,mstart:mend] = (average_arr(s_airt2,30,/nan))[metvstart:47] met[29,mstart:mend] = (average_arr(s_rh1,30,/nan))[metvstart:47] met[30,mstart:mend] = (average_arr(s_rh2,30,/nan))[metvstart:47] met[31,mstart:mend] = (average_arr(s_par1,30,/nan))[metvstart:47] met[32,mstart:mend] = (average_arr(s_par2,30,/nan))[metvstart:47] met[33,mstart:mend] = (average_arr(s_shf,30,/nan))[metvstart:47] ENDIF IF n_elements(mheader) GE 38 THEN BEGIN met[34,mstart:mend] = (average_arr(s_q10_2,30,/nan))[metvstart:47] met[35,mstart:mend] = (average_arr(s_q20_2,30,/nan))[metvstart:47] met[36,mstart:mend] = (average_arr(s_t10_2,30,/nan))[metvstart:47] met[37,mstart:mend] = (average_arr(s_t20_2,30,/nan))[metvstart:47] ENDIF flux[17,mstart:mend] = merge_array(met[7,mstart:mend],flux[17,mstart:mend]) flux[18,mstart:mend] = 0.05 flux[19,mstart:mend] = merge_array(met[8,mstart:mend],flux[19,mstart:mend]) flux[20,mstart:mend] = 0.1 flux[26,mstart:mend] = merge_array(met[5,mstart:mend],flux[26,mstart:mend]) flux[27,mstart:mend] = merge_array(met[6,mstart:mend],flux[27,mstart:mend]) flux[39,mstart:mend] = merge_array(met[15,mstart:mend],flux[39,mstart:mend]) flux[40,mstart:mend] = 21.5 ;storage flux ppmt = 3600.0*findgen(48)/2.0 ppm = make_Array(5,48,/float,value=nan()) ppm_d = ppm ppm[0,0:mend-mstart] = met[14,mstart:mend] ;0 ppm[1,0:mend-mstart] = met[14,mstart:mend] ;2.5 ppm[2,0:mend-mstart] = met[26,mstart:mend] ;10 ppm[3,0:mend-mstart] = met[15,mstart:mend] ;21.5 topco2 = flux[24,mstart:mend] ;36 badtc = where(flux[49,mstart:mend] GT 1 OR ~finite(flux[10,mstart:mend]),nbtc) IF nbtc GT 0 THEN topco2[badtc] = nan() ppm[4,0:mend-mstart] = screen_arr(topco2,350,750) ;36 m FOR i = 0,4 DO ppm[i,*] -= mean(ppm[i,*],/nan) ppm[4,*] = zapbadval(merge_Array(ppm[4,*],ppm[3,*],ppm[2,*],ppm[1,*])) ppm[3,*] = zapbadval(merge_Array(ppm[3,*],ppm[4,*],ppm[2,*],ppm[1,*])) ppm[2,*] = zapbadval(merge_Array(ppm[2,*],ppm[1,*],ppm[3,*],ppm[4,*])) ppm[1,*] = zapbadval(merge_array(ppm[1,*],ppm[2,*],ppm[3,*],ppm[4,*])) ppm[0,*] = ppm[1,*] FOR i = 0,4 DO ppm_d[i,*] = deriv(ppmt,ppm[i,*]) stor = make_array(48,/float,value=nan()) FOR i = 0,47 DO stor[i] = int_tabulated([0.,2.5,10.0,21.5,36.0],ppm_d[*,i]) m_q = mixing_ratio(met[9,mstart:mend],met[11,mstart:mend]*10.,met[10,mstart:mend]/100.) m_drho = zapbadval(ddensity(met[11,mstart:mend]*10.,m_q,met[9,mstart:mend]+273.15)) bdrho = where(~finite(m_drho),nbdrho) IF nbdrho GT 0 THEN m_drho[bdrho] = 1.1 uair = 28.97 fstor = screen_arr(stor * (m_drho/uair) * 10^3,-10,10) flux[11,mstart:mend] = merge_array(fstor,screen_arr(flux[11,mstart:mend],-10,10)) goodstor = flux[11,mstart:mend] bgs = where(~finite(goodstor),nbgs) IF nbgs GT 0 THEN goodstor[bgs] = 0.0 flux[9,mstart:mend] = flux[10,mstart:mend] + goodstor ENDIF ELSE print,'No surf_data found' ;new file wtx520 (~ one minute) wfiles = file_search(['/air/incoming/sylvania/'+yrstr+'/'+yrstr+mostr+daystr+'/','/air/incoming/sylvania/'+yrstr+'/','/air/incoming/sylvania/current/']+'*wxt520_data_'+yrstr+'_'+mostr+'_'+daystr+'*.dat',count=wcountp) ;colums are: Rain_accum_Max mm, ;Rain_intens_Avg mm/hr ;also: Wind_Dir_mean_Avg, ;Wind_Speed_mean_Avg, Air_temp_Avg, ;pres520 ;fix soil_q20 above too? IF wcountp GT 0 THEN BEGIN wfilename = strarr(n_elements(wfiles)) FOR i = 0,n_elements(wfilename)-1 DO wfilename[i] = (reverse(strsplit(wfiles[i],'/',/extract)))[0] swfiles = sort(wfilename) wfilename = wfilename[swfiles] wfiles = wfiles[swfiles] uwfiles = uniq(wfilename) wfiles = wfiles[uwfiles] wcountp = n_elements(wfiles) w_rain_max = make_array(1440,/float,value=nan()) w_rain_int = w_rain_max w_wd = w_rain_max w_ws = w_rain_max w_airt = w_rain_max w_rh = w_rain_max w_press = w_rain_max FOR k = 0,wcountp-1 DO BEGIN IF file_test(wfiles[k],/read) THEN BEGIN print,' Reading wtx520_data from '+wfiles[k] wdata = toa5_read(wfiles[k]) wnames = strlowcase(tag_names(wdata)) wlocs = long(wdata.hour) * 60L +long(wdata.minute) IF in(wnames,'rain_accum_max') THEN w_rain_max[wlocs] = wdata.rain_accum_max IF in(wnames,'rain_intens_avg') THEN w_rain_int[wlocs] = wdata.rain_intens_avg IF in(wnames,'wind_dir_mean_avg') THEN w_wd[wlocs] = wdata.wind_dir_mean_avg IF in(wnames,'wind_speed_mean_avg') THEN w_ws[wlocs] = wdata.wind_speed_mean_avg IF in(wnames,'air_temp520_avg') THEN w_airt[wlocs] = wdata.air_temp520_avg IF in(wnames,'rh520_avg') THEN w_rh[wlocs] = wdata.rh520_avg IF in(wnames,'pres520_avg') THEN w_press[wlocs] = wdata.pres520_avg ENDIF ENDFOR mstart = (doy-1l)*48l mstart-=12 mend = mstart+47l metvstart = 0 IF mstart LT 0 THEN BEGIN mstart = 0 metvstart = 12 ENDIF met[11,mstart:mend] = merge_array((average_arr(w_press/10.,30,/nan))[metvstart:47],met[11,mstart:mend]) met[12,mstart:mend] = (average_arr(w_rain_max,30,/nan,/tot))[metvstart:47] met[13,mstart:mend] = (average_arr(w_rain_int,30,/nan)/2.0)[metvstart:47] met[16,mstart:mend] = (average_arr(w_airt,30,/nan))[metvstart:47] met[17,mstart:mend] = (average_arr(w_rh,30,/nan))[metvstart:47] IF n_elements(mheader) GE 40 THEN BEGIN met[38,mstart:mend] = (average_arr(w_ws,30,/nan))[metvstart:47] met[39,mstart:mend] = (average_arr(w_wd,30,/nan))[metvstart:47] ENDIF flux[21,mstart:mend] = merge_array(met[13,mstart:mend],flux[21,mstart:mend]) flux[23,mstart:mend] = merge_array(met[11,mstart:mend],flux[23,mstart:mend]) ENDIF ;new file soil 2022 onward - 15 minute samples soilfiles = file_search(['/air/incoming/sylvania/'+yrstr+'/'+yrstr+mostr+daystr+'/','/air/incoming/sylvania/'+yrstr+'/','/air/incoming/sylvania/current/']+'*sylvania_soil_'+yrstr+'_'+mostr+'_'+daystr+'*.dat',count=soilcountp) ;"TIMESTAMP","RECORD","VWC_5cm","Ka_5cm","T_5cm","BulkEC_5cm","VWC_10cm","Ka_10cm","T_10cm","BulkEC_10cm","VWC_20cm","Ka_20cm","T_20cm","BulkEC_20cm","VWC_30cm", ;"Ka_30cm","T_30cm","BulkEC_30cm","VWC_40cm","Ka_40cm","T_40cm","BulkEC_40cm","VWC_50cm","Ka_50cm","T_50cm","BulkEC_50cm","VWC_60cm","Ka_60cm","T_60cm","BulkEC_60cm","VWC_75cm","Ka_75cm","T_75cm","BulkEC_75cm","VWC_100cm","Ka_100cm","T_100cm","BulkEC_100cm" IF soilcountp GT 0 THEN BEGIN soilfilename = strarr(n_elements(soilfiles)) FOR i = 0,n_elements(soilfilename)-1 DO soilfilename[i] = (reverse(strsplit(soilfiles[i],'/',/extract)))[0] ssoilfiles = sort(soilfilename) soilfilename = soilfilename[ssoilfiles] soilfiles = soilfiles[ssoilfiles] usoilfiles = uniq(soilfilename) soilfiles = soilfiles[usoilfiles] soilcountp = n_elements(soilfiles) soil_t = make_array(9,96,/float,value=nan()) ;5,10,20,30,40,50,60,75,100 soil_q = soil_t FOR k = 0,soilcountp-1 DO BEGIN IF file_test(soilfiles[k],/read) THEN BEGIN print,' Reading soil_data from '+soilfiles[k] soildata = toa5_read(soilfiles[k]) soilnames = strlowcase(tag_names(soildata)) soillocs = long(soildata.hour) * 4L + (long(soildata.minute/15L)) IF in(soilnames,'t_5cm') THEN soil_t[0,soillocs] = soildata.t_5cm IF in(soilnames,'t_10cm') THEN soil_t[1,soillocs] = soildata.t_10cm IF in(soilnames,'t_20cm') THEN soil_t[2,soillocs] = soildata.t_20cm IF in(soilnames,'t_30cm') THEN soil_t[3,soillocs] = soildata.t_30cm IF in(soilnames,'t_40cm') THEN soil_t[4,soillocs] = soildata.t_40cm IF in(soilnames,'t_50cm') THEN soil_t[5,soillocs] = soildata.t_50cm IF in(soilnames,'t_60cm') THEN soil_t[6,soillocs] = soildata.t_60cm IF in(soilnames,'t_75cm') THEN soil_t[7,soillocs] = soildata.t_75cm IF in(soilnames,'t_100cm') THEN soil_t[8,soillocs] = soildata.t_100cm IF in(soilnames,'vwc_5cm') THEN soil_q[0,soillocs] = soildata.vwc_5cm IF in(soilnames,'vwc_10cm') THEN soil_q[1,soillocs] = soildata.vwc_10cm IF in(soilnames,'vwc_20cm') THEN soil_q[2,soillocs] = soildata.vwc_20cm IF in(soilnames,'vwc_30cm') THEN soil_q[3,soillocs] = soildata.vwc_30cm IF in(soilnames,'vwc_40cm') THEN soil_q[4,soillocs] = soildata.vwc_40cm IF in(soilnames,'vwc_50cm') THEN soil_q[5,soillocs] = soildata.vwc_50cm IF in(soilnames,'vwc_60cm') THEN soil_q[6,soillocs] = soildata.vwc_60cm IF in(soilnames,'vwc_75cm') THEN soil_q[7,soillocs] = soildata.vwc_75cm IF in(soilnames,'vwc_100cm') THEN soil_q[8,soillocs] = soildata.vwc_100cm ENDIF ENDFOR mstart = (doy-1l)*48l mstart-=12 mend = mstart+47l metvstart = 0 IF mstart LT 0 THEN BEGIN mstart = 0 metvstart = 12 ENDIF ; mheader += ['SoilT_5_2','SoilT_10_2','SoilT_20_2','SoilT_30_2','SoilT_40_2','SoilT_50_2','SoilT_60_2','SoilT_75_2','SoilT_100_2'] ;40-48 ; mheader += ['SoilQ_5_2','SoilQ_10_2','SoilQ_20_2','SoilQ_30_2','SoilQ_40_2','SoilQ_50_2','SoilQ_60_2','SoilQ_75_2','SoilQ_100_2'] ;49-57 met[40:48,mstart:mend] = (average_cols(soil_t[0:8,*],2,/nan))[*,metvstart:47] met[49:57,mstart:mend] = (average_cols(soil_q[0:8,*],2,/nan))[*,metvstart:47] ENDIF IF yr LT 2019 THEN BEGIN mfiles = file_search(['/air/incoming/sylvania/'+yrstr+'/'+yrstr+mostr+daystr+'/','/air/incoming/sylvania/'+yrstr+'/','/air/incoming/sylvania/current/']+'*met*data_'+yrstr+'_'+mostr+'_'+daystr+'*.dat',count=mcountp) IF mcountp GT 0 THEN BEGIN mfilename = strarr(n_elements(mfiles)) FOR i = 0,n_elements(mfilename)-1 DO mfilename[i] = (reverse(strsplit(mfiles[i],'/',/extract)))[0] smfiles = sort(mfilename) mfilename = mfilename[smfiles] mfiles = mfiles[smfiles] umfiles = uniq(mfilename) mfiles = mfiles[umfiles] mcountp = n_elements(mfiles) metvars = make_array(21,1440,/float,value=nan()) FOR k = 0,mcountp-1 DO BEGIN IF file_test(mfiles[k],/read) THEN BEGIN print,' Reading met from '+mfiles[k] openr,fl,mfiles[k],/get_lun mdum = '' IF ~eof(fl) THEN readf,fl,mdum mhead = '' IF ~eof(fl) THEN readf,fl,mhead mdum = '' IF ~eof(fl) THEN readf,fl,mdum mdum = '' IF ~eof(fl) THEN readf,fl,mdum IF ~eof(fl) THEN BEGIN FOR i = 4,file_lines(mfiles[k])-1 DO BEGIN str = '' readf,fl,str str = strsplit(str,',',/extract) strel = n_elements(str) tmstr = (strsplit(strmid(str[0],1,strlen(str[0])-2),'-: ',/extract)) IF tmstr[0] EQ yrstr AND tmstr[1] EQ mostr AND tmstr[2] EQ daystr AND n_elements(tmstr) EQ 6 THEN BEGIN tloc = long(tmstr[3])*60l + long(tmstr[4]) IF tloc GE 0 AND tloc LT 1440 THEN BEGIN IF strel GT 12 THEN metvars[0,tloc] = float(strsplit(str[12],'"',/extract)) ;wc1 - already there IF strel GT 15 THEN metvars[1,tloc] = float(strsplit(str[15],'"',/extract)) ;wc2 IF strel GT 14 THEN metvars[2,tloc] = float(strsplit(str[14],'"',/extract)) ;st1 IF strel GT 17 THEN metvars[3,tloc] = float(strsplit(str[17],'"',/extract)) ;st2 IF strel GT 31 THEN metvars[4,tloc] = float(strsplit(str[31],'"',/extract)) ;T IF strel GT 32 THEN metvars[5,tloc] = float(strsplit(str[32],'"',/extract)) ;RH IF strel GT 33 THEN metvars[6,tloc] = float(strsplit(str[33],'"',/extract)) ;P (hPa) IF strel GT 34 THEN metvars[7,tloc] = float(strsplit(str[34],'"',/extract)) ;rmax IF strel GT 35 THEN metvars[8,tloc] = float(strsplit(str[36],'"',/extract)) ;rint IF strel GT 42 THEN metvars[9,tloc] = float(strsplit(str[42],'"',/extract)) ;CO2 2.5 m IF strel GT 43 THEN metvars[10,tloc] = float(strsplit(str[43],'"',/extract)) ;CO2 21.5 m IF strel GT 2 THEN metvars[11,tloc] = float(strsplit(str[2],'"',/extract)) ;T_HMP IF strel GT 3 THEN metvars[12,tloc] = float(strsplit(str[3],'"',/extract)) ;RH_HMP IF strel GT 4 THEN metvars[13,tloc] = float(strsplit(str[4],'"',/extract)) ;ea IF strel GT 7 THEN metvars[14,tloc] = float(strsplit(str[7],'"',/extract)) ;SW_in IF strel GT 8 THEN metvars[15,tloc] = float(strsplit(str[8],'"',/extract)) ;SW_out IF strel GT 9 THEN metvars[16,tloc] = float(strsplit(str[9],'"',/extract)) ;IR_in IF strel GT 10 THEN metvars[17,tloc] = float(strsplit(str[10],'"',/extract)) ;IR_out IF strel GT 11 THEN metvars[18,tloc] = float(strsplit(str[11],'"',/extract)) ;Tc (C) IF strel GT 24 THEN metvars[19,tloc] = float(strsplit(str[24],'"',/extract)) ;PAR (umol/m2/s) ENDIF ENDIF ENDFOR ;merge 2 = T, 3 = RH, 4 = Press in kPa, ;7 = swup?, 8 = swdn, 9 = irup, 10 = irdown, 11 = tc ENDIF free_lun,fl ENDIF ENDFOR mlocs = lindgen(48)+((doy-1)*48) IF (yr EQ 2012) AND (doy LT 224) THEN mlocs-=4 ;2012 TIME ZONE OFFSET IF ((yr EQ 2017) AND (doy GT 311)) OR (yr GT 2017) THEN mlocs-=12 ;2017 CORRECTION ;save averages to flux for vars not in biomet metvars[18,*] = screen_arr(metvars[18,*],-50,50) metvars[18,*] += 273.15 m_ir_in = metvars[16,*] + 5.67e-8 * metvars[18,*]^4 metvars[16,*] = m_ir_in m_ir_out = metvars[17,*] + 5.67e-8 * metvars[18,*]^4 metvars[17,*] = m_ir_out m_rnet = (metvars[14,*]+m_ir_in)-(metvars[15,*]+m_ir_out) metvars[20,*] = m_rnet ;Aug 2014 onward apply RH correction for new RH sensor IF (yr LT 2019) THEN BEGIN badrh = where(finite(metvars[5,*]) AND metvars[5,*] GT 87.0,nbrh) IF nbrh GT 0 THEN BEGIN metvars[5,badrh] -= 87.0 metvars[5,badrh] *= 1.7266296 metvars[5,badrh] += 0.12311130 metvars[5,badrh] += 87.0 badrh2 = where(finite(metvars[5,*]) AND metvars[5,*] GT 100.0,nbrh2) IF nbrh2 GT 0 THEN metvars[5,badrh2] = 100.0 ENDIF ENDIF ;save to met array mstart = (doy-1l)*48l IF (yr EQ 2012) AND (doy LT 224) THEN mstart-=4 ;2012 TIME ZONE OFFSET ;2017 correction !!! IF ((yr EQ 2017) AND (doy GT 311)) OR (yr GT 2017) THEN mstart-=12 mend = mstart+47l metvstart = 0 IF mstart LT 0 THEN BEGIN mstart = 0 metvstart = 12 ENDIF met[5:*,mstart:mend] = (average_cols(metvars[0:20,*],30,/nan))[*,metvstart:47] met[13,mstart:mend]/=2.0 ;for processing: merge in T, RH, Press only if all nan IF isallnan(metvars[4,*]) THEN metvars[4,*] = screen_arr(metvars[11,*],-39,39) IF isallnan(metvars[5,*]) THEN metvars[5,*] = screen_arr(metvars[12,*],0.1,100.1) IF isallnan(metvars[6,*]) THEN metvars[6,*] = screen_arr(metvars[13,*],80.0,110.0)*10 metvars[5,*] = screen_arr(metvars[5,*],0.1,100.1) flux[28,mlocs] = screen_arr(merge_array(average_arr(m_rnet,30,/nan),screen_arr(flux[28,mlocs],-300,1500)),-300,1500) flux[29,mlocs] = merge_array(average_arr(metvars[19,*],30,/nan),flux[29,mlocs]) flux[30,mlocs] = merge_array(average_arr(metvars[14,*],30,/nan),flux[30,mlocs]) flux[33,mlocs] = merge_array(average_arr(metvars[15,*],30,/nan),flux[33,mlocs]) flux[34,mlocs] = screen_arr(merge_array(average_arr(m_ir_in,30,/nan),screen_arr(flux[34,mlocs],150,650)),100,650) flux[35,mlocs] = screen_arr(merge_array(average_arr(m_ir_out,30,/nan),screen_arr(flux[35,mlocs],150,650)),150,650) flux[6,mlocs] = merge_array(average_arr(metvars[4,*],30.,/nan),flux[6,mlocs]) flux[17,mlocs] = merge_Array(average_arr(metvars[2,*],30.,/nan),flux[17,mlocs]) flux[18,mlocs] = 0.05 flux[19,mlocs] = average_arr(metvars[3,*],30.,/nan) flux[20,mlocs] = 0.1 flux[21,mlocs] = average_arr(metvars[8,*],30.,/nan)/2.0 flux[22,mlocs] = screen_Arr(merge_array(average_arr(metvars[5,*],30.,/nan),flux[22,mlocs]),0,100) flux[23,mlocs] = merge_array(average_arr(metvars[6,*],30.,/nan)/10.0,flux[23,mlocs]) flux[26,mlocs] = merge_array(average_arr(metvars[0,*],30.,/nan),flux[26,mlocs]) flux[27,mlocs] = average_arr(metvars[1,*],30.,/nan) flux[39,mlocs] = screen_arr(average_arr(metvars[10,*],30.,/nan),350,1000) flux[40,mlocs] = 21.5 ;storage flux ppmt = 3600.0*findgen(48)/2.0 ppm = make_Array(4,48,/float,value=nan()) ppm_d = ppm ppm[0,*] = average_arr(screen_arr(metvars[9,*],350,1000),30.,/nan) ;0 ppm[1,*] = ppm[0,*] ;2.5 ppm[2,*] = average_arr(screen_arr(metvars[10,*],350,1000),30.,/nan) ;21.5 topco2 = flux[24,mlocs] badtc = where(flux[49,mlocs] GE 1 OR ~finite(flux[10,mlocs]),nbtc) IF nbtc GT 0 THEN topco2[badtc] = nan() ppm[3,*] = screen_arr(topco2,350,500) ;37 m FOR i = 0,3 DO ppm[i,*] -= mean(ppm[i,*]) ppm[3,*] = zapbadval(merge_Array(ppm[3,*],ppm[2,*],ppm[1,*])) ppm[2,*] = zapbadval(merge_Array(ppm[2,*],ppm[1,*],ppm[3,*])) ppm[1,*] = zapbadval(merge_array(ppm[1,*],ppm[2,*],ppm[3,*])) ppm[0,*] = ppm[1,*] FOR i = 0,3 DO ppm_d[i,*] = deriv(ppmt,ppm[i,*]) stor = make_array(48,/float,value=nan()) FOR i = 0,47 DO stor[i] = int_tabulated([0.,2.5,21.5,37.0],ppm_d[*,i]) m_q = mixing_ratio(metvars[4,*],metvars[6,*],metvars[5,*]/100.) m_drho = zapbadval(ddensity(metvars[6,*],m_q,metvars[4,*]+273.15)) bdrho = where(~finite(m_drho),nbdrho) IF nbdrho GT 0 THEN m_drho[bdrho] = 1.1 uair = 28.97 fstor = screen_arr(stor * (m_drho/uair) * 10^3,-10,10) flux[11,mlocs] = merge_array(fstor,screen_arr(flux[11,mlocs],-10,10)) goodstor = flux[11,mlocs] bgs = where(~finite(goodstor),nbgs) IF nbgs GT 0 THEN goodstor[bgs] = 0.0 flux[9,mlocs] = flux[10,mlocs] + goodstor ENDIF ENDIF ;pre-2019 stuff IF keyword_set(procfile) THEN BEGIN ;for flux processing write file daily to allow for easy restart write_xdf,outfile,flux,header=fheader write_xdf,outmfile,met,header=mheader openw,fl,'/air/incoming/sylvania/'+string(yr,format='(i4.4)')+'/processed_'+jd_to_dy(doy,y=yr),/get_lun printf,fl,' ' free_lun,fl ENDIF ENDFOR ;There are two sampling locations with 2 sensors each (I believe that ;probe 3 and probe 4 are currently broken, so we only really have 1 ;sampling location). 1 sensor is at 10cm, the second is at 20cm. ;The heights are 21.5m for the GMP_upper and 2.5m for te GMP_lower. + co2 ;change -6999 -9999 -999 to nan bval = where(flux LE -6999. OR flux EQ -999.,nbv) IF nbv GT 0 THEN flux[bval] = nan() bval = where(met LE -6999. OR met EQ -999.,nbv) IF nbv GT 0 THEN met[bval] = nan() IF keyword_set(debug) THEN stop ;basic screening flux[5,*] = screen_arr(flux[5,*],0.0,10.0) flux[6,*] = screen_arr(flux[6,*],-50.0,50.0) flux[7,*] = screen_arr(flux[7,*],0.0,360.0) flux[8,*] = screen_arr(flux[8,*],0.0,50.0) flux[9,*] = screen_arr(flux[9,*],-80.0,80.0) flux[10,*] = screen_arr(flux[10,*],-80.0,80.0) flux[11,*] = screen_arr(flux[11,*],-80.0,80.0) flux[12,*] = screen_arr(flux[12,*],-200.0,600.0) flux[13,*] = screen_arr(flux[13,*],-200.0,600.0) flux[14,*] = screen_arr(flux[14,*],-200.0,600.0) flux[15,*] = screen_arr(flux[15,*],-200.0,600.0) flux[16,*] = screen_arr(flux[16,*],-200.0,600.0) flux[17,*] = screen_arr(flux[17,*],-40.0,40.0) flux[19,*] = screen_arr(flux[19,*],-40.0,40.0) flux[22,*] = screen_arr(flux[22,*],0.0,100.0) flux[23,*] = screen_arr(flux[23,*],80.0,100.0) flux[24,*] = screen_arr(flux[24,*],320.0,800.0) flux[25,*] = screen_arr(flux[25,*],0.0,10.0) flux[26,*] = screen_arr(flux[26,*],0.0,1.0) flux[27,*] = screen_arr(flux[27,*],0.0,1.0) flux[28,*] = screen_arr(flux[28,*],-300.0,1000.0) flux[29,*] = screen_arr(flux[29,*],-20.0,2500.0) flux[30,*] = screen_arr(flux[30,*],-20.0,1500.0) flux[33,*] = screen_arr(flux[33,*],-20.0,1500.0) flux[34,*] = screen_arr(flux[34,*],-20.0,1500.0) flux[35,*] = screen_arr(flux[35,*],-20.0,1500.0) ;quality flag screens badf = where(flux[49,*] GE 1 ,nbv) IF nbv GT 0 THEN flux[9,badf] = nan() IF nbv GT 0 THEN flux[10,badf] = nan() badf = where(flux[47,*] GE 1 ,nbv) IF nbv GT 0 THEN flux[12,badf] = nan() badf = where(flux[48,*] GE 1 ,nbv) IF nbv GT 0 THEN flux[14,badf] = nan() badf = where(flux[46,*] GE 1 ,nbv) IF nbv GT 0 THEN flux[5,badf] = nan() ;identify gaps flux[1,*] = ~finite(flux[10,*]) ;;stop ;write file write_xdf,outfile,flux,header=fheader write_xdf,outmfile,met,header=mheader IF keyword_set(email) THEN BEGIN ; openw,fl,'/air/incoming/sylvania/flux/lastday.txt',/get_lun openw,fl,'/air/incoming/wlef/lastday.txt',/get_lun,/append ; printf,fl,'CC: Jonathan E. Thom ' ; printf,fl,'Subject: SYLVANIA Diagnostic '+string(yr,format='(i4.4)')+' DOY '+string(Endday,format='(i3.3)') ; printf,fl,'MIME-Version: 1.0' ; printf,fl,'Content-Type: text/html' ; printf,fl,'Content-Disposition: inline' ; printf,fl,'' ; printf,fl,'' ; printf,fl,'
'
    totvar = float(lend-lstart)+1.0
    printf,fl,''
    printf,fl,'SYLVANIA MET'
    FOR i = 0,n_elements(mheader)-1 DO IF total(~finite(met[i,lstart:lend])) GT (totvar/1.8) THEN printf,fl,mheader[i],' ',100*total(~finite(met[i,lstart:lend]))/totvar,'% missing'
;    printf,fl,mheader,format='(a4," ",a3," ",a4," ",a3," ",a4,20(a10," "),a10)'
;    FOR i=lstart,lend DO printf,fl,met[0:25,i],format='(I4,1x,I3,1x,F4.1,1x,I3,1x,I4,1x,20(F10.3,1x),F10.3)'
    printf,fl,' '
    printf,fl,'SYLVANIA FLUX'
    skiphead = ['FG','Rgdif','PARout','RE','GPP','APAR','PARdif','APARpct','NEEfilled']
    FOR i = 0,n_elements(fheader)-1 DO IF ~in(skiphead,fheader[i]) THEN IF total(~finite(flux[i,lstart:lend])) GT (totvar/1.8) THEN printf,fl,fheader[i],' ',100*total(~finite(flux[i,lstart:lend]))/totvar,'% missing'
;    printf,fl,fheader,format='(a4," ",a3," ",a4," "," ",a3," ",a4,42(a10," "),4(a10," "),a10," ",a10)'
;    FOR i=lstart,lend DO printf,fl,flux[0:52,i],format='(I4,1x,I3,1x,F4.1,1x,I3,1x,I4,1x,42(f10.3,1x),4(I10,1x),F10.3,1x,F10.3)'
    printf,fl,'
' printf,fl,'' printf,fl,'' free_lun,fl ; spawn,'/usr/sbin/sendmail desai@aos.wisc.edu < /air/incoming/sylvania/flux/lastday.txt' spawn,'/usr/sbin/sendmail desai@aos.wisc.edu < /air/incoming/wlef/lastday.txt' ENDIF END PRO sylvania_fillnee,yr,mcutoff=mcutoff,cutoff=cutoff,debug=debug,noupload=noupload,nowrite=nowrite,noascii=noascii ;read flux file ;gap fill T and PAR ;save GPP and Reco into fillnee file and flux file yrstr = string(yr,format='(i4.4)') yrdr = yrstr+'/' y = yr IF yr GE 2015 THEN BEGIN IF file_test('/air/incoming/WillowCreek/alldata'+yrstr+'-processed.sav',/read) THEN restore,'/air/incoming/WillowCreek/alldata'+yrstr+'-processed.sav' ENDIF ELSE BEGIN restore,'/air/incoming/WillowCreek/alldata19982014-processed.sav' yloc = y-1998 wc_par_fill = wc_par_fill[*,yloc] wc_tsoil_fill = wc_tsoil_fill[*,yloc] ENDELSE ;wc_par_fill - extract < 2014 yr = y f = read_xdf('/air/incoming/sylvania/flux/sylvaniaflux_'+yrstr+'.xdf',header=fh) cnee_orig = f[9,*] cflag = f[49,*] qflag = f[48,*] tflag = f[47,*] cflux = f[10,*] cstor = f[11,*] co2 = f[24,*] cflux[where(cflag GE 1)]=nan() cstor[where(cflag GE 1)]=nan() cflux = localdespike(screen_arr(cflux,-40,40),nsig=3,window=48*7) cstor = localdespike(screen_Arr(cstor,-10,10),nsig=3,window=48*7) newbad = where(cflag EQ 0 AND ~finite(cflux),nnb) IF nnb GT 0 THEN cflag[newbad] = 3 co2[where(cflag GE 1)]=nan() co2 = localdespike(screen_arr(co2,300,550),nsig=3,window=48*7) tflux = localdespike(screen_arr(f[12,*],-200,600),nsig=3,window=48*7) tflux[where(tflag GE 1)]=nan() tstor = localdespike(screen_arr(f[13,*],-50,50),nsig=3,window=48*7) newbad = where(tflag EQ 0 AND ~finite(tflux),nnb) IF nnb GT 0 THEN tflag[newbad] = 3 qflux = localdespike(screen_Arr(f[14,*],-100,600),nsig=3,window=48*7) qflux[where(qflag GE 1)]=nan() qstor = localdespike(screen_arr(f[15,*],-50,50),nsig=3,window=48*7) newbad = where(qflag EQ 0 AND ~finite(qflux),nnb) IF nnb GT 0 THEN qflag[newbad] = 3 ;gap fill CO2 storage cstorfill = zapbadval(ensemble_fill(cstor,ptsperday=48,mindays=14,maxdays=56,enoughdays=14,/circular)) tstorfill = zapbadval(ensemble_fill(tstor,ptsperday=48,mindays=14,maxdays=56,enoughdays=14,/circular)) qstorfill = zapbadval(ensemble_fill(qstor,ptsperday=48,mindays=14,maxdays=56,enoughdays=14,/circular)) ;recalc NEE after gap filling storage cnee = screen_arr(cflux+cstorfill,-50,50) qnee = screen_arr(qflux+qstorfill,-100,600) tnee = screen_arr(tflux+tstorfill,-200,600) f[9,*] = cnee f[10,*] = cflux f[11,*] = cstor f[12,*] = tnee f[13,*] = tstor f[14,*] = qnee f[15,*] = qstor f[24,*] = co2 f[49,*] = cflag f[48,*] = qflag f[47,*] = tflag ;data is in local time dn = daynight(lat=46.242,lon=-89.348,interval=48,yr=yr) day = dn NE 0 night = dn EQ 0 ;u* and wind dir screen mflux = f[5,*] wdir = f[7,*] flag1 = finite(mflux) AND (mflux LT 0.325) AND night flag2 = (Wdir GE 30) AND (wdir LE 90) AND day flag = flag1 OR flag2 cpref = cnee cpref[where(flag)]=nan() f[1,*] = ~finite(cpref) ;gap fill T IF yr EQ 2013 THEN f[17,0:12600]=nan() IF yr GE 2020 THEN BEGIN badst = where((deriv(f[17,*]) eq 0) and f[17,*] gt 1,nbst) IF nbst GT 0 THEN f[17,badst]= nan() IF yr EQ 2021 THEN f[17,7300:*] = nan() sm = read_xdf('/air/incoming/sylvania/flux/sylvaniamet_'+yrstr+'.xdf',header=smh) newts = sm[36,*] IF yr EQ 2020 THEN newts[0:9430]=nan() f[17,*] = merge_array(newts,f[17,*]) ENDIF airt_s = smooth(f[6,*],48*10,/nan) cair = where(airt_s LT 0.0,nca) IF nca GT 0 THEN airt_s[cair] = 0.0 cair = where(airt_s LT 5.0,nca) IF nca GT 0 THEN airt_s[cair] *= 0.25 airt_s = fit_array(Airt_s,f[17,*]) IF n_elements(wc_tsoil_fill) NE 0 THEN t30 = merge_array(f[17,*],fit_array(f[19,*],f[17,*]),airt_s,fit_array(wc_tsoil_fill,f[17,*])) ELSE t30 = merge_array(f[17,*],fit_array(f[19,*],f[17,*]),airt_s) ;soil t ;merge f[6,*] temp = zapbadval(ensemble_fill(t30,ptsperday=48,mindays=14,maxdays=56,enoughdays=14,/circular)) ;pick one, gap fill ;gapfill par + sw_in IF yr EQ 2019 THEN f[29,11300:14500]=nan() IF n_elements(wc_par_fill) NE 0 THEN gpar = merge_array(f[29,*],fit_Array(f[30,*],f[29,*]),fit_array(wc_par_fill[*],reform(f[29,*]))) ELSE gpar = merge_array(f[29,*],fit_array(f[30,*],f[29,*])) par = reform(zapbadval(ensemble_fill(gpar,ptsperday=48,mindays=14,maxdays=56,enoughdays=14,/circular))) par[where(night)]=0.0 ;timezone shift to CST (shift by 12) ; chop2 = n_elements(cpref)-1 ; chop1 = n_elements(cpref)-1-12 ; temp = shift(temp,-12) ; temp[chop1:chop2]=nan() ; temp = zapbadval(temp) ; par = shift(par,-12) ; par[chop1:chop2]=nan() ; cpref = shift(cpref,-12) ; cpref[chop1:chop2]=nan() ;cutoff temp/par IF n_elements(mcutoff) NE 0 THEN BEGIN dd = (findgen(n_elements(temp))/48.)+1. bval_co = where(dd LT mcutoff[0] OR dd GE mcutoff[1],nbval_co) IF nbval_co GT 0 THEN BEGIN temp[bval_co]=nan() par[bval_co]=nan() ENDIF ENDIF ;stop ;gap_fill NEE IF isallnan(cpref) OR isallnan(par) OR isallnan(temp) THEN BEGIN fnee = cpref fnee[*] = !values.f_nan resp = fnee psyn = fnee modnee = fnee npts = n_elements(cpref) fn = make_array(17,npts,/float,value=nan()) doyarr = fix(findgen(npts)/48)+1 tmarr = fix(lindgen(npts) MOD 48) / 48. hharr = fix(tmarr*24.0) mmarr = fix(((tmarr*24.0)-hharr)*60.0) hhmmarr = string(hharr,format='(i2.2)')+string(mmarr,format='(i2.2)') fjdayarr = doyarr+tmarr fn[0,*] = yr fn[1,*] = doyarr fn[2,*] = hhmmarr fn[3,*] = fjdayarr fnh = ['Year','Day of Year','HHMM','fjday','A','E',$ 'Eact','Temp','Resp','b0','b1','b2','PAR','Psyn',$ 'original_NEE','model_NEE','filled_NEE'] ENDIF ELSE BEGIN CalcRespPsynFits,yr,nee=cpref,par=par,temp=temp,respfit=rf,psynfit=pf,fillednee=fn,badval=nan(),badlocs=bl,resphead=rh,psynhead=ph,filledhead=fnh,leafon=leafon,leafoff=leafoff,cutoff=cutoff,lat=46.0827,lon=-89.9792 IF n_elements(fn) NE 0 THEN BEGIN resp = fn[8,*] psyn = fn[13,*] modnee = fn[15,*] fnee = fn[16,*] ENDIF ELSE BEGIN fnee = cpref fnee[*] = !values.f_nan resp = fnee psyn = fnee modnee = fnee npts = n_elements(cpref) fn = make_array(17,npts,/float,value=nan()) doyarr = fix(findgen(npts)/48)+1 tmarr = fix(lindgen(npts) MOD 48) / 48. hharr = fix(tmarr*24.0) mmarr = fix(((tmarr*24.0)-hharr)*60.0) hhmmarr = string(hharr,format='(i2.2)')+string(mmarr,format='(i2.2)') fjdayarr = doyarr+tmarr fn[0,*] = yr fn[1,*] = doyarr fn[2,*] = hhmmarr fn[3,*] = fjdayarr fnh = ['Year','Day of Year','HHMM','fjday','A','E',$ 'Eact','Temp','Resp','b0','b1','b2','PAR','Psyn',$ 'original_NEE','model_NEE','filled_NEE'] ENDELSE ENDELSE IF keyword_set(debug) THEN stop ;copy RE and RE-FNEE into RE and GPP columns f[37,*] = resp f[38,*] = resp-fnee f[45,*] = fnee ;other vars IF yr EQ 2013 THEN f[26,0:12600]=nan() IF yr EQ 2013 THEN f[27,0:12600]=nan() IF yr EQ 2013 THEN f[17,0:12600]=nan() IF yr EQ 2013 THEN f[19,0:12600]=nan() IF yr EQ 2012 THEN f[17,12977:*]=nan() IF yr EQ 2012 THEN f[19,10912:*]=nan() f[24,*] = localdespike(F[24,*],nsig=2,window=48*7) f[39,*] = localdespike(F[39,*],nsig=2,window=48*7) IF ~keyword_set(nowrite) THEN write_xdf,'/air/incoming/sylvania/flux/sylvaniaflux_'+yrstr+'.xdf',f,header=fh yrstr2 = string(yr MOD 100,format='(i2.2)') write_xdf,'/air/incoming/sylvania/flux/sylvania'+yrstr2+'_fillednee.xdf',fn,header=fnh ; f[6:*,*] = shift(f[6:*,*],0,-12) ; f[6:*,chop1:chop2]=nan() IF ~keyword_set(noascii) THEN BEGIN bval = where(~finite(f),nbad) IF nbad GT 0 THEN f[bval] = -9999.0 fname_flux = '/air/incoming/sylvania/flux/'+'sylvaniaflux_'+yrstr+'.txt' fname_fill = '/air/incoming/sylvania/flux/'+'sylvania'+yrstr2+'_fillednee.txt' ;stop ;write ascii bval = where(~finite(f),nbv) IF nbv GT 0 THEN f[bval]=-9999.0 openw,fl,fname_flux,/get_lun printf,fl,'Ameriflux' printf,fl,'Sitename: US-Syv' printf,fl,'Location: Latitude: 46.2420170 - Longitude: -89.3476500 - Elevation (masl): 500.' printf,fl,'Principal investigator: Ankur Desai desai@aos.wisc.edu' printf,fl,'Ecosystem type: Old-growth mixed forest' printf,fl,'File creation date: '+systime() printf,fl,'Datapolicy: -- The AmeriFlux data provided on this site are freely available and were furnished by individual AmeriFlux scientists who encourage their use.' printf,fl,'Please kindly inform in writing (or e-mail) the appropriate AmeriFlux scientist(s) of how you intend to use the data and of any publication plans.' printf,fl,'It is also important to contact the AmeriFlux investigator to assure you are downloading the latest revision of the data and to prevent potential misuse or misinterpretation of the data.' printf,fl,'Please acknowledge the data source as a citation or in the acknowledgments if no citation is available.' printf,fl,'If the AmeriFlux Principal Investigators (PIs) feel that they should be acknowledged or offered participation as authors they will let you know.' printf,fl,'And we assume that an agreement on such matters will be reached before publishing and/or use of the data for publication.' printf,fl,'If your work directly competes with the PIs analysis they may ask that they have the opportunity to submit a manuscript before you submit one that uses unpublished data.' printf,fl,'In addition when publishing please acknowledge the agency that supported the research. --' printf,fl,'File Origin (4 lines) - To be compiled by AmeriFlux data management group.' printf,fl,'File Origin (4 lines) - To be compiled by AmeriFlux data management group.' printf,fl,'File Origin (4 lines) - To be compiled by AmeriFlux data management group.' printf,fl,'File Origin (4 lines) - To be compiled by AmeriFlux data management group.' printf,fl,'YEAR,GAP,DTIME,DOY,HRMIN,UST,TA,WD,WS,NEE,FC,SFC,H,SH,LE,SLE,FG,TS1,TSdepth1,TS2,TSdepth2,PREC,RH,PRESS,CO2,VPD,SWC1,SWC2,Rn,PAR,Rg,Rgdif,PARout,RgOut,Rgl,RglOut,H2O,RE,GPP,CO2top,CO2height,APAR,PARdif,APARpct,ZL,NEEfilled,Flagu*,FlagH,FlagLE,FlagC,FlagE,FetchMax,Fetch90' printf,fl,'YEAR,GAP,DTIME,DOY,HRMIN,m/s,deg C,deg,m/s,umol/m2/s,umol/m2/s,umol/m2/s,W/m2,W/m2,W/m2,W/m2,W/m2,deg C,cm,deg C,cm,mm,%,kPa,umol/mol,kPa,%,%,W/m2,umol/m2/s,W/m2,W/m2,umol/m2/s,W/m2,W/m2,W/m2,mmol/mol,umol/m2/s,umol/m2/s,umol/mol,m,umol/m2/s,umol/m2/s,%,unitless,umol/m2/s,unitless,unitless,unitless,unitless,unitless,m,m' FOR i = 0,n_elements(f[0,*])-1 DO printf,fl,f[0:52,i],format='(i4,",",i0,",",f0.14,",",i3,",",i4.4,",",39(f0.7,","),g0.7,",",f0.7,",",5(i0,","),",",f0.7,",",f0.7)' free_lun,fl IF n_elements(fn) NE 0 THEN BEGIN bval = where(~finite(fn),nbad) IF nbad GT 0 THEN fn[bval] = -999.0 openw,fl,fname_fill,/get_lun FOR k = 0,n_elements(fn[0,*])-1 DO printf,fl,fn[*,k],format='(i4.4,i4,i5.4,f8.3,13g12.3)' free_lun,fl ENDIF ENDIF IF ~keyword_set(noupload) THEN BEGIN spawn,'scp -v -i /home/adesai/ameriflux_keys/us-syv '+fname_flux+' '+fname_fill+' fluxnet@dtn01.nersc.gov:' ENDIF END PRO sylvania_fluxall,yr,startday=startday,endday=endday,noupload=noupload,noflux=noflux,nofill=nofill,nooutput=nooutput ;based on wcreek_fluxall cuttm = systime(/julian) caldat,cuttm,mon,day,curyr,hour,minute,secod curdoy = dy_to_jd(string(curyr,format='(i4.4)')+string(mon,format='(i2.2)')+string(day,format='(i2.2)')) ;default year is this year IF n_elements(yr) EQ 0 THEN yr = curyr ;default process to today IF n_elements(endday) EQ 0 THEN IF yr EQ curyr THEN endday = curdoy-1 ELSE endday = days_in_year(yr) ;(curdoy-1) > 1 IF n_elements(startday) EQ 0 THEN BEGIN procprefix = '/air/incoming/sylvania/'+string(yr,format='(i4.4)')+'/processed_'+string(yr,format='(i4.4)') procfiles = file_Search(procprefix+'*',count=nproc) IF nproc EQ 0 THEN startday = 1 ELSE BEGIN procfiles = reverse(procfiles[sort(procfiles)]) lastfile = procfiles[0] lastday = str_right(lastfile,8) ; lastday = strmid(lastday,0,4)+strmid(lastday,5,2)+strmid(lastday,8,2) startday = dy_to_jd(lastday) ENDELSE ENDIF IF endday GT startday THEN BEGIN print,'Sylvania Process Flux for day ',startday,' to ',endday,' for year',yr IF ~keyword_set(noflux) THEN sylvania_processfluxyear,yr,startday=startday,endday=endday,/email,/procfile,noupload=noupload IF ~keyword_set(nofill) THEN sylvania_fillnee,yr,mcutoff=[1,endday],cutoff=[1,endday],noupload=noupload IF ~keyword_set(nooutput) THEN sylvania_amerifluxout,yr ENDIF ELSE BEGIN print,'No new files to process' ENDELSE END