;Alterations 17 Sep 2010 to handle new trailer and surf met file conventions ;Alterations 28 Oct 2010 for new naming conventions for _slow and ;_fast files ;open CR10X,CR23X, and CR1000 data files from WLEF ;and turn into the gzipped files ;surf_met ;output filnames ;/data/incoming/wlef/new/prefixYYYYMMDDHH00.gz ;overwrite by hand to be safe ;surf_met input CR10X ;LOC,YYYY,DOY,HHMM,6 values ;surf_met output YYYY,DOY,HH,MM,6 values ;format: 4i8,6g11.4 ;one file per hour ;bot/mid/top_fast.dat input CR1000 ;1 header line ;TIME STAMP STRING,record id,u,v,w,t,c,q,t,st,p ;Time stamp = "YYYY-MM-DD HH:MM:SS.S" - 10 Hz increment ;u,v,w,t divide by 100 ;output: YYYY,DOY,HH,MM,SS.SSS,4i8,f8.3,4g8.4,5g12.7 ;one file per hour ;bot/mid/top_slow.dat input ;1 header line ;TIME STAMP STRING,record id,t,rh,s_in,s_out,sflow,rflow ;time stamp = "YYYY-MM-DD HH:MM:SS" - 1 minute increment ;output: YYYY,DOY,HH,MM,4i8,2g8.4,2g12.7,2g8.4 ;one file per hour ;trailer CR23X ;LOC,YYYY,DOY,HHMM,SS.S,16 cols ;output ;YYYY,DOY,HH,MM,SS.S,16 cols, 4i8,f8.1,16g12.5 ;one file per hour PRO wlef_split_slow,fl,fln,olddata ; print,'I got here' point_lun,fl,0 FOR i = 0,fln-1 DO BEGIN s = '' readf,fl,s dta = strsplit(s," ",/extract) badd = where(strcmp(dta,'*',1),nbadd) IF nbadd GT 0 THEN dta[badd] = '-999.0' olddata[*,i] = float(dta) ENDFOR ; stop END PRO wlef_split_hour,prefix,data,format,high=high ;output data to prefix+'YYYYMMDDHH00.gz' ;find all uniq YYYY+DOY+HH combos ;loop through all, use where statement to extract gtma = (data[0,*] GE 1995 AND data[0,*] LE 2050 AND data[1,*] GE 1 AND data[1,*] LE 366 AND data[2,*] GE 0 AND data[2,*] LT 24 AND data[3,*] GE 0 AND data[3,*] LT 60) IF keyword_set(high) THEN gtma = gtma AND data[4,*] GE 0 AND data[4,*] LT 60 goodtm = where(gtma,ngt) IF ngt GT 0 THEN BEGIN data = data[*,goodtm] tm = double(data[0,*])+(double(data[1,*])/366.)+(double(data[2,*])/(366.*24.)) mmdd = strmid(jd_to_dy(data[1,*],yr=data[0,*]),4,4) tmstr = string(data[0,*],format='(i4.4)')+mmdd+string(data[2,*],format='(i2.2)')+'00.gz' st = sort(tm) tms = tm[st] tmstrs = tmstr[st] un = uniq(tms) FOR i = 0,n_elements(un)-1 DO BEGIN thetm = (tms[un[i]]) fname = prefix+tmstrs[un[i]] gval = where(tm EQ thetm,ngval) IF ngval GT 0 THEN BEGIN wdata = data[*,gval] ;if file exists, read it, add to data, erase it IF file_test(fname,/read) THEN BEGIN openr,fl,fname,/get_lun,/compress fln = file_lines(fname,/compress) on_ioerror,bad olddata = fltarr(n_elements(data[*,0]),fln) readf,fl,olddata GOTO,done bad: wlef_split_slow,fl,fln,olddata done: free_lun,fl oldtm = olddata[3,*] IF keyword_set(high) THEN oldtm+=(double(olddata[4,*])/60.) wtm = double(wdata[3,*]) IF keyword_set(high) THEN wtm+=(double(wdata[4,gval])/60.) FOR j = 0,fln-1 DO BEGIN appd = where(wtm EQ oldtm[j],nappd) IF nappd NE 0 THEN olddata[3,j] = nan() ENDFOR goodold = where(finite(olddata[3,*]),nod) IF nod GT 0 THEN wdata = [[olddata[*,goodold]],[wdata]] IF nod GT 0 THEN print,' Updating '+fname ELSE print,' Writing '+fname ENDIF ELSE print,' Writing '+fname wtm = double(wdata[3,*]) IF keyword_set(high) THEN wtm+=(double(wdata[4,*])/60.) wdata = wdata[*,sort(wtm)] openw,fl,fname,/get_lun,/compress FOR k = 0l,n_elements(wtm)-1l DO printf,fl,wdata[*,k],format=format free_lun,fl ENDIF ENDFOR ENDIF ELSE print,' Found no valid files to write' END PRO wlef_split,dr,auto=auto,newauto=newauto,nofast=nofast,notrailer=notrailer,noslow=noslow IF n_elements(dr) EQ 0 THEN dr = '0813' indir = '/air/incoming/wlef/raw/'+dr+'/' IF keyword_set(auto) OR keyword_set(newauto) THEN indir_stub = dr+'/' outdr = '/air/incoming/wlef/new/' ;find all files in dr, each of four cases ;one file per hour ;surf_met ;LOC,YYYY,DOY,HHMM,6 values ;surf_met output YYYY,DOY,HH,MM,6 values ;format: 4i8,6g11.4 IF keyword_set(auto) THEN indir = indir_stub+'surface/' IF keyword_set(newauto) THEN indir = indir_stub sm_fl = file_search(indir+'surf*.dat',count=n) IF n GT 0 THEN BEGIN IF file_test(sm_fl[0],/read) THEN BEGIN print,'Splitting '+sm_fl[0] nl = file_lines(sm_fl[0]) IF nl GE 1 THEN BEGIN smarr = make_array(10,nl,/float,value=!values.f_nan) dum = strarr(nl) openr,fl,sm_fl[0],/get_lun readf,fl,dum free_lun,fl FOR k = 0,nl-1 DO BEGIN ln = strsplit(dum[k],',',/extract) nel = n_elements(ln) < 10 IF nel GE 5 THEN BEGIN smarr[0:1,k] = float(ln[1:2]) hhmm = string(ln[3],format='(i4.4)') smarr[2,k] = float(strmid(hhmm,0,2)) smarr[3,k] = float(strmid(hhmm,2,4)) smarr[4:(nel-1),k] = float(ln[4:(nel-1)]) ENDIF ENDFOR wlef_split_hour,outdr+'surf_met',smarr,'(4i8,5(g11.4," "),g11.4)' ENDIF ENDIF ENDIF ;slow - same as above but with for loop ;4 header line ;TIME STAMP STRING,record id,t,rh,s_in,s_out,sflow,rflow ;time stamp = "YYYY-MM-DD HH:MM:SS" - 1 minute increment ;output: YYYY,DOY,HH,MM, (4i8,2g8.4,2g12.7,2g8.4) IF ~keyword_set(noslow) THEN BEGIN IF keyword_set(auto) THEN indir = indir_stub+['top/','bot/','mid/'] IF keyword_set(newauto) THEN indir = indir_stub sl_fl = file_search(indir+'???_slow*.dat',count=n) IF n GT 0 THEN BEGIN FOR i = 0,n-1 DO BEGIN IF file_test(sl_fl[i],/read) THEN BEGIN print,'Splitting '+sl_fl[i] nl = file_lines(sl_fl[i]) IF nl GE 5 THEN BEGIN slarr = make_array(10,nl-4,/float,value=!values.f_nan) dum = strarr(nl) openr,fl,sl_fl[i],/get_lun readf,fl,dum free_lun,fl FOR k = 4l,nl-1l DO BEGIN ln = strsplit(dum[k],',',/extract) qt = where(strmatch(ln,'"NAN"'),nqt) IF nqt GT 0 THEN ln[qt] = 'NAN' nel = n_elements(ln) < 8 IF nel GE 5 THEN BEGIN slarr[0,k-4] = float(strmid(ln[0],1,4)) yyyymmdd = strmid(ln[0],1,4)+strmid(ln[0],6,2)+strmid(ln[0],9,2) doy = dy_to_jd(yyyymmdd) slarr[1,k-4] = doy slarr[2,k-4] = float(strmid(ln[0],12,2)) slarr[3,k-4] = float(strmid(ln[0],15,2)) slarr[4:(nel+1),k-4] = float(ln[2:(nel-1)]) ENDIF ENDFOR IF keyword_set(auto) THEN tp = strmid(sl_fl[i],strlen(indir[0]),8) ELSE tp = strmid(sl_fl[i],strlen(indir),8) ; tp = strmid(str_right(sl_fl[i],12),0,8) wlef_split_hour,outdr+tp,slarr,'(4i8," ",2(g9.4," "),2(g12.7," "),g10.4," ",g10.4)' ENDIF ENDIF ENDFOR ENDIF ENDIF ;fast - same as above, but set high flag ;1 header line ;TIME STAMP STRING,record id,u,v,w,t,c,q,t,st,p ;Time stamp = "YYYY-MM-DD HH:MM:SS.S" - 10 Hz increment ;u,v,w,t divide by 100 ;output: YYYY,DOY,HH,MM,SS.SSS, (4i8,f8.3,4g8.4,5g12.7) IF ~keyword_set(nofast) THEN BEGIN IF keyword_set(auto) THEN indir = indir_stub+['top/','bot/','mid/'] IF keyword_set(newauto) THEN indir = indir_stub fs_fl = file_search(indir+'???_fast_*.dat',count=n) IF n GT 0 THEN BEGIN FOR i = 0,n-1 DO BEGIN IF file_test(fs_fl[i],/read) THEN BEGIN print,'Splitting '+fs_fl[i] nl = file_lines(fs_fl[i]) IF nl GE 5 THEN BEGIN fsarr = make_array(14,nl-4,/float,value=!values.f_nan) ; dum = strarr(nl) openr,fl,fs_fl[i],/get_lun FOR k = 0l,nl-1l DO BEGIN dum = '' readf,fl,dum IF k GE 4 THEN BEGIN ln = strsplit(dum,',',/extract) qt = where(strmatch(ln,'"NAN"'),nqt) IF nqt GT 0 THEN ln[qt] = 'NAN' nel = n_elements(ln) < 11 IF nel GE 6 THEN BEGIN fsarr[0,k-4] = float(strmid(ln[0],1,4)) yyyymmdd = strmid(ln[0],1,4)+strmid(ln[0],6,2)+strmid(ln[0],9,2) doy = dy_to_jd(yyyymmdd) fsarr[1,k-4] = doy fsarr[2,k-4] = float(strmid(ln[0],12,2)) fsarr[3,k-4] = float(strmid(ln[0],15,2)) fsarr[4,k-4] = float(strmid(ln[0],18,4)) fsarr[5:(nel+2),k-4] = float(ln[2:(nel-1)]) fsarr[5:8,k-4] /= 100. ;sonic stored in m/s * 100 and K * 100 ENDIF ENDIF ENDFOR free_lun,fl IF keyword_set(auto) THEN tp = strmid(fs_fl[i],strlen(indir[0]),8) ELSE tp = strmid(fs_fl[i],strlen(indir),8) ; tp = strmid(str_right(fs_fl[i],12),0,8) wlef_split_hour,outdr+tp,fsarr,'(4i8,f8.3," ",4(g13.4," "),4(g14.7," "),g14.7)',/high ENDIF ENDIF ENDFOR ENDIF ENDIF ;trailer - same as above, but set high flag ;NEW NAME ;LOC,YYYY,DOY,HHMM,SS.S,16 cols ;output ;YYYY,DOY,HH,MM,SS.S,16 cols, 4i8,f8.1,16g12.5 ;one file per hour IF ~keyword_set(notrailer) THEN BEGIN IF keyword_set(auto) THEN indir = indir_stub+'trailer/' IF keyword_set(newauto) THEN indir = indir_stub tr_fl = file_search(indir+'trailer_final_storage_*.dat',count=n) IF n GT 0 THEN BEGIN IF file_test(tr_fl[0],/read) THEN BEGIN print,'Splitting '+tr_fl[0] nl = file_lines(tr_fl[0]) IF nl GE 1 THEN BEGIN trarr = make_array(21,nl,/float,value=!values.f_nan) ; dum = strarr(nl) openr,fl,tr_fl[0],/get_lun FOR k = 0l,nl-1l DO BEGIN dum = '' readf,fl,dum ln = strsplit(dum,',',/extract) nel = n_elements(ln) < 21 IF nel GE 6 THEN BEGIN trarr[0:1,k] = float(ln[1:2]) hhmm = string(ln[3],format='(i4.4)') trarr[2,k] = float(strmid(hhmm,0,2)) trarr[3,k] = float(strmid(hhmm,2,4)) trarr[4:(nel-1),k] = float(ln[4:(nel-1)]) ENDIF ENDFOR free_lun,fl wlef_split_hour,outdr+'trailer',trarr,'(4i8,f8.1,15(g12.5," "),g12.5)',/high ENDIF ENDIF ENDIF ;NEW TRAILER ;indir is 'newtrailer/' ;fname is newtrailer_fastData_*.dat ;"TIMESTAMP","RECORD","top_co2","top_h20","top_t","mid_co2","mid_h20","mid_t","bot_co2","bot_h20","bot_t","top_t2","mid_t2","bot_t2","top_p","mid_p","bot_p" ;"TS","RN","volts","volts","C","volts","volts","C","volts","volts","C","C","C","C","kPa","kPa","kPa" ;update ;"TOA5","newtrailer","CR3000","8727","CR3000.Std.27","CPU:TRAILER_V3.CR3","30874","fastData" ;"TIMESTAMP","RECORD","bot_co2","bot_h20","bot_t","bot_t2","bot_p","topco2","toph2o","top_t","top_p" ;"TS","RN","volts","volts","C","C","kPa","ppm","ppt","degC","kPa" ;"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp" ;sample line: "2014-01-27 17:30:04.8",4928819,4430,4045,27.42,3634,3548,25.41,898,882,1.439,21.71,24.89,24.71,97.3,48.27,41.29 ;old trailer order: ; 'LiC_6','LiQ_6','LiT_6','LiC_4','LiQ_4','LiT_4','LiC_2','LiQ_2','LiT_2','ST_6','ST_4','ST_2','SP_6','SP_4','SP_2',$ ;old version was: ;120,2014,1,1730,3.1,3704.4,4028.9,25.783,3424.3,3457.6,24.726,724.77,907.14,26.005,24.063,24.273,24.285,55.72,45.987,33.989,10.487 ;output ;new: ; 2014 26 23 30 2.1 3521.0 4086.0 ; 26.430 3616.0 3606.0 25.560 883.00 ; 932.00 1.6590 24.950 25.250 25.020 ; 41.230 47.250 40.830 NaN ; 2013 364 8 0 0.0 3637.7 3858.0 ; 25.882 3320.6 3286.0 24.554 691.86 ; 824.19 25.747 24.076 24.146 24.041 ; 54.153 44.483 32.556 10.349 ;AFTER June 14 23Z, trailer loses top/mid, and surfmet switches to metvalue IF keyword_set(auto) THEN indir = indir_stub+'newtrailer/' IF keyword_set(newauto) THEN indir = indir_stub tr_fl = file_search(indir+'newtrailer_fastData_*.dat',count=n) IF n GT 0 THEN BEGIN FOR i = 0,n-1 DO BEGIN IF file_test(tr_fl[i],/read) THEN BEGIN print,'Splitting '+tr_fl[i] nl = file_lines(tr_fl[i]) IF nl GE 5 THEN BEGIN trarr = make_array(21,nl-4l,/float,value=!values.f_nan) ; dum = strarr(nl) openr,fl,tr_fl[i],/get_lun FOR k = 0l,nl-1l DO BEGIN dum = '' readf,fl,dum IF k EQ 1 THEN BEGIN trhdr = strsplit(dum,',"',/extract) trlv = strmid(trhdr,0,3) ENDIF IF k GE 4 THEN BEGIN ln = strsplit(dum,',',/extract) qt = where(strmatch(ln,'"NAN"'),nqt) IF nqt GT 0 THEN ln[qt] = 'NAN' nel = n_elements(ln) < 17 IF nel GE 6 THEN BEGIN ;at least one level is present trarr[0,k-4] = float(strmid(ln[0],1,4)) yyyymmdd = strmid(ln[0],1,4)+strmid(ln[0],6,2)+strmid(ln[0],9,2) doy = dy_to_jd(yyyymmdd) trarr[1,k-4] = doy trarr[2,k-4] = float(strmid(ln[0],12,2)) trarr[3,k-4] = float(strmid(ln[0],15,2)) trarr[4,k-4] = float(strmid(ln[0],18,4)) ; 'LiC_6','LiQ_6','LiT_6','LiC_4','LiQ_4','LiT_4','LiC_2','LiQ_2','LiT_2',$ ;5-13 (trailer) ; 'ST_6','ST_4','ST_2','SP_6','SP_4','SP_2',$ ;14-19 ; (trailer) botlv = where(trlv EQ 'bot',nbl) IF nbl GE 3 THEN trarr[11:13,k-4] = float(ln[botlv[0:2]]) IF nbl GE 4 THEN trarr[16,k-4] = float(ln[botlv[3]]) IF nbl GE 5 THEN trarr[19,k-4] = float(ln[botlv[4]]) midlv = where(trlv EQ 'mid',nml) IF nml GE 3 THEN trarr[11:13,k-4] = float(ln[midlv[0:2]]) IF nml GE 4 THEN trarr[15,k-4] = float(ln[midlv[3]]) IF nml GE 5 THEN trarr[18,k-4] = float(ln[midlv[4]]) toplv = where(trlv EQ 'top',ntl) IF ntl GE 3 THEN trarr[5:7,k-4] = float(ln[toplv[0:2]]) IF ntl EQ 4 THEN BEGIN trarr[14,k-4] = float(ln[toplv[2]]) trarr[17,k-4] = float(ln[toplv[3]]) ENDIF IF ntl GT 4 THEN BEGIN trarr[14,k-4] = float(ln[toplv[3]]) trarr[17,k-4] = float(ln[toplv[4]]) ENDIF ENDIF ENDIF ENDFOR free_lun,fl wlef_split_hour,outdr+'trailer',trarr,'(4i8,f8.1,15(g12.5," "),g12.5)',/high ENDIF ENDIF ENDFOR ENDIF ;"TIMESTAMP","RECORD","PAR_Den_Avg","pyran_Avg","atmpres_Avg","airTC_Avg","RH_Avg","rain_mm_Tot" ;write out: PAR, SW, T, RH, Precip, Pressure (kPA) IF keyword_set(auto) THEN indir = indir_stub+'newtrailer/' IF keyword_set(newauto) THEN indir = indir_stub trm_fl = file_search(indir+'newtrailer_metvalue_*.dat',count=n) IF n GT 0 THEN BEGIN FOR i = 0,n-1 DO BEGIN IF file_test(trm_fl[i],/read) THEN BEGIN print,'Splitting '+trm_fl[i] nl = file_lines(trm_fl[i]) IF nl GE 5 THEN BEGIN smarr = make_array(10,nl,/float,value=!values.f_nan) ; dum = strarr(nl) openr,fl,trm_fl[i],/get_lun FOR k = 0l,nl-1l DO BEGIN dum = '' readf,fl,dum IF k GE 4 THEN BEGIN ln = strsplit(dum,',',/extract) qt = where(strmatch(ln,'"NAN"'),nqt) IF nqt GT 0 THEN ln[qt] = 'NAN' nel = n_elements(ln) < 9 IF nel GE 8 THEN BEGIN smarr[0,k-4] = float(strmid(ln[0],1,4)) yyyymmdd = strmid(ln[0],1,4)+strmid(ln[0],6,2)+strmid(ln[0],9,2) doy = dy_to_jd(yyyymmdd) smarr[1,k-4] = doy smarr[2,k-4] = float(strmid(ln[0],12,2)) smarr[3,k-4] = float(strmid(ln[0],15,2)) smarr[4,k-4] = float(ln[2]) ;PAR IF nel GE 9 THEN smarr[5,k-4] = float(ln[8]) ELSE smarr[5,k-4] = float(ln[3]) ;SW - take apogee over licor Sept 10, 2021 smarr[6,k-4] = float(ln[5]) ;Air TC smarr[7,k-4] = float(ln[6]) ;RH smarr[8,k-4] = float(ln[7]) ;precip smarr[9,k-4] = float(ln[4]) / 10. ;AtmP kPa ENDIF ENDIF ENDFOR free_lun,fl wlef_split_hour,outdr+'surf_met',smarr,'(4i8,5(g11.4," "),g11.4)' ENDIF ENDIF ENDFOR ENDIF ;; TIMESTAMP","RECORD","VWC_5cm","Ka_5cm","T_5cm","BulkEC_5cm","VWC_10cm","Ka_10cm ;; ","T_10cm","BulkEC_10cm","VWC_20cm","Ka_20cm","T_20cm","BulkEC_20cm","VWC_30cm", ;; "Ka_30cm","T_30cm","BulkEC_30cm","VWC_40cm","Ka_40cm","T_40cm","BulkEC_40cm","VW ;; C_50cm","Ka_50cm","T_50cm","BulkEC_50cm","VWC_60cm","Ka_60cm","T_60cm","BulkEC_6 ;; 0cm","VWC_75cm","Ka_75cm","T_75cm","BulkEC_75cm","VWC_100cm","Ka_100cm","T_100cm ;; ","BulkEC_100cm" ;; "TS","RN","m^3/m^3","unitless","Deg C","dS/m","m^3/m^3","unitless","Deg C","dS/m ;; ","m^3/m^3","unitless","Deg C","dS/m","m^3/m^3","unitless","Deg C","dS/m","m^3/m ;; ^3","unitless","Deg C","dS/m","m^3/m^3","unitless","Deg C","dS/m","m^3/m^3","uni ;; tless","Deg C","dS/m","m^3/m^3","unitless","Deg C","dS/m","m^3/m^3","unitless"," ;; Deg C","dS/m" IF keyword_set(auto) THEN indir = indir_stub+'newtrailer/' IF keyword_set(newauto) THEN indir = indir_stub trs_fl = file_search(indir+'newtrailer_Soil_*.dat',count=n) IF n GT 0 THEN BEGIN FOR i = 0,n-1 DO BEGIN IF file_test(trs_fl[i],/read) THEN BEGIN print,'Splitting '+trs_fl[i] nl = file_lines(trs_fl[i]) IF nl GE 5 THEN BEGIN soilarr = make_array(40,nl,/float,value=!values.f_nan) ; dum = strarr(nl) openr,fl,trs_fl[i],/get_lun FOR k = 0l,nl-1l DO BEGIN dum = '' readf,fl,dum IF k GE 4 THEN BEGIN ln = strsplit(dum,',',/extract) qt = where(strmatch(ln,'"NAN"'),nqt) IF nqt GT 0 THEN ln[qt] = 'NAN' nel = n_elements(ln) < 9 IF nel GE 8 THEN BEGIN soilarr[0,k-4] = float(strmid(ln[0],1,4)) yyyymmdd = strmid(ln[0],1,4)+strmid(ln[0],6,2)+strmid(ln[0],9,2) doy = dy_to_jd(yyyymmdd) soilarr[1,k-4] = doy soilarr[2,k-4] = float(strmid(ln[0],12,2)) soilarr[3,k-4] = float(strmid(ln[0],15,2)) soilarr[4:39,k-4] = float(ln[2:37]) ;soilm, ka, soilt, soilec for 5,10,20,30,40,50,60,75,100 ENDIF ENDIF ENDFOR free_lun,fl wlef_split_hour,outdr+'soil',soilarr,'(4i8,35(g11.4," "),g11.4)' ENDIF ENDIF ENDFOR ENDIF ENDIF ;new fast and slow top/mid/bot ; WLEF_bot_slow_data_2019_06_09_1730.dat WLEF_mid_ts_data_2019_06_09_1729.dat ; WLEF_bot_ts_data_2019_06_09_1729.dat WLEF_top_slow_data.dat ; WLEF_mid_slow_data_2019_06_09_1730.dat WLEF_top_ts_data.dat ;FAST ;"TOA5","WLEF_top","CR6","10479","CR6.Std.09.03","CPU:tower_v1_0_1.CR6","19119","ts_data" ;"TIMESTAMP","RECORD","u","v","w","t","co2","h2o","co2_abs","h2o_abs","rssi","p_total","Tin","Tout","Tblock","diag7200","zero_in","zero_flag" ;"TS","RN","m/s * 100","m/s * 100","m/s * 100","deg C * 100","mmol/m3","mmol/m3","%","%","%","kPa","degC","degc","degc","unitless","","" ;"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp" ;"2019-06-09 17:29:54.2",8661046,-2147483648,-2147483648,-2147483648,-2147483648,13.54652,559.8886,0.1027796,0.06648551,99.39192,88.63488,35.96603,35.94513,36.01245,8191,0,0 ;old ;TIME STAMP STRING,record id,u,v,w,t,c,q,t,st,p ;Time stamp = "YYYY-MM-DD HH:MM:SS.S" - 10 Hz increment ;u,v,w,t divide by 100 ;new TIME STAMP, record, u, v, w, t, c, q, co2_abs-8, h2o_abs-9, rssi-10, p_total-11, T_in-12, Tout-13, Tblock, diag7200, zero_in, zero_flag ; add in diag , zero_in, zero_flag, tout, tblock ;re,order to u,v,w,t,c,q,t_in,t_out,p_total,co2_Abs,h2o_abs,rssi,tblock,diag7200,zero_in,zero_flag ;new fast IF ~keyword_set(nofast) THEN BEGIN IF keyword_set(auto) THEN indir = indir_stub+['top/','bot/','mid/'] IF keyword_set(newauto) THEN indir = indir_stub fs_fl = file_search(indir+'WLEF_???_ts_data*.dat',count=n) IF n GT 0 THEN BEGIN FOR i = 0,n-1 DO BEGIN IF file_test(fs_fl[i],/read) THEN BEGIN print,'Splitting '+fs_fl[i] nl = file_lines(fs_fl[i]) IF nl GE 5 THEN BEGIN fsarr = make_array(21,nl-4,/float,value=!values.f_nan) ; dum = strarr(nl) openr,fl,fs_fl[i],/get_lun FOR k = 0l,nl-1l DO BEGIN dum = '' readf,fl,dum IF k GE 4 THEN BEGIN ln = strsplit(dum,',',/extract) qt = where(strmatch(ln,'"NAN"'),nqt) IF nqt GT 0 THEN ln[qt] = 'NAN' nel = n_elements(ln) < 11 IF nel GE 6 THEN BEGIN fsarr[0,k-4] = float(strmid(ln[0],1,4)) yyyymmdd = strmid(ln[0],1,4)+strmid(ln[0],6,2)+strmid(ln[0],9,2) doy = dy_to_jd(yyyymmdd) fsarr[1,k-4] = doy fsarr[2,k-4] = float(strmid(ln[0],12,2)) fsarr[3,k-4] = float(strmid(ln[0],15,2)) fsarr[4,k-4] = float(strmid(ln[0],18,4)) fsarr[5:8,k-4] = float(ln[2:5])/100. ;u,v,w,t fsarr[9:10,k-4] = float(ln[6:7]) ;c,q fsarr[11:12,k-4] = float(ln[12:13]) ;t_in, t_out fsarr[13,k-4] = float(ln[11]) ;p_total fsarr[14:15,k-4] = float(ln[8:9]) ;co2a,qa fsarr[16,k-4] = float(ln[10]) ; rssi fsarr[17:20,k-4] = float(ln[14:17]) ;tblock, diag7200, zero_in, zero_flag ENDIF ENDIF ENDFOR free_lun,fl ; stop ;apply diag7200, zero_flag (NE 0) and rssi < 60 ;no zero correction yet li7200flag = (long(fsarr[18,*]) MOD 16)*6.67 bliflag = where(li7200flag LE 50. OR fsarr[19,*] NE 0,nbliflag) IF nbliflag GT 0 THEN BEGIN fsarr[9,bliflag] = nan() fsarr[10,bliflag] = nan() ENDIF ;apply conversions (li7200 manual to ppm and g/kg) cellt = zapbadval(screen_arr(merge_array(0.2*fsarr[11,*]+0.8*fsarr[12,*],fsarr[11,*],fsarr[12,*]) + 273.15,225.,325.)) fsarr[10,*] = screen_arr(fsarr[10,*],0.,2000.) fsarr[9,*] = screen_arr(fsarr[9,*],0.,100.) press = zapbadval(screen_arr(fsarr[13,*],80.,110.)) bpress = where(~finite(press),nbpress) IF nbpress GT 0 THEN press[bpress] = 92.0 wf = (fsarr[10,*] * 8.314 * cellt) / (1000*press) ;h2O mmol/mol wf = rhol_w * R * (Tg+273.15) / (1000 Pg) cf = (fsarr[9,*]* 8.314 * cellt) / press ;co2 umol/mol = rhoc(mmol/m3) * R * (Tg+273.15) / Pg cd = cf / (1. - (wf/1000.)) ;co2_dry umol/mol = co2 umol/mol / (1- (wf/1000)) hd = (18./28.97) * (wf / (1. - (wf/1000.))) ;h2o dry mmol/mol wfd = wf/ (1- (wf/1000)) * (18 g H2O/mol / 28.97 g air/mol) fsarr[9,*] = cd fsarr[10,*] = hd IF keyword_set(auto) THEN tp = strmid(fs_fl[i],strlen(indir[0])+5,3)+'_fast' ELSE tp = strmid(fs_fl[i],strlen(indir)+5,3)+'_fast' wlef_split_hour,outdr+tp,fsarr,'(4i8,f8.3," ",4(g13.4," "),11(g14.7," "),g14.7)',/high ENDIF ENDIF ENDFOR ENDIF ENDIF ;NEW SLOW ;old ;TIME STAMP STRING,record id,t,rh,s_in,s_out,sflow,rflow ;time stamp = "YYYY-MM-DD HH:MM:SS" - 1 minute increment ;output: YYYY,DOY,HH,MM, (4i8,2g8.4,2g12.7,2g8.4) ;new ;TIMESTEMP, record, battery, temp, flow_v, flow_avg, air_temp, rh, ;zero-in, zero-flag, co2_avg, h2o_avg ;output as: TIMESTEMP, record, air_temp, rh, battery, temp, ;"TOA5","WLEF_top","CR6","10479","CR6.Std.09.03","CPU:tower_v1_0_1.CR6","19119","slow_data" ;"TIMESTAMP","RECORD","Batt_volt_Avg","PTemp_Avg","flow_volt_Avg","flow_Avg","air_temp_Avg","rh_Avg","zero_in","zero_flag","co2_Avg","h2o_Avg" ;"TS","RN","volts","degC","volts","lpm","degC","%","","","mmol/m3","mmol/m3" ;"","","Avg","Avg","Avg","Avg","Avg","Avg","Smp","Smp","Avg","Avg" ;"2019-06-09 17:30:00",14435,12.61,37.05,329.3,3.447,19.94,66.24,0,0,13.5464,561.2821 IF ~keyword_set(noslow) THEN BEGIN IF keyword_set(auto) THEN indir = indir_stub+['top/','bot/','mid/'] IF keyword_set(newauto) THEN indir = indir_stub sl_fl = file_search(indir+'WLEF_???_slow*.dat',count=n) IF n GT 0 THEN BEGIN FOR i = 0,n-1 DO BEGIN IF file_test(sl_fl[i],/read) THEN BEGIN print,'Splitting '+sl_fl[i] nl = file_lines(sl_fl[i]) IF nl GE 5 THEN BEGIN slarr = make_array(14,nl-4,/float,value=!values.f_nan) dum = strarr(nl) openr,fl,sl_fl[i],/get_lun readf,fl,dum free_lun,fl FOR k = 4l,nl-1l DO BEGIN ln = strsplit(dum[k],',',/extract) qt = where(strmatch(ln,'"NAN"'),nqt) IF nqt GT 0 THEN ln[qt] = 'NAN' nel = n_elements(ln) < 8 IF nel GE 5 THEN BEGIN slarr[0,k-4] = float(strmid(ln[0],1,4)) yyyymmdd = strmid(ln[0],1,4)+strmid(ln[0],6,2)+strmid(ln[0],9,2) doy = dy_to_jd(yyyymmdd) slarr[1,k-4] = doy slarr[2,k-4] = float(strmid(ln[0],12,2)) slarr[3,k-4] = float(strmid(ln[0],15,2)) slarr[4:5,k-4] = float(ln[6:7]) slarr[6:9,k-4] = float(ln[2:5]) slarr[10:13,k-4] = float(ln[8:11]) ENDIF ENDFOR IF keyword_set(auto) THEN tp = strmid(sl_fl[i],strlen(indir[0])+5,3)+'_slow' ELSE tp = strmid(sl_fl[i],strlen(indir)+5,3)+'_slow' wlef_split_hour,outdr+tp,slarr,'(4i8," ",2(g9.4," "),6(g12.7," "),g10.4," ",g10.4)' ENDIF ENDIF ENDFOR ENDIF ENDIF END PRO wlef_sm09 ;deal with giant surf_met file ;same as above, just use wlef_split_hour on the file fname = '/air/incoming/wlef/2009/raw/surf/surf_met141.dat' fln = file_lines(fname) openr,fl,fname,/get_lun data = fltarr(10,fln-1) readf,fl,data free_lun,fl outdr = '/air/incoming/wlef/new/' wlef_split_hour,outdr+'surf_met',data,'(4i8,6g11.4)' END PRO wlef_split_day,yr,mo,dy,force=force,newauto=newauto,nofast=nofast,notrailer=notrailer,noslow=noslow yrstr = string(yr,format='(i4.4)') mostr = string(mo,format='(i2.2)') dystr = string(dy,format='(i2.2)') IF keyword_set(newauto) THEN BEGIN indir = ['/air/incoming/WLEFFlux/Data/'+yrstr+'_'+mostr+'/'+dystr+'/*/','/air/incoming/WLEFFlux/Data/'+yrstr+'/'+yrstr+mostr+dystr+'/'] procfile = '/air/incoming/WLEFFlux/Data/processed_'+yrstr+'_'+mostr+'_'+dystr ENDIF ELSE BEGIN indir = '/air/incoming/WLEFFlux/FluxData/'+yrstr+'_'+mostr+'_'+dystr+'_*/' procfile = '/air/incoming/WLEFFlux/FluxData/processed_'+yrstr+'_'+mostr+'_'+dystr ENDELSE IF ~keyword_set(force) AND file_test(procfile,/read) THEN BEGIN openr,fl,procfile,/get_lun s = '' readf,fl,s free_lun,fl print,'Skipping day ',indir,' which was already split on ',s ENDIF ELSE BEGIN fs_fl = file_search(indir,count=n) print,'Splitting day ',indir,' which has ',n,' subdirectories' IF n GT 0 THEN BEGIN IF keyword_set(newauto) THEN FOR i = 0,n-1 DO wlef_split,fs_fl[i],/newauto,nofast=nofast,notrailer=notrailer,noslow=noslow ELSE FOR i = 0,n-1 DO wlef_split,fs_fl[i],/auto,nofast=nofast,notrailer=notrailer,noslow=noslow openw,fl,procfile,/get_lun printf,fl,systime() free_lun,fl ENDIF ENDELSE END