PRO convlake,fname
;Read in fname+'.txt' convert to EdiRe fname+'.dat'

;Sample input:
;2009-5-5 20:0:0.1,-3.9265,1.5755,-0.222,11.46735,722.5685,4.466752,2376.0,2512.0

;Sample output:
;2009/05/05 20:00:00.10,    -3.927,     1.576,    -0.222,    11.467,   722.568,     4.467,  2376.000,  2512.000


;The directory name
  directory = '/sap/projects/lakeflux/2009/bin/'

;Get the filename, if none provided to procedure, then choose the default one
  IF n_elements(fname) EQ 0 THEN fname = '0505200920'

;assign the input and output filename
  infname = directory+fname+'.txt'
  outfname = directory+'converted/'+fname+'.raw'

;open the infile for ASCII read, and outfile for ASCII write
  print,'Converting ',infname,' to ',outfname
  openr,ifl,infname,/get_lun
  openw,ofl,outfname,/get_lun

;write the header to the output file
  header = ['Date/Time','u','v','w','T','CO2','H2O','Gyro_1','Gyro_2']
  printf,ofl,header,format='(8(a9,","),a9)'

;set a counter
  count = 0l

;Look from start to end of file, read one line at a time
  WHILE ~eof(ifl) DO BEGIN

    IF count MOD 10000 EQ 0 THEN print,'  On line ',count

;read one line
    s = ''
    readf,ifl,s

;split the string into its components, first by comma, then by space

;First get nine columns separate by comma
    sarr = strsplit(s,',',/extract)

;convert the non date/time values to floating point
    values = float(sarr[1:*])

;Take the first column and split it by space to separate date from time
    datetime = strsplit(sarr[0],' ',/extract)

;Split first of these by - and second by :, now we have 6 elements
    date = long(strsplit(datetime[0],'-',/extract))
    time = float(strsplit(datetime[1],':',/extract))

;create datatime string in YYYY/MM/DD HH:NN:SS.ZZ
    outarr = strarr(9)
    outarr[0] = string(date[0],format='(i4.4)')+'/'+string(date[1],format='(i2.2)')+'/'+string(date[2],format='(i2.2)')+' '+$
                string(time[0],format='(i2.2)')+':'+string(time[1],format='(i2.2)')+':'+string(time[2],format='(f05.2)')

;do the rest, turn missing or bad values and overflow values into blank spaces
    FOR i = 0,n_elements(values)-1 DO BEGIN 
      IF (values[i] GT -999) AND finite(values[i]) THEN outarr[i+1]=string(values[i],format='(f10.3)') ELSE outarr[i+1]='          '
      IF strmid(outarr[i+1],0,1) EQ '*' THEN outarr[i+1]='          '
    ENDFOR 

;Write the new values to file
    printf,ofl,outarr,format='(a22,",",7(a10,","),a10)'

;keep track of how many lines processed
    count++

  ENDWHILE 

;close the files
  free_lun,ifl
  free_lun,ofl

;calculate and print the header and record size
  headersize = 8*(9+1)+9
  recordsize = 22+1+7*(10+1)+10

  print,'header size in bytes: ',headersize
  print,'record size in bytes: ',recordsize

END

PRO convlake2,fname,norec=norec
;read Trout lake TOA files
  indir = '/sap/projects/lakeflux/2009/bin/raw_data/'
  outdir = '/sap/projects/lakeflux/2009/bin/raw_data/TOB1/'
  jd1990 = julday(1,1,1990,0,0,0)

  IF n_elements(fname) EQ 0 THEN BEGIN
    fname = 'TOA5_0708.ts_data.DAT'
  ENDIF 

  outfname = 'TOB1'+strmid(fname,4)


  openr,ifl,indir+fname,/get_lun
  openw,ofl,outdir+outfname,/get_lun

  s = ''
  FOR i = 0,3 DO readf,ifl,s

  printf,ofl,'"TOB1","23193","CR1000","23193","CR1000.Std.16","CPU:michael2.cr1","43018","ts_data"'+string(13b)
  printf,ofl,'"SECONDS","NANOSECONDS","Ux","Uy","Uz","Ts","co2","h2o","diag","tiltx","tilty"'+string(13b)
  printf,ofl,'"SECONDS","NANOSECONDS","m/s","m/s","m/s","C","mg/m^3","g/m^3","unitless","degrees","degrees"'+string(13b)
  printf,ofl,'"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp"'+string(13b)
  printf,ofl,'"ULONG","ULONG","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4"'+string(13b)
  count = 0l

  WHILE ~eof(ifl) DO BEGIN
    IF count MOD 100000l EQ 0 THEN print,'On line ',count
    
    s = ''
    readf,ifl,s
    vals = strsplit(s,',"',/extract)
    dtime = vals[0]
    IF keyword_set(norec) THEN data = float(vals[1:9]) ELSE data = float(vals[2:10])    ;ux,uy,uz,ts,co2,h2o,diag,tiltx,tilty
    yr = long(strmid(dtime,0,4))
    mo = long(strmid(dtime,5,2))
    day = long(strmid(dtime,8,2))
    hr = long(strmid(dtime,11,2))
    min = long(strmid(dtime,14,2))
    sec = float(strmid(dtime,17))

    jd = julday(mo,day,yr,hr,min,long(sec))
    jd = jd - jd1990
    seconds = long(jd*86400d)
    nanosec = long(round((sec-long(sec))*100)*1e7)

    data = float(data)
    bdata = where(~finite(data) OR (data LE -99999.0),nbd)
    if nbd gt 0 then data[bdata] = nan()

    writeu,ofl,ulong(seconds)
    writeu,ofl,ulong(nanosec)
    writeu,ofl,float(data)
    count++
  ENDWHILE 

  free_lun,ifl
  free_lun,ofl

;  stop

END

PRO convlake3,fname,outfname
;from TOB3 to TOB1

;time filter: check timestamp, if within range, copy over, set range
;manually?


;9-9 file we want:
;"2009-08-19 15:32:57.6" first
;"2009-09-09 13:49:35.9"

;10-21 file we want:
;"2009-09-23 09:46:58.2" first point
;""2009-10-20 13:25:50.6" last point

  indir = '/sap/projects/lakeflux/2009/bin/raw_data/TOB3/'
  outdir = '/sap/projects/lakeflux/2009/bin/raw_data/TOB1/'
  jd1990 = julday(1,1,1990,0,0,0)

  IF n_elements(fname) EQ 0 THEN BEGIN
 ;   fname = '9-23-09tm_23193.ts_data.dat'
 ;   fname = '11-3-09tm_23193.ts_data.dat'
 ;   outfname = 'TOB1_0923.ts_data.dat'
;    fname = '10-9-09km23193.ts_data.dat'

;    fname = '9-9-09tm_23193.ts_data.dat'
;    outfname = 'TOB1_0909.ts_data.dat'
    fname = '10-21-09tm_23193.ts_data.dat'
    outfname = 'TOB1_1021.ts_data.dat'


  ENDIF 


  openr,ifl,indir+fname,/get_lun
  openw,ofl,outdir+outfname,/get_lun
;  ofl = -1

;  s = ''
;FOR i = 0,5 DO BEGIN readf,ifl,s & print,strlen(s) & ENDFOR 
  s = bytarr(512)
  readu,ifl,s

  printf,ofl,'"TOB1","23193","CR1000","23193","CR1000.Std.16","CPU:michael2.cr1","43018","ts_data"'+string(13b)
  printf,ofl,'"SECONDS","NANOSECONDS","Ux","Uy","Uz","Ts","co2","h2o","diag","tiltx","tilty"'+string(13b)
  printf,ofl,'"SECONDS","NANOSECONDS","m/s","m/s","m/s","C","mg/m^3","g/m^3","unitless","degrees","degrees"'+string(13b)
  printf,ofl,'"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp"'+string(13b)
  printf,ofl,'"ULONG","ULONG","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4","IEEE4"'+string(13b)
  count = 0l
  jd1990 = julday(1,1,1990,0,0,0)

  stamp = ulonarr(3)
  data = fltarr(6)
  byt1 = 0b
  byt2 = 0b
  pad = ulonarr(1)
  
;1006 bytes per record
 
  IF fname EQ '9-9-09tm_23193.ts_data.dat' THEN BEGIN
    s = bytarr(94)
    readu,ifl,s
    s = bytarr(358)
    readu,ifl,s
  ENDIF 

  WHILE ~eof(ifl) DO BEGIN
    IF count MOD 10000l EQ 0 THEN print,'On line ',count

    oldstamp = stamp
    readu,ifl,stamp             ;12 bytes
    
;    IF stamp[0] GT oldstamp[0] THEN BEGIN 
      FOR k = 0,32 DO BEGIN 
        IF (stamp[0] LT 600000000l) OR (stamp[0] GT 630000000l) THEN stop
	if count mod 864000l eq 0 then begin 
          jd = jd1990 + ((double(stamp[0])+double(stamp[1]/1e4))/86400d)
          caldat,jd,mon,day,yr,hr,min,sec
          print,yr,mon,day,hr,min,sec
;          IF yr NE 2009 THEN stop
        endif
        readu,ifl,data
        data = swap_Endian(data)
        readu,ifl,byt1
        readu,ifl,byt2
;        byt1 = swap_endian(byt1)
;        byt2 = swap_endian(byt2)
;        bit1 = bytebin(byt1)
;        bit2 = bytebin(byt2)
;        sign = (bit1[0]*(-2.0))+1.0
;        expo = binbyte([bit1[1],bit1[2]])
;        num = binbyte([bit1[3:7],bit2])
;        diag = sign * float(num) * (10^((-1.0)*expo))
        readu,ifl,byt1
        readu,ifl,byt2
;        byt1 = swap_endian(byt1)
;        byt2 = swap_endian(byt2)
;        bit1 = bytebin(byt1)
;        bit2 = bytebin(byt2)
;        sign = (bit1[0]*(-2.0))+1.0
;        expo = binbyte([bit1[1],bit1[2]])
;        num = binbyte([bit1[3:7],bit2])
;        tiltx = sign * float(num) * (10^((-1.0)*expo))
        readu,ifl,byt1
        readu,ifl,byt2
;        byt1 = swap_endian(byt1)
;        byt2 = swap_endian(byt2)
;        bit1 = bytebin(byt1)
;        bit2 = bytebin(byt2)
;        sign = (bit1[0]*(-2.0))+1.0
;        expo = binbyte([bit1[1],bit1[2]])
;        num = binbyte([bit1[3:7],bit2])
;        tilty = sign * float(num) * (10^((-1.0)*expo))
        count++
;        odata = [data,float(diag),float(tiltx),float(tilty)]
;        bd = where(odata LE -99999.0 OR ~finite(odata),nbd)
;        IF nbd GT 0 THEN odata[bd] = nan()
;        writeu,ofl,stamp[0]
;        writeu,ofl,stamp[1]*100000l
;        writeu,ofl,odata
;        stamp[2]++
;        stamp[1] = stamp[1] + 1000l
;        IF stamp[1] GE 10000l THEN BEGIN
;          stamp[1]-=10000l
;          stamp[0] += 1
;        ENDIF 
      ENDFOR 
;      stop
      IF ~eof(ifl) THEN readu,ifl,pad
;    ENDIF 
  ENDWHILE
  free_lun,ifl
  free_lun,ofl
END

;need code to jump to date
;for 9-9 file , should start at 2009-08-20 07:55:38 (of course it
;start 9-1)
;for 10-21 file, should start at 2009-10-09 08:10:44 (starts 9-9)



PRO convlake_fix,dt
  indir = '/sap/projects/lakeflux/2009/bin/raw_data/TOB1/'
  outdir = indir+'clean/'

  fname = indir + 'TOB1_'+dt+'.ts_data'
  find = 0b
  IF file_test(fname+'.dat',/read) THEN find = 1b
  IF file_test(fname+'.DAT',/read) THEN find = 2b
  IF find NE 0b THEN BEGIN
    outfname = outdir+'TOB1_'+dt+'.ts_data.dat'
    IF find EQ 1b THEN fname+='.dat' ELSE fname+='.DAT'
    print,'Reading ',fname
    openr,fl,fname,/get_lun
    fsize = (fstat(fl)).size
    lines = (fsize-411l)/44l
    data = { tm : ulonarr(2), vals : fltarr(9) }
    output = replicate(data,lines)
    rawd = bytarr(411)
    readu,fl,rawd
    readu,fl,output
    free_lun,fl
    stop
    output.vals[0,*] = screen_arr(output.vals[0,*],-20,20)
    output.vals[1,*] = screen_arr(output.vals[1,*],-20,20)
    output.vals[2,*] = screen_arr(output.vals[2,*],-10,10)
    output.vals[3,*] = screen_arr(output.vals[3,*],-40,40)
    IF dt EQ '1009' THEN BEGIN 
      t = output.vals[3,*]
      badt = where(t LT 1e-3 AND t GT -1e-3,nbt)
      IF nbt GT 0 THEN t[badt] = nan()
      output.vals[3,*] = t
    ENDIF 
    output.vals[4,*] = screen_arr(output.vals[4,*],500,1200)
    output.vals[5,*] = screen_arr(output.vals[5,*],0,24)
    stop
    print,'Writing ',outfname
    openw,fl,outfname,/get_lun
    writeu,fl,rawd
    writeu,fl,output
    free_lun,fl
  ENDIF ELSE print,'Cannot find ',fname

;411 byte header (4 line), then div file size by 44 to get nlines

;read data file 

;  output[6,*] = screen_arr(output[6,*],-20,20) ;u -20 to 20 m/s
;  output[7,*] = screen_arr(output[7,*],-20,20) ;v -20 to 20 m/s
;  output[8,*] = screen_arr(output[8,*],-10,10) ;w -10 to 10 m/s
;  output[9,*] = screen_arr(output[9,*],-40,40) ;Ts -40 to 40 C
;  output[10,*] = screen_arr(output[10,*],500,1200) ;C 500-1200 g/m3
;  output[11,*] = screen_arr(output[11,*],0,24) ;Q 0-24 g/m3

END

PRO convlake_testtob3

  indir = '/sap/projects/lakeflux/2009/bin/raw_data/TOB3/'
  jd1990 = julday(1,1,1990,0,0,0)

  IF n_elements(fname) EQ 0 THEN BEGIN
    fname = '9-9-09tm_23193.ts_data.dat'
;    outfname = 'TOB1_0909.ts_data.dat'
 ;   fname = '10-21-09tm_23193.ts_data.dat'
 ;   outfname = 'TOB1_1021.ts_data.dat'
 ;  fname = '11-3-09tm_23193.ts_data.dat'
 ;  fname = '10-9-09km23193.ts_data.dat'
  ENDIF 

  openr,ifl,indir+fname,/get_lun
  fsize = (fstat(ifl)).size
  records = (fsize-512l)/1006l

  s = bytarr(512)
  readu,ifl,s
 IF fname EQ '9-9-09tm_23193.ts_data.dat' THEN BEGIN
    s = bytarr(94)
    readu,ifl,s
    s = bytarr(358)
    readu,ifl,s
  ENDIF 

;9-9 file we want:
;"2009-08-19 15:32:57.6" first
;"2009-09-09 13:49:35.9"
;starts 9-1 to 9-9

;10-21 file we want:
;"2009-09-23 09:46:58.2" first point
;""2009-10-20 13:25:50.6" last point
;NO DATA AFTER 9-23



  record = { stamp:ulonarr(3), data : replicate({ vals:fltarr(6), byts:bytarr(6) },33), pad:float(0)}
  recs = replicate(record,100000l)
  readu,ifl,recs

  
  stop

END