#!/bin/tcsh -f

# Task name:  convert_sdss_plate_to_speccy.csh
# Author:     Tom Dwelly (dwelly AT mpe.mpg.de)
# Function:   Convert all of the spectra from an SDSS plate into a set of "speccy" format files.
# Inputs:     The "spPlate" and "spZbest" files that are assocated with a particular SDSS PLATE-MJD
# Outputs:    A set of 1000 (one per fiber) 1D spectral files in "speccy" format. 
#             Can be used with the "speccy" spectrum viewer: http://www.mpe.mpg.de/~tdwelly/speccy/speccy.html
# Requirements: ftlist (from FTOOLS), awk, paste

foreach HELPER ( ftlist awk paste ) 
  if ( ! -X $HELPER ) then
    echo "Error, cannot find $HELPER in your path" 
    exit 1
  endif
end

set FIRST_FIBER = 1
set LAST_FIBER  = 1000

if ( $#argv == 4 ) then
  set SPPLATE = "${argv[1]}"
  set SPZBEST = "${argv[2]}"
  set SPZALL  = "${argv[3]}"
  set OUTDIR  = "${argv[4]}"
else if ( $#argv == 6 ) then
  set SPPLATE = "${argv[1]}"
  set SPZBEST = "${argv[2]}"
  set SPZALL  = "${argv[3]}"
  set OUTDIR  = "${argv[4]}"
  set FIRST_FIBER = "${argv[5]}"
  set LAST_FIBER  = "${argv[6]}"
else
  echo "Error, incorrect number of input parameters ($#argv)"
  echo "Usage: ${0:t} SPPLATE_FILENAME SPZBEST_FILENAME SPZALL_FILENAME OUTDIR [FIRST_FIBER LAST_FIBER]"
  exit 1
endif




# check if the requisite files exist
if ( ! -e "$SPPLATE" ) then
  echo "Cannot find specified spPlate file: $SPPLATE"
  exit 1
endif

if ( ! -e "$SPZBEST" ) then
  echo "Cannot find specified spZbest file: $SPZBEST"
  exit 1
endif

if ( "$SPZALL" == "NONE" || "$SPZALL" == "-" || "$SPZALL" == "" ) then
  set SPZALL = "NONE"
else if ( ! -e "$SPZALL" ) then
  echo "Cannot find specified spZall file: $SPZALL"
  exit 1
endif

# make output directory for speccy format files:
mkdir -p "$OUTDIR"
if ( ! -d "$OUTDIR" ) then
  echo "Cannot find/make output directory: $OUTDIR"
  exit 1
endif



if ( -X gethead ) then
  set NFIBERS =  `gethead "${SPPLATE}[0]" "NAXIS2" `
  set PLATE   =  `gethead "${SPPLATE}[0]" "PLATEID"`
  set MJD     =  `gethead "${SPPLATE}[0]" "MJD"    `
  set CRVAL1  =  `gethead "${SPPLATE}[0]" "CRVAL1" `
  set CRPIX1  =  `gethead "${SPPLATE}[0]" "CRPIX1" `
  set CD1_1   =  `gethead "${SPPLATE}[0]" "CD1_1"  `
  set BUNIT   = (`gethead "${SPPLATE}[0]" "BUNIT"  `)
else
  set NFIBERS =  `ftlist "${SPPLATE}[0]" K include="NAXIS2"  | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
  set PLATE   =  `ftlist "${SPPLATE}[0]" K include="PLATEID" | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
  set MJD     =  `ftlist "${SPPLATE}[0]" K include="MJD"     | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
  set CRVAL1  =  `ftlist "${SPPLATE}[0]" K include="CRVAL1"  | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
  set CRPIX1  =  `ftlist "${SPPLATE}[0]" K include="CRPIX1"  | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
  set CD1_1   =  `ftlist "${SPPLATE}[0]" K include="CD1_1"   | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
  set BUNIT   = (`ftlist "${SPPLATE}[0]" K include="BUNIT"   | awk -v comm='/' '{print substr($0,10);}'` )
endif

set TEMP0 = "${OUTDIR}/convert_to_speccy_temp0_$$"
set TEMP1 = "${OUTDIR}/convert_to_speccy_temp1_$$"
set TEMP2 = "${OUTDIR}/convert_to_speccy_temp2_$$"
set TEMP3 = "${OUTDIR}/convert_to_speccy_temp3_$$"
set TEMP4 = "${OUTDIR}/convert_to_speccy_temp4_$$"
set TEMP5 = "${OUTDIR}/convert_to_speccy_temp5_$$"

echo "Extracting speccy files from ${SPPLATE} and ${SPZBEST}"
printf 'Output files will be at: %s/spec-%04d-%05d-{FIBERID}_speccy.txt\n'  "$OUTDIR" ${PLATE} ${MJD}
echo -n "Working on FIBERID range [${FIRST_FIBER}:${LAST_FIBER}] (total $NFIBERS): "
 
set FIBERID = $FIRST_FIBER
while ( $FIBERID <= $NFIBERS && $FIBERID <= $LAST_FIBER )
  echo -n " $FIBERID"
  set SPECCY_OUTFILE = `printf '%s/spec-%04d-%05d-%04d_speccy.txt'  "$OUTDIR" ${PLATE} ${MJD} ${FIBERID} `
  rm $TEMP0 $TEMP1 $TEMP2 $TEMP3 $TEMP4 $TEMP5 >>& /dev/null

  ftlist "${SPPLATE}[0][*,${FIBERID}:${FIBERID}]"    I rownum=no colheader=no mode=q | awk '//{for(i=1;i<=NF;i++){print $i}}' > $TEMP0
  ftlist "${SPPLATE}[IVAR][*,${FIBERID}:${FIBERID}]" I rownum=no colheader=no mode=q | awk '//{for(i=1;i<=NF;i++){print $i}}' > $TEMP1
  ftlist "${SPPLATE}[SKY][*,${FIBERID}:${FIBERID}]"  I rownum=no colheader=no mode=q | awk '//{for(i=1;i<=NF;i++){print $i}}' > $TEMP2
  ftlist "${SPZBEST}[2][*,${FIBERID}:${FIBERID}]"    I rownum=no colheader=no mode=q | awk '//{for(i=1;i<=NF;i++){print $i}}' > $TEMP3
      

  # now dump the contents of the relevant tables in the SPZBEST and SPPLATE 
  # files to ascii and format them them as speccy keywords (i.e. "# keyname = value")
  ftlist "${SPZBEST}[1]" C mode=q | tail -n +4 | awk '//{print "KEYNAME",$2}' > $TEMP4
  ftlist "${SPZBEST}[1][FIBERID==${FIBERID}]" T mode=q separator='|' rownum=no colheader=no > $TEMP5
  # deal with vector columns properly
  awk --field-separator='|' '\
function trim_string(s) {\
  for(__m=1;__m<=length(s);__m++){if(substr(s,__m,1)!=" "){break;}};\
  for(__n=length(s);__n>0;__n--) {if(substr(s,__n,1)!=" "){break;}};\
  return(substr(s,__m,__n-__m+1))}\
$0!~/^KEYNAME/ {\
  n++;nkey=NF;\
  for(i=1;i<=nkey;i++){\
    n_i=sprintf("%d_%d",n,i);\
    keyval[n_i]=trim_string($i)}}\
$0~/^KEYNAME/ {\
  k++;split($0,a," ");keyname[k]=a[2]}\
END {for(i=1;i<=nkey;i++){\
  printf("# %-20s = ", keyname[i]);\
  for(m=1;m<=n;m++){m_i=sprintf("%d_%d",m,i); printf("%s ",keyval[m_i])};printf("\n")}}' $TEMP4 $TEMP5 >  $SPECCY_OUTFILE

  ftlist "${SPPLATE}[PLUGMAP]" C mode=q | tail -n +4 | awk '//{print "KEYNAME",$2}' > $TEMP4
  ftlist "${SPPLATE}[PLUGMAP][FIBERID==${FIBERID}]" T mode=q separator='|' rownum=no colheader=no > $TEMP5
#  awk --field-separator='|' '$0!~/^KEYNAME/ {n++;if(n==1){nkey=NF;for(i=1;i<=nkey;i++){keyval[i]=$i}}} $0~/^KEYNAME/ {k++;split($0,a," ");keyname[k]=a[2]} END {for(i=1;i<=nkey;i++){printf("# %-20s = %s\n", keyname[i], keyval[i])}}' $TEMP4 $TEMP5 >>  $SPECCY_OUTFILE
  awk --field-separator='|' '\
function trim_string(s) {\
  for(__m=1;__m<=length(s);__m++){if(substr(s,__m,1)!=" "){break;}};\
  for(__n=length(s);__n>0;__n--) {if(substr(s,__n,1)!=" "){break;}};\
  return(substr(s,__m,__n-__m+1))}\
$0!~/^KEYNAME/ {\
  n++;nkey=NF;\
  for(i=1;i<=nkey;i++){\
    n_i=sprintf("%d_%d",n,i);\
    keyval[n_i]=trim_string($i)}}\
$0~/^KEYNAME/ {\
  k++;split($0,a," ");keyname[k]=a[2]}\
END {for(i=1;i<=nkey;i++){\
  printf("# %-20s = ", keyname[i]);\
  for(m=1;m<=n;m++){m_i=sprintf("%d_%d",m,i); printf("%s ",keyval[m_i])};printf("\n")}}' $TEMP4 $TEMP5 >>  $SPECCY_OUTFILE

  if ( "$SPZALL" != "NONE" ) then
    if ( -X gethead ) then
      set DIMS0 = `gethead "${SPZALL}[0]" "DIMS0" `
    else
      set DIMS0 = `ftlist "${SPZALL}[0]" K include="DIMS0" | awk -v comm='/' '{a=substr($0,10);split(a,b,comm);print b[1]}'`
    endif
    ftlist "${SPZALL}[1]" C mode=q | tail -n +4 | awk '//{print "KEYNAME",$2}' > $TEMP4
    foreach FIT ( 1 2 3 )
      ftlist "${SPZALL}[1][FIBERID==${FIBERID}&&(#row%${DIMS0}==${FIT})]" T mode=q separator='|' rownum=no colheader=no > $TEMP5
      awk --field-separator='|' -v prefix="SPZALL_FIT${FIT}_" '\
function trim_string(s) {\
  for(__m=1;__m<=length(s);__m++){if(substr(s,__m,1)!=" "){break;}};\
  for(__n=length(s);__n>0;__n--) {if(substr(s,__n,1)!=" "){break;}};\
  return(substr(s,__m,__n-__m+1))}\
$0!~/^KEYNAME/ {\
  n++;nkey=NF;\
  for(i=1;i<=nkey;i++){\
    n_i=sprintf("%d_%d",n,i);\
    keyval[n_i]=trim_string($i)}}\
$0~/^KEYNAME/ {\
  k++;split($0,a," ");keyname[k]=a[2]}\
END {for(i=1;i<=nkey;i++){\
  printf("# %s%-20s = ", prefix,keyname[i]);\
  for(m=1;m<=n;m++){m_i=sprintf("%d_%d",m,i); printf("%s ",keyval[m_i])};printf("\n")}}' $TEMP4 $TEMP5 >>  $SPECCY_OUTFILE
    end

  endif



  paste $TEMP0 $TEMP1 $TEMP2 $TEMP3 | awk -v lcrval=$CRVAL1 -v crpix=${CRPIX1} -v lcdelt=${CD1_1} -v bunit="$BUNIT" 'BEGIN {printf("#Wavelength(AA,observed,vacuum),FluxDensity,Error,Sky,bestModel (all %s)\n", bunit)} //{i++;l=10.0**(lcrval + (i-crpix)*lcdelt);printf("%.2f,%.6g,%.6g,%.6g,%.6g\n", l, $1, ($2>0.0?$2**-0.5:0.0), $3, $4)} END {print _}' >> $SPECCY_OUTFILE
  rm $TEMP0 $TEMP1 $TEMP2 $TEMP3 $TEMP4 $TEMP5 >>& /dev/null
  @ FIBERID ++
end
echo " ... Done"



exit

