#!/bin/sh
# the default shell for N1GE6 is /bin/csh
#   so we must specify this script to be /bin/sh
#$ -S /bin/sh
# the global search PATH is <NOT> automatically defined for
#   a bash script, so we can either explicitly source the
#   global definitions file (done below) or reference commands
#   using the entire path
#     (eg. mpiexec becomes /usr/local/mpich2-1.0.3/bin/mpiexec)
. /etc/bashrc

#REQUIRED PARAMETERS:
# parallel environment request a range or specific number
#$ -pe mpi 2
# estimated hard runtime. your job will get stuck waiting in the
#   queue if you do not specify this parameter
#$ -l h_rt=0:15:00   

#OPTIONAL PARAMETERS:
# submit to amdq queue to run on uniform nodes
#$ -q amdq
# job name
#$ -N MPI_Cas
# send the standard output to your current working directory
# (where you submit the job from, not where the executable exists)
#$ -cwd
# define the name of your output file
#$ -o cas.sh.log
# merge error and stdout into a single file
#$ -j y

# ---------------------------
# N1GE6 automatically defined variables:
#   $NSLOTS
#       the number of tasks to be used
#   $TMPDIR/machines
#       temporary system directory and a valid
#       machiche file to be passed to mpiexec
# ---------------------------
# User defined variables:
#   the executable you run with mpiexec will not have the benefits
#   of the globally defined PATH from /etc/bashrc so we must
#   unfortunately define it with its path included.
RUN_NUM=1
MPIR_HOME="/home/jona/casimir/mpi/courtney"
DATA_DIR="/home/jona/casimir/mpi/data"
executable="$MPIR_HOME/cas.out"

# ---------------------------
# start execution:
#   anything from stdout below is redirected to your job output file
echo "Got $NSLOTS slots."
cd $MPIR_HOME
mpif90 casimir.f90 -o $executable
mpiexec -machinefile $TMPDIR/machines -n $NSLOTS $executable
fft -x 9 2-50 < ens.log > fft.dat
mv *.log *.dat $DATA_DIR
tar -zcf cas.tgz $DATA_DIR/*
mv cas.tgz /archive/jona/casimir/mpi/archive/$RUN_NUM
rm -R $DATA_DIR