#!/bin/sh
### Set the job name
#PBS -N MEEP-Test

### Specify the number of cpus for your job (for OpenMPI not mpich).
# the following reserves 2 nodes (select=2)
# and all 8 cpus on each node (ncpus=8)
# the number of cpus used for the MPI job on each node is 8 (mpiprocs=8)
#   resulting in 2x8=16 total cpus used in the calculation
#   if mpiprocs=4, only 4 of the 8 cpus will be used 
#   resulting in 2x4=8 total cpus used in the calculation 
#   but still all 8 cpus are reserved by ncpus=8
# the amount of memory reserved on each node is 15gb (mem=15gb)
#   it is recommended to use at least mem=1gb since 256mb is the default
#PBS -l select=2:ib=true:ncpus=8:mpiprocs=8:mem=15gb

### Specify simulation runtime
#PBS -l walltime=1:00:00
 
### Combine standard error and standard out to one file.
#PBS -j oe
 
### Set the queue name, our group must always run in -q cao
#PBS -q cao

# Specify control file for Meep
executable=disk.ctl
parallel=1  #1 = true 0 = false
serial=0    #1 = true 0 = false

# -- should not have to edit below this line ------------------------------------------------
# Switch to the working directory; by default PBS launches processes from your home directory.
# Jobs should only be run from /home, /project, or /work; PBS returns results via NFS.
echo Working directory is $PBS_O_WORKDIR
cd $PBS_O_WORKDIR 

### Output node information
echo Running on host `hostname`
echo Time is `date`
echo Directory is `pwd`
echo This jobs runs on the following processors:
echo `cat $PBS_NODEFILE`
# Define number of processors
NPROCS=`wc -l < $PBS_NODEFILE`
# And the number of hosts
NHOSTS=`cat $PBS_NODEFILE|uniq|wc -l`
# And the number of processors on each host
NSLOTS=$(echo $NPROCS/$NHOSTS | bc)
echo This job has allocated $NPROCS cpus on $NHOSTS nodes
# Create machinefile for openmpi
#cat $PBS_NODEFILE | sort | uniq | sed "s/$/ slots=$NSLOTS/g" > hosts.tmp

if [ $parallel -eq 1 ] ; then
    # run in parallel 
    mpirun -n $NPROCS meep-mpi $executable | tee job-$executable.log
fi
if [ $serial -eq 1 ] ; then
    # run serial
    meep $executable | tee job-$executable.log
fi