#!/bin/sh -x
#PBS -N Sample-with-Rad
#PBS -P JUPITER
#PBS -l select=72:ncpus=36:mpiprocs=12:ompthreads=3
#PBS -l walltime=12:00:00
#PBS -j oe
#PBS -q lc192

cd ${PBS_O_WORKDIR}
. /etc/profile.d/modules.sh
module purge
module load intel/2021.4.0
module load mpt/2.23-ga
module load cuda/11.4

#export OMP_NUM_THREADS=3
export KMP_AFFINITY=disabled
#export KMP_STACKSIZE=2g 
#export MPI_IB_RAILS=2
#export MPI_BUFFER_MAX=2000
#export MPI_USE_CUDA=enabled

mpiexec -np 96 \
        omplace -nt $OMP_NUM_THREADS -vv \
            run -input param.txt \
                -flags flags.txt \
                -geom geom.txt \
                -plist plist.txt \
                -control control.txt \
      : -np 768 \
        omplace -nt $OMP_NUM_THREADS -vv \
            rad $OMP_NUM_THREADS 2 \
      > STDOUT_0 2>&1
