#!/bin/bash

#SBATCH --nodes=<NODES>
#SBATCH --ntasks=<NTASKS>
#SBATCH -p 7713
#SBATCH -J shld_global
#SBATCH --time=4:00:00
#SBATCH --exclusive
#SBATCH -c <THREADS>

ulimit -s unlimited
ulimit -c unlimited
ulimit -v unlimited
ulimit -n unlimited


exp_dir=${SLURM_SUBMIT_DIR}/global_scalability
cd v0.17.1_minerva_intel && source ./env.sh
cd $exp_dir

module list

export HDF5=/work/eduardo.inpe/usr/local/spack/github/v0.17.1_minerva_intel/opt/spack/linux-rhel8-zen2/intel-19.1.3.304/hdf5-1.10.7-zrdlnanqnuh3y3beojlqoszopf4n2sjb
export NETCDF_CXX=/work/eduardo.inpe/usr/local/spack/github/v0.17.1_minerva_intel/opt/spack/linux-rhel8-zen2/intel-19.1.3.304/netcdf-c-4.8.1-fooskkwa2sk7p5mzeirbwv7itcvmfcxf
export NETCDF_FORTRAN=/work/eduardo.inpe/usr/local/spack/github/v0.17.1_minerva_intel/opt/spack/linux-rhel8-zen2/intel-19.1.3.304/netcdf-fortran-4.5.3-pfh3lexcvi2zg7uofeabkntakpdmes7z

echo $LD_LIBRARY_PATH
echo $NETCDF_DIR $NETCDF_CXX $NETCDF_FORTRAN
echo $HDF5_DIR


echo $SLURM_JOB_NODELIST
# nodeset -e $SLURM_JOB_NODELIST    ???

echo "Entrou no diretório${exp_dir} = "$(pwd)

if [ -n "$SLURM_CPUS_PER_TASK" ]; then
  omp_threads=$SLURM_CPUS_PER_TASK
else
  omp_threads=1
fi
export OMP_NUM_THREADS=$omp_threads

echo "Running with $SLURM_NTASKS MPI processes"

exec=SHiELD_nh.prod.32bit.x
ldd $exec
date

# solu��o que eu encontrei
# export I_MPI_OFI_PROVIDER=Verbs

#solu�ão do lab
# Also due to some other performance issues we have modified some of the default parameters. You can have them try -genv UCX_TLS=all for the 16 and 32 node cases. They will have to remove the OFI=verbs setti
ng though when trying the UCX parameter.

saida_str=../${exec}_out_<NODES>nodes_<NTASKS>mpi_<THREADS>openmp_<BLOCKSIZE>blocks.txt
mpirun -genv UCX_TLS=all -np $SLURM_NTASKS $(pwd)/$exec > $saida_str

date

