Changeset 1779 for palm/trunk/SCRIPTS
- Timestamp:
- Mar 3, 2016 8:01:28 AM (9 years ago)
- Location:
- palm/trunk/SCRIPTS
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
palm/trunk/SCRIPTS/mrun
r1759 r1779 22 22 # Current revisions: 23 23 # ------------------ 24 # 24 # test: implementing an execute mechanism where the execute command is given in the 25 # configuration file 26 # ROPTS removed from execution commands 27 # Warnings about missing optional files or user code changed to informative messages 25 28 # 26 29 # Former revisions: … … 245 248 exclude="" 246 249 executable="" 250 execute_command="none" 247 251 execution_error=false 248 252 fftw_inc="" … … 671 675 while read line 672 676 do 673 if [[ "$line" != "" && $(echo $line | cut -c1) != "#" && $(echo $line | cut -d" " -s -f4) = $cond1 && $(echo $line | cut -d" " -s -f4) = $cond2 ]] 677 echo line=\"$line\" 678 if [[ "$line" != "" && $(echo $line | cut -c1) != "#" && $(echo $line | cut -d" " -s -f4) = $cond1 && $(echo $line | cut -d" " -s -f5) = $cond2 ]] 674 679 then 675 680 coupled_mode="mpi2" … … 1594 1599 if [[ ! -d $add_source_path ]] 1595 1600 then 1596 printf "\n\n +++ WARNING: additional source code directory"1601 printf "\n\n *** INFORMATIVE: additional source code directory" 1597 1602 printf "\n \"$add_source_path\" " 1598 1603 printf "\n does not exist or is not a directory." … … 1893 1898 1894 1899 1895 1896 1900 # DETERMINE COMPILE- AND LINK-OPTIONS 1897 1901 fopts="$fopts $netcdf_inc $fftw_inc $dvr_inc" 1898 1902 lopts="$lopts $netcdf_lib $fftw_lib $dvr_lib" 1899 ROPTS="$ropts" 1900 # if [[ ( $(echo $host | cut -c1-3) = nec || $(echo $host | cut -c1-3) = ibm || $host = lckyoto || $host = lctit || $host = lcflow || $host = lcxe6 || $host = lcxt5m || $host = lck || $host = lckiaps || $host = lckordi || $host = lcsb || $host ) && -n $numprocs ]] 1901 # then 1902 XOPT="-X $numprocs" 1903 # fi 1904 1903 XOPT="-X $numprocs" 1905 1904 1906 1905 … … 2875 2874 if [[ "${extin[$i]}" = "" || "${extin[$i]}" = " " ]] 2876 2875 then 2877 printf "\n +++ WARNING: input file \"${pathin[$i]}/${afname}${endin[$i]}\" "2878 printf "\n is not available!"2876 printf "\n *** INFORMATIVE: input file \"${pathin[$i]}/${afname}${endin[$i]}\" " 2877 printf "\n is not available!" 2879 2878 else 2880 printf "\n +++ WARNING: input file \"${pathin[$i]}/${afname}${endin[$i]}.${extin[$i]}\" "2881 printf "\n is not available!"2879 printf "\n *** INFORMATIVE: input file \"${pathin[$i]}/${afname}${endin[$i]}.${extin[$i]}\" " 2880 printf "\n is not available!" 2882 2881 fi 2883 2882 continue … … 3259 3258 PATH=$PATH:$TEMPDIR 3260 3259 3261 # MPI DEBUG OPTION (ARGUMENT CHECKING, SLOWS DOWN EXECUTION DUE TO INCREASED LATENCY) 3262 if [[ "$mpi_debug" = true ]] 3263 then 3264 export MPI_CHECK_ARGS=1 3265 printf "\n +++ MPI_CHECK_ARGS=$MPI_CHECK_ARGS" 3266 fi 3267 3268 if [[ "$totalview" = true ]] 3269 then 3270 printf "\n *** totalview debugger will be used" 3271 tv_opt="-tv" 3260 3261 if [[ $execute_command != "none" ]] 3262 then 3263 3264 printf "\n +++ branch still not realized" 3265 locat=execution 3266 exit 3267 3272 3268 else 3273 tv_opt="" 3274 fi 3275 3276 if [[ "$cond1" = debug || "$cond2" = debug ]] 3277 then 3278 3279 #Interactive ALLINEA DEBUG seesion 3280 if [[ "$ENVIRONMENT" != BATCH ]] 3281 then 3282 if [[ $host = lccrayb || $host = lccrayh ]] 3283 then 3284 if [[ "$allinea" = true ]] 3285 then 3286 echo "--- aprun -n $ii -N $tasks_per_node a.out $ROPTS < runfile_atmos" 3287 3288 ddt aprun -n $ii -N $tasks_per_node a.out $ROPTS 3289 wait 3290 fi 3291 fi 3292 fi 3293 3294 if [[ "$ENVIRONMENT" = BATCH ]] 3295 then 3296 continue 3297 fi 3298 if [[ $localhost = ibmh ]] 3299 then 3300 3301 # SETUP THE IBM MPI ENVIRONMENT 3302 export MP_SHARED_MEMORY=yes 3303 export AIXTHREADS_SCOPE=S 3304 export OMP_NUM_THREADS=$threads_per_task 3305 export AUTHSTATE=files 3306 export XLFRTEOPTS="nlwidth=132:err_recovery=no" # RECORD-LENGTH OF NAMELIST-OUTPUT 3307 3308 # FOLLOWING OPTIONS ARE MANDATORY FOR TOTALVIEW 3309 export MP_ADAPTER_USE=shared 3310 export MP_CPU_USE=multiple 3311 export MP_TIMEOUT=1200 3312 3313 unset MP_TASK_AFFINITY 3314 3315 # SO FAR, TOTALVIEW NEEDS HOSTFILE MECHANISM FOR EXECUTION 3316 #(( ii = 1 )) 3317 #while (( ii <= $numprocs )) 3318 #do 3319 # echo $localhost_realname >> hostfile 3320 # (( ii = ii + 1 )) 3321 #done 3322 #export MP_HOSTFILE=hostfile 3323 3324 if [[ "$LOADLBATCH" = yes ]] 3325 then 3326 totalview poe a.out $ROPTS 3327 else 3328 echo totalview poe -a a.out -procs $numprocs -rmpool 0 -nodes 1 $ROPTS 3329 export TVDSVRLAUNCHCMD=ssh 3330 totalview poe -a a.out -procs $numprocs -rmpool 0 -nodes 1 $ROPTS 3331 fi 3332 3333 fi # END DEBUG MODE 3334 3335 else 3336 3337 # NORMAL EXECUTION 3338 if [[ -n $numprocs ]] 3339 then 3340 3341 # RUNNING THE PROGRAM ON PARALLEL MACHINES 3342 if [[ $(echo $host | cut -c1-3) = ibm ]] 3269 3270 # MPI DEBUG OPTION (ARGUMENT CHECKING, SLOWS DOWN EXECUTION DUE TO INCREASED LATENCY) 3271 if [[ "$mpi_debug" = true ]] 3272 then 3273 export MPI_CHECK_ARGS=1 3274 printf "\n +++ MPI_CHECK_ARGS=$MPI_CHECK_ARGS" 3275 fi 3276 3277 if [[ "$totalview" = true ]] 3278 then 3279 printf "\n *** totalview debugger will be used" 3280 tv_opt="-tv" 3281 else 3282 tv_opt="" 3283 fi 3284 3285 if [[ "$cond1" = debug || "$cond2" = debug ]] 3286 then 3287 3288 #Interactive ALLINEA DEBUG seesion 3289 if [[ "$ENVIRONMENT" != BATCH ]] 3290 then 3291 if [[ $host = lccrayb || $host = lccrayh ]] 3292 then 3293 if [[ "$allinea" = true ]] 3294 then 3295 echo "--- aprun -n $ii -N $tasks_per_node a.out < runfile_atmos" 3296 3297 ddt aprun -n $ii -N $tasks_per_node a.out 3298 wait 3299 fi 3300 fi 3301 fi 3302 3303 if [[ $localhost = ibmh ]] 3343 3304 then 3344 3305 3345 3306 # SETUP THE IBM MPI ENVIRONMENT 3346 if [[ $host != ibmh && $host != ibmkisti ]] 3347 then 3348 export MP_SHARED_MEMORY=yes 3349 export AIXTHREAD_SCOPE=S 3350 export OMP_NUM_THREADS=$threads_per_task 3351 export XLSMPOPTS="spins=0:yields=0:stack=20000000" 3352 export AUTHSTATE=files 3353 export XLFRTEOPTS="nlwidth=132:err_recovery=no" # RECORD-LENGTH OF NAMELIST-OUTPUT 3354 # export MP_PRINTENV=yes 3355 3356 # TUNING-VARIABLES TO IMPROVE COMMUNICATION SPEED 3357 # DO NOT SHOW SIGNIFICANT EFFECTS (SEP 04, FEDERATION-SWITCH) 3358 export MP_WAIT_MODE=poll 3359 [[ $node_usage = not_shared ]] && export MP_SINGLE_THREAD=yes 3307 export MP_SHARED_MEMORY=yes 3308 export AIXTHREADS_SCOPE=S 3309 export OMP_NUM_THREADS=$threads_per_task 3310 export AUTHSTATE=files 3311 export XLFRTEOPTS="nlwidth=132:err_recovery=no" # RECORD-LENGTH OF NAMELIST-OUTPUT 3312 3313 # FOLLOWING OPTIONS ARE MANDATORY FOR TOTALVIEW 3314 export MP_ADAPTER_USE=shared 3315 export MP_CPU_USE=multiple 3316 export MP_TIMEOUT=1200 3317 3318 unset MP_TASK_AFFINITY 3319 3320 if [[ "$LOADLBATCH" = yes ]] 3321 then 3322 totalview poe a.out 3323 else 3324 echo totalview poe -a a.out -procs $numprocs -rmpool 0 -nodes 1 3325 export TVDSVRLAUNCHCMD=ssh 3326 totalview poe -a a.out -procs $numprocs -rmpool 0 -nodes 1 3360 3327 fi 3361 3328 3362 if [[ $host = ibmkisti ]] 3363 then 3364 export LANG=en_US 3365 export MP_SHARED_MEMORY=yes 3366 if [[ $threads_per_task = 1 ]] 3329 fi # END DEBUG MODE 3330 3331 else 3332 3333 # NORMAL EXECUTION 3334 if [[ -n $numprocs ]] 3335 then 3336 3337 # RUNNING THE PROGRAM ON PARALLEL MACHINES 3338 if [[ $(echo $host | cut -c1-3) = ibm ]] 3339 then 3340 3341 # SETUP THE IBM MPI ENVIRONMENT 3342 if [[ $host != ibmh && $host != ibmkisti ]] 3367 3343 then 3368 export MP_SINGLE_THREAD=yes 3369 export MEMORY_AFFINITY=MCM 3344 export MP_SHARED_MEMORY=yes 3345 export AIXTHREAD_SCOPE=S 3346 export OMP_NUM_THREADS=$threads_per_task 3347 export XLSMPOPTS="spins=0:yields=0:stack=20000000" 3348 export AUTHSTATE=files 3349 export XLFRTEOPTS="nlwidth=132:err_recovery=no" # RECORD-LENGTH OF NAMELIST-OUTPUT 3350 # export MP_PRINTENV=yes 3351 3352 # TUNING-VARIABLES TO IMPROVE COMMUNICATION SPEED 3353 # DO NOT SHOW SIGNIFICANT EFFECTS (SEP 04, FEDERATION-SWITCH) 3354 export MP_WAIT_MODE=poll 3355 [[ $node_usage = not_shared ]] && export MP_SINGLE_THREAD=yes 3356 fi 3357 3358 if [[ $host = ibmkisti ]] 3359 then 3360 export LANG=en_US 3361 export MP_SHARED_MEMORY=yes 3362 if [[ $threads_per_task = 1 ]] 3363 then 3364 export MP_SINGLE_THREAD=yes 3365 export MEMORY_AFFINITY=MCM 3366 else 3367 export OMP_NUM_THREADS=$threads_per_task 3368 fi 3369 fi 3370 3371 if [[ "$LOADLBATCH" = yes ]] 3372 then 3373 printf "\n--- Control: OMP_NUM_THREADS = \"$OMP_NUM_THREADS\" \n" 3374 if [[ "$cond1" = hpmcount || "$cond2" = hpmcount ]] 3375 then 3376 /opt/optibm/HPM_2_4_1/bin/hpmcount a.out 3377 else 3378 if [[ $run_coupled_model = false ]] 3379 then 3380 if [[ "$ocean_file_appendix" = true ]] 3381 then 3382 echo "precursor_ocean" > runfile_atmos 3383 else 3384 echo "precursor_atmos" > runfile_atmos 3385 fi 3386 else 3387 (( iia = $numprocs_atmos / $threads_per_task )) 3388 (( iio = $numprocs_ocean / $threads_per_task )) 3389 printf "\n coupled run ($iia atmosphere, $iio ocean)" 3390 printf "\n using $coupled_mode coupling" 3391 printf "\n\n" 3392 echo "coupled_run $iia $iio" > runfile_atmos 3393 fi 3394 poe ./a.out < runfile_atmos 3395 fi 3370 3396 else 3371 export OMP_NUM_THREADS=$threads_per_task 3397 if [[ $localhost = ibmh || $localhost = ibms ]] 3398 then 3399 poe a.out -procs $numprocs -nodes 1 -rmpool 0 3400 elif [[ $localhost = ibmkisti || $localhost = ibmku ]] 3401 then 3402 if [[ -f $hostfile ]] 3403 then 3404 cp $hostfile hostfile 3405 else 3406 (( ii = 1 )) 3407 while (( ii <= $numprocs )) 3408 do 3409 echo $localhost_realname >> hostfile 3410 (( ii = ii + 1 )) 3411 done 3412 fi 3413 export MP_HOSTFILE=hostfile 3414 if [[ $run_coupled_model = false ]] 3415 then 3416 if [[ "$ocean_file_appendix" = true ]] 3417 then 3418 echo "precursor_ocean" > runfile_atmos 3419 else 3420 echo "precursor_atmos" > runfile_atmos 3421 fi 3422 else 3423 (( iia = $numprocs_atmos / $threads_per_task )) 3424 (( iio = $numprocs_ocean / $threads_per_task )) 3425 printf "\n coupled run ($iia atmosphere, $iio ocean)" 3426 printf "\n using $coupled_mode coupling" 3427 printf "\n\n" 3428 echo "coupled_run $iia $iio" > runfile_atmos 3429 fi 3430 3431 poe ./a.out -procs $numprocs < runfile_atmos 3432 3433 else 3434 if [[ "$host_file" = "" ]] 3435 then 3436 printf "\n +++ no hostfile given in configuration file" 3437 locat=config_file 3438 exit 3439 else 3440 eval host_file=$host_file 3441 fi 3442 export MP_HOSTFILE=$host_file 3443 poe a.out -procs $numprocs -tasks_per_node $numprocs 3444 fi 3372 3445 fi 3373 fi 3374 3375 if [[ "$LOADLBATCH" = yes ]]3376 then 3377 printf "\n--- Control: OMP_NUM_THREADS = \"$OMP_NUM_THREADS\" \n"3378 if [[ "$cond1" = hpmcount || "$cond2" = hpmcount]]3446 3447 elif [[ $host = nech ]] # running on NEC machines 3448 then 3449 3450 (( ii = nodes )) 3451 if [[ $ii = 1 ]] 3379 3452 then 3380 /opt/optibm/HPM_2_4_1/bin/hpmcount a.out $ROPTS 3453 export F_ERRCNT=0 # acceptable number of errors before program is stopped 3454 export MPIPROGINF=YES 3455 # export F_TRACE=YES|FMT1|FMT2 # output of ftrace informations to job protocol 3456 echo "*** execution on single node with mpirun" 3457 mpirun -np $numprocs ./a.out 3381 3458 else 3382 if [[ $run_coupled_model = false ]] 3383 then 3384 if [[ "$ocean_file_appendix" = true ]] 3385 then 3386 echo "precursor_ocean" > runfile_atmos 3387 else 3388 echo "precursor_atmos" > runfile_atmos 3389 fi 3390 else 3391 (( iia = $numprocs_atmos / $threads_per_task )) 3392 (( iio = $numprocs_ocean / $threads_per_task )) 3393 printf "\n coupled run ($iia atmosphere, $iio ocean)" 3394 printf "\n using $coupled_mode coupling" 3395 printf "\n\n" 3396 echo "coupled_run $iia $iio" > runfile_atmos 3397 fi 3398 poe ./a.out $ROPTS < runfile_atmos 3459 (( i = 0 )) 3460 while (( i < ii )) 3461 do 3462 echo "-h $i -p $tasks_per_node -e ./mpi_exec_shell" >> multinode_config 3463 (( i = i + 1 )) 3464 done 3465 3466 echo "#!/bin/sh" > mpi_exec_shell 3467 echo " " >> mpi_exec_shell 3468 echo "set -u" >> mpi_exec_shell 3469 echo "F_ERRCNT=0" >> mpi_exec_shell 3470 echo "MPIPROGINV=YES" >> mpi_exec_shell 3471 echo "OMP_NUM_THREADS=$threads_per_task" >> mpi_exec_shell 3472 echo "cpurest=$cpurest" >> mpi_exec_shell 3473 echo "fname=$fname" >> mpi_exec_shell 3474 echo "localhost=$localhost" >> mpi_exec_shell 3475 echo "return_address=$return_address" >> mpi_exec_shell 3476 echo "return_username=$return_username" >> mpi_exec_shell 3477 echo "tasks_per_node=$tasks_per_node" >> mpi_exec_shell 3478 echo "write_binary=$write_binary" >> mpi_exec_shell 3479 echo "use_seperate_pe_for_dvrp_output=$use_seperate_pe_for_dvrp_output" >> mpi_exec_shell 3480 echo " " >> mpi_exec_shell 3481 echo "export F_ERRCNT" >> mpi_exec_shell 3482 echo "export MPIPROGINV" >> mpi_exec_shell 3483 echo "export OMP_NUM_THREADS" >> mpi_exec_shell 3484 echo "export cpurest" >> mpi_exec_shell 3485 echo "export fname" >> mpi_exec_shell 3486 echo "export localhost" >> mpi_exec_shell 3487 echo "export return_address" >> mpi_exec_shell 3488 echo "export return_username" >> mpi_exec_shell 3489 echo "export tasks_per_node" >> mpi_exec_shell 3490 echo "export write_binary" >> mpi_exec_shell 3491 echo "export use_seperate_pe_for_dvrp_output" >> mpi_exec_shell 3492 echo " " >> mpi_exec_shell 3493 echo "exec ./a.out" >> mpi_exec_shell 3494 3495 chmod u+x mpi_exec_shell 3496 export MPIPROGINF=YES 3497 mpirun -f multinode_config & 3498 wait 3499 3399 3500 fi 3400 else 3401 if [[ $localhost = ibmh || $localhost = ibms ]] 3402 then 3403 poe a.out -procs $numprocs -nodes 1 -rmpool 0 $ROPTS 3404 elif [[ $localhost = ibmkisti || $localhost = ibmku ]] 3501 3502 elif [[ $(echo $host | cut -c1-2) = lc && $host != lckyoto && $host != lctit ]] 3503 then 3504 3505 # COPY HOSTFILE FROM SOURCE DIRECTORY OR CREATE IT, IF IT 3506 # DOES NOT EXIST 3507 if [[ $host != lcbullhh && $host != lccrayb && $host != lccrayf && $host != lccrayh && $host != lckyuh && $host != lckyut ]] 3405 3508 then 3406 3509 if [[ -f $hostfile ]] 3407 3510 then 3408 3511 cp $hostfile hostfile 3512 (( ii = $numprocs / $threads_per_task )) 3513 [[ $ii = 0 ]] && (( ii = 1 )) 3409 3514 else 3410 3515 (( ii = 1 )) 3411 while (( ii <= $numprocs ))3516 while (( ii <= $numprocs / $threads_per_task )) 3412 3517 do 3413 3518 echo $localhost_realname >> hostfile 3414 3519 (( ii = ii + 1 )) 3415 3520 done 3521 if (( $numprocs / $threads_per_task == 0 )) 3522 then 3523 echo $localhost_realname >> hostfile 3524 fi 3416 3525 fi 3417 export MP_HOSTFILE=hostfile 3418 if [[ $run_coupled_model = false ]] 3526 eval zeile=\"`head -n $ii hostfile`\" 3527 printf "\n *** running on: $zeile" 3528 fi 3529 3530 (( ii = $numprocs / $threads_per_task )) 3531 [[ $ii = 0 ]] && (( ii = 1 )) 3532 export OMP_NUM_THREADS=$threads_per_task 3533 3534 if [[ $threads_per_task != 1 ]] 3535 then 3536 # INCREASE STACK SIZE TO UNLIMITED, BECAUSE OTHERWISE LARGE RUNS 3537 # MAY ABORT 3538 ulimit -s unlimited 3539 printf "\n threads per task: $threads_per_task stacksize: unlimited" 3540 fi 3541 if [[ $run_coupled_model = false ]] 3542 then 3543 if [[ "$ocean_file_appendix" = true ]] 3419 3544 then 3420 if [[ "$ocean_file_appendix" = true ]] 3545 echo "precursor_ocean" > runfile_atmos 3546 else 3547 echo "precursor_atmos" > runfile_atmos 3548 fi 3549 printf "\n\n" 3550 3551 if [[ $host = lccrayb || $host = lccrayh ]] 3552 then 3553 echo "--- aprun -n $ii -N $tasks_per_node a.out < runfile_atmos" 3554 aprun -n $ii -N $tasks_per_node a.out < runfile_atmos 3555 elif [[ $host = lcbullhh ]] 3556 then 3557 export OMPI_MCA_pml=cm 3558 export OMPI_MCA_mtl=mxm 3559 export OMPI_MCA_coll=^ghc 3560 export OMPI_MCA_mtl_mxm_np=0 3561 export MXM_RDMA_PORTS=mlx5_0:1 3562 export MXM_LOG_LEVEL=ERROR 3563 export OMP_NUM_THREADS=$threads_per_task 3564 export KMP_AFFINITY=verbose,granularity=core,compact,1 3565 export KMP_STACKSIZE=64m 3566 3567 srun --nodes=$nodes --ntasks-per-node=$tasks_per_node ./a.out < runfile_atmos 3568 3569 elif [[ $host = lccrayf ]] 3570 then 3571 aprun -j1 -n $ii -N $tasks_per_node -m ${memory}M a.out < runfile_atmos 3572 elif [[ $host = lcxe6 || $host = lcxt5m ]] 3573 then 3574 aprun -n $ii -N $tasks_per_node a.out < runfile_atmos 3575 elif [[ $host = lcflow ]] 3576 then 3577 mpirun -np $ii a.out < runfile_atmos 3578 elif [[ $host = lcsb ]] 3579 then 3580 mpirun_rsh -hostfile $PBS_NODEFILE -np `cat $PBS_NODEFILE | wc -l` a.out < runfile_atmos 3581 elif [[ $host = lckiaps ]] 3582 then 3583 mpirun -np $ii -f $PBS_NODEFILE a.out < runfile_atmos 3584 elif [[ $host = lckyu* ]] 3585 then 3586 mpiexec -n $ii --stdin runfile_atmos ./a.out 3587 else 3588 mpiexec -machinefile hostfile -n $ii a.out < runfile_atmos 3589 fi 3590 3591 else 3592 3593 # CURRENTLY THERE IS NO FULL MPI-2 SUPPORT ON ICE AND XT4 3594 (( iia = $numprocs_atmos / $threads_per_task )) 3595 (( iio = $numprocs_ocean / $threads_per_task )) 3596 printf "\n coupled run ($iia atmosphere, $iio ocean)" 3597 printf "\n using $coupled_mode coupling" 3598 printf "\n\n" 3599 3600 if [[ $coupled_mode = "mpi2" ]] 3601 then 3602 echo "atmosphere_to_ocean $iia $iio" > runfile_atmos 3603 echo "ocean_to_atmosphere $iia $iio" > runfile_ocean 3604 3605 if [[ $host = lccrayf || $host = lcxe6 || $host = lcxt5m ]] 3421 3606 then 3422 echo "precursor_ocean" > runfile_atmos 3607 3608 aprun -n $iia -N $tasks_per_node a.out < runfile_atmos & 3609 aprun -n $iio -N $tasks_per_node a.out < runfile_ocean & 3610 3423 3611 else 3424 echo "precursor_atmos" > runfile_atmos 3612 # WORKAROUND BECAUSE mpiexec WITH -env option IS NOT AVAILABLE ON SOME SYSTEMS 3613 mpiexec -machinefile hostfile -n $iia a.out < runfile_atmos & 3614 mpiexec -machinefile hostfile -n $iio a.out < runfile_ocean & 3615 # mpiexec -machinefile hostfile -n $iia -env coupling_mode atmosphere_to_ocean a.out & 3616 # mpiexec -machinefile hostfile -n $iio -env coupling_mode ocean_to_atmosphere a.out & 3425 3617 fi 3618 wait 3619 3426 3620 else 3427 (( iia = $numprocs_atmos / $threads_per_task )) 3428 (( iio = $numprocs_ocean / $threads_per_task )) 3429 printf "\n coupled run ($iia atmosphere, $iio ocean)" 3430 printf "\n using $coupled_mode coupling" 3431 printf "\n\n" 3621 3432 3622 echo "coupled_run $iia $iio" > runfile_atmos 3623 3624 if [[ $host = lccrayf || $host = lcxe6 || $host = lcxt5m ]] 3625 then 3626 3627 aprun -n $ii -N $tasks_per_node a.out < runfile_atmos 3628 3629 elif [[ $host = lck || $host = lckordi ]] 3630 then 3631 3632 mpiexec -n $ii ./a.out < runfile_atmos & 3633 3634 elif [[ $host = lckyu* ]] 3635 then 3636 3637 mpiexec -n $ii --stdin runfile_atmos ./a.out 3638 3639 elif [[ $host = lcmuk ]] 3640 then 3641 3642 mpiexec -machinefile hostfile -n $ii a.out < runfile_atmos 3643 3644 fi 3645 wait 3433 3646 fi 3434 3647 3435 poe ./a.out -procs $numprocs $ROPTS < runfile_atmos 3436 3648 fi 3649 3650 elif [[ $host = lckyoto ]] 3651 then 3652 set -xv 3653 export P4_RSHCOMMAND=plesh 3654 echo " P4_RSHCOMMAND = $P4_RSHCOMMAND" 3655 if [[ "$ENVIRONMENT" = BATCH ]] 3656 then 3657 if [[ "$cond2" = fujitsu ]] 3658 then 3659 mpiexec -n $numprocs ./a.out # for fujitsu-compiler 3660 elif [[ "cond2" = pgi ]] 3661 then 3662 mpirun -np $numprocs -machinefile ${QSUB_NODEINF} ./a.out 3663 else 3664 mpirun_rsh -np $numprocs -hostfile ${QSUB_NODEINF} MV2_USE_SRQ=0 ./a.out || /bin/true 3665 fi 3437 3666 else 3438 if [[ "$ host_file" = "" ]]3667 if [[ "$cond2" = "" ]] 3439 3668 then 3440 printf "\n +++ no hostfile given in configuration file" 3441 locat=config_file 3442 exit 3669 mpiruni_rsh -np $numprocs ./a.out # for intel 3443 3670 else 3444 eval host_file=$host_file 3445 fi 3446 export MP_HOSTFILE=$host_file 3447 poe a.out -procs $numprocs -tasks_per_node $numprocs $ROPTS 3448 fi 3449 fi 3450 elif [[ $host = nech ]] 3451 then 3452 (( ii = nodes )) 3453 if [[ $ii = 1 ]] 3454 then 3455 export F_ERRCNT=0 # acceptable number of errors before program is stopped 3456 export MPIPROGINF=YES 3457 # export F_TRACE=YES|FMT1|FMT2 # output of ftrace informations to job protocol 3458 echo "*** execution on single node with mpirun" 3459 mpirun -np $numprocs ./a.out $ROPTS 3460 else 3461 (( i = 0 )) 3462 while (( i < ii )) 3463 do 3464 echo "-h $i -p $tasks_per_node -e ./mpi_exec_shell" >> multinode_config 3465 (( i = i + 1 )) 3466 done 3467 3468 echo "#!/bin/sh" > mpi_exec_shell 3469 echo " " >> mpi_exec_shell 3470 echo "set -u" >> mpi_exec_shell 3471 echo "F_ERRCNT=0" >> mpi_exec_shell 3472 echo "MPIPROGINV=YES" >> mpi_exec_shell 3473 echo "OMP_NUM_THREADS=$threads_per_task" >> mpi_exec_shell 3474 echo "cpurest=$cpurest" >> mpi_exec_shell 3475 echo "fname=$fname" >> mpi_exec_shell 3476 echo "localhost=$localhost" >> mpi_exec_shell 3477 echo "return_address=$return_address" >> mpi_exec_shell 3478 echo "return_username=$return_username" >> mpi_exec_shell 3479 echo "tasks_per_node=$tasks_per_node" >> mpi_exec_shell 3480 echo "write_binary=$write_binary" >> mpi_exec_shell 3481 echo "use_seperate_pe_for_dvrp_output=$use_seperate_pe_for_dvrp_output" >> mpi_exec_shell 3482 echo " " >> mpi_exec_shell 3483 echo "export F_ERRCNT" >> mpi_exec_shell 3484 echo "export MPIPROGINV" >> mpi_exec_shell 3485 echo "export OMP_NUM_THREADS" >> mpi_exec_shell 3486 echo "export cpurest" >> mpi_exec_shell 3487 echo "export fname" >> mpi_exec_shell 3488 echo "export localhost" >> mpi_exec_shell 3489 echo "export return_address" >> mpi_exec_shell 3490 echo "export return_username" >> mpi_exec_shell 3491 echo "export tasks_per_node" >> mpi_exec_shell 3492 echo "export write_binary" >> mpi_exec_shell 3493 echo "export use_seperate_pe_for_dvrp_output" >> mpi_exec_shell 3494 echo " " >> mpi_exec_shell 3495 echo "exec ./a.out" >> mpi_exec_shell 3496 3497 chmod u+x mpi_exec_shell 3498 export MPIPROGINF=YES 3499 mpirun -f multinode_config & 3500 wait 3501 3502 fi 3503 elif [[ $(echo $host | cut -c1-2) = lc && $host != lckyoto && $host != lctit ]] 3504 then 3505 3506 # COPY HOSTFILE FROM SOURCE DIRECTORY OR CREATE IT, IF IT 3507 # DOES NOT EXIST 3508 if [[ $host != lcbullhh && $host != lccrayb && $host != lccrayf && $host != lccrayh && $host != lckyuh && $host != lckyut ]] 3509 3510 then 3511 if [[ -f $hostfile ]] 3512 then 3513 cp $hostfile hostfile 3514 (( ii = $numprocs / $threads_per_task )) 3515 [[ $ii = 0 ]] && (( ii = 1 )) 3516 else 3517 (( ii = 1 )) 3518 while (( ii <= $numprocs / $threads_per_task )) 3519 do 3520 echo $localhost_realname >> hostfile 3521 (( ii = ii + 1 )) 3522 done 3523 if (( $numprocs / $threads_per_task == 0 )) 3524 then 3525 echo $localhost_realname >> hostfile 3671 mpirun -np $numprocs ./a.out 3526 3672 fi 3527 3673 fi 3528 eval zeile=\"`head -n $ii hostfile`\" 3529 printf "\n *** running on: $zeile" 3674 set +xv 3675 3676 elif [[ $host = lctit ]] 3677 then 3678 export OMP_NUM_THREADS=$threads_per_task 3679 echo "OMP_NUM_THREADS=$OMP_NUM_THREADS" 3680 if [[ "$threads_per_task" != 1 ]] 3681 then 3682 export MV2_ENABLE_AFFINITY=0 3683 fi 3684 echo "----- PBS_NODEFILE content:" 3685 cat $PBS_NODEFILE 3686 echo "-----" 3687 (( ii = $numprocs / $threads_per_task )) 3688 echo "mpirun -np $ii -hostfile $PBS_NODEFILE ./a.out" 3689 mpirun -np $ii -hostfile $PBS_NODEFILE ./a.out 3690 3691 else 3692 mpprun -n $numprocs a.out 3530 3693 fi 3531 3532 (( ii = $numprocs / $threads_per_task ))3533 [[ $ii = 0 ]] && (( ii = 1 ))3534 export OMP_NUM_THREADS=$threads_per_task3535 3536 if [[ $threads_per_task != 1 ]]3537 then3538 # INCREASE STACK SIZE TO UNLIMITED, BECAUSE OTHERWISE LARGE RUNS3539 # MAY ABORT3540 ulimit -s unlimited3541 printf "\n threads per task: $threads_per_task stacksize: unlimited"3542 fi3543 if [[ $run_coupled_model = false ]]3544 then3545 if [[ "$ocean_file_appendix" = true ]]3546 then3547 echo "precursor_ocean" > runfile_atmos3548 else3549 echo "precursor_atmos" > runfile_atmos3550 fi3551 printf "\n\n"3552 3553 if [[ $host = lccrayb || $host = lccrayh ]]3554 then3555 echo "--- aprun -n $ii -N $tasks_per_node a.out $ROPTS < runfile_atmos"3556 aprun -n $ii -N $tasks_per_node a.out $ROPTS < runfile_atmos3557 # aprun -n $ii -ss -r2 -j1 a.out $ROPTS < runfile_atmos3558 3559 elif [[ $host = lcbullhh ]]3560 then3561 export OMPI_MCA_pml=cm3562 export OMPI_MCA_mtl=mxm3563 export OMPI_MCA_coll=^ghc3564 export OMPI_MCA_mtl_mxm_np=03565 export MXM_RDMA_PORTS=mlx5_0:13566 export MXM_LOG_LEVEL=ERROR3567 export OMP_NUM_THREADS=$threads_per_task3568 export KMP_AFFINITY=verbose,granularity=core,compact,13569 export KMP_STACKSIZE=64m3570 3571 srun --nodes=$nodes --ntasks-per-node=$tasks_per_node ./a.out3572 3573 elif [[ $host = lccrayf ]]3574 then3575 aprun -j1 -n $ii -N $tasks_per_node -m ${memory}M a.out $ROPTS < runfile_atmos3576 elif [[ $host = lcxe6 || $host = lcxt5m ]]3577 then3578 aprun -n $ii -N $tasks_per_node a.out $ROPTS < runfile_atmos3579 elif [[ $host = lcflow ]]3580 then3581 mpirun -np $ii a.out < runfile_atmos $ROPTS3582 elif [[ $host = lcsb ]]3583 then3584 mpirun_rsh -hostfile $PBS_NODEFILE -np `cat $PBS_NODEFILE | wc -l` a.out < runfile_atmos $ROPTS3585 elif [[ $host = lckiaps ]]3586 then3587 mpirun -np $ii -f $PBS_NODEFILE a.out < runfile_atmos $ROPTS3588 elif [[ $host = lckyu* ]]3589 then3590 mpiexec -n $ii --stdin runfile_atmos ./a.out $ROPTS3591 else3592 mpiexec -machinefile hostfile -n $ii a.out < runfile_atmos $ROPTS3593 fi3594 else3595 3596 # CURRENTLY THERE IS NO FULL MPI-2 SUPPORT ON ICE AND XT43597 (( iia = $numprocs_atmos / $threads_per_task ))3598 (( iio = $numprocs_ocean / $threads_per_task ))3599 printf "\n coupled run ($iia atmosphere, $iio ocean)"3600 printf "\n using $coupled_mode coupling"3601 printf "\n\n"3602 3603 if [[ $coupled_mode = "mpi2" ]]3604 then3605 echo "atmosphere_to_ocean $iia $iio" > runfile_atmos3606 echo "ocean_to_atmosphere $iia $iio" > runfile_ocean3607 3608 if [[ $host = lccrayf || $host = lcxe6 || $host = lcxt5m ]]3609 then3610 3611 aprun -n $iia -N $tasks_per_node a.out < runfile_atmos $ROPTS &3612 aprun -n $iio -N $tasks_per_node a.out < runfile_ocean $ROPTS &3613 3614 else3615 # WORKAROUND BECAUSE mpiexec WITH -env option IS NOT AVAILABLE ON SOME SYSTEMS3616 mpiexec -machinefile hostfile -n $iia a.out $ROPTS < runfile_atmos &3617 mpiexec -machinefile hostfile -n $iio a.out $ROPTS < runfile_ocean &3618 # mpiexec -machinefile hostfile -n $iia -env coupling_mode atmosphere_to_ocean a.out $ROPTS &3619 # mpiexec -machinefile hostfile -n $iio -env coupling_mode ocean_to_atmosphere a.out $ROPTS &3620 fi3621 wait3622 3623 else3624 3625 echo "coupled_run $iia $iio" > runfile_atmos3626 3627 if [[ $host = lccrayf || $host = lcxe6 || $host = lcxt5m ]]3628 then3629 3630 aprun -n $ii -N $tasks_per_node a.out < runfile_atmos $ROPTS3631 3632 elif [[ $host = lck || $host = lckordi ]]3633 then3634 3635 mpiexec -n $ii ./a.out $ROPTS < runfile_atmos &3636 3637 elif [[ $host = lckyu* ]]3638 then3639 3640 mpiexec -n $ii --stdin runfile_atmos ./a.out $ROPTS3641 3642 elif [[ $host = lcmuk ]]3643 then3644 3645 mpiexec -machinefile hostfile -n $ii a.out < runfile_atmos $ROPTS3646 3647 fi3648 wait3649 fi3650 3651 fi3652 3653 elif [[ $host = lckyoto ]]3654 then3655 set -xv3656 export P4_RSHCOMMAND=plesh3657 echo " P4_RSHCOMMAND = $P4_RSHCOMMAND"3658 if [[ "$ENVIRONMENT" = BATCH ]]3659 then3660 if [[ "$cond2" = fujitsu ]]3661 then3662 mpiexec -n $numprocs ./a.out $ROPTS # for fujitsu-compiler3663 elif [[ "cond2" = pgi ]]3664 then3665 mpirun -np $numprocs -machinefile ${QSUB_NODEINF} ./a.out $ROPTS3666 else3667 mpirun_rsh -np $numprocs -hostfile ${QSUB_NODEINF} MV2_USE_SRQ=0 ./a.out ${ROPTS} || /bin/true3668 fi3669 else3670 if [[ "$cond2" = "" ]]3671 then3672 mpiruni_rsh -np $numprocs ./a.out $ROPTS # for intel3673 else3674 mpirun -np $numprocs ./a.out $ROPTS3675 fi3676 fi3677 set +xv3678 3679 elif [[ $host = lctit ]]3680 then3681 export OMP_NUM_THREADS=$threads_per_task3682 echo "OMP_NUM_THREADS=$OMP_NUM_THREADS"3683 if [[ "$threads_per_task" != 1 ]]3684 then3685 export MV2_ENABLE_AFFINITY=03686 fi3687 echo "----- PBS_NODEFILE content:"3688 cat $PBS_NODEFILE3689 echo "-----"3690 (( ii = $numprocs / $threads_per_task ))3691 echo "mpirun -np $ii -hostfile $PBS_NODEFILE ./a.out $ROPTS"3692 mpirun -np $ii -hostfile $PBS_NODEFILE ./a.out $ROPTS3693 3694 3694 else 3695 mpprun -n $numprocs a.out $ROPTS3696 fi 3697 else 3698 a.out $ROPTS3699 fi 3700 fi 3695 a.out 3696 fi 3697 3698 fi # end normal (non-debug) execution 3699 3700 fi # end explicit execute_command or host-specific execute actions 3701 3701 3702 3702 if [[ $? != 0 ]] … … 3762 3762 then 3763 3763 printf "\n\n\n *** post-processing: now executing \"aprun -n 1 -N 1 combine_plot_fields${block}.x\" ..." 3764 aprun -n 1 -N 1 combine_plot_fields${block}.x #$ROPTS < runfile_atmos3764 aprun -n 1 -N 1 combine_plot_fields${block}.x 3765 3765 else 3766 3766 printf "\n\n\n *** post-processing: now executing \"combine_plot_fields${block}.x\" ..." -
palm/trunk/SCRIPTS/subjob
r1702 r1779 152 152 153 153 typeset -i cputime=0 memory=0 Memory=0 minuten resttime sekunden stunden 154 typeset -i inumprocs mpi_tasks=0 nodes=0 processes_per_node=0 tasks_per_node=0 threads_per_task=1154 typeset -i numprocs mpi_tasks=0 nodes=0 processes_per_node=0 tasks_per_node=0 threads_per_task=1 155 155 156 156
Note: See TracChangeset
for help on using the changeset viewer.
