Changeset 759 for palm/trunk
- Timestamp:
- Sep 15, 2011 1:58:31 PM (13 years ago)
- Location:
- palm/trunk
- Files:
-
- 14 edited
-
SCRIPTS/mrun (modified) (14 diffs)
-
SOURCE/advec_particles.f90 (modified) (3 diffs)
-
SOURCE/data_output_2d.f90 (modified) (5 diffs)
-
SOURCE/data_output_3d.f90 (modified) (3 diffs)
-
SOURCE/header.f90 (modified) (3 diffs)
-
SOURCE/init_3d_model.f90 (modified) (3 diffs)
-
SOURCE/init_grid.f90 (modified) (4 diffs)
-
SOURCE/init_pegrid.f90 (modified) (2 diffs)
-
SOURCE/modules.f90 (modified) (4 diffs)
-
SOURCE/palm.f90 (modified) (3 diffs)
-
SOURCE/parin.f90 (modified) (3 diffs)
-
SOURCE/prandtl_fluxes.f90 (modified) (2 diffs)
-
SOURCE/production_e.f90 (modified) (2 diffs)
-
SOURCE/write_3d_binary.f90 (modified) (3 diffs)
Legend:
- Unmodified
- Added
- Removed
-
palm/trunk/SCRIPTS/mrun
r757 r759 215 215 # 14/12/10 - Siggi - adjustments for new Tsubame system at Tokyo 216 216 # institute of technology (lctit) 217 # 23/12/10 - Micha - different number of processors in ocean and atmosphere218 # is now allowed217 # 23/12/10 - Micha - different number of processors in ocean and 218 # atmosphere is now allowed 219 219 # 02/02/10 - Siggi - further adjustments on Tsubame and concerning openMP 220 220 # usage … … 225 225 # 06/04/11 - BjornM - bugfix for runs with mpt on lcsgi 226 226 # 17/08/11 - Siggi - extensions for impi library 227 # 18/08/11 - Siggi - bugfix for local append of output files with suffix (.nc) 227 # 18/08/11 - Siggi - bugfix for local append of output files with suffix 228 # (.nc) 228 229 # 18/08/11 - Marcus - support for Linux OS with German locale 229 # - properly report hosts on general Linux clusters 230 # - properly report hosts on general Linux clusters 230 231 # 29/08/11 - BjornW - adapted for lcflow (ForWind cluster in Oldenburg) 231 # 29/08/11 - Carolin- initiating restart-run: adjustment of the path at IMUK 232 # 29/08/11 - Carolin- initiating restart-run: adjustment of the path at 233 # IMUK 234 # 15/09/11 - Siggi - new option -w tp set the maximum number of parallel 235 # io streams, option -T is obligatory from now on 232 236 233 237 … … 287 291 mainprog="" 288 292 makefile="" 293 max_par_io_str="" 289 294 mc=$0 290 295 while [[ $(echo $mc | grep -c "/") != 0 ]] … … 474 479 # SHELLSCRIPT-OPTIONEN EINLESEN UND KOMMANDO NEU ZUSAMMENSETZEN, FALLS ES 475 480 # FUER FOLGEJOBS BENOETIGT WIRD 476 while getopts :a:AbBc:Cd:D:Fg:G:h:H:i:IkK:m:M:n:o:O:p:P:q:r:R:s:St:T:u:U:v xX:yY: option481 while getopts :a:AbBc:Cd:D:Fg:G:h:H:i:IkK:m:M:n:o:O:p:P:q:r:R:s:St:T:u:U:vw:xX:yY: option 477 482 do 478 483 case $option in … … 507 512 (S) read_from_config=false; mc="$mc -S";; 508 513 (t) cpumax=$OPTARG; mc="$mc -t$OPTARG";; 509 (T) tasks_per_node=$OPTARG; mc="$mc -T$OPTARG";;514 (T) mrun_tasks_per_node=$OPTARG; mc="$mc -T$OPTARG";; 510 515 (u) remote_username=$OPTARG; mc="$mc -u$OPTARG";; 511 516 (U) return_username=$OPTARG; mc="$mc -U$OPTARG";; 512 517 (v) silent=true; mc="$mc -v";; 518 (w) max_par_io_str=$OPTARG; mc="$mc -w$OPTARG";; 513 519 (x) do_trace=true;set -x; mc="$mc -x";; 514 520 (X) numprocs=$OPTARG; mc="$mc -X$OPTARG";; … … 567 573 printf "\n -u username on remote machine \"\" " 568 574 printf "\n -v no prompt for confirmation ---" 575 printf "\n -w maximum parallel io streams as given by -X" 569 576 printf "\n -x tracing of mrun for debug purposes ---" 570 577 printf "\n -X # of processors (on parallel machines) 1" … … 1036 1043 1037 1044 # OPTIONSWERTE UEBERSTEUERN KONFIGURATIONSDATEI 1038 [[ $mrun_memory != 0 ]] && memory=$mrun_memory1045 [[ $mrun_memory != 0 ]] && memory=$mrun_memory 1039 1046 [[ "$mrun_group_number" != "none" ]] && group_number=$mrun_group_number 1040 [[ $mrun_cpumax != 0 ]] && cpumax=$mrun_cpumax 1041 [[ "$mrun_numprocs" != "" ]] && numprocs=$mrun_numprocs 1047 [[ $mrun_cpumax != 0 ]] && cpumax=$mrun_cpumax 1048 [[ "$mrun_numprocs" != "" ]] && numprocs=$mrun_numprocs 1049 [[ "$max_par_io_str" != "" ]] && maximum_parallel_io_streams=$max_par_io_str 1050 [[ "$mrun_tasks_per_node" != "" ]] && tasks_per_node=$mrun_tasks_per_node 1042 1051 1043 1052 fi … … 1157 1166 # DEFAULT-WERT SETZEN) UND OB SIE EIN GANZZAHLIGER TEILER DER 1158 1167 # GESAMTPROZESSORANZAHL IST 1159 if [[ $host = nech || $host = necriam || $host = ibmh || $host = ibmkisti || $host = ibms ]] 1160 then 1161 [[ "$tasks_per_node" = "" ]] && tasks_per_node=6 1162 (( ival = $tasks_per_node )) 1163 (( pes = numprocs )) 1168 if [[ "$tasks_per_node" = "" ]] 1169 then 1170 printf "\n" 1171 printf "\n +++ option \"-T\" (tasks per node) is missing" 1172 printf "\n set -T option or define tasks_per_node in the config file" 1173 locat=tasks_per_node; (( iec = 0 )); exit 1174 fi 1175 (( ival = $tasks_per_node )) 1176 (( pes = numprocs )) 1164 1177 # if [[ $(echo $package_list | grep -c dvrp_graphics+1PE) = 1 ]] 1165 1178 # then 1166 1179 # (( pes = pes - 1 )) 1167 1180 # fi 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 fi 1181 (( ii = pes / ival )) 1182 if (( pes - ii * ival > 0 )) 1183 then 1184 printf "\n" 1185 printf "\n +++ tasks per node (option \"-T\") must be an integral" 1186 printf "\n divisor of the total number of processors (option \"-X\")" 1187 printf "\n values of this mrun-call: \"-T $tasks_per_node\" \"-X $numprocs\"" 1188 locat=tasks_per_node; (( iec = 0 )); exit 1189 fi 1190 1178 1191 1179 1192 # IBMY HAT NUR EINEN KNOTEN … … 1190 1203 fi 1191 1204 1192 # FALLS OPENMP PARALLELISIERUNG VERWENDET WERDEN SOLL, ANZAHL VON THREADS1193 # SETZEN UND ZAHL DER TASKS PRO KNOTEN AUF 1 SETZEN1194 # if [[ $use_openmp = true ]]1195 # then1196 # threads_per_task=$tasks_per_node1197 # tasks_per_node=11198 # fi1199 1205 1200 1206 # SETTINGS FOR SUBJOB-COMMAND … … 1263 1269 fi 1264 1270 1271 1272 # Set default value for the maximum number of parallel io streams 1273 if [[ "$maximum_parallel_io_streams" = "" ]] 1274 then 1275 maximum_parallel_io_streams=$numprocs 1276 fi 1265 1277 1266 1278 … … 2194 2206 then 2195 2207 spalte1="tasks per node:"; spalte2="$tasks_per_node (number of nodes: $nodes)" 2208 printf "| $spalte1$spalte2 | \n" 2209 fi 2210 if [[ $maximum_parallel_io_streams != $numprocs ]] 2211 then 2212 spalte1="max par io streams:"; spalte2="$maximum_parallel_io_streams" 2196 2213 printf "| $spalte1$spalte2 | \n" 2197 2214 fi … … 2996 3013 &envpar run_identifier = '$fname', host = '$localhost', 2997 3014 write_binary = '$write_binary', tasks_per_node = $tasks_per_node, 3015 maximum_parallel_io_streams = $maximum_parallel_io_streams, 2998 3016 maximum_cpu_time_allowed = ${cpumax}., 2999 3017 revision = '$global_revision', … … 4422 4440 mrun_com=${mrun_com}" -O $threads_per_task" 4423 4441 fi 4424 [[ "$tasks_per_node" != "" ]] && mrun_com=${mrun_com}" -T $tasks_per_node"4442 [[ "$tasks_per_node" != "" ]] && mrun_com=${mrun_com}" -T $tasks_per_node" 4425 4443 [[ $store_on_archive_system = true ]] && mrun_com=${mrun_com}" -A" 4426 4444 [[ $package_list != "" ]] && mrun_com=${mrun_com}" -p \"$package_list\"" … … 4430 4448 [[ "$ocean_file_appendix" = true ]] && mrun_com=${mrun_com}" -y" 4431 4449 [[ $run_coupled_model = true ]] && mrun_com=${mrun_com}" -Y \"$coupled_dist\"" 4450 [[ "$max_par_io_str" != "" ]] && mrun_com=${mrun_com}" -w $max_par_io_str" 4432 4451 if [[ $do_remote = true ]] 4433 4452 then -
palm/trunk/SOURCE/advec_particles.f90
r668 r759 3 3 !------------------------------------------------------------------------------! 4 4 ! Current revisions: 5 ! ----------------- 5 ! ------------------ 6 ! Splitting of parallel I/O (routine write_particles) 6 7 ! 7 8 ! Former revisions: … … 3811 3812 CHARACTER (LEN=10) :: particle_binary_version 3812 3813 3814 INTEGER :: i 3815 3813 3816 ! 3814 3817 !-- First open the output unit. … … 3828 3831 ENDIF 3829 3832 3830 ! 3831 !-- Write the version number of the binary format. 3832 !-- Attention: After changes to the following output commands the version 3833 !-- --------- number of the variable particle_binary_version must be changed! 3834 !-- Also, the version number and the list of arrays to be read in 3835 !-- init_particles must be adjusted accordingly. 3836 particle_binary_version = '3.0' 3837 WRITE ( 90 ) particle_binary_version 3838 3839 ! 3840 !-- Write some particle parameters, the size of the particle arrays as well as 3841 !-- other dvrp-plot variables. 3842 WRITE ( 90 ) bc_par_b, bc_par_lr, bc_par_ns, bc_par_t, & 3843 maximum_number_of_particles, maximum_number_of_tailpoints, & 3844 maximum_number_of_tails, number_of_initial_particles, & 3845 number_of_particles, number_of_particle_groups, & 3846 number_of_tails, particle_groups, time_prel, & 3847 time_write_particle_data, uniform_particles 3848 3849 IF ( number_of_initial_particles /= 0 ) WRITE ( 90 ) initial_particles 3850 3851 WRITE ( 90 ) prt_count, prt_start_index 3852 WRITE ( 90 ) particles 3853 3854 IF ( use_particle_tails ) THEN 3855 WRITE ( 90 ) particle_tail_coordinates 3856 ENDIF 3857 3858 CLOSE ( 90 ) 3833 DO i = 0, io_blocks-1 3834 3835 IF ( i == io_group ) THEN 3836 3837 ! 3838 !-- Write the version number of the binary format. 3839 !-- Attention: After changes to the following output commands the version 3840 !-- --------- number of the variable particle_binary_version must be 3841 !-- changed! Also, the version number and the list of arrays 3842 !-- to be read in init_particles must be adjusted accordingly. 3843 particle_binary_version = '3.0' 3844 WRITE ( 90 ) particle_binary_version 3845 3846 ! 3847 !-- Write some particle parameters, the size of the particle arrays as 3848 !-- well as other dvrp-plot variables. 3849 WRITE ( 90 ) bc_par_b, bc_par_lr, bc_par_ns, bc_par_t, & 3850 maximum_number_of_particles, & 3851 maximum_number_of_tailpoints, maximum_number_of_tails, & 3852 number_of_initial_particles, number_of_particles, & 3853 number_of_particle_groups, number_of_tails, & 3854 particle_groups, time_prel, time_write_particle_data, & 3855 uniform_particles 3856 3857 IF ( number_of_initial_particles /= 0 ) WRITE ( 90 ) initial_particles 3858 3859 WRITE ( 90 ) prt_count, prt_start_index 3860 WRITE ( 90 ) particles 3861 3862 IF ( use_particle_tails ) THEN 3863 WRITE ( 90 ) particle_tail_coordinates 3864 ENDIF 3865 3866 CLOSE ( 90 ) 3867 3868 ENDIF 3869 3870 #if defined( __parallel ) 3871 CALL MPI_BARRIER( comm2d, ierr ) 3872 #endif 3873 3874 ENDDO 3859 3875 3860 3876 END SUBROUTINE write_particles -
palm/trunk/SOURCE/data_output_2d.f90
r730 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! Splitting of parallel I/O 6 7 ! 7 8 ! Former revisions: … … 737 738 ENDIF 738 739 #endif 739 WRITE ( 21 ) nxlg, nxrg, nysg, nyng 740 WRITE ( 21 ) local_2d 740 DO i = 0, io_blocks-1 741 IF ( i == io_group ) THEN 742 WRITE ( 21 ) nxlg, nxrg, nysg, nyng 743 WRITE ( 21 ) local_2d 744 ENDIF 745 #if defined( __parallel ) 746 CALL MPI_BARRIER( comm2d, ierr ) 747 #endif 748 ENDDO 741 749 742 750 ELSE … … 1033 1041 ENDIF 1034 1042 #endif 1035 IF ( ( section(is,s) >= nys .AND. & 1036 section(is,s) <= nyn ) .OR. & 1037 ( section(is,s) == -1 .AND. nys-1 == -1 ) ) & 1038 THEN 1039 WRITE (22) nxlg, nxrg, nzb, nzt+1 1040 WRITE (22) local_2d 1041 ELSE 1042 WRITE (22) -1, -1, -1, -1 1043 ENDIF 1043 DO i = 0, io_blocks-1 1044 IF ( i == io_group ) THEN 1045 IF ( ( section(is,s) >= nys .AND. & 1046 section(is,s) <= nyn ) .OR. & 1047 ( section(is,s) == -1 .AND. & 1048 nys-1 == -1 ) ) & 1049 THEN 1050 WRITE (22) nxlg, nxrg, nzb, nzt+1 1051 WRITE (22) local_2d 1052 ELSE 1053 WRITE (22) -1, -1, -1, -1 1054 ENDIF 1055 ENDIF 1056 #if defined( __parallel ) 1057 CALL MPI_BARRIER( comm2d, ierr ) 1058 #endif 1059 ENDDO 1044 1060 1045 1061 ELSE … … 1334 1350 ENDIF 1335 1351 #endif 1336 IF ( ( section(is,s) >= nxl .AND. & 1337 section(is,s) <= nxr ) .OR. & 1338 ( section(is,s) == -1 .AND. nxl-1 == -1 ) ) & 1339 THEN 1340 WRITE (23) nysg, nyng, nzb, nzt+1 1341 WRITE (23) local_2d 1342 ELSE 1343 WRITE (23) -1, -1, -1, -1 1344 ENDIF 1352 DO i = 0, io_blocks-1 1353 IF ( i == io_group ) THEN 1354 IF ( ( section(is,s) >= nxl .AND. & 1355 section(is,s) <= nxr ) .OR. & 1356 ( section(is,s) == -1 .AND. & 1357 nxl-1 == -1 ) ) & 1358 THEN 1359 WRITE (23) nysg, nyng, nzb, nzt+1 1360 WRITE (23) local_2d 1361 ELSE 1362 WRITE (23) -1, -1, -1, -1 1363 ENDIF 1364 ENDIF 1365 #if defined( __parallel ) 1366 CALL MPI_BARRIER( comm2d, ierr ) 1367 #endif 1368 ENDDO 1345 1369 1346 1370 ELSE … … 1504 1528 1505 1529 IF ( data_output_2d_on_each_pe ) THEN 1506 CALL close_file( file_id ) 1530 DO i = 0, io_blocks-1 1531 IF ( i == io_group ) THEN 1532 CALL close_file( file_id ) 1533 ENDIF 1534 #if defined( __parallel ) 1535 CALL MPI_BARRIER( comm2d, ierr ) 1536 #endif 1537 ENDDO 1507 1538 ELSE 1508 1539 IF ( myid == 0 ) CALL close_file( file_id ) -
palm/trunk/SOURCE/data_output_3d.f90
r728 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! Splitting of parallel I/O 6 7 ! 7 8 ! Former revisions: … … 395 396 !-- Determine the Skip-value for the next array. Record end and start 396 397 !-- require 4 byte each. 397 skip_do_avs = skip_do_avs + ( ((nx+2*nbgp)*(ny+2*nbgp)*(nz_do3d+1)) * 4 + 8 ) 398 skip_do_avs = skip_do_avs + ( ( ( nx+2*nbgp ) * ( ny+2*nbgp ) * & 399 ( nz_do3d+1 ) ) * 4 + 8 ) 398 400 ENDIF 399 401 … … 420 422 do3d_time_count(av), av 421 423 ENDIF 422 WRITE ( 30 ) nxlg, nxrg, nysg, nyng, nzb, nz_do3d 423 WRITE ( 30 ) local_pf 424 DO i = 0, io_blocks-1 425 IF ( i == io_group ) THEN 426 WRITE ( 30 ) nxlg, nxrg, nysg, nyng, nzb, nz_do3d 427 WRITE ( 30 ) local_pf 428 ENDIF 429 #if defined( __parallel ) 430 CALL MPI_BARRIER( comm2d, ierr ) 431 #endif 432 ENDDO 424 433 425 434 ELSE -
palm/trunk/SOURCE/header.f90
r708 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! 6 ! output of maximum number of parallel io streams 7 7 ! 8 8 ! Former revisions: … … 220 220 ENDIF 221 221 IF ( use_seperate_pe_for_dvrp_output ) WRITE ( io, 105 ) 222 IF ( numprocs /= maximum_parallel_io_streams ) THEN 223 WRITE ( io, 108 ) maximum_parallel_io_streams 224 ENDIF 222 225 #endif 223 226 WRITE ( io, 99 ) … … 1596 1599 37X,'because the job is running on an SMP-cluster') 1597 1600 107 FORMAT (37X,'A 1d-decomposition along ',A,' is used') 1601 108 FORMAT (37X,'Max. # of parallel I/O streams is ',I5) 1598 1602 #endif 1599 1603 110 FORMAT (/' Numerical Schemes:'/ & -
palm/trunk/SOURCE/init_3d_model.f90
r732 r759 7 7 ! Current revisions: 8 8 ! ------------------ 9 ! Splitting of parallel I/O 9 ! Splitting of parallel I/O in blocks of PEs 10 10 ! Bugfix: No zero assignments to volume_flow_initial and volume_flow_area in 11 11 ! case of normal restart runs. … … 938 938 IF ( TRIM( initializing_actions ) == 'cyclic_fill' ) THEN 939 939 940 CALL read_parts_of_var_list 941 CALL close_file( 13 ) 940 DO i = 0, io_blocks-1 941 IF ( i == io_group ) THEN 942 CALL read_parts_of_var_list 943 CALL close_file( 13 ) 944 ENDIF 945 #if defined( __parallel ) 946 CALL MPI_BARRIER( comm2d, ierr ) 947 #endif 948 ENDDO 942 949 943 950 ! … … 1020 1027 ! 1021 1028 !-- Read binary data from restart file 1022 !DO i = 0, io_blocks-11023 !IF ( i == io_group ) THEN1029 DO i = 0, io_blocks-1 1030 IF ( i == io_group ) THEN 1024 1031 CALL read_3d_binary 1025 !ENDIF1026 !#if defined( __parallel )1027 !CALL MPI_BARRIER( comm2d, ierr )1028 !#endif1029 !ENDDO1032 ENDIF 1033 #if defined( __parallel ) 1034 CALL MPI_BARRIER( comm2d, ierr ) 1035 #endif 1036 ENDDO 1030 1037 1031 1038 ! -
palm/trunk/SOURCE/init_grid.f90
r723 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! 6 ! Splitting of parallel I/O in blocks of PEs 7 7 ! 8 8 ! Former revisions: … … 80 80 81 81 INTEGER :: bh, blx, bly, bxl, bxr, byn, bys, ch, cwx, cwy, cxl, cxr, cyn, & 82 cys, gls, i, i nc, i_center, j, j_center, k, l, nxl_l, nxr_l,&83 n yn_l, nys_l, nzb_si, nzt_l, vi82 cys, gls, i, ii, inc, i_center, j, j_center, k, l, nxl_l, & 83 nxr_l, nyn_l, nys_l, nzb_si, nzt_l, vi 84 84 85 85 INTEGER, DIMENSION(:), ALLOCATABLE :: vertical_influence … … 507 507 508 508 CASE ( 'read_from_file' ) 509 ! 510 !-- Arbitrary irregular topography data in PALM format (exactly matching 511 !-- the grid size and total domain size) 512 OPEN( 90, FILE='TOPOGRAPHY_DATA', STATUS='OLD', FORM='FORMATTED', & 513 ERR=10 ) 514 DO j = ny, 0, -1 515 READ( 90, *, ERR=11, END=11 ) ( topo_height(j,i), i = 0, nx ) 516 ENDDO 509 510 DO ii = 0, io_blocks-1 511 IF ( ii == io_group ) THEN 512 513 ! 514 !-- Arbitrary irregular topography data in PALM format (exactly 515 !-- matching the grid size and total domain size) 516 OPEN( 90, FILE='TOPOGRAPHY_DATA', STATUS='OLD', & 517 FORM='FORMATTED', ERR=10 ) 518 DO j = ny, 0, -1 519 READ( 90, *, ERR=11, END=11 ) ( topo_height(j,i), i = 0,nx ) 520 ENDDO 521 522 GOTO 12 523 524 10 message_string = 'file TOPOGRAPHY_DATA does not exist' 525 CALL message( 'init_grid', 'PA0208', 1, 2, 0, 6, 0 ) 526 527 11 message_string = 'errors in file TOPOGRAPHY_DATA' 528 CALL message( 'init_grid', 'PA0209', 1, 2, 0, 6, 0 ) 529 530 12 CLOSE( 90 ) 531 532 ENDIF 533 #if defined( __parallel ) 534 CALL MPI_BARRIER( comm2d, ierr ) 535 #endif 536 ENDDO 537 517 538 ! 518 539 !-- Calculate the index height of the topography … … 523 544 ENDDO 524 545 ! 525 !-- Add cyclic boundaries (additional layers are for calculating flag526 !-- arrays needed for the multigrid sover)546 !-- Add cyclic boundaries (additional layers are for calculating 547 !-- flag arrays needed for the multigrid sover) 527 548 nzb_local(-gls:-1,0:nx) = nzb_local(ny-gls+1:ny,0:nx) 528 549 nzb_local(ny+1:ny+gls,0:nx) = nzb_local(0:gls-1,0:nx) 529 550 nzb_local(:,-gls:-1) = nzb_local(:,nx-gls+1:nx) 530 551 nzb_local(:,nx+1:nx+gls) = nzb_local(:,0:gls-1) 531 532 533 534 GOTO 12535 536 10 message_string = 'file TOPOGRAPHY_DATA does not exist'537 CALL message( 'init_grid', 'PA0208', 1, 2, 0, 6, 0 )538 539 11 message_string = 'errors in file TOPOGRAPHY_DATA'540 CALL message( 'init_grid', 'PA0209', 1, 2, 0, 6, 0 )541 542 12 CLOSE( 90 )543 552 544 553 CASE DEFAULT -
palm/trunk/SOURCE/init_pegrid.f90
r756 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! calculation of number of io_blocks and the io_group to which the respective 7 ! PE belongs 6 8 ! 7 9 ! ATTENTION: nnz_x undefined problem still has to be solved!!!!!!!! … … 1237 1239 ENDIF 1238 1240 1241 ! 1242 !-- Calculate the number of groups into which parallel I/O is split. 1243 !-- The default for files which are opened by all PEs (or where each 1244 !-- PE opens his own independent file) is, that all PEs are doing input/output 1245 !-- in parallel at the same time. This might cause performance or even more 1246 !-- severe problems depending on the configuration of the underlying file 1247 !-- system. 1248 !-- First, set the default: 1249 IF ( maximum_parallel_io_streams == -1 .OR. & 1250 maximum_parallel_io_streams > numprocs ) THEN 1251 maximum_parallel_io_streams = numprocs 1252 ENDIF 1253 1254 ! 1255 !-- Now calculate the number of io_blocks and the io_group to which the 1256 !-- respective PE belongs. I/O of the groups is done in serial, but in parallel 1257 !-- for all PEs belonging to the same group. A preliminary setting with myid 1258 !-- based on MPI_COMM_WORLD has been done in parin. 1259 io_blocks = numprocs / maximum_parallel_io_streams 1260 io_group = MOD( myid+1, io_blocks ) 1261 1262 1239 1263 END SUBROUTINE init_pegrid -
palm/trunk/SOURCE/modules.f90
r744 r759 5 5 ! Current revisions: 6 6 ! ----------------- 7 ! +io_blocks, io_group, maximum_parallel_io_streams, 8 ! synchronous_exchange moved to control_parameters 7 9 ! 8 10 ! Former revisions: … … 462 464 inflow_disturbance_begin = -1, inflow_disturbance_end = -1, & 463 465 intermediate_timestep_count, intermediate_timestep_count_max, & 464 iran = -1234567, last_dt_change = 0, masks = 0, & 465 maximum_grid_level, max_pr_user = 0, mgcycles = 0, & 466 mg_cycles = -1, mg_switch_to_pe0_level = 0, mid, & 466 io_group = 0, io_blocks = 1, iran = -1234567, & 467 last_dt_change = 0, masks = 0, maximum_grid_level, & 468 maximum_parallel_io_streams = -1, max_pr_user = 0, & 469 mgcycles = 0, mg_cycles = -1, mg_switch_to_pe0_level = 0, mid, & 467 470 netcdf_data_format = 2, ngsrb = 2, nsor = 20, & 468 471 nsor_ini = 100, n_sor, normalizing_region = 0, & … … 529 532 random_heatflux = .FALSE., run_control_header = .FALSE., & 530 533 run_coupled = .TRUE., sloping_surface = .FALSE., & 531 stop_dt = .FALSE., terminate_run= .FALSE., &532 t urbulent_inflow = .FALSE., &534 stop_dt = .FALSE., synchronous_exchange = .FALSE., & 535 terminate_run = .FALSE., turbulent_inflow = .FALSE., & 533 536 use_prior_plot1d_parameters = .FALSE., use_reference = .FALSE.,& 534 537 use_surface_fluxes = .FALSE., use_top_fluxes = .FALSE., & … … 1250 1253 LOGICAL :: collective_wait = .FALSE., left_border_pe = .FALSE., & 1251 1254 north_border_pe = .FALSE., reorder = .TRUE., & 1252 right_border_pe = .FALSE., south_border_pe = .FALSE., & 1253 synchronous_exchange = .FALSE. 1255 right_border_pe = .FALSE., south_border_pe = .FALSE. 1254 1256 1255 1257 LOGICAL, DIMENSION(2) :: cyclic = (/ .TRUE. , .TRUE. /), & -
palm/trunk/SOURCE/palm.f90
r715 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! 6 ! Splitting of parallel I/O, cpu measurement for write_3d_binary and opening 7 ! of unit 14 moved to here 7 8 ! 8 9 ! Former revisions: … … 175 176 !-- If required, write binary data for restart runs 176 177 IF ( write_binary(1:4) == 'true' ) THEN 177 ! 178 !-- Write flow field data 179 CALL write_3d_binary 178 179 CALL cpu_log( log_point(22), 'write_3d_binary', 'start' ) 180 181 CALL check_open( 14 ) 182 183 DO i = 0, io_blocks-1 184 IF ( i == io_group ) THEN 185 ! 186 !-- Write flow field data 187 CALL write_3d_binary 188 ENDIF 189 #if defined( __parallel ) 190 CALL MPI_BARRIER( comm2d, ierr ) 191 #endif 192 ENDDO 193 194 CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' ) 195 180 196 ! 181 197 !-- If required, write particle data … … 192 208 !-- unit in routine user_last_actions. 193 209 CALL cpu_log( log_point(4), 'last actions', 'start' ) 194 CALL user_last_actions 195 IF ( write_binary(1:4) == 'true' ) CALL close_file( 14 ) 210 DO i = 0, io_blocks-1 211 IF ( i == io_group ) THEN 212 CALL user_last_actions 213 IF ( write_binary(1:4) == 'true' ) CALL close_file( 14 ) 214 ENDIF 215 #if defined( __parallel ) 216 CALL MPI_BARRIER( comm2d, ierr ) 217 #endif 218 ENDDO 196 219 CALL close_file( 0 ) 197 220 CALL close_dvrp -
palm/trunk/SOURCE/parin.f90
r684 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! 6 ! +maximum_parallel_io_streams in envpar, 7 ! splitting of parallel I/O in blocks of PEs 7 8 ! 8 9 ! Former revisions: … … 117 118 IMPLICIT NONE 118 119 119 INTEGER :: i dum120 INTEGER :: i, idum 120 121 121 122 … … 183 184 dt_run_control,end_time, force_print_header, mask_scale_x, & 184 185 mask_scale_y, mask_scale_z, mask_x, mask_y, mask_z, mask_x_loop, & 185 mask_y_loop, mask_z_loop, netcdf_data_format, & 186 normalizing_region, npex, npey, nz_do3d, & 187 precipitation_amount_interval, profile_columns, profile_rows, & 188 restart_time, section_xy, section_xz, section_yz, & 189 skip_time_data_output, skip_time_data_output_av, skip_time_dopr, & 190 skip_time_do2d_xy, skip_time_do2d_xz, skip_time_do2d_yz, & 191 skip_time_do3d, skip_time_domask, synchronous_exchange, & 192 termination_time_needed, use_prior_plot1d_parameters, z_max_do1d, & 193 z_max_do1d_normalized, z_max_do2d 194 195 196 NAMELIST /envpar/ host, local_dvrserver_running, maximum_cpu_time_allowed, & 197 revision, return_addres, return_username, run_identifier, & 198 tasks_per_node, write_binary 199 200 ! 201 !-- Open the NAMELIST-file which is send with this job 202 CALL check_open( 11 ) 203 204 ! 205 !-- Read the control parameters for initialization. 206 !-- The namelist "inipar" must be provided in the NAMELIST-file. If this is 207 !-- not the case and the file contains - instead of "inipar" - any other 208 !-- namelist, a read error is created on t3e and control is transferred 209 !-- to the statement with label 10. Therefore, on t3e machines one can not 210 !-- distinguish between errors produced by a wrong "inipar" namelist or 211 !-- because this namelist is totally missing. 212 READ ( 11, inipar, ERR=10, END=11 ) 213 GOTO 12 214 10 message_string = 'errors in \$inipar &or no \$inipar-namelist ' // & 215 'found (CRAY-machines only)' 216 CALL message( 'parin', 'PA0271', 1, 2, 0, 6, 0 ) 217 218 11 message_string = 'no \$inipar-namelist found' 219 CALL message( 'parin', 'PA0272', 1, 2, 0, 6, 0 ) 220 221 ! 222 !-- If required, read control parameters from restart file (produced by 223 !-- a prior run). All PEs are reading from file created by PE0 (see check_open) 224 12 IF ( TRIM( initializing_actions ) == 'read_restart_data' ) THEN 225 226 CALL read_var_list 227 ! 228 !-- The restart file will be reopened when reading the subdomain data 229 CALL close_file( 13 ) 230 231 ! 232 !-- Increment the run count 233 runnr = runnr + 1 234 235 ENDIF 236 237 ! 238 !-- Definition of names of areas used for computing statistics. They must 239 !-- be defined at this place, because they are allowed to be redefined by 240 !-- the user in user_parin. 241 region = 'total domain' 242 243 ! 244 !-- Read runtime parameters given by the user for this run (namelist "d3par"). 245 !-- The namelist "d3par" can be omitted. In that case, default values are 246 !-- used for the parameters. 247 READ ( 11, d3par, END=20 ) 248 249 ! 250 !-- Read control parameters for optionally used model software packages 251 20 CALL package_parin 252 253 ! 254 !-- Read user-defined variables 255 CALL user_parin 256 257 ! 258 !-- Check in case of initial run, if the grid point numbers are well defined 259 !-- and allocate some arrays which are already needed in init_pegrid or 260 !-- check_parameters. During restart jobs, these arrays will be allocated 261 !-- in read_var_list. All other arrays are allocated in init_3d_model. 262 IF ( TRIM( initializing_actions ) /= 'read_restart_data' ) THEN 263 264 IF ( nx <= 0 ) THEN 265 WRITE( message_string, * ) 'no value or wrong value given for nx: ',& 266 'nx=', nx 267 CALL message( 'parin', 'PA0273', 1, 2, 0, 6, 0 ) 268 ENDIF 269 IF ( ny <= 0 ) THEN 270 WRITE( message_string, * ) 'no value or wrong value given for ny: ',& 271 'ny=', ny 272 CALL message( 'parin', 'PA0274', 1, 2, 0, 6, 0 ) 273 ENDIF 274 IF ( nz <= 0 ) THEN 275 WRITE( message_string, * ) 'no value or wrong value given for nz: ',& 276 'nz=', nz 277 CALL message( 'parin', 'PA0275', 1, 2, 0, 6, 0 ) 278 ENDIF 279 ! 280 !-- ATTENTION: in case of changes to the following statement please also 281 !-- check the allocate statement in routine read_var_list 282 ALLOCATE( lad(0:nz+1),pt_init(0:nz+1), q_init(0:nz+1), sa_init(0:nz+1), & 283 ug(0:nz+1), u_init(0:nz+1), v_init(0:nz+1), vg(0:nz+1), & 284 hom(0:nz+1,2,pr_palm+max_pr_user,0:statistic_regions), & 285 hom_sum(0:nz+1,pr_palm+max_pr_user,0:statistic_regions) ) 286 287 hom = 0.0 288 289 ENDIF 290 291 ! 292 !-- NAMELIST-file is not needed anymore 293 CALL close_file( 11 ) 294 295 ! 296 !-- Read values of environment variables (this NAMELIST file is generated by 297 !-- mrun) 186 mask_y_loop, mask_z_loop, netcdf_data_format, normalizing_region, & 187 npex, npey, nz_do3d, precipitation_amount_interval, & 188 profile_columns, profile_rows, restart_time, section_xy, & 189 section_xz, section_yz, skip_time_data_output, & 190 skip_time_data_output_av, skip_time_dopr, skip_time_do2d_xy, & 191 skip_time_do2d_xz, skip_time_do2d_yz, skip_time_do3d, & 192 skip_time_domask, synchronous_exchange, termination_time_needed, & 193 use_prior_plot1d_parameters, z_max_do1d, z_max_do1d_normalized, & 194 z_max_do2d 195 196 197 NAMELIST /envpar/ host, local_dvrserver_running, maximum_cpu_time_allowed,& 198 maximum_parallel_io_streams, revision, return_addres, & 199 return_username, run_identifier, tasks_per_node, & 200 write_binary 201 202 ! 203 !-- First read values of environment variables (this NAMELIST file is 204 !-- generated by mrun) 298 205 OPEN ( 90, FILE='ENVPAR', STATUS='OLD', FORM='FORMATTED', ERR=30 ) 299 206 READ ( 90, envpar, ERR=31, END=32 ) 300 207 CLOSE ( 90 ) 208 209 ! 210 !-- Calculate the number of groups into which parallel I/O is split. 211 !-- The default for files which are opened by all PEs (or where each 212 !-- PE opens his own independent file) is, that all PEs are doing input/output 213 !-- in parallel at the same time. This might cause performance or even more 214 !-- severe problems depending on the configuration of the underlying file 215 !-- system. 216 !-- First, set the default: 217 IF ( maximum_parallel_io_streams == -1 .OR. & 218 maximum_parallel_io_streams > numprocs ) THEN 219 maximum_parallel_io_streams = numprocs 220 ENDIF 221 ! 222 !-- Now calculate the number of io_blocks and the io_group to which the 223 !-- respective PE belongs. I/O of the groups is done in serial, but in parallel 224 !-- for all PEs belonging to the same group. 225 !-- These settings are repeated in init_pegrid for the communicator comm2d, 226 !-- which is not available here 227 io_blocks = numprocs / maximum_parallel_io_streams 228 io_group = MOD( myid+1, io_blocks ) 229 230 ! 231 !-- Data is read in parallel by groups of PEs 232 DO i = 0, io_blocks-1 233 IF ( i == io_group ) THEN 234 235 ! 236 !-- Open the NAMELIST-file which is send with this job 237 CALL check_open( 11 ) 238 239 ! 240 !-- Read the control parameters for initialization. 241 !-- The namelist "inipar" must be provided in the NAMELIST-file. If this 242 !-- is not the case and the file contains - instead of "inipar" - any 243 !-- other namelist, a read error is created on t3e and control is 244 !-- transferred to the statement with label 10. Therefore, on t3e 245 !-- machines one can not distinguish between errors produced by a wrong 246 !-- "inipar" namelist or because this namelist is totally missing. 247 READ ( 11, inipar, ERR=10, END=11 ) 248 GOTO 12 249 10 message_string = 'errors in \$inipar &or no \$inipar-namelist ' // & 250 'found (CRAY-machines only)' 251 CALL message( 'parin', 'PA0271', 1, 2, 0, 6, 0 ) 252 253 11 message_string = 'no \$inipar-namelist found' 254 CALL message( 'parin', 'PA0272', 1, 2, 0, 6, 0 ) 255 256 ! 257 !-- If required, read control parameters from restart file (produced by 258 !-- a prior run). All PEs are reading from file created by PE0 (see 259 !-- check_open) 260 12 IF ( TRIM( initializing_actions ) == 'read_restart_data' ) THEN 261 262 CALL read_var_list 263 ! 264 !-- The restart file will be reopened when reading the subdomain data 265 CALL close_file( 13 ) 266 267 ! 268 !-- Increment the run count 269 runnr = runnr + 1 270 271 ENDIF 272 273 ! 274 !-- Definition of names of areas used for computing statistics. They must 275 !-- be defined at this place, because they are allowed to be redefined by 276 !-- the user in user_parin. 277 region = 'total domain' 278 279 ! 280 !-- Read runtime parameters given by the user for this run (namelist 281 !-- "d3par"). The namelist "d3par" can be omitted. In that case, default 282 !-- values are used for the parameters. 283 READ ( 11, d3par, END=20 ) 284 285 ! 286 !-- Read control parameters for optionally used model software packages 287 20 CALL package_parin 288 289 ! 290 !-- Read user-defined variables 291 CALL user_parin 292 293 ! 294 !-- Check in case of initial run, if the grid point numbers are well 295 !-- defined and allocate some arrays which are already needed in 296 !-- init_pegrid or check_parameters. During restart jobs, these arrays 297 !-- will be allocated in read_var_list. All other arrays are allocated 298 !-- in init_3d_model. 299 IF ( TRIM( initializing_actions ) /= 'read_restart_data' ) THEN 300 301 IF ( nx <= 0 ) THEN 302 WRITE( message_string, * ) 'no value or wrong value given', & 303 ' for nx: nx=', nx 304 CALL message( 'parin', 'PA0273', 1, 2, 0, 6, 0 ) 305 ENDIF 306 IF ( ny <= 0 ) THEN 307 WRITE( message_string, * ) 'no value or wrong value given', & 308 ' for ny: ny=', ny 309 CALL message( 'parin', 'PA0274', 1, 2, 0, 6, 0 ) 310 ENDIF 311 IF ( nz <= 0 ) THEN 312 WRITE( message_string, * ) 'no value or wrong value given', & 313 ' for nz: nz=', nz 314 CALL message( 'parin', 'PA0275', 1, 2, 0, 6, 0 ) 315 ENDIF 316 ! 317 !-- ATTENTION: in case of changes to the following statement please 318 !-- also check the allocate statement in routine read_var_list 319 ALLOCATE( lad(0:nz+1),pt_init(0:nz+1), q_init(0:nz+1), & 320 sa_init(0:nz+1), ug(0:nz+1), u_init(0:nz+1), & 321 v_init(0:nz+1), vg(0:nz+1), & 322 hom(0:nz+1,2,pr_palm+max_pr_user,0:statistic_regions), & 323 hom_sum(0:nz+1,pr_palm+max_pr_user,0:statistic_regions) ) 324 325 hom = 0.0 326 327 ENDIF 328 329 ! 330 !-- NAMELIST-file is not needed anymore 331 CALL close_file( 11 ) 332 333 ENDIF 334 #if defined( __parallel ) 335 CALL MPI_BARRIER( MPI_COMM_WORLD, ierr ) 336 #endif 337 ENDDO 338 301 339 RETURN 302 340 -
palm/trunk/SOURCE/prandtl_fluxes.f90
r710 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! 6 ! Bugfix for ts limitation 7 7 ! 8 8 ! Former revisions: … … 74 74 !-- ts must be limited, because otherwise overflow may occur in case of 75 75 !-- us=0 when computing rif further below 76 IF ( ts(j,i) < -1.05E5 ) ts = -1.0E577 IF ( ts(j,i) > 1.0E5 ) ts = 1.0E576 IF ( ts(j,i) < -1.05E5 ) ts(j,i) = -1.0E5 77 IF ( ts(j,i) > 1.0E5 ) ts(j,i) = 1.0E5 78 78 ENDDO 79 79 ENDDO -
palm/trunk/SOURCE/production_e.f90
r668 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! initialization of u_0, v_0 6 7 ! 7 8 ! Former revisions: … … 1111 1112 1112 1113 IF ( first_call ) THEN 1113 ALLOCATE( u_0(nysg:nyng,nxlg:nxrg), & 1114 v_0(nysg:nyng,nxlg:nxrg) ) 1114 ALLOCATE( u_0(nysg:nyng,nxlg:nxrg), v_0(nysg:nyng,nxlg:nxrg) ) 1115 u_0 = 0.0 ! just to avoid access of uninitialized memory 1116 v_0 = 0.0 ! within exchange_horiz_2d 1115 1117 first_call = .FALSE. 1116 1118 ENDIF -
palm/trunk/SOURCE/write_3d_binary.f90
r588 r759 4 4 ! Current revisions: 5 5 ! ----------------- 6 ! 6 ! cpu measurement and file opening moved to main program 7 7 ! 8 8 ! Former revisions: … … 67 67 68 68 69 CALL cpu_log( log_point(22), 'write_3d_binary', 'start' )70 71 CALL check_open( 14 )72 73 69 ! 74 70 !-- Write control parameters and other variables for restart. … … 292 288 WRITE ( 14 ) '*** end *** ' 293 289 294 295 CALL cpu_log( log_point(22), 'write_3d_binary', 'stop' )296 297 298 290 END SUBROUTINE write_3d_binary
Note: See TracChangeset
for help on using the changeset viewer.
