[1029] | 1 | #!/bin/ksh |
---|
[1046] | 2 | #--------------------------------------------------------------------------------# |
---|
| 3 | # This file is part of PALM. |
---|
| 4 | # |
---|
| 5 | # PALM is free software: you can redistribute it and/or modify it under the terms |
---|
| 6 | # of the GNU General Public License as published by the Free Software Foundation, |
---|
| 7 | # either version 3 of the License, or (at your option) any later version. |
---|
| 8 | # |
---|
| 9 | # PALM is distributed in the hope that it will be useful, but WITHOUT ANY |
---|
| 10 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
---|
| 11 | # A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
---|
| 12 | # |
---|
| 13 | # You should have received a copy of the GNU General Public License along with |
---|
| 14 | # PALM. If not, see <http://www.gnu.org/licenses/>. |
---|
| 15 | # |
---|
[1310] | 16 | # Copyright 1997-2014 Leibniz Universitaet Hannover |
---|
[1046] | 17 | #--------------------------------------------------------------------------------# |
---|
| 18 | # |
---|
[1029] | 19 | # Current revisions: |
---|
| 20 | # ----------------- |
---|
[1446] | 21 | # Adapted for HLRN III. Added windows if no runs are queued. |
---|
| 22 | # |
---|
[1029] | 23 | # Former revisions: |
---|
| 24 | # ----------------- |
---|
| 25 | # $Id: hlrn_watchdog 1446 2014-08-07 10:08:56Z maronga $ |
---|
| 26 | # |
---|
[1047] | 27 | # 1046 2012-11-09 14:38:45Z maronga |
---|
| 28 | # code put under GPL (PALM 3.9) |
---|
| 29 | # |
---|
[1030] | 30 | # 1029 2012-10-17 15:33:56Z maronga |
---|
| 31 | # Initial revision |
---|
[1029] | 32 | # |
---|
| 33 | # Description: |
---|
| 34 | # ------------ |
---|
| 35 | # The hlrn_watchdog works for jobs on HLRN in Hannover/Berlin and with KDE. |
---|
| 36 | # It can be used for monitoring currently submitted jobs and will display |
---|
| 37 | # Running, Idle and Blocked jobs in a separate window, which is updated every |
---|
| 38 | # 10 minutes. |
---|
| 39 | # Starting: "hlrn_watchdog start <your hlrn username>" |
---|
| 40 | # Stopping: "hlrn_watchdog stop" |
---|
| 41 | #------------------------------------------------------------------------------! |
---|
| 42 | |
---|
[1446] | 43 | check_hannover=false |
---|
| 44 | check_berlin=true |
---|
| 45 | debug=true |
---|
| 46 | PALM_BIN="/home/maronga/palm/tmp_version/trunk/SCRIPTS" |
---|
| 47 | |
---|
[1029] | 48 | cd $PALM_BIN |
---|
| 49 | |
---|
[1446] | 50 | gate_h="hlogin.hlrn.de" |
---|
| 51 | gate_b="blogin.hlrn.de" |
---|
[1029] | 52 | update_frequency=600 |
---|
| 53 | |
---|
| 54 | # trap strg+c |
---|
[1446] | 55 | trap 'kill -9 $infoPID > /dev/null; kill -9 $dialogPID > /dev/null; exit' 2 |
---|
[1029] | 56 | |
---|
| 57 | |
---|
| 58 | # start/stop routine |
---|
| 59 | if [[ $1 == "stop" ]] |
---|
| 60 | then |
---|
| 61 | result=`ps aux|grep -c "./hlrn_watchdog"` |
---|
| 62 | if (( $result > 2 )) |
---|
| 63 | then |
---|
| 64 | result=`ps aux|grep -m1 "./hlrn_watchdog"` |
---|
| 65 | killid=`echo $result | tr -s " " | cut -d" " -s -f2` |
---|
| 66 | kill -9 $killid > /dev/null |
---|
| 67 | killall kdialog |
---|
[1446] | 68 | |
---|
| 69 | if [[ -f .watchdog_report.x ]] then |
---|
| 70 | rm .watchdog_report.x |
---|
| 71 | fi |
---|
| 72 | if [[ -f .watchdog_status.x ]] then |
---|
| 73 | rm .watchdog_status.x |
---|
| 74 | fi |
---|
[1029] | 75 | echo "*** hlrn_watchdog stopped." |
---|
| 76 | else |
---|
| 77 | echo "+++ hlrn_watchdog is not running." |
---|
| 78 | fi |
---|
| 79 | exit |
---|
| 80 | elif [[ $1 == "start" ]] |
---|
| 81 | then |
---|
| 82 | result=`ps aux|grep -c "hlrn_watchdog"` |
---|
| 83 | if (( $result > 2 )) |
---|
| 84 | then |
---|
| 85 | echo "+++ hlrn_watchdog is already running." |
---|
| 86 | else |
---|
[1446] | 87 | if [[ $debug = true ]] then |
---|
| 88 | ./hlrn_watchdog $2 & |
---|
| 89 | else |
---|
| 90 | nohup ./hlrn_watchdog $2 1> /dev/null 2> /dev/null & |
---|
| 91 | fi |
---|
[1029] | 92 | echo "\n*** hlrn_watchdog starting..." |
---|
| 93 | fi |
---|
| 94 | exit |
---|
| 95 | else |
---|
| 96 | # login via ssh and collect information in .watchdog_report.x |
---|
| 97 | while true |
---|
| 98 | do |
---|
[1446] | 99 | touch .watchdog_report.x |
---|
| 100 | if [[ $check_hannover == true ]] then |
---|
| 101 | ssh $gate_h -l $1 "showq | egrep \"($1)\"" > .watchdog_report.x |
---|
| 102 | fi |
---|
| 103 | if [[ $check_berlin == true ]] then |
---|
[1029] | 104 | ssh $gate_b -l $1 "showq | egrep \"($1)\"" >> .watchdog_report.x |
---|
[1446] | 105 | fi |
---|
[1029] | 106 | |
---|
| 107 | i=0 |
---|
| 108 | j=0 |
---|
| 109 | cat .watchdog_report.x|while read variable |
---|
| 110 | do |
---|
| 111 | # analyze output |
---|
| 112 | comid[$i]=`echo $variable | tr -s " " | cut -d" " -s -f1` |
---|
| 113 | jobid[$i]=`echo ${comid[$i]} | tr -s " " | cut -d"." -s -f2` |
---|
| 114 | complex[$i]=`echo ${comid[$i]} | tr -s " " | cut -d"." -s -f1` |
---|
| 115 | username[$i]=`echo $variable | tr -s " " | cut -d" " -s -f2` |
---|
| 116 | status[$i]=`echo $variable | tr -s " " | cut -d" " -s -f3` |
---|
| 117 | nodes[$i]=`echo $variable | tr -s " " | cut -d" " -s -f4` |
---|
| 118 | walltime[$i]=`echo $variable | tr -s " " | cut -d" " -s -f5` |
---|
| 119 | day[$i]=`echo $variable | tr -s " " | cut -d" " -s -f6` |
---|
| 120 | month[$i]=`echo $variable | tr -s " " | cut -d" " -s -f7` |
---|
| 121 | date[$i]=`echo $variable | tr -s " " | cut -d" " -s -f8` |
---|
| 122 | subtime[$i]=`echo $variable | tr -s " " | cut -d" " -s -f9` |
---|
| 123 | ((i = i + 1)) |
---|
| 124 | done |
---|
| 125 | rm .watchdog_report.x |
---|
[1446] | 126 | touch .watchdog_status.x |
---|
[1029] | 127 | |
---|
| 128 | # check for terminated jobs and status changes |
---|
| 129 | k=0 |
---|
| 130 | cat .watchdog_status.x|while read variable |
---|
| 131 | do |
---|
| 132 | # analyze output |
---|
| 133 | old_comid[$k]=`echo $variable | tr -s " " | cut -d" " -s -f1` |
---|
| 134 | old_status[$k]=`echo $variable | tr -s " " | cut -d" " -s -f2` |
---|
| 135 | ((k = k + 1)) |
---|
| 136 | done |
---|
| 137 | rm .watchdog_status.x |
---|
| 138 | |
---|
| 139 | info="" |
---|
| 140 | for ((m=0;m<$k;m++)) |
---|
| 141 | do |
---|
| 142 | found=0 |
---|
| 143 | for ((n=0;n<$i;n++)) |
---|
| 144 | do |
---|
| 145 | if [[ ${old_comid[$m]} == ${comid[$n]} ]] |
---|
| 146 | then |
---|
| 147 | if [[ ${old_status[$m]} != ${status[$n]} ]] |
---|
| 148 | then |
---|
| 149 | info="$info ${old_comid[$m]} has changed status from ${old_status[$m]} to ${status[$n]}.\n" |
---|
| 150 | fi |
---|
| 151 | found=1 |
---|
| 152 | break |
---|
| 153 | fi |
---|
| 154 | done |
---|
| 155 | if (( $found == 0 )) |
---|
| 156 | then |
---|
| 157 | info="$info ${old_comid[$m]} has been terminated (Status was ${old_status[$m]}).\n" |
---|
| 158 | fi |
---|
| 159 | done |
---|
| 160 | |
---|
[1446] | 161 | # check whether any jobs are queued |
---|
| 162 | touch .watchdog_report.x |
---|
| 163 | file_size=`ls -l .watchdog_report.x | tr -s " " | cut -d " " -f 5` |
---|
| 164 | if [[ $file_size == 0 ]] then |
---|
| 165 | printf "No jobs queued.\n" >> .watchdog_report.x |
---|
| 166 | fi |
---|
| 167 | |
---|
[1029] | 168 | # get estimated starting time for all idle jobs and write watchdog output in .watchdog_report.x |
---|
| 169 | while (( $j < $i )) |
---|
| 170 | do |
---|
| 171 | if [[ ${status[$j]} == "Idle" ]] |
---|
| 172 | then |
---|
| 173 | if [[ ${complex[$j]} == "hannover" ]] |
---|
| 174 | then |
---|
| 175 | eststart[$j]=`ssh $gate_h -l $1 "showstart ${comid[$j]}|grep \"based start in\""` |
---|
| 176 | else |
---|
| 177 | eststart[$j]=`ssh $gate_b -l $1 "showstart ${comid[$j]}|grep \"based start in\""` |
---|
| 178 | fi |
---|
| 179 | eststart[$j]="Start in: "`echo ${eststart[$j]} | tr -s " " | cut -d" " -s -f6` |
---|
| 180 | else |
---|
| 181 | eststart[$j]="" |
---|
| 182 | fi |
---|
| 183 | |
---|
| 184 | # write final output line |
---|
| 185 | ((k = j + 1)) |
---|
| 186 | printf "%-9s%8s%3s%04i%13s%11s%2s%-s%3s%-20s\n" "Job: $k:" "${status[$j]}" " @ " "${nodes[$j]}" " nodes, time:" "${walltime[$j]}" " (" "${comid[$j]}" "). " "${eststart[$j]}" >> .watchdog_report.x |
---|
| 187 | printf "${comid[$j]} ${status[$j]}\n" >> .watchdog_status.x |
---|
| 188 | ((j = j + 1)) |
---|
| 189 | done |
---|
| 190 | |
---|
| 191 | # kill all windows |
---|
[1446] | 192 | if [[ "$infoPID" -ne "" ]] then |
---|
| 193 | kill -9 $infoPID > /dev/null |
---|
| 194 | unset $infoPID |
---|
| 195 | fi |
---|
| 196 | if [[ "$dialogPID" -ne "" ]] then |
---|
| 197 | kill -9 $dialogPID > /dev/null |
---|
| 198 | unset $dialogPID |
---|
| 199 | fi |
---|
[1029] | 200 | |
---|
| 201 | timestamp=`date` |
---|
| 202 | |
---|
| 203 | # create window and show information |
---|
[1446] | 204 | kdialog --textbox .watchdog_report.x 550 150 --title "HLRN watchdog (last update: $timestamp)" & dialogPID=$! |
---|
[1029] | 205 | |
---|
| 206 | # in case of status changes and terminated jobs, inform the user |
---|
| 207 | if [[ $info != "" ]] |
---|
| 208 | then |
---|
[1446] | 209 | kdialog --msgbox "$info" --title "HLRN Job Information" & infoPID=$! |
---|
[1029] | 210 | fi |
---|
| 211 | |
---|
| 212 | sleep $update_frequency |
---|
| 213 | done |
---|
| 214 | |
---|
| 215 | fi |
---|