1 | #!/bin/ksh |
---|
2 | #------------------------------------------------------------------------------! |
---|
3 | # Current revisions: |
---|
4 | # ----------------- |
---|
5 | # |
---|
6 | # Former revisions: |
---|
7 | # ----------------- |
---|
8 | # $Id: hlrn_watchdog 1030 2012-10-17 15:46:46Z knoop $ |
---|
9 | # |
---|
10 | # 1029 2012-10-17 15:33:56Z maronga |
---|
11 | # Initial revision |
---|
12 | # |
---|
13 | # Description: |
---|
14 | # ------------ |
---|
15 | # The hlrn_watchdog works for jobs on HLRN in Hannover/Berlin and with KDE. |
---|
16 | # It can be used for monitoring currently submitted jobs and will display |
---|
17 | # Running, Idle and Blocked jobs in a separate window, which is updated every |
---|
18 | # 10 minutes. |
---|
19 | # Starting: "hlrn_watchdog start <your hlrn username>" |
---|
20 | # Stopping: "hlrn_watchdog stop" |
---|
21 | #------------------------------------------------------------------------------! |
---|
22 | |
---|
23 | cd $PALM_BIN |
---|
24 | |
---|
25 | gate_h="hicegate.hlrn.de" |
---|
26 | gate_b="bicegate.hlrn.de" |
---|
27 | update_frequency=600 |
---|
28 | |
---|
29 | # trap strg+c |
---|
30 | trap 'killall kdialog; exit' 2 |
---|
31 | |
---|
32 | |
---|
33 | # start/stop routine |
---|
34 | if [[ $1 == "stop" ]] |
---|
35 | then |
---|
36 | result=`ps aux|grep -c "./hlrn_watchdog"` |
---|
37 | if (( $result > 2 )) |
---|
38 | then |
---|
39 | result=`ps aux|grep -m1 "./hlrn_watchdog"` |
---|
40 | killid=`echo $result | tr -s " " | cut -d" " -s -f2` |
---|
41 | kill -9 $killid > /dev/null |
---|
42 | killall kdialog |
---|
43 | echo "*** hlrn_watchdog stopped." |
---|
44 | else |
---|
45 | echo "+++ hlrn_watchdog is not running." |
---|
46 | fi |
---|
47 | exit |
---|
48 | elif [[ $1 == "start" ]] |
---|
49 | then |
---|
50 | result=`ps aux|grep -c "hlrn_watchdog"` |
---|
51 | if (( $result > 2 )) |
---|
52 | then |
---|
53 | echo "+++ hlrn_watchdog is already running." |
---|
54 | else |
---|
55 | nohup ./hlrn_watchdog $2 1> /dev/null 2> /dev/null & |
---|
56 | echo "\n*** hlrn_watchdog starting..." |
---|
57 | fi |
---|
58 | exit |
---|
59 | else |
---|
60 | |
---|
61 | # login via ssh and collect information in .watchdog_report.x |
---|
62 | while true |
---|
63 | do |
---|
64 | ssh $gate_h -l $1 "showq | egrep \"($1)\"" > .watchdog_report.x |
---|
65 | ssh $gate_b -l $1 "showq | egrep \"($1)\"" >> .watchdog_report.x |
---|
66 | |
---|
67 | i=0 |
---|
68 | j=0 |
---|
69 | cat .watchdog_report.x|while read variable |
---|
70 | do |
---|
71 | # analyze output |
---|
72 | comid[$i]=`echo $variable | tr -s " " | cut -d" " -s -f1` |
---|
73 | jobid[$i]=`echo ${comid[$i]} | tr -s " " | cut -d"." -s -f2` |
---|
74 | complex[$i]=`echo ${comid[$i]} | tr -s " " | cut -d"." -s -f1` |
---|
75 | username[$i]=`echo $variable | tr -s " " | cut -d" " -s -f2` |
---|
76 | status[$i]=`echo $variable | tr -s " " | cut -d" " -s -f3` |
---|
77 | nodes[$i]=`echo $variable | tr -s " " | cut -d" " -s -f4` |
---|
78 | walltime[$i]=`echo $variable | tr -s " " | cut -d" " -s -f5` |
---|
79 | day[$i]=`echo $variable | tr -s " " | cut -d" " -s -f6` |
---|
80 | month[$i]=`echo $variable | tr -s " " | cut -d" " -s -f7` |
---|
81 | date[$i]=`echo $variable | tr -s " " | cut -d" " -s -f8` |
---|
82 | subtime[$i]=`echo $variable | tr -s " " | cut -d" " -s -f9` |
---|
83 | ((i = i + 1)) |
---|
84 | done |
---|
85 | rm .watchdog_report.x |
---|
86 | |
---|
87 | # check for terminated jobs and status changes |
---|
88 | k=0 |
---|
89 | cat .watchdog_status.x|while read variable |
---|
90 | do |
---|
91 | # analyze output |
---|
92 | old_comid[$k]=`echo $variable | tr -s " " | cut -d" " -s -f1` |
---|
93 | old_status[$k]=`echo $variable | tr -s " " | cut -d" " -s -f2` |
---|
94 | ((k = k + 1)) |
---|
95 | done |
---|
96 | rm .watchdog_status.x |
---|
97 | |
---|
98 | info="" |
---|
99 | for ((m=0;m<$k;m++)) |
---|
100 | do |
---|
101 | found=0 |
---|
102 | for ((n=0;n<$i;n++)) |
---|
103 | do |
---|
104 | if [[ ${old_comid[$m]} == ${comid[$n]} ]] |
---|
105 | then |
---|
106 | if [[ ${old_status[$m]} != ${status[$n]} ]] |
---|
107 | then |
---|
108 | info="$info ${old_comid[$m]} has changed status from ${old_status[$m]} to ${status[$n]}.\n" |
---|
109 | fi |
---|
110 | found=1 |
---|
111 | break |
---|
112 | fi |
---|
113 | done |
---|
114 | if (( $found == 0 )) |
---|
115 | then |
---|
116 | info="$info ${old_comid[$m]} has been terminated (Status was ${old_status[$m]}).\n" |
---|
117 | fi |
---|
118 | done |
---|
119 | |
---|
120 | # get estimated starting time for all idle jobs and write watchdog output in .watchdog_report.x |
---|
121 | while (( $j < $i )) |
---|
122 | do |
---|
123 | if [[ ${status[$j]} == "Idle" ]] |
---|
124 | then |
---|
125 | if [[ ${complex[$j]} == "hannover" ]] |
---|
126 | then |
---|
127 | eststart[$j]=`ssh $gate_h -l $1 "showstart ${comid[$j]}|grep \"based start in\""` |
---|
128 | else |
---|
129 | eststart[$j]=`ssh $gate_b -l $1 "showstart ${comid[$j]}|grep \"based start in\""` |
---|
130 | fi |
---|
131 | eststart[$j]="Start in: "`echo ${eststart[$j]} | tr -s " " | cut -d" " -s -f6` |
---|
132 | else |
---|
133 | eststart[$j]="" |
---|
134 | fi |
---|
135 | |
---|
136 | # write final output line |
---|
137 | ((k = j + 1)) |
---|
138 | printf "%-9s%8s%3s%04i%13s%11s%2s%-s%3s%-20s\n" "Job: $k:" "${status[$j]}" " @ " "${nodes[$j]}" " nodes, time:" "${walltime[$j]}" " (" "${comid[$j]}" "). " "${eststart[$j]}" >> .watchdog_report.x |
---|
139 | printf "${comid[$j]} ${status[$j]}\n" >> .watchdog_status.x |
---|
140 | ((j = j + 1)) |
---|
141 | done |
---|
142 | |
---|
143 | # kill all windows |
---|
144 | killall kdialog |
---|
145 | |
---|
146 | timestamp=`date` |
---|
147 | |
---|
148 | # create window and show information |
---|
149 | kdialog --textbox .watchdog_report.x 550 150 --title "HLRN watchdog (last update: $timestamp)" & |
---|
150 | |
---|
151 | # in case of status changes and terminated jobs, inform the user |
---|
152 | if [[ $info != "" ]] |
---|
153 | then |
---|
154 | kdialog --msgbox "$info" --title "HLRN Job Information" & |
---|
155 | fi |
---|
156 | |
---|
157 | sleep $update_frequency |
---|
158 | done |
---|
159 | |
---|
160 | fi |
---|