source: palm/trunk/SCRIPTS/hlrn_watchdog @ 1540

Last change on this file since 1540 was 1449, checked in by maronga, 10 years ago

last commit documented

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 7.5 KB
Line 
1#!/bin/ksh
2#--------------------------------------------------------------------------------#
3# This file is part of PALM.
4#
5# PALM is free software: you can redistribute it and/or modify it under the terms
6# of the GNU General Public License as published by the Free Software Foundation,
7# either version 3 of the License, or (at your option) any later version.
8#
9# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
10# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
11# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License along with
14# PALM. If not, see <http://www.gnu.org/licenses/>.
15#
16# Copyright 1997-2014  Leibniz Universitaet Hannover
17#--------------------------------------------------------------------------------#
18#
19# Current revisions:
20# -----------------
21#
22#
23# Former revisions:
24# -----------------
25# $Id: hlrn_watchdog 1449 2014-08-07 10:47:26Z raasch $
26#
27# 1448 2014-08-07 10:47:05Z maronga
28# Bugfix: removed debug mode
29#
30# 1446 2014-08-07 10:08:56Z maronga
31# Adapted for HLRN III. Added windows if no runs are queued.
32#
33# 1046 2012-11-09 14:38:45Z maronga
34# code put under GPL (PALM 3.9)
35#
36# 1029 2012-10-17 15:33:56Z maronga
37# Initial revision
38#
39# Description:
40# ------------
41# The hlrn_watchdog works for jobs on HLRN in Hannover/Berlin and with KDE.
42# It can be used for monitoring currently submitted jobs and will display
43# Running, Idle and Blocked jobs in a separate window, which is updated every
44# 10 minutes.
45# Starting: "hlrn_watchdog start <your hlrn username>"
46# Stopping: "hlrn_watchdog stop"
47#------------------------------------------------------------------------------!
48
49    check_hannover=true
50    check_berlin=true
51    debug=false
52
53    cd $PALM_BIN
54
55    gate_h="hlogin.hlrn.de"
56    gate_b="blogin.hlrn.de"
57    update_frequency=600
58
59#   trap strg+c
60    trap 'kill -9 $infoPID > /dev/null; kill -9 $dialogPID > /dev/null; exit' 2
61
62
63#   start/stop routine
64    if [[ $1 == "stop" ]]
65    then
66       result=`ps aux|grep -c "./hlrn_watchdog"`
67       if (( $result > 2 ))
68       then
69          result=`ps aux|grep -m1 "./hlrn_watchdog"`
70          killid=`echo $result | tr -s " " | cut -d" " -s -f2`
71          kill -9 $killid > /dev/null
72          killall kdialog
73
74          if [[ -f .watchdog_report.x ]] then
75             rm .watchdog_report.x
76          fi
77          if [[ -f .watchdog_status.x ]] then
78             rm .watchdog_status.x
79          fi
80          echo "*** hlrn_watchdog stopped."
81       else
82          echo "+++ hlrn_watchdog is not running."
83       fi
84       exit
85    elif [[ $1 == "start" ]]
86    then
87        result=`ps aux|grep -c "hlrn_watchdog"`
88        if (( $result > 2 ))
89        then
90           echo "+++ hlrn_watchdog is already running."
91        else
92           if [[ $debug = true ]] then
93              ./hlrn_watchdog $2 &
94           else
95              nohup ./hlrn_watchdog $2 1> /dev/null 2> /dev/null &
96           fi
97           echo "\n*** hlrn_watchdog starting..."
98        fi
99        exit
100    else
101#      login via ssh and collect information in .watchdog_report.x
102       while true
103       do
104          touch .watchdog_report.x
105          if [[ $check_hannover == true ]] then
106             ssh  $gate_h -l $1 "showq | egrep \"($1)\"" > .watchdog_report.x
107          fi
108          if [[ $check_berlin == true ]] then
109          ssh  $gate_b -l $1 "showq | egrep \"($1)\"" >> .watchdog_report.x
110          fi
111
112          i=0
113          j=0
114          cat .watchdog_report.x|while read variable
115          do
116#            analyze output
117             comid[$i]=`echo $variable | tr -s " " | cut -d" " -s -f1`
118             jobid[$i]=`echo ${comid[$i]} | tr -s " " | cut -d"." -s -f2`
119             complex[$i]=`echo ${comid[$i]} | tr -s " " | cut -d"." -s -f1`
120             username[$i]=`echo $variable | tr -s " " | cut -d" " -s -f2`
121             status[$i]=`echo $variable | tr -s " " | cut -d" " -s -f3`
122             nodes[$i]=`echo $variable | tr -s " " | cut -d" " -s -f4`
123             walltime[$i]=`echo $variable | tr -s " " | cut -d" " -s -f5` 
124             day[$i]=`echo $variable | tr -s " " | cut -d" " -s -f6`   
125             month[$i]=`echo $variable | tr -s " " | cut -d" " -s -f7`   
126             date[$i]=`echo $variable | tr -s " " | cut -d" " -s -f8`
127             subtime[$i]=`echo $variable | tr -s " " | cut -d" " -s -f9`   
128             ((i = i + 1))
129          done
130          rm .watchdog_report.x
131          touch .watchdog_status.x
132
133#         check for terminated jobs and status changes
134          k=0
135          cat .watchdog_status.x|while read variable
136          do
137#            analyze output
138             old_comid[$k]=`echo $variable | tr -s " " | cut -d" " -s -f1`
139             old_status[$k]=`echo $variable | tr -s " " | cut -d" " -s -f2` 
140             ((k = k + 1))
141          done
142          rm .watchdog_status.x
143
144          info=""
145          for ((m=0;m<$k;m++))
146          do
147             found=0
148             for ((n=0;n<$i;n++))
149             do       
150                if [[ ${old_comid[$m]} == ${comid[$n]} ]]
151                then
152                   if [[ ${old_status[$m]} != ${status[$n]} ]]
153                   then
154                       info="$info ${old_comid[$m]} has changed status from ${old_status[$m]} to ${status[$n]}.\n"
155                   fi
156                   found=1
157                   break
158                fi
159             done
160             if (( $found == 0 ))
161             then
162                info="$info ${old_comid[$m]} has been terminated (Status was ${old_status[$m]}).\n"
163             fi
164          done
165
166#         check whether any jobs are queued
167          touch .watchdog_report.x
168          file_size=`ls -l .watchdog_report.x | tr -s " " | cut -d " " -f 5`
169          if [[ $file_size == 0 ]] then
170            printf "No jobs queued.\n" >> .watchdog_report.x
171          fi
172
173#         get estimated starting time for all idle jobs and write watchdog output in .watchdog_report.x
174          while (( $j < $i ))
175          do
176             if [[ ${status[$j]} == "Idle" ]]
177             then
178                if [[ ${complex[$j]} == "hannover" ]]
179                then
180                   eststart[$j]=`ssh $gate_h -l $1 "showstart ${comid[$j]}|grep \"based start in\""`
181                else
182                   eststart[$j]=`ssh $gate_b -l $1 "showstart ${comid[$j]}|grep \"based start in\""`
183                fi
184                eststart[$j]="Start in: "`echo ${eststart[$j]} | tr -s " " | cut -d" " -s -f6`   
185             else
186                eststart[$j]=""
187             fi
188
189#            write final output line
190             ((k = j + 1))
191             printf "%-9s%8s%3s%04i%13s%11s%2s%-s%3s%-20s\n" "Job: $k:" "${status[$j]}" " @ " "${nodes[$j]}" " nodes, time:" "${walltime[$j]}" " (" "${comid[$j]}" "). " "${eststart[$j]}" >> .watchdog_report.x
192             printf "${comid[$j]} ${status[$j]}\n" >> .watchdog_status.x
193             ((j = j + 1))
194          done
195
196#         kill all windows
197          if  [[ "$infoPID" -ne "" ]] then
198             kill -9 $infoPID > /dev/null
199             unset $infoPID
200          fi
201          if  [[ "$dialogPID" -ne "" ]] then
202             kill -9 $dialogPID > /dev/null
203             unset $dialogPID
204          fi
205
206          timestamp=`date`
207
208#         create window and show information
209          kdialog --textbox .watchdog_report.x 550 150 --title "HLRN watchdog  (last update: $timestamp)" & dialogPID=$!
210
211#         in case of status changes and terminated jobs, inform the user
212          if [[ $info != "" ]]
213          then
214             kdialog --msgbox "$info" --title "HLRN Job Information" & infoPID=$!
215          fi
216
217          sleep $update_frequency
218       done
219
220    fi
Note: See TracBrowser for help on using the repository browser.