source: palm/trunk/SCRIPTS/hlrn_watchdog @ 1080

Last change on this file since 1080 was 1047, checked in by maronga, 12 years ago

last commit documented / added nc2vdf

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 6.3 KB
Line 
1#!/bin/ksh
2
3#--------------------------------------------------------------------------------#
4# This file is part of PALM.
5#
6# PALM is free software: you can redistribute it and/or modify it under the terms
7# of the GNU General Public License as published by the Free Software Foundation,
8# either version 3 of the License, or (at your option) any later version.
9#
10# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
11# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along with
15# PALM. If not, see <http://www.gnu.org/licenses/>.
16#
17# Copyright 1997-2012  Leibniz University Hannover
18#--------------------------------------------------------------------------------#
19#
20# Current revisions:
21# -----------------
22#
23# Former revisions:
24# -----------------
25# $Id: hlrn_watchdog 1047 2012-11-09 15:32:58Z maronga $
26#
27# 1046 2012-11-09 14:38:45Z maronga
28# code put under GPL (PALM 3.9)
29#
30# 1029 2012-10-17 15:33:56Z maronga
31# Initial revision
32#
33# Description:
34# ------------
35# The hlrn_watchdog works for jobs on HLRN in Hannover/Berlin and with KDE.
36# It can be used for monitoring currently submitted jobs and will display
37# Running, Idle and Blocked jobs in a separate window, which is updated every
38# 10 minutes.
39# Starting: "hlrn_watchdog start <your hlrn username>"
40# Stopping: "hlrn_watchdog stop"
41#------------------------------------------------------------------------------!
42
43    cd $PALM_BIN
44
45    gate_h="hicegate.hlrn.de"
46    gate_b="bicegate.hlrn.de"
47    update_frequency=600
48
49#   trap strg+c
50    trap 'killall kdialog; exit' 2
51
52
53#   start/stop routine
54    if [[ $1 == "stop" ]]
55    then
56       result=`ps aux|grep -c "./hlrn_watchdog"`
57       if (( $result > 2 ))
58       then
59          result=`ps aux|grep -m1 "./hlrn_watchdog"`
60          killid=`echo $result | tr -s " " | cut -d" " -s -f2`
61          kill -9 $killid > /dev/null
62          killall kdialog
63          echo "*** hlrn_watchdog stopped."
64       else
65          echo "+++ hlrn_watchdog is not running."
66       fi
67       exit
68    elif [[ $1 == "start" ]]
69    then
70        result=`ps aux|grep -c "hlrn_watchdog"`
71        if (( $result > 2 ))
72        then
73           echo "+++ hlrn_watchdog is already running."
74        else
75           nohup ./hlrn_watchdog $2 1> /dev/null 2> /dev/null &
76           echo "\n*** hlrn_watchdog starting..."
77        fi
78        exit
79    else
80
81#      login via ssh and collect information in .watchdog_report.x
82       while true
83       do
84          ssh  $gate_h -l $1 "showq | egrep \"($1)\"" > .watchdog_report.x
85          ssh  $gate_b -l $1 "showq | egrep \"($1)\"" >> .watchdog_report.x
86
87          i=0
88          j=0
89          cat .watchdog_report.x|while read variable
90          do
91#            analyze output
92             comid[$i]=`echo $variable | tr -s " " | cut -d" " -s -f1`
93             jobid[$i]=`echo ${comid[$i]} | tr -s " " | cut -d"." -s -f2`
94             complex[$i]=`echo ${comid[$i]} | tr -s " " | cut -d"." -s -f1`
95             username[$i]=`echo $variable | tr -s " " | cut -d" " -s -f2`
96             status[$i]=`echo $variable | tr -s " " | cut -d" " -s -f3`
97             nodes[$i]=`echo $variable | tr -s " " | cut -d" " -s -f4`
98             walltime[$i]=`echo $variable | tr -s " " | cut -d" " -s -f5` 
99             day[$i]=`echo $variable | tr -s " " | cut -d" " -s -f6`   
100             month[$i]=`echo $variable | tr -s " " | cut -d" " -s -f7`   
101             date[$i]=`echo $variable | tr -s " " | cut -d" " -s -f8`
102             subtime[$i]=`echo $variable | tr -s " " | cut -d" " -s -f9`   
103             ((i = i + 1))
104          done
105          rm .watchdog_report.x
106
107#         check for terminated jobs and status changes
108          k=0
109          cat .watchdog_status.x|while read variable
110          do
111#            analyze output
112             old_comid[$k]=`echo $variable | tr -s " " | cut -d" " -s -f1`
113             old_status[$k]=`echo $variable | tr -s " " | cut -d" " -s -f2` 
114             ((k = k + 1))
115          done
116          rm .watchdog_status.x
117
118          info=""
119          for ((m=0;m<$k;m++))
120          do
121             found=0
122             for ((n=0;n<$i;n++))
123             do       
124                if [[ ${old_comid[$m]} == ${comid[$n]} ]]
125                then
126                   if [[ ${old_status[$m]} != ${status[$n]} ]]
127                   then
128                       info="$info ${old_comid[$m]} has changed status from ${old_status[$m]} to ${status[$n]}.\n"
129                   fi
130                   found=1
131                   break
132                fi
133             done
134             if (( $found == 0 ))
135             then
136                info="$info ${old_comid[$m]} has been terminated (Status was ${old_status[$m]}).\n"
137             fi
138          done
139
140#         get estimated starting time for all idle jobs and write watchdog output in .watchdog_report.x
141          while (( $j < $i ))
142          do
143             if [[ ${status[$j]} == "Idle" ]]
144             then
145                if [[ ${complex[$j]} == "hannover" ]]
146                then
147                   eststart[$j]=`ssh $gate_h -l $1 "showstart ${comid[$j]}|grep \"based start in\""`
148                else
149                   eststart[$j]=`ssh $gate_b -l $1 "showstart ${comid[$j]}|grep \"based start in\""`
150                fi
151                eststart[$j]="Start in: "`echo ${eststart[$j]} | tr -s " " | cut -d" " -s -f6`   
152             else
153                eststart[$j]=""
154             fi
155
156#            write final output line
157             ((k = j + 1))
158             printf "%-9s%8s%3s%04i%13s%11s%2s%-s%3s%-20s\n" "Job: $k:" "${status[$j]}" " @ " "${nodes[$j]}" " nodes, time:" "${walltime[$j]}" " (" "${comid[$j]}" "). " "${eststart[$j]}" >> .watchdog_report.x
159             printf "${comid[$j]} ${status[$j]}\n" >> .watchdog_status.x
160             ((j = j + 1))
161          done
162
163#         kill all windows
164          killall kdialog
165
166          timestamp=`date`
167
168#         create window and show information
169          kdialog --textbox .watchdog_report.x 550 150 --title "HLRN watchdog  (last update: $timestamp)" &
170
171#         in case of status changes and terminated jobs, inform the user
172          if [[ $info != "" ]]
173          then
174             kdialog --msgbox "$info" --title "HLRN Job Information" &
175          fi
176
177          sleep $update_frequency
178       done
179
180    fi
Note: See TracBrowser for help on using the repository browser.