#!/usr/bin/python # -*- coding: utf-8 -*- #--------------------------------------------------------------------------------# # This file is part of PALM. # # PALM is free software: you can redistribute it and/or modify it under the terms # of the GNU General Public License as published by the Free Software Foundation, # either version 3 of the License, or (at your option) any later version. # # PALM is distributed in the hope that it will be useful, but WITHOUT ANY # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along with # PALM. If not, see . # # Copyright 1997-2015 Leibniz Universitaet Hannover #--------------------------------------------------------------------------------# # # Current revisions: # ----------------- # Bugfix: tooltip for queuing name did not show up on first update. # New: added contect menu for showing the parameter file and the run control # output # # Former revisions: # ----------------- # $Id: palm_wdd 1613 2015-07-08 14:53:29Z maronga $ # # 1611 2015-07-07 12:23:22Z maronga # Initial revision # # # Description: # ------------ # PALM watchdog client for monitoring batch jobs on a variety of hosts specified # by the user. The watchdog server requires python 2.7 or higher installed on # host to be monitored. # # Instructions: # ------------- # 1) Modify the header section of palm_wd # 2) Move .wd.olddata and .wd.newdata to your palm directory # (e.g. /home/user/current_version/.wd.newdata etc.) # 3) Modify a copy of palm_wdd for each host to be monitored and move it to the # respective hosts # 4) Start the client either from mrungui or from shell by "nohup palm_wd&" # # To do: # ------ # 1) Add "Options", "Help" and "Manual" # 2) Move user settings to a configuration file #------------------------------------------------------------------------------! import os import sys from subprocess import check_output # START OF HEADER # configuration for host cmd_readqueue = "showq | egrep " cmd_tmpdir = "/gfs1/tmp/" cmd_canceljob = "canceljob" cmd_checkjob = "checkjob" cmd_realname_grep = "AName" cmd_starttime = "showstart" cmd_starttime_grep = "start in" # END OF HEADER action = str(sys.argv[1]) data = str(sys.argv[2]) if ( len(sys.argv) > 3 ): data2 = str(sys.argv[3]) cmd_readqueue = cmd_readqueue + data cmd_tmpdir = cmd_tmpdir + data # reading queuing system def ReadQueue(username): # collect queuing information try: out = check_output(cmd_readqueue, shell=True) job_list = out.splitlines() out = None # do nothing for empty results list except: job_list = [] job_data_tmp = [] for j in range(0,len(job_list)): # Write temporary data array containing the job information. job_data_tmp.append(j) job_data_tmp[j] = job_list[j].split(" ") job_data_tmp[j] = filter(None, job_data_tmp[j]) cmd_realname = cmd_checkjob + " " + job_data_tmp[j][0] + "|grep " + cmd_realname_grep # retrieve real job name for all jobs try: out = check_output(cmd_realname, shell=True) job_realname = out.split(" ")[1].rstrip() except: job_realname = "error" # for running jobs, determine progress if ( job_data_tmp[j][2] == "Running" ): # collect progress information cmd_progress = "cat " + cmd_tmpdir + "/" + username + "." + job_data_tmp[j][0].partition(".")[2] + "/PROGRESS" try: devnull = open(os.devnull, 'w') out = check_output(cmd_progress, shell=True, stderr=devnull) progress_lines = out.splitlines() job_progress = progress_lines[1].split(" ")[1] out = None except: job_progress = "0" else: job_progress = "0" # return the job data job_starttime = GetStartTime(job_data_tmp[j][0]) print job_realname + " " + job_data_tmp[j][0] + " " + job_data_tmp[j][3] + " " + job_data_tmp[j][2] + " " + job_progress + " " + job_data_tmp[j][4] + " " + job_starttime # check details of specific job def CheckJob(jobid): cmd_checkjob_tmp = cmd_checkjob + " " + jobid try: out = check_output(cmd_checkjob_tmp, shell=True) job_details = out except: job_details = "No details available." return job_details # cancel a specific job def CancelJob(jobid): cmd_canceljob_tmp = cmd_canceljob + " " + jobid try: out = check_output(cmd_canceljob_tmp, shell=True) job_canceled = out except: job_canceled = "Action failed." return job_canceled # retrieve estimated start time of job def GetStartTime(jobid): cmd_starttime_tmp = cmd_starttime + " " + jobid + "|grep \"" + cmd_starttime_grep + "\"" try: out = check_output(cmd_starttime_tmp, shell=True) job_starttime = out.split()[5] except: job_starttime = "Action failed." return job_starttime def DoStopNow(username,jobid): # collect progress information cmd_dostop = "touch " + cmd_tmpdir + "/" + username + "." + jobid.partition(".")[2] + "/DO_STOP_NOW" try: devnull = open(os.devnull, 'w') out = check_output(cmd_dostop, shell=True, stderr=devnull) out = None except: return_message = "Action failed." return return_message def DoRestartNow(username,jobid): # collect progress information cmd_dorestart = "touch " + cmd_tmpdir + "/" + username + "." + jobid.partition(".")[2] + "/DO_RESTART_NOW" try: devnull = open(os.devnull, 'w') out = check_output(cmd_dorestart, shell=True, stderr=devnull) out = None except: return_message = "Action failed." return return_message def GetPARIN(username,jobid): # collect progress information cmd_dorestart = "cat " + cmd_tmpdir + "/" + username + "." + jobid.partition(".")[2] + "/PARIN" try: devnull = open(os.devnull, 'w') out = check_output(cmd_dorestart, shell=True, stderr=devnull) return_message = out out = None except: return_message = "Action failed." return return_message def GetRC(username,jobid): # collect progress information cmd_dorestart = "cat " + cmd_tmpdir + "/" + username + "." + jobid.partition(".")[2] + "/RUN_CONTROL" try: devnull = open(os.devnull, 'w') out = check_output(cmd_dorestart, shell=True, stderr=devnull) return_message = out out = None except: return_message = "Action failed." return return_message # START OF MAIN if ( action == "queue" ): ReadQueue(data) elif ( action == "check"): print CheckJob(data) elif ( action == "cancel"): print CancelJob(data) elif ( action == "start"): print GetStartTime(data) elif ( action == "stop"): print DoStopNow(data,data2) elif ( action == "restart"): print DoRestartNow(data,data2) elif ( action == "parin"): print GetPARIN(data,data2) elif ( action == "rc"): print GetRC(data,data2) else: print "Error. Action " + action + " unknown."