Home

Context Navigation

source: palm/trunk/SCRIPTS/palm_wd @ 2670

Last change on this file since 2670 was 2416, checked in by maronga, 7 years ago
watchdog adapted for palmrun
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 39.7 KB

Rev	Line
[1611]	1	#!/usr/bin/python
	2	# -- coding: utf-8 --
	3	#--------------------------------------------------------------------------------#
	4	# This file is part of PALM.
	5	#
	6	# PALM is free software: you can redistribute it and/or modify it under the terms
	7	# of the GNU General Public License as published by the Free Software Foundation,
	8	# either version 3 of the License, or (at your option) any later version.
	9	#
	10	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License along with
	15	# PALM. If not, see <http://www.gnu.org/licenses/>.
	16	#
	17	# Copyright 1997-2015 Leibniz Universitaet Hannover
	18	#--------------------------------------------------------------------------------#
	19	#
	20	# Current revisions:
	21	# -----------------
[1752]	22	#
[1754]	23	#
[1752]	24	# Former revisions:
	25	# -----------------
	26	# $Id: palm_wd 2416 2017-09-06 14:28:14Z raasch $
[2416]	27	# Adapted for palmrun
	28	#
	29	# 1754 2016-02-22 13:50:22Z maronga
[1752]	30	#
[1754]	31	# 1753 2016-02-22 13:49:49Z maronga
	32	# Bugfix: use of global variables is required after updating configuration file
	33	#
[1752]	34	# 1751 2016-02-15 07:44:16Z maronga
[1751]	35	# Bugfixes: runs on multiple hosts caused crash of the watchdog. Progress bar
	36	# occasionally showed wrong progress.
	37	# New: Hosts can be switched on/off and update_frequency can be modified via
	38	# File->Options. Security check for cancelation of jobs.
[1611]	39	#
[1619]	40	# 1618 2015-07-13 06:52:15Z maronga
	41	# Added steering via configuration file .wd.config, to be place in the local
	42	# palm directory. Watchdog save files are automatically created upon first start.
	43	#
[1614]	44	# 1613 2015-07-08 14:53:29Z maronga
	45	# Bugfix: tooltip for queuing name did not show up on first update.
	46	# New: added contect menu for showing the parameter file and the run control
	47	# output
	48	#
[1612]	49	# 1611 2015-07-07 12:23:22Z maronga
	50	# Initial revision
	51	#
[1611]	52	#
	53	# Description:
	54	# ------------
	55	# PALM watchdog client for monitoring batch jobs on a variety of hosts specified
	56	# by the user
	57	#
	58	# Instructions:
	59	# -------------
	60	# 1) Modify the header section of palm_wd
	61	# 2) Move .wd.olddata and .wd.newdata to your palm directory
	62	# (e.g. /home/user/current_version/.wd.newdata etc.)
	63	# 3) Modify a copy of palm_wdd for each host to be monitored and move it to the
	64	# respective hosts
	65	# 4) Start the client either from mrungui or from shell by "nohup palm_wd&"
	66
	67	# To do:
	68	# ------
	69	# 1) Add "Options", "Help" and "Manual"
	70	# 2) Move user settings to a configuration file
	71	#------------------------------------------------------------------------------!
	72
[1618]	73	import ConfigParser
	74	import datetime
	75	import os
[1611]	76	from PyQt4 import QtGui, QtCore, uic
	77	from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT
[1618]	78	import shutil
	79	import subprocess as sub
	80	import sys
[1611]	81	import time
	82
	83	# Determine PALM directories
	84	try:
	85	devnull = open(os.devnull, 'w')
	86	out = sub.check_output("echo $PALM_BIN", shell=True, stderr=sub.STDOUT)
	87	palm_bin = out.rstrip()
	88	palm_dir = out.split("palm")[0] + "palm/" + out.split("palm")[1].split("/")[1]
	89	out = None
	90	except:
	91	print "Error. $PALM_BIN is not set."
	92	raise SystemExit
	93
	94
[1618]	95	# Read configuration file
	96	# First check if the configuration file exists
	97	if ( os.path.exists(palm_dir + '/.wd.config') == False ):
	98	print "Error. No configuration file .wd.config found in " + palm_dir
	99	raise SystemExit
	100
	101	config = ConfigParser.RawConfigParser()
	102	config.read(palm_dir + '/.wd.config')
	103
	104	description = []
	105	hostname = []
	106	username = []
	107
	108	for i in range(0,len(config.sections())):
	109
	110	description_tmp = config.sections()[i]
	111
	112	if ( description_tmp != 'Settings' ):
[1751]	113
	114	if ( config.get(description_tmp, 'enabled') == 'true' ):
	115	description.append(description_tmp)
	116	hostname.append(config.get(description_tmp, 'hostname'))
	117	username.append(config.get(description_tmp, 'username'))
[1618]	118	else:
	119	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	120
	121	# Check if .wd.olddata and .wd.newdata exist. Otherwise create the files
	122	if ( os.path.exists(palm_dir + '/.wd.olddata') == False ):
	123	print "No .wd.olddata found. Creating..."
	124	file1 = open(palm_dir + '/.wd.olddata', 'a')
	125	file1.close()
	126
	127
	128	if ( os.path.exists(palm_dir + '/.wd.newdata') == False ):
	129	print "No .wd.newdata found. Creating..."
	130	file1 = open(palm_dir + '/.wd.newdata', 'a')
	131	file1.close()
	132
[1611]	133	# Dummy variables for the jobname and the progressbars
	134	jobname = ""
	135	timestamp = ""
	136	pbars = []
	137	job_data_list = []
	138
[1751]	139	# Message box for showing RUN_CONTROL output
[1613]	140	class MessageBoxScroll(QtGui.QMessageBox):
	141	def __init__(self):
	142	QtGui.QMessageBox.__init__(self)
	143	self.setSizeGripEnabled(True)
	144
	145	def event(self, e):
	146	result = QtGui.QMessageBox.event(self, e)
	147
	148	self.setMinimumHeight(100)
	149	self.setMaximumHeight(16777215)
	150	self.setMinimumWidth(400)
	151	self.setMaximumWidth(16777215)
	152	self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	153
	154	textEdit = self.findChild(QtGui.QTextEdit)
	155	if textEdit != None :
	156	textEdit.setMinimumHeight(800)
	157	textEdit.setMaximumHeight(16777215)
	158	textEdit.setMinimumWidth(1000)
	159	textEdit.setMaximumWidth(16777215)
	160	textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	161	f = QtGui.QFont('not_a_font') #
	162	f.setStyleHint(f.TypeWriter, f.PreferDefault)
	163	textEdit.setFont(f)
	164
	165	return result
	166
	167
[1751]	168	# Message box for showing RUN_CONTROL output
	169	class OptionBox(QtGui.QDialog):
	170	def __init__(self):
	171
	172	super(OptionBox, self).__init__()
	173
	174	uic.loadUi(palm_bin + '/palm_wd_files/wdoptions.ui', self)
	175
	176	hostname = []
	177	self.checkbox = []
	178	j = -1
	179	ypos = 0
	180
	181	self.vbox = QtGui.QVBoxLayout(self)
	182	self.vbox.setSpacing(0)
	183	self.vbox.setMargin(0)
	184
	185	config.read(palm_dir + '/.wd.config')
	186
	187	for i in range(0,len(config.sections())):
	188
	189	description_tmp = config.sections()[i]
	190
	191	if ( description_tmp != 'Settings' ):
	192	description.append(description_tmp)
	193	hostname.append(config.get(description_tmp, 'hostname'))
	194
	195	j = j + 1
	196	self.checkbox.append(j)
	197	self.checkbox[j] = QtGui.QCheckBox(description_tmp, self)
	198	ypos = ypos + 20
	199	self.checkbox[j].move(10,0)
	200	if ( config.get(description_tmp, 'enabled') == 'true' ):
	201	self.checkbox[j].toggle()
	202
	203	self.vbox.addWidget(self.checkbox[j])
	204	else:
	205	self.update_spin.setValue(int(config.get(description_tmp, 'update_frequency')))
	206
	207	self.hostBox.setLayout(self.vbox)
	208	self.hostBox.setGeometry(0, 0, 272, 50+ypos)
	209	self.settingBox.setGeometry(281, 0, 214, 50+ypos)
	210	self.ok.setGeometry(385, 50+ypos, 111, 24)
	211	self.cancel.setGeometry(270, 50+ypos, 111, 24)
	212
	213	self.setGeometry(0, 0, 500, 80+ypos)
	214	self.show()
	215
	216	return
	217
	218	# save updtes to file
	219	def SaveAndClose(self):
	220
	221	for i in range(0,len(self.checkbox)):
	222
	223	if ( self.checkbox[i].checkState() == 0 ):
	224	config.set(str(self.checkbox[i].text()),'enabled','false')
	225	else:
	226	config.set(str(self.checkbox[i].text()),'enabled','true')
	227
	228	config.set('Settings','update_frequency',self.update_spin.text())
	229
	230	with open(palm_dir + '/.wd.config', 'wb') as configfile:
	231	config.write(configfile)
	232
	233
	234	self.close()
	235
	236	return
	237
	238
	239
[1611]	240	# MainWindow class
	241	class Watchdog(QtGui.QMainWindow):
	242
	243	def __init__(self):
	244	super(Watchdog, self).__init__()
	245
	246	self.InitUi()
	247
	248
	249	# Initialized MainWindow UI
	250	def InitUi(self):
	251
	252	# Load predefined mainwindow
	253	uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self)
	254
	255	# Resize columns and set number of rows to zero. Column 6 is hidden and
	256	# contains the remote jobname (e.g. hannover.174610)
	257	self.table.setColumnWidth(0,230)
	258	self.table.setColumnWidth(1,80)
	259	self.table.setColumnWidth(2,50)
	260	self.table.setColumnWidth(3,80)
	261	self.table.setColumnWidth(4,180)
	262	self.table.setColumnWidth(5,115)
	263	self.table.setColumnHidden(6, True)
	264	self.table.setRowCount(0)
[1613]	265
[1611]	266	# Display MainWindow
	267	self.show()
	268	QtGui.QApplication.processEvents()
	269
	270	# Start refresh timer. On timeout perform update
	271	self.timer = QtCore.QTimer(self)
	272	self.timer.timeout.connect(self.Refresh)
	273	self.timer.setSingleShot(False)
	274	self.timer.start(update_frequency)
	275
	276	# The timetimer counts the time since last update
	277	self.timetimer= QtCore.QElapsedTimer()
	278	self.timetimer.start()
	279
	280	# The labeltimer induces the update of the remaining time in the UI
	281	self.labeltimer = QtCore.QTimer(self)
	282	self.labeltimer.timeout.connect(self.UpdateLabel)
	283	self.labeltimer.setSingleShot(False)
	284
	285	# Update in the UI will be performed at each 1/10 of the update interval
	286	self.labeltimer.start(update_frequency/10)
	287	self.label.setText("Next update in " + str(update_frequency/1000/60) + "min")
	288
	289	# Update the list now
	290	self.Refresh()
	291
	292
	293	# Add a custom context menu
	294	def OpenContextMenu(self, position):
	295	menu = QtGui.QMenu()
	296
	297	# "Show details" -> fetch details for the selected job
[1613]	298	detailsjobAction = menu.addAction('Show job details')
[1611]	299	detailsjobAction.setStatusTip('Display detailed info for job')
	300	detailsjobAction.triggered.connect(self.ShowDetails)
	301
[1613]	302	# "Show parameter file" -> show the contents of PARIN
	303	parinAction = menu.addAction('Show parameter file')
	304	parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)')
	305	parinAction.triggered.connect(self.ShowPARIN)
	306
	307	rcAction = menu.addAction('Show run control file')
	308	rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)')
	309	rcAction.triggered.connect(self.ShowRC)
	310
[1611]	311	# "Cancel job" -> remove job from queuing system
	312	canceljobAction = menu.addAction('Cancel job')
	313	canceljobAction.setStatusTip('Remove job from queueing system')
	314	canceljobAction.triggered.connect(self.CancelJob)
	315
[1613]	316	# "Force stop" -> force termination of job
[1611]	317	stopjobAction = menu.addAction('Force stop')
	318	stopjobAction.setStatusTip('Terminate job properly')
	319	stopjobAction.triggered.connect(self.DoStop)
	320
[1613]	321	# "Force restart" -> force rermination and restart of job
[1611]	322	restartjobAction = menu.addAction('Force restart')
	323	restartjobAction.setStatusTip('Terminate job properly and initiate restart')
	324	restartjobAction.triggered.connect(self.DoRestart)
	325
	326	# "Remove from list" -> remove a completed job from the list
	327	removefromlistAction = menu.addAction('Remove from list')
	328	removefromlistAction.setStatusTip('Remove finished job')
	329	removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1))
	330
	331
	332	# Activate/deactive contect menu items based on the status of the runs
	333	state = self.table.item(self.table.currentRow(),3).text()
	334	if (state == "Canceled" or state == "Completed" or state == "Terminated"):
	335	detailsjobAction.setEnabled(False)
	336	canceljobAction.setEnabled(False)
	337	removefromlistAction.setEnabled(True)
	338	else:
	339	detailsjobAction.setEnabled(True)
	340	canceljobAction.setEnabled(True)
	341	removefromlistAction.setEnabled(False)
	342
	343	if ( state == "Running" ):
	344	stopjobAction.setEnabled(True)
	345	restartjobAction.setEnabled(True)
[1613]	346	parinAction.setEnabled(True)
	347	rcAction.setEnabled(True)
[1611]	348	else:
	349	stopjobAction.setEnabled(False)
	350	restartjobAction.setEnabled(False)
[1613]	351	parinAction.setEnabled(False)
	352	rcAction.setEnabled(False)
	353
[1611]	354	# Show the context menu
	355	action = menu.exec_(self.table.mapToGlobal(position))
	356
	357
	358	# Collecting watchdog data and display in the UI
	359	def Refresh(self):
	360
	361	# Use global variables
	362	global pbars
[1751]	363	global update_frequency
[1611]	364
	365	# Deactivate timer processes until update is finished
	366	# QtGui.QApplication.processEvents() updates the UI whenever needed
	367	self.labeltimer.stop()
	368	self.label.setText("Updating...")
	369	QtGui.QApplication.processEvents()
	370	self.setEnabled(False)
	371	QtGui.QApplication.processEvents()
	372
	373	# Get current timestamp
	374	timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
	375
	376	# Open an empty file for the watchdog data
	377	file = open(palm_dir + '/.wd.newdata', 'w')
	378
	379
	380	# Set all required variables to their initial values. Sorting must be
	381	# disabled.
	382	self.table.setRowCount(0)
	383	self.table.setSortingEnabled(False)
	384	i = -1
	385	job_data_list = []
	386	pbars = []
	387
	388	# Scan all hosts in the variable hostname. For each host perform the
	389	# update loop
	390	for h in range(0,len(hostname)):
	391
	392	# Perform ssh command
	393	host = username[h] + "@" + hostname[h]
	394	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd queue " + username[h]],
	395	shell=False,
	396	stdout=sub.PIPE,
	397	stderr=sub.PIPE)
	398	result = ssh.stdout.readlines()
	399	result = filter(None, result)
	400
	401
	402	# Delete all job data
	403	job_data_tmp = []
	404	job_data = []
	405	job_data_n = []
	406	job_progress = 0
	407
	408	# In case of errors display error message
	409	if ( len(result) < 1 ):
	410	error = ssh.stderr.readlines()
	411	if ( error != [] ):
	412	notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error))
	413
	414	# No error -> save job data
	415	else:
	416
	417	# Successively write to job_data
	418	for j in range(0,len(result)):
	419
	420	job_data_tmp.append(j)
	421	job_data_tmp[j] = result[j].split(" ")
	422	job_data_tmp[j] = filter(None, job_data_tmp[j])
	423
	424	job_data.append(j)
	425	job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2],
	426	job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5],
	427	job_data_tmp[j][1], job_data_tmp[j][6].rstrip()]
	428	job_data_n.append(j)
	429	job_data_n[j] = job_data_tmp[j][1]
	430
	431	del(result)
	432
	433
	434
	435	# Now read the data from the last update from file. These data are
	436	# already in the prefered format
	437	file2 = open(palm_dir + '/.wd.olddata', 'r')
	438	result = file2.readlines()
	439	file2.close()
[1751]	440
	441	job_data_old = []
	442	job_data_old_n = []
	443	k = -1
[1611]	444	for j in range(0,len(result)):
	445	if ( result[j].split()[1] == description[h] ):
[1751]	446	k = k + 1
	447	job_data_old.append(k)
	448	job_data_old[k] = result[j].split(" ")
	449	job_data_old[k] = filter(None, job_data_old[k])
	450	job_data_old[k] = [line.rstrip('\n') for line in job_data_old[k]]
	451	job_data_old_n.append(k)
	452	job_data_old_n[k] = job_data_old[k][6]
[1611]	453
	454	# Merge the job_data and job_data_old to find completed, new and
	455	# still queued jobs
	456	if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ):
	457	jobs_full = list(set(job_data_old_n) \| set(job_data_n))
	458	jobs_known = list(set(job_data_old_n) & set(job_data_n))
	459	jobs_complete = set(job_data_old_n) - set(job_data_n)
	460	jobs_new = set(job_data_n) - set(job_data_old_n)
	461	elif ( len(job_data_n) > 0 ):
	462	jobs_full = job_data_n
	463	jobs_known = []
	464	jobs_new = job_data_n
	465	jobs_complete = []
	466	elif ( len(job_data_old_n) > 0 ):
	467	jobs_full = job_data_old_n
	468	jobs_known = []
	469	jobs_new = []
	470	jobs_complete = job_data_old_n
	471	else:
	472	jobs_full = []
	473	jobs_known = []
	474	jobs_new = []
	475	jobs_complete = []
	476
	477	# Display all known jobs
	478	to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)]
	479	for j in range(0,len(to_display)):
	480	i = i + 1
	481	self.table.insertRow(i)
	482	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	483	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	484	self.table.setItem(i, 0, item)
	485	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	486	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	487	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	488	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	489	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	490	self.table.setItem(i, 5, item)
	491	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	492	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	493	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	494	pbars.append(i)
	495	pbars[i] = QtGui.QProgressBar(self)
	496	pbars[i].setValue(int(job_data[to_display[j]][4]))
	497
	498	# Apply a color depending on the status of the job
	499	if ( job_data[to_display[j]][3] == "Running" ):
[1751]	500	self.table.setCellWidget(i,4,pbars[i])
[1611]	501	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	502	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	503	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	504	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	505	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	506	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	507	else:
	508	pbars[j].setEnabled(False)
[1751]	509	self.table.setCellWidget(i,4,pbars[i])
[1611]	510	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	511	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	512	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	513	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	514	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	515	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	516
	517	# Save the job data to file
	518	job_data_list.append(i)
	519	job_data_list[i] = job_data[to_display[j]][0]
	520
	521	printstring = str(job_data[to_display[j]][0]) + " " + \
	522	str(job_data[to_display[j]][1]) + " " + \
	523	str(job_data[to_display[j]][2]) + " " + \
	524	str(job_data[to_display[j]][3]) + " " + \
	525	str(job_data[to_display[j]][4]) + " " + \
	526	str(job_data[to_display[j]][5]) + " " + \
	527	str(job_data[to_display[j]][6]) + " " + \
	528	str(job_data[to_display[j]][7]) + "\n"
	529	file.write(printstring)
	530
	531
	532	# Display all new jobs
	533	to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)]
	534	for j in range(0,len(to_display)):
	535	i = i + 1
	536	self.table.insertRow(i)
	537	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	538	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	539	self.table.setItem(i, 0, item)
	540	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	541	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	542	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	543	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	544	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	545	self.table.setItem(i, 5, item)
	546	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	547	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	548	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	549
	550	pbars.append(i)
	551	pbars[i] = QtGui.QProgressBar(self)
	552	pbars[i].setValue(int(job_data[to_display[j]][4]))
	553
	554	# Apply a color depending on the status of the job
	555	if ( job_data[to_display[j]][3] == "Running" ):
	556	self.table.setCellWidget(i,4,pbars[i])
	557	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	558	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	559	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	560	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	561	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	562	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	563	else:
	564	pbars[j].setEnabled(False)
	565	self.table.setCellWidget(i,4,pbars[i])
	566	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	567	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	568	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	569	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	570	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	571	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	572
	573	# Save job data to file
	574	job_data_list.append(i)
	575	job_data_list[i] = job_data[to_display[j]][0]
	576
	577	printstring = str(job_data[to_display[j]][0]) + " " + \
	578	str(job_data[to_display[j]][1]) + " " + \
	579	str(job_data[to_display[j]][2]) + " " + \
	580	str(job_data[to_display[j]][3]) + " " + \
	581	str(job_data[to_display[j]][4]) + " " + \
	582	str(job_data[to_display[j]][5]) + " " + \
	583	str(job_data[to_display[j]][6]) + " " + \
	584	str(job_data[to_display[j]][7]) + "\n"
	585	file.write(printstring)
	586
	587
	588	# Display all completed/canceled/aborted jobs. The watchdog cannot
	589	# distinguish why the job has been finished
	590	to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)]
	591	for j in range(0,len(to_display)):
	592	i = i + 1
	593	self.table.insertRow(i)
	594	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0])
	595	item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6])
	596	self.table.setItem(i, 0, item)
	597	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1]))
	598	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2]))
	599	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3]))
	600	pbars.append(i)
	601	pbars[j] = QtGui.QProgressBar(self)
	602	pbars[j].setValue(int(job_data_old[to_display[j]][4]))
	603	pbars[j].setEnabled(False)
	604	self.table.setCellWidget(i,4,pbars[j])
	605	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5])
	606	item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7])
	607	self.table.setItem(i, 5, item)
	608	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6]))
	609	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	610	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	611	self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed"))
	612
	613	# Apply a color depending on the status of the job
	614	self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175))
	615	self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175))
	616	self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175))
	617	self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175))
	618	self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175))
	619	self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175))
	620
	621
	622	# Save job data to file
	623	job_data_list.append(i)
	624	job_data_list[i] = job_data_old[to_display[j]][0]
	625
	626	printstring = str(job_data_old[to_display[j]][0]) + " " + \
	627	str(job_data_old[to_display[j]][1]) + " " + \
	628	str(job_data_old[to_display[j]][2]) + " " + \
	629	str(job_data_old[to_display[j]][3]) + " " + \
	630	str(job_data_old[to_display[j]][4]) + " " + \
	631	str(job_data_old[to_display[j]][5]) + " " + \
	632	str(job_data_old[to_display[j]][6]) + " " + \
	633	str(job_data_old[to_display[j]][7]) + "\n"
	634	file.write(printstring)
	635
	636	del(jobs_complete)
	637	del(jobs_new)
	638	del(result)
[1751]	639	del(job_data_old)
	640	del(job_data_old_n)
	641
[1611]	642	file.close()
	643
	644
	645	# Update is complete, sorting can thus be re-enabled
	646	self.table.setSortingEnabled(True)
	647
	648	# Update the logfile
	649	if ( os.path.isfile(palm_dir + '/.wd.newdata') ):
	650	shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata')
	651
	652	# Re-enable timer processes
	653	self.setEnabled(True)
	654	self.label2.setText("Last update: " + timestamp)
	655	self.label.setText("Update complete.")
[1751]	656	self.timetimer.start()
	657	self.timer.start(update_frequency)
	658	self.labeltimer.start(update_frequency/10)
	659	self.UpdateLabel()
[1611]	660	QtGui.QApplication.processEvents()
	661
	662	# Canceling selected job from table
	663	def CancelJob(self):
	664
	665	# Return internal jobname
	666	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	667	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	668
	669	# Check description of job in order to login on the correct host
	670	descr = self.table.item(self.table.currentRow(),1).text()
[1751]	671
	672	querybox = QtGui.QMessageBox()
	673	querybox.setText("Attention!\nYou are trying to cancel a job. It will not be able to terminate properly.")
	674	querybox.setInformativeText("Do you really want to cancel " + jobname + " on host " + descr + "?")
	675	querybox.setStandardButtons(QtGui.QMessageBox.Yes \| QtGui.QMessageBox.No)
	676	querybox.setDefaultButton(QtGui.QMessageBox.No)
	677	returnvalue = querybox.exec_()
	678
	679	if ( returnvalue == QtGui.QMessageBox.Yes ):
	680
	681	for h in range(0,len(description)):
	682	if ( descr == description[h] ):
	683	host = username[h] + "@" + hostname[h]
[1611]	684
[1751]	685	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd cancel " + jobname],
	686	shell=False,
	687	stdout=sub.PIPE,
	688	stderr=sub.PIPE)
	689	result = ssh.stdout.readlines()
	690	result = filter(None, result)
[1611]	691
[1751]	692	# In case of error display a warning message
	693	if ( len(result) == 0 ):
	694	error = ssh.stderr.readlines()
	695	notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error))
[1611]	696
[1751]	697	# Otherwise inform the user and mark the job in the table
	698	else:
	699	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled"))
	700	notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result))
[1611]	701
	702
	703	# Show detailed information on job. For documentation see above
	704	def ShowDetails(self):
	705
[2416]	706	jobname = self.table.item(self.table.currentRow(),6).text()
[1611]	707	descr = self.table.item(self.table.currentRow(),1).text()
	708	for h in range(0,len(description)):
	709	if ( descr == description[h] ):
	710	host = username[h] + "@" + hostname[h]
	711
	712	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd check " + jobname],
	713	shell=False,
	714	stdout=sub.PIPE,
	715	stderr=sub.PIPE)
	716	result = ssh.stdout.readlines()
	717	result = filter(None, result)
	718
	719	if ( len(result) == 0 ):
	720	error = ssh.stderr.readlines()
	721	notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error))
	722	else:
	723	notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result))
	724
	725
	726	# Perform a forced stop on the job
	727	def DoStop(self):
	728
	729	# Return internal jobname
	730	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	731	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	732	# Check description of job in order to login on the correct host
	733	descr = self.table.item(self.table.currentRow(),1).text()
	734	for h in range(0,len(description)):
	735	if ( descr == description[h] ):
	736	host = username[h] + "@" + hostname[h]
	737	user = username[h]
	738
[2416]	739	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd stop " + jobrealname],
[1611]	740	shell=False,
	741	stdout=sub.PIPE,
	742	stderr=sub.PIPE)
	743	result = ssh.stdout.readlines()
	744	result = filter(None, result)
	745
	746	# In case of error display a warning message
	747	if ( len(result) == 0 ):
	748	error = ssh.stderr.readlines()
	749	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	750
	751	# Otherwise inform the user and mark the job in the table
	752	else:
	753	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
[1751]	754	notify = QtGui.QMessageBox.information(self,'Terminate job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
[1611]	755
	756
	757	# Perform a forced stop on the job
	758	def DoRestart(self):
	759
	760	# Return internal jobname
	761	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	762	jobrealname = self.table.item(self.table.currentRow(),0).text()
	763
[1611]	764	# Check description of job in order to login on the correct host
	765	descr = self.table.item(self.table.currentRow(),1).text()
	766	for h in range(0,len(description)):
	767	if ( descr == description[h] ):
	768	host = username[h] + "@" + hostname[h]
	769	user = username[h]
	770
[2416]	771	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd restart " + jobrealname],
[1611]	772	shell=False,
	773	stdout=sub.PIPE,
	774	stderr=sub.PIPE)
	775	result = ssh.stdout.readlines()
	776	result = filter(None, result)
	777
	778	# In case of error display a warning message
	779	if ( len(result) == 0 ):
	780	error = ssh.stderr.readlines()
	781	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	782
	783	# Otherwise inform the user and mark the job in the table
	784	else:
	785	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	786	notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	787
	788
[1613]	789	# Read the PARIN file of the job
	790	def ShowPARIN(self):
	791
	792	# Return internal jobname
	793	jobname = self.table.item(self.table.currentRow(),6).text()
	794	jobrealname = self.table.item(self.table.currentRow(),0).text()
	795
	796	# Check description of job in order to login on the correct host
	797	descr = self.table.item(self.table.currentRow(),1).text()
	798	for h in range(0,len(description)):
	799	if ( descr == description[h] ):
	800	host = username[h] + "@" + hostname[h]
	801	user = username[h]
	802
[2416]	803	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd parin " + jobrealname],
[1613]	804	shell=False,
	805	stdout=sub.PIPE,
	806	stderr=sub.PIPE)
	807	result = ssh.stdout.readlines()
	808	result = filter(None, result)
	809
	810	# In case of error display a warning message
	811	if ( len(result) == 0 ):
	812	error = ssh.stderr.readlines()
	813	notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error))
	814
	815	# Otherwise inform the user and mark the job in the table
	816	else:
	817	mb = MessageBoxScroll()
	818	mb.setText("Parameter file for job: " + jobrealname + "")
	819	mb.setDetailedText(''.join(result))
	820	mb.exec_()
	821
	822	# Read the PARIN file of the job
	823	def ShowRC(self):
	824
	825	# Return internal jobname and real job name
	826	jobname = self.table.item(self.table.currentRow(),6).text()
	827	jobrealname = self.table.item(self.table.currentRow(),0).text()
	828
	829	# Check description of job in order to login on the correct host
	830	descr = self.table.item(self.table.currentRow(),1).text()
	831	for h in range(0,len(description)):
	832	if ( descr == description[h] ):
	833	host = username[h] + "@" + hostname[h]
	834	user = username[h]
	835
[2416]	836	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd rc " + jobrealname],
[1613]	837	shell=False,
	838	stdout=sub.PIPE,
	839	stderr=sub.PIPE)
	840	result = ssh.stdout.readlines()
	841	result = filter(None, result)
	842
	843	# In case of error display a warning message
	844	if ( len(result) == 0 ):
	845	error = ssh.stderr.readlines()
	846	notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error))
	847
	848	# Otherwise inform the user and mark the job in the table
	849	else:
	850	mb = MessageBoxScroll()
	851	lastline = result[len(result)-2].split()[2]
	852	mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline)
	853	mb.setDetailedText(''.join(result))
	854	mb.exec_()
	855
[1611]	856	# Remove a job from list - removes the current row from the table and from
	857	# save file
	858	def RemoveFromList(self, row):
	859	if ( row == -1 ):
	860	row = self.table.currentRow()
	861
	862	# Read data from save file
	863	job_to_delete = self.table.item(row,6).text()
	864	self.table.removeRow(row)
	865	file = open(palm_dir + '/.wd.olddata', 'r')
	866	result = file.readlines()
	867	result = filter(None, result)
	868	file.close()
	869	file = open(palm_dir + '/.wd.olddata', 'w')
	870
	871	job_data_old = []
	872
	873	if ( len(result) == 0 ):
	874	notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.")
	875	else:
	876	# Save data in array job_data_old
	877	for j in range(0,len(result)):
	878	job_data_old.append(j)
	879	job_data_old[j] = result[j].split(" ")
	880	job_data_old[j] = filter(None, job_data_old[j])
	881	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	882
	883	# Check if line j is the selected job, if not -> save to file
	884	if ( job_data_old[j][6] != job_to_delete ):
	885	printstring = str(job_data_old[j][0]) + " " + \
	886	str(job_data_old[j][1]) + " " + \
	887	str(job_data_old[j][2]) + " " + \
	888	str(job_data_old[j][3]) + " " + \
	889	str(job_data_old[j][4]) + " " + \
	890	str(job_data_old[j][5]) + " " + \
	891	str(job_data_old[j][6]) + " " + \
	892	str(job_data_old[j][7]) + "\n"
	893	file.write(printstring)
	894
	895	file.close()
	896
	897	# Remove all completed jobs from list
	898	def ClearList(self):
	899
	900	num_of_lines = self.table.rowCount()
	901
	902	# Delete all lines with completed/canceled jobs from list. The counter
	903	# must decrease as the line numbers would be messed up otherwise
	904	for j in range(num_of_lines-1,-1,-1):
	905	state = self.table.item(j,3).text()
	906	if ( state == "Completed" or state == "Canceled" or state == "Terminated"):
	907	self.RemoveFromList(j)
	908
	909	# Update the label
	910	def UpdateLabel(self):
	911	remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60
	912	self.label.setText("Next update in " + str(remaining_time) + " min")
	913
	914
[1751]	915	# Enter Options menu
	916	def Options(self):
	917
	918	# disable mainwindow
	919	self.setEnabled(False)
	920
	921	# show Options Dialog
	922	opt = OptionBox()
	923	opt.exec_()
	924
	925	self.UpdateHosts()
	926	self.setEnabled(True)
	927
	928	# Update settings
	929	def UpdateHosts(self):
	930
	931	global update_frequency
[1753]	932	global description
	933	global hostname
	934	global username
	935
[1751]	936	description = []
	937	hostname = []
	938	username = []
	939
	940	for i in range(0,len(config.sections())):
	941
	942	description_tmp = config.sections()[i]
	943
	944	if ( description_tmp != 'Settings' ):
	945
	946	if ( config.get(description_tmp, 'enabled') == 'true' ):
	947	description.append(description_tmp)
	948	hostname.append(config.get(description_tmp, 'hostname'))
	949	username.append(config.get(description_tmp, 'username'))
	950	else:
	951	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	952
	953
[1611]	954	# Main loop
	955	def main():
	956
	957	app = QtGui.QApplication(sys.argv)
	958	res = Watchdog()
	959	sys.exit(app.exec_())
	960
	961
	962	if __name__ == '__main__':
[1613]	963	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |