Home

Context Navigation

source: palm/trunk/SCRIPTS/palm_wd @ 1914

Last change on this file since 1914 was 1754, checked in by maronga, 9 years ago
last commit documented
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 39.4 KB

Rev	Line
[1611]	1	#!/usr/bin/python
	2	# -- coding: utf-8 --
	3	#--------------------------------------------------------------------------------#
	4	# This file is part of PALM.
	5	#
	6	# PALM is free software: you can redistribute it and/or modify it under the terms
	7	# of the GNU General Public License as published by the Free Software Foundation,
	8	# either version 3 of the License, or (at your option) any later version.
	9	#
	10	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License along with
	15	# PALM. If not, see <http://www.gnu.org/licenses/>.
	16	#
	17	# Copyright 1997-2015 Leibniz Universitaet Hannover
	18	#--------------------------------------------------------------------------------#
	19	#
	20	# Current revisions:
	21	# -----------------
[1752]	22	#
[1754]	23	#
[1752]	24	# Former revisions:
	25	# -----------------
	26	# $Id: palm_wd 1754 2016-02-22 13:50:22Z witha $
	27	#
[1754]	28	# 1753 2016-02-22 13:49:49Z maronga
	29	# Bugfix: use of global variables is required after updating configuration file
	30	#
[1752]	31	# 1751 2016-02-15 07:44:16Z maronga
[1751]	32	# Bugfixes: runs on multiple hosts caused crash of the watchdog. Progress bar
	33	# occasionally showed wrong progress.
	34	# New: Hosts can be switched on/off and update_frequency can be modified via
	35	# File->Options. Security check for cancelation of jobs.
[1611]	36	#
[1619]	37	# 1618 2015-07-13 06:52:15Z maronga
	38	# Added steering via configuration file .wd.config, to be place in the local
	39	# palm directory. Watchdog save files are automatically created upon first start.
	40	#
[1614]	41	# 1613 2015-07-08 14:53:29Z maronga
	42	# Bugfix: tooltip for queuing name did not show up on first update.
	43	# New: added contect menu for showing the parameter file and the run control
	44	# output
	45	#
[1612]	46	# 1611 2015-07-07 12:23:22Z maronga
	47	# Initial revision
	48	#
[1611]	49	#
	50	# Description:
	51	# ------------
	52	# PALM watchdog client for monitoring batch jobs on a variety of hosts specified
	53	# by the user
	54	#
	55	# Instructions:
	56	# -------------
	57	# 1) Modify the header section of palm_wd
	58	# 2) Move .wd.olddata and .wd.newdata to your palm directory
	59	# (e.g. /home/user/current_version/.wd.newdata etc.)
	60	# 3) Modify a copy of palm_wdd for each host to be monitored and move it to the
	61	# respective hosts
	62	# 4) Start the client either from mrungui or from shell by "nohup palm_wd&"
	63
	64	# To do:
	65	# ------
	66	# 1) Add "Options", "Help" and "Manual"
	67	# 2) Move user settings to a configuration file
	68	#------------------------------------------------------------------------------!
	69
[1618]	70	import ConfigParser
	71	import datetime
	72	import os
[1611]	73	from PyQt4 import QtGui, QtCore, uic
	74	from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT
[1618]	75	import shutil
	76	import subprocess as sub
	77	import sys
[1611]	78	import time
	79
	80	# Determine PALM directories
	81	try:
	82	devnull = open(os.devnull, 'w')
	83	out = sub.check_output("echo $PALM_BIN", shell=True, stderr=sub.STDOUT)
	84	palm_bin = out.rstrip()
	85	palm_dir = out.split("palm")[0] + "palm/" + out.split("palm")[1].split("/")[1]
	86	out = None
	87	except:
	88	print "Error. $PALM_BIN is not set."
	89	raise SystemExit
	90
	91
[1618]	92	# Read configuration file
	93	# First check if the configuration file exists
	94	if ( os.path.exists(palm_dir + '/.wd.config') == False ):
	95	print "Error. No configuration file .wd.config found in " + palm_dir
	96	raise SystemExit
	97
	98	config = ConfigParser.RawConfigParser()
	99	config.read(palm_dir + '/.wd.config')
	100
	101	description = []
	102	hostname = []
	103	username = []
	104
	105	for i in range(0,len(config.sections())):
	106
	107	description_tmp = config.sections()[i]
	108
	109	if ( description_tmp != 'Settings' ):
[1751]	110
	111	if ( config.get(description_tmp, 'enabled') == 'true' ):
	112	description.append(description_tmp)
	113	hostname.append(config.get(description_tmp, 'hostname'))
	114	username.append(config.get(description_tmp, 'username'))
[1618]	115	else:
	116	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	117
	118	# Check if .wd.olddata and .wd.newdata exist. Otherwise create the files
	119	if ( os.path.exists(palm_dir + '/.wd.olddata') == False ):
	120	print "No .wd.olddata found. Creating..."
	121	file1 = open(palm_dir + '/.wd.olddata', 'a')
	122	file1.close()
	123
	124
	125	if ( os.path.exists(palm_dir + '/.wd.newdata') == False ):
	126	print "No .wd.newdata found. Creating..."
	127	file1 = open(palm_dir + '/.wd.newdata', 'a')
	128	file1.close()
	129
[1611]	130	# Dummy variables for the jobname and the progressbars
	131	jobname = ""
	132	timestamp = ""
	133	pbars = []
	134	job_data_list = []
	135
[1751]	136	# Message box for showing RUN_CONTROL output
[1613]	137	class MessageBoxScroll(QtGui.QMessageBox):
	138	def __init__(self):
	139	QtGui.QMessageBox.__init__(self)
	140	self.setSizeGripEnabled(True)
	141
	142	def event(self, e):
	143	result = QtGui.QMessageBox.event(self, e)
	144
	145	self.setMinimumHeight(100)
	146	self.setMaximumHeight(16777215)
	147	self.setMinimumWidth(400)
	148	self.setMaximumWidth(16777215)
	149	self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	150
	151	textEdit = self.findChild(QtGui.QTextEdit)
	152	if textEdit != None :
	153	textEdit.setMinimumHeight(800)
	154	textEdit.setMaximumHeight(16777215)
	155	textEdit.setMinimumWidth(1000)
	156	textEdit.setMaximumWidth(16777215)
	157	textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	158	f = QtGui.QFont('not_a_font') #
	159	f.setStyleHint(f.TypeWriter, f.PreferDefault)
	160	textEdit.setFont(f)
	161
	162	return result
	163
	164
[1751]	165	# Message box for showing RUN_CONTROL output
	166	class OptionBox(QtGui.QDialog):
	167	def __init__(self):
	168
	169	super(OptionBox, self).__init__()
	170
	171	uic.loadUi(palm_bin + '/palm_wd_files/wdoptions.ui', self)
	172
	173	hostname = []
	174	self.checkbox = []
	175	j = -1
	176	ypos = 0
	177
	178	self.vbox = QtGui.QVBoxLayout(self)
	179	self.vbox.setSpacing(0)
	180	self.vbox.setMargin(0)
	181
	182	config.read(palm_dir + '/.wd.config')
	183
	184	for i in range(0,len(config.sections())):
	185
	186	description_tmp = config.sections()[i]
	187
	188	if ( description_tmp != 'Settings' ):
	189	description.append(description_tmp)
	190	hostname.append(config.get(description_tmp, 'hostname'))
	191
	192	j = j + 1
	193	self.checkbox.append(j)
	194	self.checkbox[j] = QtGui.QCheckBox(description_tmp, self)
	195	ypos = ypos + 20
	196	self.checkbox[j].move(10,0)
	197	if ( config.get(description_tmp, 'enabled') == 'true' ):
	198	self.checkbox[j].toggle()
	199
	200	self.vbox.addWidget(self.checkbox[j])
	201	else:
	202	self.update_spin.setValue(int(config.get(description_tmp, 'update_frequency')))
	203
	204	self.hostBox.setLayout(self.vbox)
	205	self.hostBox.setGeometry(0, 0, 272, 50+ypos)
	206	self.settingBox.setGeometry(281, 0, 214, 50+ypos)
	207	self.ok.setGeometry(385, 50+ypos, 111, 24)
	208	self.cancel.setGeometry(270, 50+ypos, 111, 24)
	209
	210	self.setGeometry(0, 0, 500, 80+ypos)
	211	self.show()
	212
	213	return
	214
	215	# save updtes to file
	216	def SaveAndClose(self):
	217
	218	for i in range(0,len(self.checkbox)):
	219
	220	if ( self.checkbox[i].checkState() == 0 ):
	221	config.set(str(self.checkbox[i].text()),'enabled','false')
	222	else:
	223	config.set(str(self.checkbox[i].text()),'enabled','true')
	224
	225	config.set('Settings','update_frequency',self.update_spin.text())
	226
	227	with open(palm_dir + '/.wd.config', 'wb') as configfile:
	228	config.write(configfile)
	229
	230
	231	self.close()
	232
	233	return
	234
	235
	236
[1611]	237	# MainWindow class
	238	class Watchdog(QtGui.QMainWindow):
	239
	240	def __init__(self):
	241	super(Watchdog, self).__init__()
	242
	243	self.InitUi()
	244
	245
	246	# Initialized MainWindow UI
	247	def InitUi(self):
	248
	249	# Load predefined mainwindow
	250	uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self)
	251
	252	# Resize columns and set number of rows to zero. Column 6 is hidden and
	253	# contains the remote jobname (e.g. hannover.174610)
	254	self.table.setColumnWidth(0,230)
	255	self.table.setColumnWidth(1,80)
	256	self.table.setColumnWidth(2,50)
	257	self.table.setColumnWidth(3,80)
	258	self.table.setColumnWidth(4,180)
	259	self.table.setColumnWidth(5,115)
	260	self.table.setColumnHidden(6, True)
	261	self.table.setRowCount(0)
[1613]	262
[1611]	263	# Display MainWindow
	264	self.show()
	265	QtGui.QApplication.processEvents()
	266
	267	# Start refresh timer. On timeout perform update
	268	self.timer = QtCore.QTimer(self)
	269	self.timer.timeout.connect(self.Refresh)
	270	self.timer.setSingleShot(False)
	271	self.timer.start(update_frequency)
	272
	273	# The timetimer counts the time since last update
	274	self.timetimer= QtCore.QElapsedTimer()
	275	self.timetimer.start()
	276
	277	# The labeltimer induces the update of the remaining time in the UI
	278	self.labeltimer = QtCore.QTimer(self)
	279	self.labeltimer.timeout.connect(self.UpdateLabel)
	280	self.labeltimer.setSingleShot(False)
	281
	282	# Update in the UI will be performed at each 1/10 of the update interval
	283	self.labeltimer.start(update_frequency/10)
	284	self.label.setText("Next update in " + str(update_frequency/1000/60) + "min")
	285
	286	# Update the list now
	287	self.Refresh()
	288
	289
	290	# Add a custom context menu
	291	def OpenContextMenu(self, position):
	292	menu = QtGui.QMenu()
	293
	294	# "Show details" -> fetch details for the selected job
[1613]	295	detailsjobAction = menu.addAction('Show job details')
[1611]	296	detailsjobAction.setStatusTip('Display detailed info for job')
	297	detailsjobAction.triggered.connect(self.ShowDetails)
	298
[1613]	299	# "Show parameter file" -> show the contents of PARIN
	300	parinAction = menu.addAction('Show parameter file')
	301	parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)')
	302	parinAction.triggered.connect(self.ShowPARIN)
	303
	304	rcAction = menu.addAction('Show run control file')
	305	rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)')
	306	rcAction.triggered.connect(self.ShowRC)
	307
[1611]	308	# "Cancel job" -> remove job from queuing system
	309	canceljobAction = menu.addAction('Cancel job')
	310	canceljobAction.setStatusTip('Remove job from queueing system')
	311	canceljobAction.triggered.connect(self.CancelJob)
	312
[1613]	313	# "Force stop" -> force termination of job
[1611]	314	stopjobAction = menu.addAction('Force stop')
	315	stopjobAction.setStatusTip('Terminate job properly')
	316	stopjobAction.triggered.connect(self.DoStop)
	317
[1613]	318	# "Force restart" -> force rermination and restart of job
[1611]	319	restartjobAction = menu.addAction('Force restart')
	320	restartjobAction.setStatusTip('Terminate job properly and initiate restart')
	321	restartjobAction.triggered.connect(self.DoRestart)
	322
	323	# "Remove from list" -> remove a completed job from the list
	324	removefromlistAction = menu.addAction('Remove from list')
	325	removefromlistAction.setStatusTip('Remove finished job')
	326	removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1))
	327
	328
	329	# Activate/deactive contect menu items based on the status of the runs
	330	state = self.table.item(self.table.currentRow(),3).text()
	331	if (state == "Canceled" or state == "Completed" or state == "Terminated"):
	332	detailsjobAction.setEnabled(False)
	333	canceljobAction.setEnabled(False)
	334	removefromlistAction.setEnabled(True)
	335	else:
	336	detailsjobAction.setEnabled(True)
	337	canceljobAction.setEnabled(True)
	338	removefromlistAction.setEnabled(False)
	339
	340	if ( state == "Running" ):
	341	stopjobAction.setEnabled(True)
	342	restartjobAction.setEnabled(True)
[1613]	343	parinAction.setEnabled(True)
	344	rcAction.setEnabled(True)
[1611]	345	else:
	346	stopjobAction.setEnabled(False)
	347	restartjobAction.setEnabled(False)
[1613]	348	parinAction.setEnabled(False)
	349	rcAction.setEnabled(False)
	350
[1611]	351	# Show the context menu
	352	action = menu.exec_(self.table.mapToGlobal(position))
	353
	354
	355	# Collecting watchdog data and display in the UI
	356	def Refresh(self):
	357
	358	# Use global variables
	359	global pbars
[1751]	360	global update_frequency
[1611]	361
	362	# Deactivate timer processes until update is finished
	363	# QtGui.QApplication.processEvents() updates the UI whenever needed
	364	self.labeltimer.stop()
	365	self.label.setText("Updating...")
	366	QtGui.QApplication.processEvents()
	367	self.setEnabled(False)
	368	QtGui.QApplication.processEvents()
	369
	370	# Get current timestamp
	371	timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
	372
	373	# Open an empty file for the watchdog data
	374	file = open(palm_dir + '/.wd.newdata', 'w')
	375
	376
	377	# Set all required variables to their initial values. Sorting must be
	378	# disabled.
	379	self.table.setRowCount(0)
	380	self.table.setSortingEnabled(False)
	381	i = -1
	382	job_data_list = []
	383	pbars = []
	384
	385	# Scan all hosts in the variable hostname. For each host perform the
	386	# update loop
	387	for h in range(0,len(hostname)):
	388
	389	# Perform ssh command
	390	host = username[h] + "@" + hostname[h]
	391	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd queue " + username[h]],
	392	shell=False,
	393	stdout=sub.PIPE,
	394	stderr=sub.PIPE)
	395	result = ssh.stdout.readlines()
	396	result = filter(None, result)
	397
	398
	399	# Delete all job data
	400	job_data_tmp = []
	401	job_data = []
	402	job_data_n = []
	403	job_progress = 0
	404
	405	# In case of errors display error message
	406	if ( len(result) < 1 ):
	407	error = ssh.stderr.readlines()
	408	if ( error != [] ):
	409	notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error))
	410
	411	# No error -> save job data
	412	else:
	413
	414	# Successively write to job_data
	415	for j in range(0,len(result)):
	416
	417	job_data_tmp.append(j)
	418	job_data_tmp[j] = result[j].split(" ")
	419	job_data_tmp[j] = filter(None, job_data_tmp[j])
	420
	421	job_data.append(j)
	422	job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2],
	423	job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5],
	424	job_data_tmp[j][1], job_data_tmp[j][6].rstrip()]
	425	job_data_n.append(j)
	426	job_data_n[j] = job_data_tmp[j][1]
	427
	428	del(result)
	429
	430
	431
	432	# Now read the data from the last update from file. These data are
	433	# already in the prefered format
	434	file2 = open(palm_dir + '/.wd.olddata', 'r')
	435	result = file2.readlines()
	436	file2.close()
[1751]	437
	438	job_data_old = []
	439	job_data_old_n = []
	440	k = -1
[1611]	441	for j in range(0,len(result)):
	442	if ( result[j].split()[1] == description[h] ):
[1751]	443	k = k + 1
	444	job_data_old.append(k)
	445	job_data_old[k] = result[j].split(" ")
	446	job_data_old[k] = filter(None, job_data_old[k])
	447	job_data_old[k] = [line.rstrip('\n') for line in job_data_old[k]]
	448	job_data_old_n.append(k)
	449	job_data_old_n[k] = job_data_old[k][6]
[1611]	450
	451	# Merge the job_data and job_data_old to find completed, new and
	452	# still queued jobs
	453	if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ):
	454	jobs_full = list(set(job_data_old_n) \| set(job_data_n))
	455	jobs_known = list(set(job_data_old_n) & set(job_data_n))
	456	jobs_complete = set(job_data_old_n) - set(job_data_n)
	457	jobs_new = set(job_data_n) - set(job_data_old_n)
	458	elif ( len(job_data_n) > 0 ):
	459	jobs_full = job_data_n
	460	jobs_known = []
	461	jobs_new = job_data_n
	462	jobs_complete = []
	463	elif ( len(job_data_old_n) > 0 ):
	464	jobs_full = job_data_old_n
	465	jobs_known = []
	466	jobs_new = []
	467	jobs_complete = job_data_old_n
	468	else:
	469	jobs_full = []
	470	jobs_known = []
	471	jobs_new = []
	472	jobs_complete = []
	473
	474	# Display all known jobs
	475	to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)]
	476	for j in range(0,len(to_display)):
	477	i = i + 1
	478	self.table.insertRow(i)
	479	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	480	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	481	self.table.setItem(i, 0, item)
	482	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	483	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	484	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	485	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	486	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	487	self.table.setItem(i, 5, item)
	488	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	489	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	490	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	491	pbars.append(i)
	492	pbars[i] = QtGui.QProgressBar(self)
	493	pbars[i].setValue(int(job_data[to_display[j]][4]))
	494
	495	# Apply a color depending on the status of the job
	496	if ( job_data[to_display[j]][3] == "Running" ):
[1751]	497	self.table.setCellWidget(i,4,pbars[i])
[1611]	498	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	499	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	500	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	501	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	502	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	503	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	504	else:
	505	pbars[j].setEnabled(False)
[1751]	506	self.table.setCellWidget(i,4,pbars[i])
[1611]	507	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	508	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	509	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	510	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	511	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	512	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	513
	514	# Save the job data to file
	515	job_data_list.append(i)
	516	job_data_list[i] = job_data[to_display[j]][0]
	517
	518	printstring = str(job_data[to_display[j]][0]) + " " + \
	519	str(job_data[to_display[j]][1]) + " " + \
	520	str(job_data[to_display[j]][2]) + " " + \
	521	str(job_data[to_display[j]][3]) + " " + \
	522	str(job_data[to_display[j]][4]) + " " + \
	523	str(job_data[to_display[j]][5]) + " " + \
	524	str(job_data[to_display[j]][6]) + " " + \
	525	str(job_data[to_display[j]][7]) + "\n"
	526	file.write(printstring)
	527
	528
	529	# Display all new jobs
	530	to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)]
	531	for j in range(0,len(to_display)):
	532	i = i + 1
	533	self.table.insertRow(i)
	534	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	535	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	536	self.table.setItem(i, 0, item)
	537	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	538	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	539	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	540	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	541	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	542	self.table.setItem(i, 5, item)
	543	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	544	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	545	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	546
	547	pbars.append(i)
	548	pbars[i] = QtGui.QProgressBar(self)
	549	pbars[i].setValue(int(job_data[to_display[j]][4]))
	550
	551	# Apply a color depending on the status of the job
	552	if ( job_data[to_display[j]][3] == "Running" ):
	553	self.table.setCellWidget(i,4,pbars[i])
	554	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	555	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	556	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	557	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	558	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	559	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	560	else:
	561	pbars[j].setEnabled(False)
	562	self.table.setCellWidget(i,4,pbars[i])
	563	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	564	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	565	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	566	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	567	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	568	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	569
	570	# Save job data to file
	571	job_data_list.append(i)
	572	job_data_list[i] = job_data[to_display[j]][0]
	573
	574	printstring = str(job_data[to_display[j]][0]) + " " + \
	575	str(job_data[to_display[j]][1]) + " " + \
	576	str(job_data[to_display[j]][2]) + " " + \
	577	str(job_data[to_display[j]][3]) + " " + \
	578	str(job_data[to_display[j]][4]) + " " + \
	579	str(job_data[to_display[j]][5]) + " " + \
	580	str(job_data[to_display[j]][6]) + " " + \
	581	str(job_data[to_display[j]][7]) + "\n"
	582	file.write(printstring)
	583
	584
	585	# Display all completed/canceled/aborted jobs. The watchdog cannot
	586	# distinguish why the job has been finished
	587	to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)]
	588	for j in range(0,len(to_display)):
	589	i = i + 1
	590	self.table.insertRow(i)
	591	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0])
	592	item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6])
	593	self.table.setItem(i, 0, item)
	594	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1]))
	595	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2]))
	596	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3]))
	597	pbars.append(i)
	598	pbars[j] = QtGui.QProgressBar(self)
	599	pbars[j].setValue(int(job_data_old[to_display[j]][4]))
	600	pbars[j].setEnabled(False)
	601	self.table.setCellWidget(i,4,pbars[j])
	602	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5])
	603	item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7])
	604	self.table.setItem(i, 5, item)
	605	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6]))
	606	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	607	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	608	self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed"))
	609
	610	# Apply a color depending on the status of the job
	611	self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175))
	612	self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175))
	613	self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175))
	614	self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175))
	615	self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175))
	616	self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175))
	617
	618
	619	# Save job data to file
	620	job_data_list.append(i)
	621	job_data_list[i] = job_data_old[to_display[j]][0]
	622
	623	printstring = str(job_data_old[to_display[j]][0]) + " " + \
	624	str(job_data_old[to_display[j]][1]) + " " + \
	625	str(job_data_old[to_display[j]][2]) + " " + \
	626	str(job_data_old[to_display[j]][3]) + " " + \
	627	str(job_data_old[to_display[j]][4]) + " " + \
	628	str(job_data_old[to_display[j]][5]) + " " + \
	629	str(job_data_old[to_display[j]][6]) + " " + \
	630	str(job_data_old[to_display[j]][7]) + "\n"
	631	file.write(printstring)
	632
	633	del(jobs_complete)
	634	del(jobs_new)
	635	del(result)
[1751]	636	del(job_data_old)
	637	del(job_data_old_n)
	638
[1611]	639	file.close()
	640
	641
	642	# Update is complete, sorting can thus be re-enabled
	643	self.table.setSortingEnabled(True)
	644
	645	# Update the logfile
	646	if ( os.path.isfile(palm_dir + '/.wd.newdata') ):
	647	shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata')
	648
	649	# Re-enable timer processes
	650	self.setEnabled(True)
	651	self.label2.setText("Last update: " + timestamp)
	652	self.label.setText("Update complete.")
[1751]	653	self.timetimer.start()
	654	self.timer.start(update_frequency)
	655	self.labeltimer.start(update_frequency/10)
	656	self.UpdateLabel()
[1611]	657	QtGui.QApplication.processEvents()
	658
	659	# Canceling selected job from table
	660	def CancelJob(self):
	661
	662	# Return internal jobname
	663	jobname = self.table.item(self.table.currentRow(),6).text()
	664
	665	# Check description of job in order to login on the correct host
	666	descr = self.table.item(self.table.currentRow(),1).text()
[1751]	667
	668	querybox = QtGui.QMessageBox()
	669	querybox.setText("Attention!\nYou are trying to cancel a job. It will not be able to terminate properly.")
	670	querybox.setInformativeText("Do you really want to cancel " + jobname + " on host " + descr + "?")
	671	querybox.setStandardButtons(QtGui.QMessageBox.Yes \| QtGui.QMessageBox.No)
	672	querybox.setDefaultButton(QtGui.QMessageBox.No)
	673	returnvalue = querybox.exec_()
	674
	675	if ( returnvalue == QtGui.QMessageBox.Yes ):
	676
	677	for h in range(0,len(description)):
	678	if ( descr == description[h] ):
	679	host = username[h] + "@" + hostname[h]
[1611]	680
[1751]	681	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd cancel " + jobname],
	682	shell=False,
	683	stdout=sub.PIPE,
	684	stderr=sub.PIPE)
	685	result = ssh.stdout.readlines()
	686	result = filter(None, result)
[1611]	687
[1751]	688	# In case of error display a warning message
	689	if ( len(result) == 0 ):
	690	error = ssh.stderr.readlines()
	691	notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error))
[1611]	692
[1751]	693	# Otherwise inform the user and mark the job in the table
	694	else:
	695	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled"))
	696	notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result))
[1611]	697
	698
	699	# Show detailed information on job. For documentation see above
	700	def ShowDetails(self):
	701
	702	jobname = self.table.item(self.table.currentRow(),6).text()
	703	descr = self.table.item(self.table.currentRow(),1).text()
	704	for h in range(0,len(description)):
	705	if ( descr == description[h] ):
	706	host = username[h] + "@" + hostname[h]
	707
	708	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd check " + jobname],
	709	shell=False,
	710	stdout=sub.PIPE,
	711	stderr=sub.PIPE)
	712	result = ssh.stdout.readlines()
	713	result = filter(None, result)
	714
	715	if ( len(result) == 0 ):
	716	error = ssh.stderr.readlines()
	717	notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error))
	718	else:
	719	notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result))
	720
	721
	722	# Perform a forced stop on the job
	723	def DoStop(self):
	724
	725	# Return internal jobname
	726	jobname = self.table.item(self.table.currentRow(),6).text()
	727
	728	# Check description of job in order to login on the correct host
	729	descr = self.table.item(self.table.currentRow(),1).text()
	730	for h in range(0,len(description)):
	731	if ( descr == description[h] ):
	732	host = username[h] + "@" + hostname[h]
	733	user = username[h]
	734
	735	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd stop " + user + " " + jobname],
	736	shell=False,
	737	stdout=sub.PIPE,
	738	stderr=sub.PIPE)
	739	result = ssh.stdout.readlines()
	740	result = filter(None, result)
	741
	742	# In case of error display a warning message
	743	if ( len(result) == 0 ):
	744	error = ssh.stderr.readlines()
	745	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	746
	747	# Otherwise inform the user and mark the job in the table
	748	else:
	749	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
[1751]	750	notify = QtGui.QMessageBox.information(self,'Terminate job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
[1611]	751
	752
	753	# Perform a forced stop on the job
	754	def DoRestart(self):
	755
	756	# Return internal jobname
	757	jobname = self.table.item(self.table.currentRow(),6).text()
	758
	759	# Check description of job in order to login on the correct host
	760	descr = self.table.item(self.table.currentRow(),1).text()
	761	for h in range(0,len(description)):
	762	if ( descr == description[h] ):
	763	host = username[h] + "@" + hostname[h]
	764	user = username[h]
	765
	766	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd restart " + user + " " + jobname],
	767	shell=False,
	768	stdout=sub.PIPE,
	769	stderr=sub.PIPE)
	770	result = ssh.stdout.readlines()
	771	result = filter(None, result)
	772
	773	# In case of error display a warning message
	774	if ( len(result) == 0 ):
	775	error = ssh.stderr.readlines()
	776	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	777
	778	# Otherwise inform the user and mark the job in the table
	779	else:
	780	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	781	notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	782
	783
[1613]	784	# Read the PARIN file of the job
	785	def ShowPARIN(self):
	786
	787	# Return internal jobname
	788	jobname = self.table.item(self.table.currentRow(),6).text()
	789	jobrealname = self.table.item(self.table.currentRow(),0).text()
	790
	791	# Check description of job in order to login on the correct host
	792	descr = self.table.item(self.table.currentRow(),1).text()
	793	for h in range(0,len(description)):
	794	if ( descr == description[h] ):
	795	host = username[h] + "@" + hostname[h]
	796	user = username[h]
	797
	798	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd parin " + user + " " + jobname],
	799	shell=False,
	800	stdout=sub.PIPE,
	801	stderr=sub.PIPE)
	802	result = ssh.stdout.readlines()
	803	result = filter(None, result)
	804
	805	# In case of error display a warning message
	806	if ( len(result) == 0 ):
	807	error = ssh.stderr.readlines()
	808	notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error))
	809
	810	# Otherwise inform the user and mark the job in the table
	811	else:
	812	mb = MessageBoxScroll()
	813	mb.setText("Parameter file for job: " + jobrealname + "")
	814	mb.setDetailedText(''.join(result))
	815	mb.exec_()
	816
	817	# Read the PARIN file of the job
	818	def ShowRC(self):
	819
	820	# Return internal jobname and real job name
	821	jobname = self.table.item(self.table.currentRow(),6).text()
	822	jobrealname = self.table.item(self.table.currentRow(),0).text()
	823
	824	# Check description of job in order to login on the correct host
	825	descr = self.table.item(self.table.currentRow(),1).text()
	826	for h in range(0,len(description)):
	827	if ( descr == description[h] ):
	828	host = username[h] + "@" + hostname[h]
	829	user = username[h]
	830
	831	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd rc " + user + " " + jobname],
	832	shell=False,
	833	stdout=sub.PIPE,
	834	stderr=sub.PIPE)
	835	result = ssh.stdout.readlines()
	836	result = filter(None, result)
	837
	838	# In case of error display a warning message
	839	if ( len(result) == 0 ):
	840	error = ssh.stderr.readlines()
	841	notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error))
	842
	843	# Otherwise inform the user and mark the job in the table
	844	else:
	845	mb = MessageBoxScroll()
	846	lastline = result[len(result)-2].split()[2]
	847	mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline)
	848	mb.setDetailedText(''.join(result))
	849	mb.exec_()
	850
[1611]	851	# Remove a job from list - removes the current row from the table and from
	852	# save file
	853	def RemoveFromList(self, row):
	854	if ( row == -1 ):
	855	row = self.table.currentRow()
	856
	857	# Read data from save file
	858	job_to_delete = self.table.item(row,6).text()
	859	self.table.removeRow(row)
	860	file = open(palm_dir + '/.wd.olddata', 'r')
	861	result = file.readlines()
	862	result = filter(None, result)
	863	file.close()
	864	file = open(palm_dir + '/.wd.olddata', 'w')
	865
	866	job_data_old = []
	867
	868	if ( len(result) == 0 ):
	869	notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.")
	870	else:
	871	# Save data in array job_data_old
	872	for j in range(0,len(result)):
	873	job_data_old.append(j)
	874	job_data_old[j] = result[j].split(" ")
	875	job_data_old[j] = filter(None, job_data_old[j])
	876	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	877
	878	# Check if line j is the selected job, if not -> save to file
	879	if ( job_data_old[j][6] != job_to_delete ):
	880	printstring = str(job_data_old[j][0]) + " " + \
	881	str(job_data_old[j][1]) + " " + \
	882	str(job_data_old[j][2]) + " " + \
	883	str(job_data_old[j][3]) + " " + \
	884	str(job_data_old[j][4]) + " " + \
	885	str(job_data_old[j][5]) + " " + \
	886	str(job_data_old[j][6]) + " " + \
	887	str(job_data_old[j][7]) + "\n"
	888	file.write(printstring)
	889
	890	file.close()
	891
	892	# Remove all completed jobs from list
	893	def ClearList(self):
	894
	895	num_of_lines = self.table.rowCount()
	896
	897	# Delete all lines with completed/canceled jobs from list. The counter
	898	# must decrease as the line numbers would be messed up otherwise
	899	for j in range(num_of_lines-1,-1,-1):
	900	state = self.table.item(j,3).text()
	901	if ( state == "Completed" or state == "Canceled" or state == "Terminated"):
	902	self.RemoveFromList(j)
	903
	904	# Update the label
	905	def UpdateLabel(self):
	906	remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60
	907	self.label.setText("Next update in " + str(remaining_time) + " min")
	908
	909
[1751]	910	# Enter Options menu
	911	def Options(self):
	912
	913	# disable mainwindow
	914	self.setEnabled(False)
	915
	916	# show Options Dialog
	917	opt = OptionBox()
	918	opt.exec_()
	919
	920	self.UpdateHosts()
	921	self.setEnabled(True)
	922
	923	# Update settings
	924	def UpdateHosts(self):
	925
	926	global update_frequency
[1753]	927	global description
	928	global hostname
	929	global username
	930
[1751]	931	description = []
	932	hostname = []
	933	username = []
	934
	935	for i in range(0,len(config.sections())):
	936
	937	description_tmp = config.sections()[i]
	938
	939	if ( description_tmp != 'Settings' ):
	940
	941	if ( config.get(description_tmp, 'enabled') == 'true' ):
	942	description.append(description_tmp)
	943	hostname.append(config.get(description_tmp, 'hostname'))
	944	username.append(config.get(description_tmp, 'username'))
	945	else:
	946	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	947
	948
[1611]	949	# Main loop
	950	def main():
	951
	952	app = QtGui.QApplication(sys.argv)
	953	res = Watchdog()
	954	sys.exit(app.exec_())
	955
	956
	957	if __name__ == '__main__':
[1613]	958	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |