Home

Context Navigation

source: palm/trunk/SCRIPTS/palm_wd @ 2809

Last change on this file since 2809 was 2718, checked in by maronga, 7 years ago
deleting of deprecated files; headers updated where needed
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 39.8 KB

Rev	Line
[1611]	1	#!/usr/bin/python
	2	# -- coding: utf-8 --
	3	#--------------------------------------------------------------------------------#
[2696]	4	# This file is part of the PALM model system.
[1611]	5	#
	6	# PALM is free software: you can redistribute it and/or modify it under the terms
	7	# of the GNU General Public License as published by the Free Software Foundation,
	8	# either version 3 of the License, or (at your option) any later version.
	9	#
	10	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License along with
	15	# PALM. If not, see <http://www.gnu.org/licenses/>.
	16	#
[2718]	17	# Copyright 1997-2018 Leibniz Universitaet Hannover
[1611]	18	#--------------------------------------------------------------------------------#
	19	#
	20	# Current revisions:
	21	# -----------------
[1752]	22	#
[1754]	23	#
[1752]	24	# Former revisions:
	25	# -----------------
	26	# $Id: palm_wd 2718 2018-01-02 08:49:38Z schwenkel $
[2716]	27	# Corrected "Former revisions" section
	28	#
	29	# 2696 2017-12-14 17:12:51Z kanani
	30	# Change in file header (GPL part)
	31	#
	32	# 2416 2017-09-06 14:28:14Z maronga
[2416]	33	# Adapted for palmrun
	34	#
	35	# 1754 2016-02-22 13:50:22Z maronga
[1752]	36	#
[1754]	37	# 1753 2016-02-22 13:49:49Z maronga
	38	# Bugfix: use of global variables is required after updating configuration file
	39	#
[1752]	40	# 1751 2016-02-15 07:44:16Z maronga
[1751]	41	# Bugfixes: runs on multiple hosts caused crash of the watchdog. Progress bar
	42	# occasionally showed wrong progress.
	43	# New: Hosts can be switched on/off and update_frequency can be modified via
	44	# File->Options. Security check for cancelation of jobs.
[1611]	45	#
[1619]	46	# 1618 2015-07-13 06:52:15Z maronga
	47	# Added steering via configuration file .wd.config, to be place in the local
	48	# palm directory. Watchdog save files are automatically created upon first start.
	49	#
[1614]	50	# 1613 2015-07-08 14:53:29Z maronga
	51	# Bugfix: tooltip for queuing name did not show up on first update.
	52	# New: added contect menu for showing the parameter file and the run control
	53	# output
	54	#
[1612]	55	# 1611 2015-07-07 12:23:22Z maronga
	56	# Initial revision
	57	#
[1611]	58	#
	59	# Description:
	60	# ------------
	61	# PALM watchdog client for monitoring batch jobs on a variety of hosts specified
	62	# by the user
	63	#
	64	# Instructions:
	65	# -------------
	66	# 1) Modify the header section of palm_wd
	67	# 2) Move .wd.olddata and .wd.newdata to your palm directory
	68	# (e.g. /home/user/current_version/.wd.newdata etc.)
	69	# 3) Modify a copy of palm_wdd for each host to be monitored and move it to the
	70	# respective hosts
	71	# 4) Start the client either from mrungui or from shell by "nohup palm_wd&"
	72
	73	# To do:
	74	# ------
	75	# 1) Add "Options", "Help" and "Manual"
	76	# 2) Move user settings to a configuration file
	77	#------------------------------------------------------------------------------!
	78
[1618]	79	import ConfigParser
	80	import datetime
	81	import os
[1611]	82	from PyQt4 import QtGui, QtCore, uic
	83	from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT
[1618]	84	import shutil
	85	import subprocess as sub
	86	import sys
[1611]	87	import time
	88
	89	# Determine PALM directories
	90	try:
	91	devnull = open(os.devnull, 'w')
	92	out = sub.check_output("echo $PALM_BIN", shell=True, stderr=sub.STDOUT)
	93	palm_bin = out.rstrip()
	94	palm_dir = out.split("palm")[0] + "palm/" + out.split("palm")[1].split("/")[1]
	95	out = None
	96	except:
	97	print "Error. $PALM_BIN is not set."
	98	raise SystemExit
	99
	100
[1618]	101	# Read configuration file
	102	# First check if the configuration file exists
	103	if ( os.path.exists(palm_dir + '/.wd.config') == False ):
	104	print "Error. No configuration file .wd.config found in " + palm_dir
	105	raise SystemExit
	106
	107	config = ConfigParser.RawConfigParser()
	108	config.read(palm_dir + '/.wd.config')
	109
	110	description = []
	111	hostname = []
	112	username = []
	113
	114	for i in range(0,len(config.sections())):
	115
	116	description_tmp = config.sections()[i]
	117
	118	if ( description_tmp != 'Settings' ):
[1751]	119
	120	if ( config.get(description_tmp, 'enabled') == 'true' ):
	121	description.append(description_tmp)
	122	hostname.append(config.get(description_tmp, 'hostname'))
	123	username.append(config.get(description_tmp, 'username'))
[1618]	124	else:
	125	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	126
	127	# Check if .wd.olddata and .wd.newdata exist. Otherwise create the files
	128	if ( os.path.exists(palm_dir + '/.wd.olddata') == False ):
	129	print "No .wd.olddata found. Creating..."
	130	file1 = open(palm_dir + '/.wd.olddata', 'a')
	131	file1.close()
	132
	133
	134	if ( os.path.exists(palm_dir + '/.wd.newdata') == False ):
	135	print "No .wd.newdata found. Creating..."
	136	file1 = open(palm_dir + '/.wd.newdata', 'a')
	137	file1.close()
	138
[1611]	139	# Dummy variables for the jobname and the progressbars
	140	jobname = ""
	141	timestamp = ""
	142	pbars = []
	143	job_data_list = []
	144
[1751]	145	# Message box for showing RUN_CONTROL output
[1613]	146	class MessageBoxScroll(QtGui.QMessageBox):
	147	def __init__(self):
	148	QtGui.QMessageBox.__init__(self)
	149	self.setSizeGripEnabled(True)
	150
	151	def event(self, e):
	152	result = QtGui.QMessageBox.event(self, e)
	153
	154	self.setMinimumHeight(100)
	155	self.setMaximumHeight(16777215)
	156	self.setMinimumWidth(400)
	157	self.setMaximumWidth(16777215)
	158	self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	159
	160	textEdit = self.findChild(QtGui.QTextEdit)
	161	if textEdit != None :
	162	textEdit.setMinimumHeight(800)
	163	textEdit.setMaximumHeight(16777215)
	164	textEdit.setMinimumWidth(1000)
	165	textEdit.setMaximumWidth(16777215)
	166	textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	167	f = QtGui.QFont('not_a_font') #
	168	f.setStyleHint(f.TypeWriter, f.PreferDefault)
	169	textEdit.setFont(f)
	170
	171	return result
	172
	173
[1751]	174	# Message box for showing RUN_CONTROL output
	175	class OptionBox(QtGui.QDialog):
	176	def __init__(self):
	177
	178	super(OptionBox, self).__init__()
	179
	180	uic.loadUi(palm_bin + '/palm_wd_files/wdoptions.ui', self)
	181
	182	hostname = []
	183	self.checkbox = []
	184	j = -1
	185	ypos = 0
	186
	187	self.vbox = QtGui.QVBoxLayout(self)
	188	self.vbox.setSpacing(0)
	189	self.vbox.setMargin(0)
	190
	191	config.read(palm_dir + '/.wd.config')
	192
	193	for i in range(0,len(config.sections())):
	194
	195	description_tmp = config.sections()[i]
	196
	197	if ( description_tmp != 'Settings' ):
	198	description.append(description_tmp)
	199	hostname.append(config.get(description_tmp, 'hostname'))
	200
	201	j = j + 1
	202	self.checkbox.append(j)
	203	self.checkbox[j] = QtGui.QCheckBox(description_tmp, self)
	204	ypos = ypos + 20
	205	self.checkbox[j].move(10,0)
	206	if ( config.get(description_tmp, 'enabled') == 'true' ):
	207	self.checkbox[j].toggle()
	208
	209	self.vbox.addWidget(self.checkbox[j])
	210	else:
	211	self.update_spin.setValue(int(config.get(description_tmp, 'update_frequency')))
	212
	213	self.hostBox.setLayout(self.vbox)
	214	self.hostBox.setGeometry(0, 0, 272, 50+ypos)
	215	self.settingBox.setGeometry(281, 0, 214, 50+ypos)
	216	self.ok.setGeometry(385, 50+ypos, 111, 24)
	217	self.cancel.setGeometry(270, 50+ypos, 111, 24)
	218
	219	self.setGeometry(0, 0, 500, 80+ypos)
	220	self.show()
	221
	222	return
	223
	224	# save updtes to file
	225	def SaveAndClose(self):
	226
	227	for i in range(0,len(self.checkbox)):
	228
	229	if ( self.checkbox[i].checkState() == 0 ):
	230	config.set(str(self.checkbox[i].text()),'enabled','false')
	231	else:
	232	config.set(str(self.checkbox[i].text()),'enabled','true')
	233
	234	config.set('Settings','update_frequency',self.update_spin.text())
	235
	236	with open(palm_dir + '/.wd.config', 'wb') as configfile:
	237	config.write(configfile)
	238
	239
	240	self.close()
	241
	242	return
	243
	244
	245
[1611]	246	# MainWindow class
	247	class Watchdog(QtGui.QMainWindow):
	248
	249	def __init__(self):
	250	super(Watchdog, self).__init__()
	251
	252	self.InitUi()
	253
	254
	255	# Initialized MainWindow UI
	256	def InitUi(self):
	257
	258	# Load predefined mainwindow
	259	uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self)
	260
	261	# Resize columns and set number of rows to zero. Column 6 is hidden and
	262	# contains the remote jobname (e.g. hannover.174610)
	263	self.table.setColumnWidth(0,230)
	264	self.table.setColumnWidth(1,80)
	265	self.table.setColumnWidth(2,50)
	266	self.table.setColumnWidth(3,80)
	267	self.table.setColumnWidth(4,180)
	268	self.table.setColumnWidth(5,115)
	269	self.table.setColumnHidden(6, True)
	270	self.table.setRowCount(0)
[1613]	271
[1611]	272	# Display MainWindow
	273	self.show()
	274	QtGui.QApplication.processEvents()
	275
	276	# Start refresh timer. On timeout perform update
	277	self.timer = QtCore.QTimer(self)
	278	self.timer.timeout.connect(self.Refresh)
	279	self.timer.setSingleShot(False)
	280	self.timer.start(update_frequency)
	281
	282	# The timetimer counts the time since last update
	283	self.timetimer= QtCore.QElapsedTimer()
	284	self.timetimer.start()
	285
	286	# The labeltimer induces the update of the remaining time in the UI
	287	self.labeltimer = QtCore.QTimer(self)
	288	self.labeltimer.timeout.connect(self.UpdateLabel)
	289	self.labeltimer.setSingleShot(False)
	290
	291	# Update in the UI will be performed at each 1/10 of the update interval
	292	self.labeltimer.start(update_frequency/10)
	293	self.label.setText("Next update in " + str(update_frequency/1000/60) + "min")
	294
	295	# Update the list now
	296	self.Refresh()
	297
	298
	299	# Add a custom context menu
	300	def OpenContextMenu(self, position):
	301	menu = QtGui.QMenu()
	302
	303	# "Show details" -> fetch details for the selected job
[1613]	304	detailsjobAction = menu.addAction('Show job details')
[1611]	305	detailsjobAction.setStatusTip('Display detailed info for job')
	306	detailsjobAction.triggered.connect(self.ShowDetails)
	307
[1613]	308	# "Show parameter file" -> show the contents of PARIN
	309	parinAction = menu.addAction('Show parameter file')
	310	parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)')
	311	parinAction.triggered.connect(self.ShowPARIN)
	312
	313	rcAction = menu.addAction('Show run control file')
	314	rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)')
	315	rcAction.triggered.connect(self.ShowRC)
	316
[1611]	317	# "Cancel job" -> remove job from queuing system
	318	canceljobAction = menu.addAction('Cancel job')
	319	canceljobAction.setStatusTip('Remove job from queueing system')
	320	canceljobAction.triggered.connect(self.CancelJob)
	321
[1613]	322	# "Force stop" -> force termination of job
[1611]	323	stopjobAction = menu.addAction('Force stop')
	324	stopjobAction.setStatusTip('Terminate job properly')
	325	stopjobAction.triggered.connect(self.DoStop)
	326
[1613]	327	# "Force restart" -> force rermination and restart of job
[1611]	328	restartjobAction = menu.addAction('Force restart')
	329	restartjobAction.setStatusTip('Terminate job properly and initiate restart')
	330	restartjobAction.triggered.connect(self.DoRestart)
	331
	332	# "Remove from list" -> remove a completed job from the list
	333	removefromlistAction = menu.addAction('Remove from list')
	334	removefromlistAction.setStatusTip('Remove finished job')
	335	removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1))
	336
	337
	338	# Activate/deactive contect menu items based on the status of the runs
	339	state = self.table.item(self.table.currentRow(),3).text()
	340	if (state == "Canceled" or state == "Completed" or state == "Terminated"):
	341	detailsjobAction.setEnabled(False)
	342	canceljobAction.setEnabled(False)
	343	removefromlistAction.setEnabled(True)
	344	else:
	345	detailsjobAction.setEnabled(True)
	346	canceljobAction.setEnabled(True)
	347	removefromlistAction.setEnabled(False)
	348
	349	if ( state == "Running" ):
	350	stopjobAction.setEnabled(True)
	351	restartjobAction.setEnabled(True)
[1613]	352	parinAction.setEnabled(True)
	353	rcAction.setEnabled(True)
[1611]	354	else:
	355	stopjobAction.setEnabled(False)
	356	restartjobAction.setEnabled(False)
[1613]	357	parinAction.setEnabled(False)
	358	rcAction.setEnabled(False)
	359
[1611]	360	# Show the context menu
	361	action = menu.exec_(self.table.mapToGlobal(position))
	362
	363
	364	# Collecting watchdog data and display in the UI
	365	def Refresh(self):
	366
	367	# Use global variables
	368	global pbars
[1751]	369	global update_frequency
[1611]	370
	371	# Deactivate timer processes until update is finished
	372	# QtGui.QApplication.processEvents() updates the UI whenever needed
	373	self.labeltimer.stop()
	374	self.label.setText("Updating...")
	375	QtGui.QApplication.processEvents()
	376	self.setEnabled(False)
	377	QtGui.QApplication.processEvents()
	378
	379	# Get current timestamp
	380	timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
	381
	382	# Open an empty file for the watchdog data
	383	file = open(palm_dir + '/.wd.newdata', 'w')
	384
	385
	386	# Set all required variables to their initial values. Sorting must be
	387	# disabled.
	388	self.table.setRowCount(0)
	389	self.table.setSortingEnabled(False)
	390	i = -1
	391	job_data_list = []
	392	pbars = []
	393
	394	# Scan all hosts in the variable hostname. For each host perform the
	395	# update loop
	396	for h in range(0,len(hostname)):
	397
	398	# Perform ssh command
	399	host = username[h] + "@" + hostname[h]
	400	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd queue " + username[h]],
	401	shell=False,
	402	stdout=sub.PIPE,
	403	stderr=sub.PIPE)
	404	result = ssh.stdout.readlines()
	405	result = filter(None, result)
	406
	407
	408	# Delete all job data
	409	job_data_tmp = []
	410	job_data = []
	411	job_data_n = []
	412	job_progress = 0
	413
	414	# In case of errors display error message
	415	if ( len(result) < 1 ):
	416	error = ssh.stderr.readlines()
	417	if ( error != [] ):
	418	notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error))
	419
	420	# No error -> save job data
	421	else:
	422
	423	# Successively write to job_data
	424	for j in range(0,len(result)):
	425
	426	job_data_tmp.append(j)
	427	job_data_tmp[j] = result[j].split(" ")
	428	job_data_tmp[j] = filter(None, job_data_tmp[j])
	429
	430	job_data.append(j)
	431	job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2],
	432	job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5],
	433	job_data_tmp[j][1], job_data_tmp[j][6].rstrip()]
	434	job_data_n.append(j)
	435	job_data_n[j] = job_data_tmp[j][1]
	436
	437	del(result)
	438
	439
	440
	441	# Now read the data from the last update from file. These data are
	442	# already in the prefered format
	443	file2 = open(palm_dir + '/.wd.olddata', 'r')
	444	result = file2.readlines()
	445	file2.close()
[1751]	446
	447	job_data_old = []
	448	job_data_old_n = []
	449	k = -1
[1611]	450	for j in range(0,len(result)):
	451	if ( result[j].split()[1] == description[h] ):
[1751]	452	k = k + 1
	453	job_data_old.append(k)
	454	job_data_old[k] = result[j].split(" ")
	455	job_data_old[k] = filter(None, job_data_old[k])
	456	job_data_old[k] = [line.rstrip('\n') for line in job_data_old[k]]
	457	job_data_old_n.append(k)
	458	job_data_old_n[k] = job_data_old[k][6]
[1611]	459
	460	# Merge the job_data and job_data_old to find completed, new and
	461	# still queued jobs
	462	if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ):
	463	jobs_full = list(set(job_data_old_n) \| set(job_data_n))
	464	jobs_known = list(set(job_data_old_n) & set(job_data_n))
	465	jobs_complete = set(job_data_old_n) - set(job_data_n)
	466	jobs_new = set(job_data_n) - set(job_data_old_n)
	467	elif ( len(job_data_n) > 0 ):
	468	jobs_full = job_data_n
	469	jobs_known = []
	470	jobs_new = job_data_n
	471	jobs_complete = []
	472	elif ( len(job_data_old_n) > 0 ):
	473	jobs_full = job_data_old_n
	474	jobs_known = []
	475	jobs_new = []
	476	jobs_complete = job_data_old_n
	477	else:
	478	jobs_full = []
	479	jobs_known = []
	480	jobs_new = []
	481	jobs_complete = []
	482
	483	# Display all known jobs
	484	to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)]
	485	for j in range(0,len(to_display)):
	486	i = i + 1
	487	self.table.insertRow(i)
	488	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	489	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	490	self.table.setItem(i, 0, item)
	491	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	492	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	493	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	494	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	495	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	496	self.table.setItem(i, 5, item)
	497	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	498	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	499	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	500	pbars.append(i)
	501	pbars[i] = QtGui.QProgressBar(self)
	502	pbars[i].setValue(int(job_data[to_display[j]][4]))
	503
	504	# Apply a color depending on the status of the job
	505	if ( job_data[to_display[j]][3] == "Running" ):
[1751]	506	self.table.setCellWidget(i,4,pbars[i])
[1611]	507	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	508	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	509	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	510	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	511	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	512	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	513	else:
	514	pbars[j].setEnabled(False)
[1751]	515	self.table.setCellWidget(i,4,pbars[i])
[1611]	516	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	517	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	518	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	519	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	520	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	521	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	522
	523	# Save the job data to file
	524	job_data_list.append(i)
	525	job_data_list[i] = job_data[to_display[j]][0]
	526
	527	printstring = str(job_data[to_display[j]][0]) + " " + \
	528	str(job_data[to_display[j]][1]) + " " + \
	529	str(job_data[to_display[j]][2]) + " " + \
	530	str(job_data[to_display[j]][3]) + " " + \
	531	str(job_data[to_display[j]][4]) + " " + \
	532	str(job_data[to_display[j]][5]) + " " + \
	533	str(job_data[to_display[j]][6]) + " " + \
	534	str(job_data[to_display[j]][7]) + "\n"
	535	file.write(printstring)
	536
	537
	538	# Display all new jobs
	539	to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)]
	540	for j in range(0,len(to_display)):
	541	i = i + 1
	542	self.table.insertRow(i)
	543	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	544	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	545	self.table.setItem(i, 0, item)
	546	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	547	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	548	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	549	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	550	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	551	self.table.setItem(i, 5, item)
	552	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	553	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	554	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	555
	556	pbars.append(i)
	557	pbars[i] = QtGui.QProgressBar(self)
	558	pbars[i].setValue(int(job_data[to_display[j]][4]))
	559
	560	# Apply a color depending on the status of the job
	561	if ( job_data[to_display[j]][3] == "Running" ):
	562	self.table.setCellWidget(i,4,pbars[i])
	563	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	564	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	565	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	566	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	567	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	568	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	569	else:
	570	pbars[j].setEnabled(False)
	571	self.table.setCellWidget(i,4,pbars[i])
	572	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	573	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	574	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	575	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	576	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	577	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	578
	579	# Save job data to file
	580	job_data_list.append(i)
	581	job_data_list[i] = job_data[to_display[j]][0]
	582
	583	printstring = str(job_data[to_display[j]][0]) + " " + \
	584	str(job_data[to_display[j]][1]) + " " + \
	585	str(job_data[to_display[j]][2]) + " " + \
	586	str(job_data[to_display[j]][3]) + " " + \
	587	str(job_data[to_display[j]][4]) + " " + \
	588	str(job_data[to_display[j]][5]) + " " + \
	589	str(job_data[to_display[j]][6]) + " " + \
	590	str(job_data[to_display[j]][7]) + "\n"
	591	file.write(printstring)
	592
	593
	594	# Display all completed/canceled/aborted jobs. The watchdog cannot
	595	# distinguish why the job has been finished
	596	to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)]
	597	for j in range(0,len(to_display)):
	598	i = i + 1
	599	self.table.insertRow(i)
	600	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0])
	601	item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6])
	602	self.table.setItem(i, 0, item)
	603	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1]))
	604	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2]))
	605	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3]))
	606	pbars.append(i)
	607	pbars[j] = QtGui.QProgressBar(self)
	608	pbars[j].setValue(int(job_data_old[to_display[j]][4]))
	609	pbars[j].setEnabled(False)
	610	self.table.setCellWidget(i,4,pbars[j])
	611	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5])
	612	item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7])
	613	self.table.setItem(i, 5, item)
	614	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6]))
	615	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	616	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	617	self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed"))
	618
	619	# Apply a color depending on the status of the job
	620	self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175))
	621	self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175))
	622	self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175))
	623	self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175))
	624	self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175))
	625	self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175))
	626
	627
	628	# Save job data to file
	629	job_data_list.append(i)
	630	job_data_list[i] = job_data_old[to_display[j]][0]
	631
	632	printstring = str(job_data_old[to_display[j]][0]) + " " + \
	633	str(job_data_old[to_display[j]][1]) + " " + \
	634	str(job_data_old[to_display[j]][2]) + " " + \
	635	str(job_data_old[to_display[j]][3]) + " " + \
	636	str(job_data_old[to_display[j]][4]) + " " + \
	637	str(job_data_old[to_display[j]][5]) + " " + \
	638	str(job_data_old[to_display[j]][6]) + " " + \
	639	str(job_data_old[to_display[j]][7]) + "\n"
	640	file.write(printstring)
	641
	642	del(jobs_complete)
	643	del(jobs_new)
	644	del(result)
[1751]	645	del(job_data_old)
	646	del(job_data_old_n)
	647
[1611]	648	file.close()
	649
	650
	651	# Update is complete, sorting can thus be re-enabled
	652	self.table.setSortingEnabled(True)
	653
	654	# Update the logfile
	655	if ( os.path.isfile(palm_dir + '/.wd.newdata') ):
	656	shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata')
	657
	658	# Re-enable timer processes
	659	self.setEnabled(True)
	660	self.label2.setText("Last update: " + timestamp)
	661	self.label.setText("Update complete.")
[1751]	662	self.timetimer.start()
	663	self.timer.start(update_frequency)
	664	self.labeltimer.start(update_frequency/10)
	665	self.UpdateLabel()
[1611]	666	QtGui.QApplication.processEvents()
	667
	668	# Canceling selected job from table
	669	def CancelJob(self):
	670
	671	# Return internal jobname
	672	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	673	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	674
	675	# Check description of job in order to login on the correct host
	676	descr = self.table.item(self.table.currentRow(),1).text()
[1751]	677
	678	querybox = QtGui.QMessageBox()
	679	querybox.setText("Attention!\nYou are trying to cancel a job. It will not be able to terminate properly.")
	680	querybox.setInformativeText("Do you really want to cancel " + jobname + " on host " + descr + "?")
	681	querybox.setStandardButtons(QtGui.QMessageBox.Yes \| QtGui.QMessageBox.No)
	682	querybox.setDefaultButton(QtGui.QMessageBox.No)
	683	returnvalue = querybox.exec_()
	684
	685	if ( returnvalue == QtGui.QMessageBox.Yes ):
	686
	687	for h in range(0,len(description)):
	688	if ( descr == description[h] ):
	689	host = username[h] + "@" + hostname[h]
[1611]	690
[1751]	691	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd cancel " + jobname],
	692	shell=False,
	693	stdout=sub.PIPE,
	694	stderr=sub.PIPE)
	695	result = ssh.stdout.readlines()
	696	result = filter(None, result)
[1611]	697
[1751]	698	# In case of error display a warning message
	699	if ( len(result) == 0 ):
	700	error = ssh.stderr.readlines()
	701	notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error))
[1611]	702
[1751]	703	# Otherwise inform the user and mark the job in the table
	704	else:
	705	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled"))
	706	notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result))
[1611]	707
	708
	709	# Show detailed information on job. For documentation see above
	710	def ShowDetails(self):
	711
[2416]	712	jobname = self.table.item(self.table.currentRow(),6).text()
[1611]	713	descr = self.table.item(self.table.currentRow(),1).text()
	714	for h in range(0,len(description)):
	715	if ( descr == description[h] ):
	716	host = username[h] + "@" + hostname[h]
	717
	718	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd check " + jobname],
	719	shell=False,
	720	stdout=sub.PIPE,
	721	stderr=sub.PIPE)
	722	result = ssh.stdout.readlines()
	723	result = filter(None, result)
	724
	725	if ( len(result) == 0 ):
	726	error = ssh.stderr.readlines()
	727	notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error))
	728	else:
	729	notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result))
	730
	731
	732	# Perform a forced stop on the job
	733	def DoStop(self):
	734
	735	# Return internal jobname
	736	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	737	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	738	# Check description of job in order to login on the correct host
	739	descr = self.table.item(self.table.currentRow(),1).text()
	740	for h in range(0,len(description)):
	741	if ( descr == description[h] ):
	742	host = username[h] + "@" + hostname[h]
	743	user = username[h]
	744
[2416]	745	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd stop " + jobrealname],
[1611]	746	shell=False,
	747	stdout=sub.PIPE,
	748	stderr=sub.PIPE)
	749	result = ssh.stdout.readlines()
	750	result = filter(None, result)
	751
	752	# In case of error display a warning message
	753	if ( len(result) == 0 ):
	754	error = ssh.stderr.readlines()
	755	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	756
	757	# Otherwise inform the user and mark the job in the table
	758	else:
	759	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
[1751]	760	notify = QtGui.QMessageBox.information(self,'Terminate job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
[1611]	761
	762
	763	# Perform a forced stop on the job
	764	def DoRestart(self):
	765
	766	# Return internal jobname
	767	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	768	jobrealname = self.table.item(self.table.currentRow(),0).text()
	769
[1611]	770	# Check description of job in order to login on the correct host
	771	descr = self.table.item(self.table.currentRow(),1).text()
	772	for h in range(0,len(description)):
	773	if ( descr == description[h] ):
	774	host = username[h] + "@" + hostname[h]
	775	user = username[h]
	776
[2416]	777	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd restart " + jobrealname],
[1611]	778	shell=False,
	779	stdout=sub.PIPE,
	780	stderr=sub.PIPE)
	781	result = ssh.stdout.readlines()
	782	result = filter(None, result)
	783
	784	# In case of error display a warning message
	785	if ( len(result) == 0 ):
	786	error = ssh.stderr.readlines()
	787	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	788
	789	# Otherwise inform the user and mark the job in the table
	790	else:
	791	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	792	notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	793
	794
[1613]	795	# Read the PARIN file of the job
	796	def ShowPARIN(self):
	797
	798	# Return internal jobname
	799	jobname = self.table.item(self.table.currentRow(),6).text()
	800	jobrealname = self.table.item(self.table.currentRow(),0).text()
	801
	802	# Check description of job in order to login on the correct host
	803	descr = self.table.item(self.table.currentRow(),1).text()
	804	for h in range(0,len(description)):
	805	if ( descr == description[h] ):
	806	host = username[h] + "@" + hostname[h]
	807	user = username[h]
	808
[2416]	809	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd parin " + jobrealname],
[1613]	810	shell=False,
	811	stdout=sub.PIPE,
	812	stderr=sub.PIPE)
	813	result = ssh.stdout.readlines()
	814	result = filter(None, result)
	815
	816	# In case of error display a warning message
	817	if ( len(result) == 0 ):
	818	error = ssh.stderr.readlines()
	819	notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error))
	820
	821	# Otherwise inform the user and mark the job in the table
	822	else:
	823	mb = MessageBoxScroll()
	824	mb.setText("Parameter file for job: " + jobrealname + "")
	825	mb.setDetailedText(''.join(result))
	826	mb.exec_()
	827
	828	# Read the PARIN file of the job
	829	def ShowRC(self):
	830
	831	# Return internal jobname and real job name
	832	jobname = self.table.item(self.table.currentRow(),6).text()
	833	jobrealname = self.table.item(self.table.currentRow(),0).text()
	834
	835	# Check description of job in order to login on the correct host
	836	descr = self.table.item(self.table.currentRow(),1).text()
	837	for h in range(0,len(description)):
	838	if ( descr == description[h] ):
	839	host = username[h] + "@" + hostname[h]
	840	user = username[h]
	841
[2416]	842	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd rc " + jobrealname],
[1613]	843	shell=False,
	844	stdout=sub.PIPE,
	845	stderr=sub.PIPE)
	846	result = ssh.stdout.readlines()
	847	result = filter(None, result)
	848
	849	# In case of error display a warning message
	850	if ( len(result) == 0 ):
	851	error = ssh.stderr.readlines()
	852	notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error))
	853
	854	# Otherwise inform the user and mark the job in the table
	855	else:
	856	mb = MessageBoxScroll()
	857	lastline = result[len(result)-2].split()[2]
	858	mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline)
	859	mb.setDetailedText(''.join(result))
	860	mb.exec_()
	861
[1611]	862	# Remove a job from list - removes the current row from the table and from
	863	# save file
	864	def RemoveFromList(self, row):
	865	if ( row == -1 ):
	866	row = self.table.currentRow()
	867
	868	# Read data from save file
	869	job_to_delete = self.table.item(row,6).text()
	870	self.table.removeRow(row)
	871	file = open(palm_dir + '/.wd.olddata', 'r')
	872	result = file.readlines()
	873	result = filter(None, result)
	874	file.close()
	875	file = open(palm_dir + '/.wd.olddata', 'w')
	876
	877	job_data_old = []
	878
	879	if ( len(result) == 0 ):
	880	notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.")
	881	else:
	882	# Save data in array job_data_old
	883	for j in range(0,len(result)):
	884	job_data_old.append(j)
	885	job_data_old[j] = result[j].split(" ")
	886	job_data_old[j] = filter(None, job_data_old[j])
	887	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	888
	889	# Check if line j is the selected job, if not -> save to file
	890	if ( job_data_old[j][6] != job_to_delete ):
	891	printstring = str(job_data_old[j][0]) + " " + \
	892	str(job_data_old[j][1]) + " " + \
	893	str(job_data_old[j][2]) + " " + \
	894	str(job_data_old[j][3]) + " " + \
	895	str(job_data_old[j][4]) + " " + \
	896	str(job_data_old[j][5]) + " " + \
	897	str(job_data_old[j][6]) + " " + \
	898	str(job_data_old[j][7]) + "\n"
	899	file.write(printstring)
	900
	901	file.close()
	902
	903	# Remove all completed jobs from list
	904	def ClearList(self):
	905
	906	num_of_lines = self.table.rowCount()
	907
	908	# Delete all lines with completed/canceled jobs from list. The counter
	909	# must decrease as the line numbers would be messed up otherwise
	910	for j in range(num_of_lines-1,-1,-1):
	911	state = self.table.item(j,3).text()
	912	if ( state == "Completed" or state == "Canceled" or state == "Terminated"):
	913	self.RemoveFromList(j)
	914
	915	# Update the label
	916	def UpdateLabel(self):
	917	remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60
	918	self.label.setText("Next update in " + str(remaining_time) + " min")
	919
	920
[1751]	921	# Enter Options menu
	922	def Options(self):
	923
	924	# disable mainwindow
	925	self.setEnabled(False)
	926
	927	# show Options Dialog
	928	opt = OptionBox()
	929	opt.exec_()
	930
	931	self.UpdateHosts()
	932	self.setEnabled(True)
	933
	934	# Update settings
	935	def UpdateHosts(self):
	936
	937	global update_frequency
[1753]	938	global description
	939	global hostname
	940	global username
	941
[1751]	942	description = []
	943	hostname = []
	944	username = []
	945
	946	for i in range(0,len(config.sections())):
	947
	948	description_tmp = config.sections()[i]
	949
	950	if ( description_tmp != 'Settings' ):
	951
	952	if ( config.get(description_tmp, 'enabled') == 'true' ):
	953	description.append(description_tmp)
	954	hostname.append(config.get(description_tmp, 'hostname'))
	955	username.append(config.get(description_tmp, 'username'))
	956	else:
	957	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	958
	959
[1611]	960	# Main loop
	961	def main():
	962
	963	app = QtGui.QApplication(sys.argv)
	964	res = Watchdog()
	965	sys.exit(app.exec_())
	966
	967
	968	if __name__ == '__main__':
[1613]	969	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |