Home

Context Navigation

source: palm/trunk/SCRIPTS/palm_wd @ 3499

Last change on this file since 3499 was 2825, checked in by maronga, 7 years ago
adjustments in gui tools
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 39.9 KB

Rev	Line
[2825]	1	#!/usr/bin/env python
[1611]	2	# -- coding: utf-8 --
	3	#--------------------------------------------------------------------------------#
[2696]	4	# This file is part of the PALM model system.
[1611]	5	#
	6	# PALM is free software: you can redistribute it and/or modify it under the terms
	7	# of the GNU General Public License as published by the Free Software Foundation,
	8	# either version 3 of the License, or (at your option) any later version.
	9	#
	10	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License along with
	15	# PALM. If not, see <http://www.gnu.org/licenses/>.
	16	#
[2718]	17	# Copyright 1997-2018 Leibniz Universitaet Hannover
[1611]	18	#--------------------------------------------------------------------------------#
	19	#
	20	# Current revisions:
	21	# -----------------
[1752]	22	#
[1754]	23	#
[1752]	24	# Former revisions:
	25	# -----------------
	26	# $Id: palm_wd 2825 2018-02-20 21:48:27Z suehring $
[2825]	27	# Modified header
	28	#
	29	# 2718 2018-01-02 08:49:38Z maronga
[2716]	30	# Corrected "Former revisions" section
	31	#
	32	# 2696 2017-12-14 17:12:51Z kanani
	33	# Change in file header (GPL part)
	34	#
	35	# 2416 2017-09-06 14:28:14Z maronga
[2416]	36	# Adapted for palmrun
	37	#
	38	# 1754 2016-02-22 13:50:22Z maronga
[1752]	39	#
[1754]	40	# 1753 2016-02-22 13:49:49Z maronga
	41	# Bugfix: use of global variables is required after updating configuration file
	42	#
[1752]	43	# 1751 2016-02-15 07:44:16Z maronga
[1751]	44	# Bugfixes: runs on multiple hosts caused crash of the watchdog. Progress bar
	45	# occasionally showed wrong progress.
	46	# New: Hosts can be switched on/off and update_frequency can be modified via
	47	# File->Options. Security check for cancelation of jobs.
[1611]	48	#
[1619]	49	# 1618 2015-07-13 06:52:15Z maronga
	50	# Added steering via configuration file .wd.config, to be place in the local
	51	# palm directory. Watchdog save files are automatically created upon first start.
	52	#
[1614]	53	# 1613 2015-07-08 14:53:29Z maronga
	54	# Bugfix: tooltip for queuing name did not show up on first update.
	55	# New: added contect menu for showing the parameter file and the run control
	56	# output
	57	#
[1612]	58	# 1611 2015-07-07 12:23:22Z maronga
	59	# Initial revision
	60	#
[1611]	61	#
	62	# Description:
	63	# ------------
	64	# PALM watchdog client for monitoring batch jobs on a variety of hosts specified
	65	# by the user
	66	#
	67	# Instructions:
	68	# -------------
	69	# 1) Modify the header section of palm_wd
	70	# 2) Move .wd.olddata and .wd.newdata to your palm directory
	71	# (e.g. /home/user/current_version/.wd.newdata etc.)
	72	# 3) Modify a copy of palm_wdd for each host to be monitored and move it to the
	73	# respective hosts
	74	# 4) Start the client either from mrungui or from shell by "nohup palm_wd&"
	75
	76	# To do:
	77	# ------
	78	# 1) Add "Options", "Help" and "Manual"
	79	# 2) Move user settings to a configuration file
	80	#------------------------------------------------------------------------------!
	81
[1618]	82	import ConfigParser
	83	import datetime
	84	import os
[1611]	85	from PyQt4 import QtGui, QtCore, uic
	86	from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT
[1618]	87	import shutil
	88	import subprocess as sub
	89	import sys
[1611]	90	import time
	91
	92	# Determine PALM directories
	93	try:
	94	devnull = open(os.devnull, 'w')
	95	out = sub.check_output("echo $PALM_BIN", shell=True, stderr=sub.STDOUT)
	96	palm_bin = out.rstrip()
	97	palm_dir = out.split("palm")[0] + "palm/" + out.split("palm")[1].split("/")[1]
	98	out = None
	99	except:
	100	print "Error. $PALM_BIN is not set."
	101	raise SystemExit
	102
	103
[1618]	104	# Read configuration file
	105	# First check if the configuration file exists
	106	if ( os.path.exists(palm_dir + '/.wd.config') == False ):
	107	print "Error. No configuration file .wd.config found in " + palm_dir
	108	raise SystemExit
	109
	110	config = ConfigParser.RawConfigParser()
	111	config.read(palm_dir + '/.wd.config')
	112
	113	description = []
	114	hostname = []
	115	username = []
	116
	117	for i in range(0,len(config.sections())):
	118
	119	description_tmp = config.sections()[i]
	120
	121	if ( description_tmp != 'Settings' ):
[1751]	122
	123	if ( config.get(description_tmp, 'enabled') == 'true' ):
	124	description.append(description_tmp)
	125	hostname.append(config.get(description_tmp, 'hostname'))
	126	username.append(config.get(description_tmp, 'username'))
[1618]	127	else:
	128	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	129
	130	# Check if .wd.olddata and .wd.newdata exist. Otherwise create the files
	131	if ( os.path.exists(palm_dir + '/.wd.olddata') == False ):
	132	print "No .wd.olddata found. Creating..."
	133	file1 = open(palm_dir + '/.wd.olddata', 'a')
	134	file1.close()
	135
	136
	137	if ( os.path.exists(palm_dir + '/.wd.newdata') == False ):
	138	print "No .wd.newdata found. Creating..."
	139	file1 = open(palm_dir + '/.wd.newdata', 'a')
	140	file1.close()
	141
[1611]	142	# Dummy variables for the jobname and the progressbars
	143	jobname = ""
	144	timestamp = ""
	145	pbars = []
	146	job_data_list = []
	147
[1751]	148	# Message box for showing RUN_CONTROL output
[1613]	149	class MessageBoxScroll(QtGui.QMessageBox):
	150	def __init__(self):
	151	QtGui.QMessageBox.__init__(self)
	152	self.setSizeGripEnabled(True)
	153
	154	def event(self, e):
	155	result = QtGui.QMessageBox.event(self, e)
	156
	157	self.setMinimumHeight(100)
	158	self.setMaximumHeight(16777215)
	159	self.setMinimumWidth(400)
	160	self.setMaximumWidth(16777215)
	161	self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	162
	163	textEdit = self.findChild(QtGui.QTextEdit)
	164	if textEdit != None :
	165	textEdit.setMinimumHeight(800)
	166	textEdit.setMaximumHeight(16777215)
	167	textEdit.setMinimumWidth(1000)
	168	textEdit.setMaximumWidth(16777215)
	169	textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	170	f = QtGui.QFont('not_a_font') #
	171	f.setStyleHint(f.TypeWriter, f.PreferDefault)
	172	textEdit.setFont(f)
	173
	174	return result
	175
	176
[1751]	177	# Message box for showing RUN_CONTROL output
	178	class OptionBox(QtGui.QDialog):
	179	def __init__(self):
	180
	181	super(OptionBox, self).__init__()
	182
	183	uic.loadUi(palm_bin + '/palm_wd_files/wdoptions.ui', self)
	184
	185	hostname = []
	186	self.checkbox = []
	187	j = -1
	188	ypos = 0
	189
	190	self.vbox = QtGui.QVBoxLayout(self)
	191	self.vbox.setSpacing(0)
	192	self.vbox.setMargin(0)
	193
	194	config.read(palm_dir + '/.wd.config')
	195
	196	for i in range(0,len(config.sections())):
	197
	198	description_tmp = config.sections()[i]
	199
	200	if ( description_tmp != 'Settings' ):
	201	description.append(description_tmp)
	202	hostname.append(config.get(description_tmp, 'hostname'))
	203
	204	j = j + 1
	205	self.checkbox.append(j)
	206	self.checkbox[j] = QtGui.QCheckBox(description_tmp, self)
	207	ypos = ypos + 20
	208	self.checkbox[j].move(10,0)
	209	if ( config.get(description_tmp, 'enabled') == 'true' ):
	210	self.checkbox[j].toggle()
	211
	212	self.vbox.addWidget(self.checkbox[j])
	213	else:
	214	self.update_spin.setValue(int(config.get(description_tmp, 'update_frequency')))
	215
	216	self.hostBox.setLayout(self.vbox)
	217	self.hostBox.setGeometry(0, 0, 272, 50+ypos)
	218	self.settingBox.setGeometry(281, 0, 214, 50+ypos)
	219	self.ok.setGeometry(385, 50+ypos, 111, 24)
	220	self.cancel.setGeometry(270, 50+ypos, 111, 24)
	221
	222	self.setGeometry(0, 0, 500, 80+ypos)
	223	self.show()
	224
	225	return
	226
	227	# save updtes to file
	228	def SaveAndClose(self):
	229
	230	for i in range(0,len(self.checkbox)):
	231
	232	if ( self.checkbox[i].checkState() == 0 ):
	233	config.set(str(self.checkbox[i].text()),'enabled','false')
	234	else:
	235	config.set(str(self.checkbox[i].text()),'enabled','true')
	236
	237	config.set('Settings','update_frequency',self.update_spin.text())
	238
	239	with open(palm_dir + '/.wd.config', 'wb') as configfile:
	240	config.write(configfile)
	241
	242
	243	self.close()
	244
	245	return
	246
	247
	248
[1611]	249	# MainWindow class
	250	class Watchdog(QtGui.QMainWindow):
	251
	252	def __init__(self):
	253	super(Watchdog, self).__init__()
	254
	255	self.InitUi()
	256
	257
	258	# Initialized MainWindow UI
	259	def InitUi(self):
	260
	261	# Load predefined mainwindow
	262	uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self)
	263
	264	# Resize columns and set number of rows to zero. Column 6 is hidden and
	265	# contains the remote jobname (e.g. hannover.174610)
	266	self.table.setColumnWidth(0,230)
	267	self.table.setColumnWidth(1,80)
	268	self.table.setColumnWidth(2,50)
	269	self.table.setColumnWidth(3,80)
	270	self.table.setColumnWidth(4,180)
	271	self.table.setColumnWidth(5,115)
	272	self.table.setColumnHidden(6, True)
	273	self.table.setRowCount(0)
[1613]	274
[1611]	275	# Display MainWindow
	276	self.show()
	277	QtGui.QApplication.processEvents()
	278
	279	# Start refresh timer. On timeout perform update
	280	self.timer = QtCore.QTimer(self)
	281	self.timer.timeout.connect(self.Refresh)
	282	self.timer.setSingleShot(False)
	283	self.timer.start(update_frequency)
	284
	285	# The timetimer counts the time since last update
	286	self.timetimer= QtCore.QElapsedTimer()
	287	self.timetimer.start()
	288
	289	# The labeltimer induces the update of the remaining time in the UI
	290	self.labeltimer = QtCore.QTimer(self)
	291	self.labeltimer.timeout.connect(self.UpdateLabel)
	292	self.labeltimer.setSingleShot(False)
	293
	294	# Update in the UI will be performed at each 1/10 of the update interval
	295	self.labeltimer.start(update_frequency/10)
	296	self.label.setText("Next update in " + str(update_frequency/1000/60) + "min")
	297
	298	# Update the list now
	299	self.Refresh()
	300
	301
	302	# Add a custom context menu
	303	def OpenContextMenu(self, position):
	304	menu = QtGui.QMenu()
	305
	306	# "Show details" -> fetch details for the selected job
[1613]	307	detailsjobAction = menu.addAction('Show job details')
[1611]	308	detailsjobAction.setStatusTip('Display detailed info for job')
	309	detailsjobAction.triggered.connect(self.ShowDetails)
	310
[1613]	311	# "Show parameter file" -> show the contents of PARIN
	312	parinAction = menu.addAction('Show parameter file')
	313	parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)')
	314	parinAction.triggered.connect(self.ShowPARIN)
	315
	316	rcAction = menu.addAction('Show run control file')
	317	rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)')
	318	rcAction.triggered.connect(self.ShowRC)
	319
[1611]	320	# "Cancel job" -> remove job from queuing system
	321	canceljobAction = menu.addAction('Cancel job')
	322	canceljobAction.setStatusTip('Remove job from queueing system')
	323	canceljobAction.triggered.connect(self.CancelJob)
	324
[1613]	325	# "Force stop" -> force termination of job
[1611]	326	stopjobAction = menu.addAction('Force stop')
	327	stopjobAction.setStatusTip('Terminate job properly')
	328	stopjobAction.triggered.connect(self.DoStop)
	329
[1613]	330	# "Force restart" -> force rermination and restart of job
[1611]	331	restartjobAction = menu.addAction('Force restart')
	332	restartjobAction.setStatusTip('Terminate job properly and initiate restart')
	333	restartjobAction.triggered.connect(self.DoRestart)
	334
	335	# "Remove from list" -> remove a completed job from the list
	336	removefromlistAction = menu.addAction('Remove from list')
	337	removefromlistAction.setStatusTip('Remove finished job')
	338	removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1))
	339
	340
	341	# Activate/deactive contect menu items based on the status of the runs
	342	state = self.table.item(self.table.currentRow(),3).text()
	343	if (state == "Canceled" or state == "Completed" or state == "Terminated"):
	344	detailsjobAction.setEnabled(False)
	345	canceljobAction.setEnabled(False)
	346	removefromlistAction.setEnabled(True)
	347	else:
	348	detailsjobAction.setEnabled(True)
	349	canceljobAction.setEnabled(True)
	350	removefromlistAction.setEnabled(False)
	351
	352	if ( state == "Running" ):
	353	stopjobAction.setEnabled(True)
	354	restartjobAction.setEnabled(True)
[1613]	355	parinAction.setEnabled(True)
	356	rcAction.setEnabled(True)
[1611]	357	else:
	358	stopjobAction.setEnabled(False)
	359	restartjobAction.setEnabled(False)
[1613]	360	parinAction.setEnabled(False)
	361	rcAction.setEnabled(False)
	362
[1611]	363	# Show the context menu
	364	action = menu.exec_(self.table.mapToGlobal(position))
	365
	366
	367	# Collecting watchdog data and display in the UI
	368	def Refresh(self):
	369
	370	# Use global variables
	371	global pbars
[1751]	372	global update_frequency
[1611]	373
	374	# Deactivate timer processes until update is finished
	375	# QtGui.QApplication.processEvents() updates the UI whenever needed
	376	self.labeltimer.stop()
	377	self.label.setText("Updating...")
	378	QtGui.QApplication.processEvents()
	379	self.setEnabled(False)
	380	QtGui.QApplication.processEvents()
	381
	382	# Get current timestamp
	383	timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
	384
	385	# Open an empty file for the watchdog data
	386	file = open(palm_dir + '/.wd.newdata', 'w')
	387
	388
	389	# Set all required variables to their initial values. Sorting must be
	390	# disabled.
	391	self.table.setRowCount(0)
	392	self.table.setSortingEnabled(False)
	393	i = -1
	394	job_data_list = []
	395	pbars = []
	396
	397	# Scan all hosts in the variable hostname. For each host perform the
	398	# update loop
	399	for h in range(0,len(hostname)):
	400
	401	# Perform ssh command
	402	host = username[h] + "@" + hostname[h]
	403	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd queue " + username[h]],
	404	shell=False,
	405	stdout=sub.PIPE,
	406	stderr=sub.PIPE)
	407	result = ssh.stdout.readlines()
	408	result = filter(None, result)
	409
	410
	411	# Delete all job data
	412	job_data_tmp = []
	413	job_data = []
	414	job_data_n = []
	415	job_progress = 0
	416
	417	# In case of errors display error message
	418	if ( len(result) < 1 ):
	419	error = ssh.stderr.readlines()
	420	if ( error != [] ):
	421	notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error))
	422
	423	# No error -> save job data
	424	else:
	425
	426	# Successively write to job_data
	427	for j in range(0,len(result)):
	428
	429	job_data_tmp.append(j)
	430	job_data_tmp[j] = result[j].split(" ")
	431	job_data_tmp[j] = filter(None, job_data_tmp[j])
	432
	433	job_data.append(j)
	434	job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2],
	435	job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5],
	436	job_data_tmp[j][1], job_data_tmp[j][6].rstrip()]
	437	job_data_n.append(j)
	438	job_data_n[j] = job_data_tmp[j][1]
	439
	440	del(result)
	441
	442
	443
	444	# Now read the data from the last update from file. These data are
	445	# already in the prefered format
	446	file2 = open(palm_dir + '/.wd.olddata', 'r')
	447	result = file2.readlines()
	448	file2.close()
[1751]	449
	450	job_data_old = []
	451	job_data_old_n = []
	452	k = -1
[1611]	453	for j in range(0,len(result)):
	454	if ( result[j].split()[1] == description[h] ):
[1751]	455	k = k + 1
	456	job_data_old.append(k)
	457	job_data_old[k] = result[j].split(" ")
	458	job_data_old[k] = filter(None, job_data_old[k])
	459	job_data_old[k] = [line.rstrip('\n') for line in job_data_old[k]]
	460	job_data_old_n.append(k)
	461	job_data_old_n[k] = job_data_old[k][6]
[1611]	462
	463	# Merge the job_data and job_data_old to find completed, new and
	464	# still queued jobs
	465	if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ):
	466	jobs_full = list(set(job_data_old_n) \| set(job_data_n))
	467	jobs_known = list(set(job_data_old_n) & set(job_data_n))
	468	jobs_complete = set(job_data_old_n) - set(job_data_n)
	469	jobs_new = set(job_data_n) - set(job_data_old_n)
	470	elif ( len(job_data_n) > 0 ):
	471	jobs_full = job_data_n
	472	jobs_known = []
	473	jobs_new = job_data_n
	474	jobs_complete = []
	475	elif ( len(job_data_old_n) > 0 ):
	476	jobs_full = job_data_old_n
	477	jobs_known = []
	478	jobs_new = []
	479	jobs_complete = job_data_old_n
	480	else:
	481	jobs_full = []
	482	jobs_known = []
	483	jobs_new = []
	484	jobs_complete = []
	485
	486	# Display all known jobs
	487	to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)]
	488	for j in range(0,len(to_display)):
	489	i = i + 1
	490	self.table.insertRow(i)
	491	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	492	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	493	self.table.setItem(i, 0, item)
	494	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	495	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	496	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	497	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	498	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	499	self.table.setItem(i, 5, item)
	500	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	501	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	502	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	503	pbars.append(i)
	504	pbars[i] = QtGui.QProgressBar(self)
	505	pbars[i].setValue(int(job_data[to_display[j]][4]))
	506
	507	# Apply a color depending on the status of the job
	508	if ( job_data[to_display[j]][3] == "Running" ):
[1751]	509	self.table.setCellWidget(i,4,pbars[i])
[1611]	510	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	511	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	512	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	513	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	514	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	515	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	516	else:
	517	pbars[j].setEnabled(False)
[1751]	518	self.table.setCellWidget(i,4,pbars[i])
[1611]	519	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	520	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	521	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	522	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	523	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	524	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	525
	526	# Save the job data to file
	527	job_data_list.append(i)
	528	job_data_list[i] = job_data[to_display[j]][0]
	529
	530	printstring = str(job_data[to_display[j]][0]) + " " + \
	531	str(job_data[to_display[j]][1]) + " " + \
	532	str(job_data[to_display[j]][2]) + " " + \
	533	str(job_data[to_display[j]][3]) + " " + \
	534	str(job_data[to_display[j]][4]) + " " + \
	535	str(job_data[to_display[j]][5]) + " " + \
	536	str(job_data[to_display[j]][6]) + " " + \
	537	str(job_data[to_display[j]][7]) + "\n"
	538	file.write(printstring)
	539
	540
	541	# Display all new jobs
	542	to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)]
	543	for j in range(0,len(to_display)):
	544	i = i + 1
	545	self.table.insertRow(i)
	546	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	547	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	548	self.table.setItem(i, 0, item)
	549	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	550	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	551	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	552	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	553	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	554	self.table.setItem(i, 5, item)
	555	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	556	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	557	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	558
	559	pbars.append(i)
	560	pbars[i] = QtGui.QProgressBar(self)
	561	pbars[i].setValue(int(job_data[to_display[j]][4]))
	562
	563	# Apply a color depending on the status of the job
	564	if ( job_data[to_display[j]][3] == "Running" ):
	565	self.table.setCellWidget(i,4,pbars[i])
	566	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	567	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	568	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	569	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	570	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	571	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	572	else:
	573	pbars[j].setEnabled(False)
	574	self.table.setCellWidget(i,4,pbars[i])
	575	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	576	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	577	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	578	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	579	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	580	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	581
	582	# Save job data to file
	583	job_data_list.append(i)
	584	job_data_list[i] = job_data[to_display[j]][0]
	585
	586	printstring = str(job_data[to_display[j]][0]) + " " + \
	587	str(job_data[to_display[j]][1]) + " " + \
	588	str(job_data[to_display[j]][2]) + " " + \
	589	str(job_data[to_display[j]][3]) + " " + \
	590	str(job_data[to_display[j]][4]) + " " + \
	591	str(job_data[to_display[j]][5]) + " " + \
	592	str(job_data[to_display[j]][6]) + " " + \
	593	str(job_data[to_display[j]][7]) + "\n"
	594	file.write(printstring)
	595
	596
	597	# Display all completed/canceled/aborted jobs. The watchdog cannot
	598	# distinguish why the job has been finished
	599	to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)]
	600	for j in range(0,len(to_display)):
	601	i = i + 1
	602	self.table.insertRow(i)
	603	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0])
	604	item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6])
	605	self.table.setItem(i, 0, item)
	606	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1]))
	607	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2]))
	608	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3]))
	609	pbars.append(i)
	610	pbars[j] = QtGui.QProgressBar(self)
	611	pbars[j].setValue(int(job_data_old[to_display[j]][4]))
	612	pbars[j].setEnabled(False)
	613	self.table.setCellWidget(i,4,pbars[j])
	614	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5])
	615	item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7])
	616	self.table.setItem(i, 5, item)
	617	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6]))
	618	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	619	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	620	self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed"))
	621
	622	# Apply a color depending on the status of the job
	623	self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175))
	624	self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175))
	625	self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175))
	626	self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175))
	627	self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175))
	628	self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175))
	629
	630
	631	# Save job data to file
	632	job_data_list.append(i)
	633	job_data_list[i] = job_data_old[to_display[j]][0]
	634
	635	printstring = str(job_data_old[to_display[j]][0]) + " " + \
	636	str(job_data_old[to_display[j]][1]) + " " + \
	637	str(job_data_old[to_display[j]][2]) + " " + \
	638	str(job_data_old[to_display[j]][3]) + " " + \
	639	str(job_data_old[to_display[j]][4]) + " " + \
	640	str(job_data_old[to_display[j]][5]) + " " + \
	641	str(job_data_old[to_display[j]][6]) + " " + \
	642	str(job_data_old[to_display[j]][7]) + "\n"
	643	file.write(printstring)
	644
	645	del(jobs_complete)
	646	del(jobs_new)
	647	del(result)
[1751]	648	del(job_data_old)
	649	del(job_data_old_n)
	650
[1611]	651	file.close()
	652
	653
	654	# Update is complete, sorting can thus be re-enabled
	655	self.table.setSortingEnabled(True)
	656
	657	# Update the logfile
	658	if ( os.path.isfile(palm_dir + '/.wd.newdata') ):
	659	shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata')
	660
	661	# Re-enable timer processes
	662	self.setEnabled(True)
	663	self.label2.setText("Last update: " + timestamp)
	664	self.label.setText("Update complete.")
[1751]	665	self.timetimer.start()
	666	self.timer.start(update_frequency)
	667	self.labeltimer.start(update_frequency/10)
	668	self.UpdateLabel()
[1611]	669	QtGui.QApplication.processEvents()
	670
	671	# Canceling selected job from table
	672	def CancelJob(self):
	673
	674	# Return internal jobname
	675	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	676	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	677
	678	# Check description of job in order to login on the correct host
	679	descr = self.table.item(self.table.currentRow(),1).text()
[1751]	680
	681	querybox = QtGui.QMessageBox()
	682	querybox.setText("Attention!\nYou are trying to cancel a job. It will not be able to terminate properly.")
	683	querybox.setInformativeText("Do you really want to cancel " + jobname + " on host " + descr + "?")
	684	querybox.setStandardButtons(QtGui.QMessageBox.Yes \| QtGui.QMessageBox.No)
	685	querybox.setDefaultButton(QtGui.QMessageBox.No)
	686	returnvalue = querybox.exec_()
	687
	688	if ( returnvalue == QtGui.QMessageBox.Yes ):
	689
	690	for h in range(0,len(description)):
	691	if ( descr == description[h] ):
	692	host = username[h] + "@" + hostname[h]
[1611]	693
[1751]	694	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd cancel " + jobname],
	695	shell=False,
	696	stdout=sub.PIPE,
	697	stderr=sub.PIPE)
	698	result = ssh.stdout.readlines()
	699	result = filter(None, result)
[1611]	700
[1751]	701	# In case of error display a warning message
	702	if ( len(result) == 0 ):
	703	error = ssh.stderr.readlines()
	704	notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error))
[1611]	705
[1751]	706	# Otherwise inform the user and mark the job in the table
	707	else:
	708	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled"))
	709	notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result))
[1611]	710
	711
	712	# Show detailed information on job. For documentation see above
	713	def ShowDetails(self):
	714
[2416]	715	jobname = self.table.item(self.table.currentRow(),6).text()
[1611]	716	descr = self.table.item(self.table.currentRow(),1).text()
	717	for h in range(0,len(description)):
	718	if ( descr == description[h] ):
	719	host = username[h] + "@" + hostname[h]
	720
	721	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd check " + jobname],
	722	shell=False,
	723	stdout=sub.PIPE,
	724	stderr=sub.PIPE)
	725	result = ssh.stdout.readlines()
	726	result = filter(None, result)
	727
	728	if ( len(result) == 0 ):
	729	error = ssh.stderr.readlines()
	730	notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error))
	731	else:
	732	notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result))
	733
	734
	735	# Perform a forced stop on the job
	736	def DoStop(self):
	737
	738	# Return internal jobname
	739	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	740	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	741	# Check description of job in order to login on the correct host
	742	descr = self.table.item(self.table.currentRow(),1).text()
	743	for h in range(0,len(description)):
	744	if ( descr == description[h] ):
	745	host = username[h] + "@" + hostname[h]
	746	user = username[h]
	747
[2416]	748	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd stop " + jobrealname],
[1611]	749	shell=False,
	750	stdout=sub.PIPE,
	751	stderr=sub.PIPE)
	752	result = ssh.stdout.readlines()
	753	result = filter(None, result)
	754
	755	# In case of error display a warning message
	756	if ( len(result) == 0 ):
	757	error = ssh.stderr.readlines()
	758	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	759
	760	# Otherwise inform the user and mark the job in the table
	761	else:
	762	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
[1751]	763	notify = QtGui.QMessageBox.information(self,'Terminate job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
[1611]	764
	765
	766	# Perform a forced stop on the job
	767	def DoRestart(self):
	768
	769	# Return internal jobname
	770	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	771	jobrealname = self.table.item(self.table.currentRow(),0).text()
	772
[1611]	773	# Check description of job in order to login on the correct host
	774	descr = self.table.item(self.table.currentRow(),1).text()
	775	for h in range(0,len(description)):
	776	if ( descr == description[h] ):
	777	host = username[h] + "@" + hostname[h]
	778	user = username[h]
	779
[2416]	780	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd restart " + jobrealname],
[1611]	781	shell=False,
	782	stdout=sub.PIPE,
	783	stderr=sub.PIPE)
	784	result = ssh.stdout.readlines()
	785	result = filter(None, result)
	786
	787	# In case of error display a warning message
	788	if ( len(result) == 0 ):
	789	error = ssh.stderr.readlines()
	790	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	791
	792	# Otherwise inform the user and mark the job in the table
	793	else:
	794	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	795	notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	796
	797
[1613]	798	# Read the PARIN file of the job
	799	def ShowPARIN(self):
	800
	801	# Return internal jobname
	802	jobname = self.table.item(self.table.currentRow(),6).text()
	803	jobrealname = self.table.item(self.table.currentRow(),0).text()
	804
	805	# Check description of job in order to login on the correct host
	806	descr = self.table.item(self.table.currentRow(),1).text()
	807	for h in range(0,len(description)):
	808	if ( descr == description[h] ):
	809	host = username[h] + "@" + hostname[h]
	810	user = username[h]
	811
[2416]	812	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd parin " + jobrealname],
[1613]	813	shell=False,
	814	stdout=sub.PIPE,
	815	stderr=sub.PIPE)
	816	result = ssh.stdout.readlines()
	817	result = filter(None, result)
	818
	819	# In case of error display a warning message
	820	if ( len(result) == 0 ):
	821	error = ssh.stderr.readlines()
	822	notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error))
	823
	824	# Otherwise inform the user and mark the job in the table
	825	else:
	826	mb = MessageBoxScroll()
	827	mb.setText("Parameter file for job: " + jobrealname + "")
	828	mb.setDetailedText(''.join(result))
	829	mb.exec_()
	830
	831	# Read the PARIN file of the job
	832	def ShowRC(self):
	833
	834	# Return internal jobname and real job name
	835	jobname = self.table.item(self.table.currentRow(),6).text()
	836	jobrealname = self.table.item(self.table.currentRow(),0).text()
	837
	838	# Check description of job in order to login on the correct host
	839	descr = self.table.item(self.table.currentRow(),1).text()
	840	for h in range(0,len(description)):
	841	if ( descr == description[h] ):
	842	host = username[h] + "@" + hostname[h]
	843	user = username[h]
	844
[2416]	845	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd rc " + jobrealname],
[1613]	846	shell=False,
	847	stdout=sub.PIPE,
	848	stderr=sub.PIPE)
	849	result = ssh.stdout.readlines()
	850	result = filter(None, result)
	851
	852	# In case of error display a warning message
	853	if ( len(result) == 0 ):
	854	error = ssh.stderr.readlines()
	855	notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error))
	856
	857	# Otherwise inform the user and mark the job in the table
	858	else:
	859	mb = MessageBoxScroll()
	860	lastline = result[len(result)-2].split()[2]
	861	mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline)
	862	mb.setDetailedText(''.join(result))
	863	mb.exec_()
	864
[1611]	865	# Remove a job from list - removes the current row from the table and from
	866	# save file
	867	def RemoveFromList(self, row):
	868	if ( row == -1 ):
	869	row = self.table.currentRow()
	870
	871	# Read data from save file
	872	job_to_delete = self.table.item(row,6).text()
	873	self.table.removeRow(row)
	874	file = open(palm_dir + '/.wd.olddata', 'r')
	875	result = file.readlines()
	876	result = filter(None, result)
	877	file.close()
	878	file = open(palm_dir + '/.wd.olddata', 'w')
	879
	880	job_data_old = []
	881
	882	if ( len(result) == 0 ):
	883	notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.")
	884	else:
	885	# Save data in array job_data_old
	886	for j in range(0,len(result)):
	887	job_data_old.append(j)
	888	job_data_old[j] = result[j].split(" ")
	889	job_data_old[j] = filter(None, job_data_old[j])
	890	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	891
	892	# Check if line j is the selected job, if not -> save to file
	893	if ( job_data_old[j][6] != job_to_delete ):
	894	printstring = str(job_data_old[j][0]) + " " + \
	895	str(job_data_old[j][1]) + " " + \
	896	str(job_data_old[j][2]) + " " + \
	897	str(job_data_old[j][3]) + " " + \
	898	str(job_data_old[j][4]) + " " + \
	899	str(job_data_old[j][5]) + " " + \
	900	str(job_data_old[j][6]) + " " + \
	901	str(job_data_old[j][7]) + "\n"
	902	file.write(printstring)
	903
	904	file.close()
	905
	906	# Remove all completed jobs from list
	907	def ClearList(self):
	908
	909	num_of_lines = self.table.rowCount()
	910
	911	# Delete all lines with completed/canceled jobs from list. The counter
	912	# must decrease as the line numbers would be messed up otherwise
	913	for j in range(num_of_lines-1,-1,-1):
	914	state = self.table.item(j,3).text()
	915	if ( state == "Completed" or state == "Canceled" or state == "Terminated"):
	916	self.RemoveFromList(j)
	917
	918	# Update the label
	919	def UpdateLabel(self):
	920	remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60
	921	self.label.setText("Next update in " + str(remaining_time) + " min")
	922
	923
[1751]	924	# Enter Options menu
	925	def Options(self):
	926
	927	# disable mainwindow
	928	self.setEnabled(False)
	929
	930	# show Options Dialog
	931	opt = OptionBox()
	932	opt.exec_()
	933
	934	self.UpdateHosts()
	935	self.setEnabled(True)
	936
	937	# Update settings
	938	def UpdateHosts(self):
	939
	940	global update_frequency
[1753]	941	global description
	942	global hostname
	943	global username
	944
[1751]	945	description = []
	946	hostname = []
	947	username = []
	948
	949	for i in range(0,len(config.sections())):
	950
	951	description_tmp = config.sections()[i]
	952
	953	if ( description_tmp != 'Settings' ):
	954
	955	if ( config.get(description_tmp, 'enabled') == 'true' ):
	956	description.append(description_tmp)
	957	hostname.append(config.get(description_tmp, 'hostname'))
	958	username.append(config.get(description_tmp, 'username'))
	959	else:
	960	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	961
	962
[1611]	963	# Main loop
	964	def main():
	965
	966	app = QtGui.QApplication(sys.argv)
	967	res = Watchdog()
	968	sys.exit(app.exec_())
	969
	970
	971	if __name__ == '__main__':
[1613]	972	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |