Home

Context Navigation

source: palm/trunk/SCRIPTS/palm_wd @ 4751

Last change on this file since 4751 was 4481, checked in by maronga, 5 years ago
Bugfix for copyright updates in document_changes; copyright update applied to all files
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 39.9 KB

Rev	Line
[2825]	1	#!/usr/bin/env python
[1611]	2	# -- coding: utf-8 --
	3	#--------------------------------------------------------------------------------#
[2696]	4	# This file is part of the PALM model system.
[1611]	5	#
	6	# PALM is free software: you can redistribute it and/or modify it under the terms
	7	# of the GNU General Public License as published by the Free Software Foundation,
	8	# either version 3 of the License, or (at your option) any later version.
	9	#
	10	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License along with
	15	# PALM. If not, see <http://www.gnu.org/licenses/>.
	16	#
[4481]	17	# Copyright 1997-2020 Leibniz Universitaet Hannover
[1611]	18	#--------------------------------------------------------------------------------#
	19	#
	20	# Current revisions:
	21	# -----------------
[1752]	22	#
[1754]	23	#
[1752]	24	# Former revisions:
	25	# -----------------
	26	# $Id: palm_wd 4481 2020-03-31 18:55:54Z knoop $
[4393]	27	# Removed PALM_BIN dependency
	28	#
	29	# 3943 2019-05-02 09:50:41Z maronga
[3567]	30	# Adjustments and bugfixes
	31	#
	32	#
	33	# 2825 2018-02-20 21:48:27Z maronga
[2825]	34	# Modified header
	35	#
	36	# 2718 2018-01-02 08:49:38Z maronga
[2716]	37	# Corrected "Former revisions" section
	38	#
	39	# 2696 2017-12-14 17:12:51Z kanani
	40	# Change in file header (GPL part)
	41	#
	42	# 2416 2017-09-06 14:28:14Z maronga
[2416]	43	# Adapted for palmrun
	44	#
	45	# 1754 2016-02-22 13:50:22Z maronga
[1752]	46	#
[1754]	47	# 1753 2016-02-22 13:49:49Z maronga
	48	# Bugfix: use of global variables is required after updating configuration file
	49	#
[1752]	50	# 1751 2016-02-15 07:44:16Z maronga
[1751]	51	# Bugfixes: runs on multiple hosts caused crash of the watchdog. Progress bar
	52	# occasionally showed wrong progress.
	53	# New: Hosts can be switched on/off and update_frequency can be modified via
	54	# File->Options. Security check for cancelation of jobs.
[1611]	55	#
[1619]	56	# 1618 2015-07-13 06:52:15Z maronga
	57	# Added steering via configuration file .wd.config, to be place in the local
	58	# palm directory. Watchdog save files are automatically created upon first start.
	59	#
[1614]	60	# 1613 2015-07-08 14:53:29Z maronga
	61	# Bugfix: tooltip for queuing name did not show up on first update.
	62	# New: added contect menu for showing the parameter file and the run control
	63	# output
	64	#
[1612]	65	# 1611 2015-07-07 12:23:22Z maronga
	66	# Initial revision
	67	#
[1611]	68	#
	69	# Description:
	70	# ------------
	71	# PALM watchdog client for monitoring batch jobs on a variety of hosts specified
	72	# by the user
	73	#
	74	# Instructions:
	75	# -------------
	76	# 1) Modify the header section of palm_wd
	77	# 2) Move .wd.olddata and .wd.newdata to your palm directory
	78	# (e.g. /home/user/current_version/.wd.newdata etc.)
	79	# 3) Modify a copy of palm_wdd for each host to be monitored and move it to the
	80	# respective hosts
	81	# 4) Start the client either from mrungui or from shell by "nohup palm_wd&"
	82
	83	# To do:
	84	# ------
	85	# 1) Add "Options", "Help" and "Manual"
	86	# 2) Move user settings to a configuration file
	87	#------------------------------------------------------------------------------!
	88
[3943]	89	import ConfigParser as configparser
[1618]	90	import datetime
	91	import os
[1611]	92	from PyQt4 import QtGui, QtCore, uic
	93	from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT
[1618]	94	import shutil
	95	import subprocess as sub
	96	import sys
[1611]	97	import time
	98
	99	# Determine PALM directories
[3567]	100	# Check if configuration files exists and quit otherwise
	101	input_config = ".wd.config"
[3943]	102
	103	for i in range(0,len(sys.argv)):
[3567]	104	input_config = str(sys.argv[i])
[4393]	105	palm_dir = os.getcwd()
	106	palm_bin = palm_dir + '/trunk/SCRIPTS'
[3567]	107
	108	config = configparser.RawConfigParser(allow_no_value=True)
[1611]	109
[3567]	110	if ( os.path.isfile(input_config) == False ):
	111	print ("Error. No configuration file " + input_config + " found.")
	112	raise SystemExit
	113	else:
	114	print(os.path.isfile(input_config))
[1611]	115
[3943]	116	print(input_config)
	117
[3567]	118	config.read(input_config)
[1618]	119
[3943]	120	config = ConfigParser.RawConfigParser()
	121	config.read(palm_dir + '/.wd.config')
[1618]	122
	123	description = []
	124	hostname = []
	125	username = []
[3567]	126	pycall = []
[1618]	127
[3943]	128
[1618]	129	for i in range(0,len(config.sections())):
	130
	131	description_tmp = config.sections()[i]
	132
	133	if ( description_tmp != 'Settings' ):
[1751]	134
	135	if ( config.get(description_tmp, 'enabled') == 'true' ):
	136	description.append(description_tmp)
	137	hostname.append(config.get(description_tmp, 'hostname'))
	138	username.append(config.get(description_tmp, 'username'))
[3567]	139	pycall.append(config.get(description_tmp, 'pycall'))
[1618]	140	else:
	141	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	142
	143	# Check if .wd.olddata and .wd.newdata exist. Otherwise create the files
	144	if ( os.path.exists(palm_dir + '/.wd.olddata') == False ):
	145	print "No .wd.olddata found. Creating..."
	146	file1 = open(palm_dir + '/.wd.olddata', 'a')
	147	file1.close()
	148
	149
	150	if ( os.path.exists(palm_dir + '/.wd.newdata') == False ):
	151	print "No .wd.newdata found. Creating..."
	152	file1 = open(palm_dir + '/.wd.newdata', 'a')
	153	file1.close()
	154
[1611]	155	# Dummy variables for the jobname and the progressbars
	156	jobname = ""
	157	timestamp = ""
	158	pbars = []
	159	job_data_list = []
	160
[1751]	161	# Message box for showing RUN_CONTROL output
[1613]	162	class MessageBoxScroll(QtGui.QMessageBox):
	163	def __init__(self):
	164	QtGui.QMessageBox.__init__(self)
	165	self.setSizeGripEnabled(True)
	166
	167	def event(self, e):
	168	result = QtGui.QMessageBox.event(self, e)
	169
	170	self.setMinimumHeight(100)
	171	self.setMaximumHeight(16777215)
	172	self.setMinimumWidth(400)
	173	self.setMaximumWidth(16777215)
	174	self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	175
	176	textEdit = self.findChild(QtGui.QTextEdit)
	177	if textEdit != None :
	178	textEdit.setMinimumHeight(800)
	179	textEdit.setMaximumHeight(16777215)
	180	textEdit.setMinimumWidth(1000)
	181	textEdit.setMaximumWidth(16777215)
	182	textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	183	f = QtGui.QFont('not_a_font') #
	184	f.setStyleHint(f.TypeWriter, f.PreferDefault)
	185	textEdit.setFont(f)
	186
	187	return result
	188
	189
[1751]	190	# Message box for showing RUN_CONTROL output
	191	class OptionBox(QtGui.QDialog):
	192	def __init__(self):
	193
	194	super(OptionBox, self).__init__()
	195
	196	uic.loadUi(palm_bin + '/palm_wd_files/wdoptions.ui', self)
	197
	198	hostname = []
	199	self.checkbox = []
	200	j = -1
	201	ypos = 0
	202
	203	self.vbox = QtGui.QVBoxLayout(self)
	204	self.vbox.setSpacing(0)
	205	self.vbox.setMargin(0)
	206
	207	config.read(palm_dir + '/.wd.config')
	208
	209	for i in range(0,len(config.sections())):
	210
	211	description_tmp = config.sections()[i]
	212
	213	if ( description_tmp != 'Settings' ):
	214	description.append(description_tmp)
	215	hostname.append(config.get(description_tmp, 'hostname'))
	216
	217	j = j + 1
	218	self.checkbox.append(j)
	219	self.checkbox[j] = QtGui.QCheckBox(description_tmp, self)
	220	ypos = ypos + 20
	221	self.checkbox[j].move(10,0)
	222	if ( config.get(description_tmp, 'enabled') == 'true' ):
	223	self.checkbox[j].toggle()
	224
	225	self.vbox.addWidget(self.checkbox[j])
	226	else:
	227	self.update_spin.setValue(int(config.get(description_tmp, 'update_frequency')))
	228
	229	self.hostBox.setLayout(self.vbox)
	230	self.hostBox.setGeometry(0, 0, 272, 50+ypos)
	231	self.settingBox.setGeometry(281, 0, 214, 50+ypos)
	232	self.ok.setGeometry(385, 50+ypos, 111, 24)
	233	self.cancel.setGeometry(270, 50+ypos, 111, 24)
	234
	235	self.setGeometry(0, 0, 500, 80+ypos)
	236	self.show()
	237
	238	return
	239
	240	# save updtes to file
	241	def SaveAndClose(self):
	242
	243	for i in range(0,len(self.checkbox)):
	244
	245	if ( self.checkbox[i].checkState() == 0 ):
	246	config.set(str(self.checkbox[i].text()),'enabled','false')
	247	else:
	248	config.set(str(self.checkbox[i].text()),'enabled','true')
	249
	250	config.set('Settings','update_frequency',self.update_spin.text())
	251
	252	with open(palm_dir + '/.wd.config', 'wb') as configfile:
	253	config.write(configfile)
	254
	255
	256	self.close()
	257
	258	return
	259
	260
	261
[1611]	262	# MainWindow class
	263	class Watchdog(QtGui.QMainWindow):
	264
	265	def __init__(self):
	266	super(Watchdog, self).__init__()
	267
	268	self.InitUi()
	269
	270
	271	# Initialized MainWindow UI
	272	def InitUi(self):
	273
	274	# Load predefined mainwindow
	275	uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self)
	276
	277	# Resize columns and set number of rows to zero. Column 6 is hidden and
	278	# contains the remote jobname (e.g. hannover.174610)
	279	self.table.setColumnWidth(0,230)
	280	self.table.setColumnWidth(1,80)
	281	self.table.setColumnWidth(2,50)
	282	self.table.setColumnWidth(3,80)
	283	self.table.setColumnWidth(4,180)
	284	self.table.setColumnWidth(5,115)
	285	self.table.setColumnHidden(6, True)
	286	self.table.setRowCount(0)
[1613]	287
[1611]	288	# Display MainWindow
	289	self.show()
	290	QtGui.QApplication.processEvents()
	291
	292	# Start refresh timer. On timeout perform update
	293	self.timer = QtCore.QTimer(self)
	294	self.timer.timeout.connect(self.Refresh)
	295	self.timer.setSingleShot(False)
	296	self.timer.start(update_frequency)
	297
	298	# The timetimer counts the time since last update
	299	self.timetimer= QtCore.QElapsedTimer()
	300	self.timetimer.start()
	301
	302	# The labeltimer induces the update of the remaining time in the UI
	303	self.labeltimer = QtCore.QTimer(self)
	304	self.labeltimer.timeout.connect(self.UpdateLabel)
	305	self.labeltimer.setSingleShot(False)
	306
	307	# Update in the UI will be performed at each 1/10 of the update interval
	308	self.labeltimer.start(update_frequency/10)
	309	self.label.setText("Next update in " + str(update_frequency/1000/60) + "min")
	310
	311	# Update the list now
	312	self.Refresh()
	313
	314
	315	# Add a custom context menu
	316	def OpenContextMenu(self, position):
	317	menu = QtGui.QMenu()
	318
	319	# "Show details" -> fetch details for the selected job
[1613]	320	detailsjobAction = menu.addAction('Show job details')
[1611]	321	detailsjobAction.setStatusTip('Display detailed info for job')
	322	detailsjobAction.triggered.connect(self.ShowDetails)
	323
[1613]	324	# "Show parameter file" -> show the contents of PARIN
	325	parinAction = menu.addAction('Show parameter file')
	326	parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)')
	327	parinAction.triggered.connect(self.ShowPARIN)
	328
	329	rcAction = menu.addAction('Show run control file')
	330	rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)')
	331	rcAction.triggered.connect(self.ShowRC)
	332
[1611]	333	# "Cancel job" -> remove job from queuing system
	334	canceljobAction = menu.addAction('Cancel job')
	335	canceljobAction.setStatusTip('Remove job from queueing system')
	336	canceljobAction.triggered.connect(self.CancelJob)
	337
[1613]	338	# "Force stop" -> force termination of job
[1611]	339	stopjobAction = menu.addAction('Force stop')
	340	stopjobAction.setStatusTip('Terminate job properly')
	341	stopjobAction.triggered.connect(self.DoStop)
	342
[1613]	343	# "Force restart" -> force rermination and restart of job
[1611]	344	restartjobAction = menu.addAction('Force restart')
	345	restartjobAction.setStatusTip('Terminate job properly and initiate restart')
	346	restartjobAction.triggered.connect(self.DoRestart)
	347
	348	# "Remove from list" -> remove a completed job from the list
	349	removefromlistAction = menu.addAction('Remove from list')
	350	removefromlistAction.setStatusTip('Remove finished job')
	351	removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1))
	352
	353
	354	# Activate/deactive contect menu items based on the status of the runs
	355	state = self.table.item(self.table.currentRow(),3).text()
	356	if (state == "Canceled" or state == "Completed" or state == "Terminated"):
	357	detailsjobAction.setEnabled(False)
	358	canceljobAction.setEnabled(False)
	359	removefromlistAction.setEnabled(True)
	360	else:
	361	detailsjobAction.setEnabled(True)
	362	canceljobAction.setEnabled(True)
	363	removefromlistAction.setEnabled(False)
	364
	365	if ( state == "Running" ):
	366	stopjobAction.setEnabled(True)
	367	restartjobAction.setEnabled(True)
[1613]	368	parinAction.setEnabled(True)
	369	rcAction.setEnabled(True)
[1611]	370	else:
	371	stopjobAction.setEnabled(False)
	372	restartjobAction.setEnabled(False)
[1613]	373	parinAction.setEnabled(False)
	374	rcAction.setEnabled(False)
	375
[1611]	376	# Show the context menu
	377	action = menu.exec_(self.table.mapToGlobal(position))
	378
	379
	380	# Collecting watchdog data and display in the UI
	381	def Refresh(self):
	382
	383	# Use global variables
	384	global pbars
[1751]	385	global update_frequency
[1611]	386
	387	# Deactivate timer processes until update is finished
	388	# QtGui.QApplication.processEvents() updates the UI whenever needed
	389	self.labeltimer.stop()
	390	self.label.setText("Updating...")
	391	QtGui.QApplication.processEvents()
	392	self.setEnabled(False)
	393	QtGui.QApplication.processEvents()
	394
	395	# Get current timestamp
	396	timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
	397
	398	# Open an empty file for the watchdog data
	399	file = open(palm_dir + '/.wd.newdata', 'w')
	400
	401
	402	# Set all required variables to their initial values. Sorting must be
	403	# disabled.
	404	self.table.setRowCount(0)
	405	self.table.setSortingEnabled(False)
	406	i = -1
	407	job_data_list = []
	408	pbars = []
	409
	410	# Scan all hosts in the variable hostname. For each host perform the
	411	# update loop
	412	for h in range(0,len(hostname)):
	413
	414	# Perform ssh command
	415	host = username[h] + "@" + hostname[h]
[3567]	416	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd queue " + username[h]],
[1611]	417	shell=False,
	418	stdout=sub.PIPE,
	419	stderr=sub.PIPE)
	420	result = ssh.stdout.readlines()
	421	result = filter(None, result)
	422
	423
	424	# Delete all job data
	425	job_data_tmp = []
	426	job_data = []
	427	job_data_n = []
	428	job_progress = 0
	429
	430	# In case of errors display error message
	431	if ( len(result) < 1 ):
	432	error = ssh.stderr.readlines()
	433	if ( error != [] ):
	434	notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error))
	435
	436	# No error -> save job data
	437	else:
	438
	439	# Successively write to job_data
	440	for j in range(0,len(result)):
	441
	442	job_data_tmp.append(j)
	443	job_data_tmp[j] = result[j].split(" ")
	444	job_data_tmp[j] = filter(None, job_data_tmp[j])
	445
	446	job_data.append(j)
	447	job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2],
	448	job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5],
	449	job_data_tmp[j][1], job_data_tmp[j][6].rstrip()]
	450	job_data_n.append(j)
	451	job_data_n[j] = job_data_tmp[j][1]
	452
	453	del(result)
	454
	455
	456
	457	# Now read the data from the last update from file. These data are
	458	# already in the prefered format
	459	file2 = open(palm_dir + '/.wd.olddata', 'r')
	460	result = file2.readlines()
	461	file2.close()
[1751]	462
	463	job_data_old = []
	464	job_data_old_n = []
	465	k = -1
[1611]	466	for j in range(0,len(result)):
	467	if ( result[j].split()[1] == description[h] ):
[1751]	468	k = k + 1
	469	job_data_old.append(k)
	470	job_data_old[k] = result[j].split(" ")
	471	job_data_old[k] = filter(None, job_data_old[k])
	472	job_data_old[k] = [line.rstrip('\n') for line in job_data_old[k]]
	473	job_data_old_n.append(k)
	474	job_data_old_n[k] = job_data_old[k][6]
[1611]	475
	476	# Merge the job_data and job_data_old to find completed, new and
	477	# still queued jobs
	478	if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ):
	479	jobs_full = list(set(job_data_old_n) \| set(job_data_n))
	480	jobs_known = list(set(job_data_old_n) & set(job_data_n))
	481	jobs_complete = set(job_data_old_n) - set(job_data_n)
	482	jobs_new = set(job_data_n) - set(job_data_old_n)
	483	elif ( len(job_data_n) > 0 ):
	484	jobs_full = job_data_n
	485	jobs_known = []
	486	jobs_new = job_data_n
	487	jobs_complete = []
	488	elif ( len(job_data_old_n) > 0 ):
	489	jobs_full = job_data_old_n
	490	jobs_known = []
	491	jobs_new = []
	492	jobs_complete = job_data_old_n
	493	else:
	494	jobs_full = []
	495	jobs_known = []
	496	jobs_new = []
	497	jobs_complete = []
	498
	499	# Display all known jobs
	500	to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)]
	501	for j in range(0,len(to_display)):
	502	i = i + 1
	503	self.table.insertRow(i)
	504	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	505	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	506	self.table.setItem(i, 0, item)
	507	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	508	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	509	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	510	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	511	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	512	self.table.setItem(i, 5, item)
	513	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	514	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	515	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	516	pbars.append(i)
	517	pbars[i] = QtGui.QProgressBar(self)
	518	pbars[i].setValue(int(job_data[to_display[j]][4]))
	519
	520	# Apply a color depending on the status of the job
	521	if ( job_data[to_display[j]][3] == "Running" ):
[1751]	522	self.table.setCellWidget(i,4,pbars[i])
[1611]	523	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	524	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	525	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	526	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	527	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	528	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	529	else:
	530	pbars[j].setEnabled(False)
[1751]	531	self.table.setCellWidget(i,4,pbars[i])
[1611]	532	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	533	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	534	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	535	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	536	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	537	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	538
	539	# Save the job data to file
	540	job_data_list.append(i)
	541	job_data_list[i] = job_data[to_display[j]][0]
	542
	543	printstring = str(job_data[to_display[j]][0]) + " " + \
	544	str(job_data[to_display[j]][1]) + " " + \
	545	str(job_data[to_display[j]][2]) + " " + \
	546	str(job_data[to_display[j]][3]) + " " + \
	547	str(job_data[to_display[j]][4]) + " " + \
	548	str(job_data[to_display[j]][5]) + " " + \
	549	str(job_data[to_display[j]][6]) + " " + \
	550	str(job_data[to_display[j]][7]) + "\n"
	551	file.write(printstring)
	552
	553
	554	# Display all new jobs
	555	to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)]
	556	for j in range(0,len(to_display)):
	557	i = i + 1
	558	self.table.insertRow(i)
	559	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	560	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	561	self.table.setItem(i, 0, item)
	562	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	563	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	564	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	565	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	566	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	567	self.table.setItem(i, 5, item)
	568	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	569	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	570	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	571
	572	pbars.append(i)
	573	pbars[i] = QtGui.QProgressBar(self)
	574	pbars[i].setValue(int(job_data[to_display[j]][4]))
	575
	576	# Apply a color depending on the status of the job
	577	if ( job_data[to_display[j]][3] == "Running" ):
	578	self.table.setCellWidget(i,4,pbars[i])
	579	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	580	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	581	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	582	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	583	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	584	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	585	else:
	586	pbars[j].setEnabled(False)
	587	self.table.setCellWidget(i,4,pbars[i])
	588	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	589	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	590	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	591	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	592	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	593	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	594
	595	# Save job data to file
	596	job_data_list.append(i)
	597	job_data_list[i] = job_data[to_display[j]][0]
	598
	599	printstring = str(job_data[to_display[j]][0]) + " " + \
	600	str(job_data[to_display[j]][1]) + " " + \
	601	str(job_data[to_display[j]][2]) + " " + \
	602	str(job_data[to_display[j]][3]) + " " + \
	603	str(job_data[to_display[j]][4]) + " " + \
	604	str(job_data[to_display[j]][5]) + " " + \
	605	str(job_data[to_display[j]][6]) + " " + \
	606	str(job_data[to_display[j]][7]) + "\n"
	607	file.write(printstring)
	608
	609
	610	# Display all completed/canceled/aborted jobs. The watchdog cannot
	611	# distinguish why the job has been finished
	612	to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)]
	613	for j in range(0,len(to_display)):
	614	i = i + 1
	615	self.table.insertRow(i)
	616	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0])
	617	item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6])
	618	self.table.setItem(i, 0, item)
	619	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1]))
	620	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2]))
	621	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3]))
	622	pbars.append(i)
	623	pbars[j] = QtGui.QProgressBar(self)
	624	pbars[j].setValue(int(job_data_old[to_display[j]][4]))
	625	pbars[j].setEnabled(False)
	626	self.table.setCellWidget(i,4,pbars[j])
	627	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5])
	628	item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7])
	629	self.table.setItem(i, 5, item)
	630	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6]))
	631	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	632	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	633	self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed"))
	634
	635	# Apply a color depending on the status of the job
	636	self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175))
	637	self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175))
	638	self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175))
	639	self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175))
	640	self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175))
	641	self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175))
	642
	643
	644	# Save job data to file
	645	job_data_list.append(i)
	646	job_data_list[i] = job_data_old[to_display[j]][0]
	647
	648	printstring = str(job_data_old[to_display[j]][0]) + " " + \
	649	str(job_data_old[to_display[j]][1]) + " " + \
	650	str(job_data_old[to_display[j]][2]) + " " + \
	651	str(job_data_old[to_display[j]][3]) + " " + \
	652	str(job_data_old[to_display[j]][4]) + " " + \
	653	str(job_data_old[to_display[j]][5]) + " " + \
	654	str(job_data_old[to_display[j]][6]) + " " + \
	655	str(job_data_old[to_display[j]][7]) + "\n"
	656	file.write(printstring)
	657
	658	del(jobs_complete)
	659	del(jobs_new)
	660	del(result)
[1751]	661	del(job_data_old)
	662	del(job_data_old_n)
	663
[1611]	664	file.close()
	665
	666
	667	# Update is complete, sorting can thus be re-enabled
	668	self.table.setSortingEnabled(True)
	669
	670	# Update the logfile
	671	if ( os.path.isfile(palm_dir + '/.wd.newdata') ):
	672	shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata')
	673
	674	# Re-enable timer processes
	675	self.setEnabled(True)
	676	self.label2.setText("Last update: " + timestamp)
	677	self.label.setText("Update complete.")
[1751]	678	self.timetimer.start()
	679	self.timer.start(update_frequency)
	680	self.labeltimer.start(update_frequency/10)
	681	self.UpdateLabel()
[1611]	682	QtGui.QApplication.processEvents()
	683
	684	# Canceling selected job from table
	685	def CancelJob(self):
	686
	687	# Return internal jobname
	688	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	689	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	690
	691	# Check description of job in order to login on the correct host
	692	descr = self.table.item(self.table.currentRow(),1).text()
[1751]	693
	694	querybox = QtGui.QMessageBox()
	695	querybox.setText("Attention!\nYou are trying to cancel a job. It will not be able to terminate properly.")
	696	querybox.setInformativeText("Do you really want to cancel " + jobname + " on host " + descr + "?")
	697	querybox.setStandardButtons(QtGui.QMessageBox.Yes \| QtGui.QMessageBox.No)
	698	querybox.setDefaultButton(QtGui.QMessageBox.No)
	699	returnvalue = querybox.exec_()
	700
	701	if ( returnvalue == QtGui.QMessageBox.Yes ):
	702
	703	for h in range(0,len(description)):
	704	if ( descr == description[h] ):
	705	host = username[h] + "@" + hostname[h]
[1611]	706
[3567]	707	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd cancel " + jobname],
[1751]	708	shell=False,
	709	stdout=sub.PIPE,
	710	stderr=sub.PIPE)
	711	result = ssh.stdout.readlines()
	712	result = filter(None, result)
[1611]	713
[1751]	714	# In case of error display a warning message
	715	if ( len(result) == 0 ):
	716	error = ssh.stderr.readlines()
	717	notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error))
[1611]	718
[1751]	719	# Otherwise inform the user and mark the job in the table
	720	else:
	721	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled"))
	722	notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result))
[1611]	723
	724
	725	# Show detailed information on job. For documentation see above
	726	def ShowDetails(self):
	727
[2416]	728	jobname = self.table.item(self.table.currentRow(),6).text()
[1611]	729	descr = self.table.item(self.table.currentRow(),1).text()
	730	for h in range(0,len(description)):
	731	if ( descr == description[h] ):
	732	host = username[h] + "@" + hostname[h]
	733
[3567]	734	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd check " + jobname],
[1611]	735	shell=False,
	736	stdout=sub.PIPE,
	737	stderr=sub.PIPE)
	738	result = ssh.stdout.readlines()
	739	result = filter(None, result)
	740
	741	if ( len(result) == 0 ):
	742	error = ssh.stderr.readlines()
	743	notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error))
	744	else:
	745	notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result))
	746
	747
	748	# Perform a forced stop on the job
	749	def DoStop(self):
	750
	751	# Return internal jobname
	752	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	753	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	754	# Check description of job in order to login on the correct host
	755	descr = self.table.item(self.table.currentRow(),1).text()
	756	for h in range(0,len(description)):
	757	if ( descr == description[h] ):
	758	host = username[h] + "@" + hostname[h]
	759	user = username[h]
	760
[3567]	761	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd stop " + jobrealname],
[1611]	762	shell=False,
	763	stdout=sub.PIPE,
	764	stderr=sub.PIPE)
	765	result = ssh.stdout.readlines()
	766	result = filter(None, result)
	767
	768	# In case of error display a warning message
	769	if ( len(result) == 0 ):
	770	error = ssh.stderr.readlines()
	771	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	772
	773	# Otherwise inform the user and mark the job in the table
	774	else:
	775	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
[1751]	776	notify = QtGui.QMessageBox.information(self,'Terminate job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
[1611]	777
	778
	779	# Perform a forced stop on the job
	780	def DoRestart(self):
	781
	782	# Return internal jobname
	783	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	784	jobrealname = self.table.item(self.table.currentRow(),0).text()
	785
[1611]	786	# Check description of job in order to login on the correct host
	787	descr = self.table.item(self.table.currentRow(),1).text()
	788	for h in range(0,len(description)):
	789	if ( descr == description[h] ):
	790	host = username[h] + "@" + hostname[h]
	791	user = username[h]
	792
[3567]	793	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd restart " + jobrealname],
[1611]	794	shell=False,
	795	stdout=sub.PIPE,
	796	stderr=sub.PIPE)
	797	result = ssh.stdout.readlines()
	798	result = filter(None, result)
	799
	800	# In case of error display a warning message
	801	if ( len(result) == 0 ):
	802	error = ssh.stderr.readlines()
	803	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	804
	805	# Otherwise inform the user and mark the job in the table
	806	else:
	807	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	808	notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	809
	810
[1613]	811	# Read the PARIN file of the job
	812	def ShowPARIN(self):
	813
	814	# Return internal jobname
	815	jobname = self.table.item(self.table.currentRow(),6).text()
	816	jobrealname = self.table.item(self.table.currentRow(),0).text()
	817
	818	# Check description of job in order to login on the correct host
	819	descr = self.table.item(self.table.currentRow(),1).text()
	820	for h in range(0,len(description)):
	821	if ( descr == description[h] ):
	822	host = username[h] + "@" + hostname[h]
	823	user = username[h]
	824
[3567]	825	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd parin " + jobrealname],
[1613]	826	shell=False,
	827	stdout=sub.PIPE,
	828	stderr=sub.PIPE)
	829	result = ssh.stdout.readlines()
	830	result = filter(None, result)
	831
	832	# In case of error display a warning message
	833	if ( len(result) == 0 ):
	834	error = ssh.stderr.readlines()
	835	notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error))
	836
	837	# Otherwise inform the user and mark the job in the table
	838	else:
	839	mb = MessageBoxScroll()
	840	mb.setText("Parameter file for job: " + jobrealname + "")
	841	mb.setDetailedText(''.join(result))
	842	mb.exec_()
	843
	844	# Read the PARIN file of the job
	845	def ShowRC(self):
	846
	847	# Return internal jobname and real job name
	848	jobname = self.table.item(self.table.currentRow(),6).text()
	849	jobrealname = self.table.item(self.table.currentRow(),0).text()
	850
	851	# Check description of job in order to login on the correct host
	852	descr = self.table.item(self.table.currentRow(),1).text()
	853	for h in range(0,len(description)):
	854	if ( descr == description[h] ):
	855	host = username[h] + "@" + hostname[h]
	856	user = username[h]
	857
[3567]	858	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd rc " + jobrealname],
[1613]	859	shell=False,
	860	stdout=sub.PIPE,
	861	stderr=sub.PIPE)
	862	result = ssh.stdout.readlines()
	863	result = filter(None, result)
	864
	865	# In case of error display a warning message
	866	if ( len(result) == 0 ):
	867	error = ssh.stderr.readlines()
	868	notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error))
	869
	870	# Otherwise inform the user and mark the job in the table
	871	else:
	872	mb = MessageBoxScroll()
	873	lastline = result[len(result)-2].split()[2]
	874	mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline)
	875	mb.setDetailedText(''.join(result))
	876	mb.exec_()
	877
[1611]	878	# Remove a job from list - removes the current row from the table and from
	879	# save file
	880	def RemoveFromList(self, row):
	881	if ( row == -1 ):
	882	row = self.table.currentRow()
	883
	884	# Read data from save file
	885	job_to_delete = self.table.item(row,6).text()
	886	self.table.removeRow(row)
	887	file = open(palm_dir + '/.wd.olddata', 'r')
	888	result = file.readlines()
	889	result = filter(None, result)
	890	file.close()
	891	file = open(palm_dir + '/.wd.olddata', 'w')
	892
	893	job_data_old = []
	894
	895	if ( len(result) == 0 ):
	896	notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.")
	897	else:
	898	# Save data in array job_data_old
	899	for j in range(0,len(result)):
	900	job_data_old.append(j)
	901	job_data_old[j] = result[j].split(" ")
	902	job_data_old[j] = filter(None, job_data_old[j])
	903	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	904
	905	# Check if line j is the selected job, if not -> save to file
	906	if ( job_data_old[j][6] != job_to_delete ):
	907	printstring = str(job_data_old[j][0]) + " " + \
	908	str(job_data_old[j][1]) + " " + \
	909	str(job_data_old[j][2]) + " " + \
	910	str(job_data_old[j][3]) + " " + \
	911	str(job_data_old[j][4]) + " " + \
	912	str(job_data_old[j][5]) + " " + \
	913	str(job_data_old[j][6]) + " " + \
	914	str(job_data_old[j][7]) + "\n"
	915	file.write(printstring)
	916
	917	file.close()
	918
	919	# Remove all completed jobs from list
	920	def ClearList(self):
	921
	922	num_of_lines = self.table.rowCount()
	923
	924	# Delete all lines with completed/canceled jobs from list. The counter
	925	# must decrease as the line numbers would be messed up otherwise
	926	for j in range(num_of_lines-1,-1,-1):
	927	state = self.table.item(j,3).text()
	928	if ( state == "Completed" or state == "Canceled" or state == "Terminated"):
	929	self.RemoveFromList(j)
	930
	931	# Update the label
	932	def UpdateLabel(self):
	933	remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60
	934	self.label.setText("Next update in " + str(remaining_time) + " min")
	935
	936
[1751]	937	# Enter Options menu
	938	def Options(self):
	939
	940	# disable mainwindow
	941	self.setEnabled(False)
	942
	943	# show Options Dialog
	944	opt = OptionBox()
	945	opt.exec_()
	946
	947	self.UpdateHosts()
	948	self.setEnabled(True)
	949
	950	# Update settings
	951	def UpdateHosts(self):
	952
	953	global update_frequency
[1753]	954	global description
	955	global hostname
	956	global username
	957
[1751]	958	description = []
	959	hostname = []
	960	username = []
	961
	962	for i in range(0,len(config.sections())):
	963
	964	description_tmp = config.sections()[i]
	965
	966	if ( description_tmp != 'Settings' ):
	967
	968	if ( config.get(description_tmp, 'enabled') == 'true' ):
	969	description.append(description_tmp)
	970	hostname.append(config.get(description_tmp, 'hostname'))
	971	username.append(config.get(description_tmp, 'username'))
	972	else:
	973	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	974
	975
[1611]	976	# Main loop
	977	def main():
	978
	979	app = QtGui.QApplication(sys.argv)
	980	res = Watchdog()
	981	sys.exit(app.exec_())
	982
	983
	984	if __name__ == '__main__':
[1613]	985	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |