Home

Context Navigation

source: palm/trunk/SCRIPTS/palm_wd @ 3899

Last change on this file since 3899 was 3567, checked in by maronga, 6 years ago
added first version of palm_csd (preprocessing tool for creating static drivers)
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 39.7 KB

Rev	Line
[2825]	1	#!/usr/bin/env python
[1611]	2	# -- coding: utf-8 --
	3	#--------------------------------------------------------------------------------#
[2696]	4	# This file is part of the PALM model system.
[1611]	5	#
	6	# PALM is free software: you can redistribute it and/or modify it under the terms
	7	# of the GNU General Public License as published by the Free Software Foundation,
	8	# either version 3 of the License, or (at your option) any later version.
	9	#
	10	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License along with
	15	# PALM. If not, see <http://www.gnu.org/licenses/>.
	16	#
[2718]	17	# Copyright 1997-2018 Leibniz Universitaet Hannover
[1611]	18	#--------------------------------------------------------------------------------#
	19	#
	20	# Current revisions:
	21	# -----------------
[1752]	22	#
[1754]	23	#
[1752]	24	# Former revisions:
	25	# -----------------
	26	# $Id: palm_wd 3567 2018-11-27 13:59:21Z monakurppa $
[3567]	27	# Adjustments and bugfixes
	28	#
	29	#
	30	# 2825 2018-02-20 21:48:27Z maronga
[2825]	31	# Modified header
	32	#
	33	# 2718 2018-01-02 08:49:38Z maronga
[2716]	34	# Corrected "Former revisions" section
	35	#
	36	# 2696 2017-12-14 17:12:51Z kanani
	37	# Change in file header (GPL part)
	38	#
	39	# 2416 2017-09-06 14:28:14Z maronga
[2416]	40	# Adapted for palmrun
	41	#
	42	# 1754 2016-02-22 13:50:22Z maronga
[1752]	43	#
[1754]	44	# 1753 2016-02-22 13:49:49Z maronga
	45	# Bugfix: use of global variables is required after updating configuration file
	46	#
[1752]	47	# 1751 2016-02-15 07:44:16Z maronga
[1751]	48	# Bugfixes: runs on multiple hosts caused crash of the watchdog. Progress bar
	49	# occasionally showed wrong progress.
	50	# New: Hosts can be switched on/off and update_frequency can be modified via
	51	# File->Options. Security check for cancelation of jobs.
[1611]	52	#
[1619]	53	# 1618 2015-07-13 06:52:15Z maronga
	54	# Added steering via configuration file .wd.config, to be place in the local
	55	# palm directory. Watchdog save files are automatically created upon first start.
	56	#
[1614]	57	# 1613 2015-07-08 14:53:29Z maronga
	58	# Bugfix: tooltip for queuing name did not show up on first update.
	59	# New: added contect menu for showing the parameter file and the run control
	60	# output
	61	#
[1612]	62	# 1611 2015-07-07 12:23:22Z maronga
	63	# Initial revision
	64	#
[1611]	65	#
	66	# Description:
	67	# ------------
	68	# PALM watchdog client for monitoring batch jobs on a variety of hosts specified
	69	# by the user
	70	#
	71	# Instructions:
	72	# -------------
	73	# 1) Modify the header section of palm_wd
	74	# 2) Move .wd.olddata and .wd.newdata to your palm directory
	75	# (e.g. /home/user/current_version/.wd.newdata etc.)
	76	# 3) Modify a copy of palm_wdd for each host to be monitored and move it to the
	77	# respective hosts
	78	# 4) Start the client either from mrungui or from shell by "nohup palm_wd&"
	79
	80	# To do:
	81	# ------
	82	# 1) Add "Options", "Help" and "Manual"
	83	# 2) Move user settings to a configuration file
	84	#------------------------------------------------------------------------------!
	85
[1618]	86	import ConfigParser
	87	import datetime
	88	import os
[1611]	89	from PyQt4 import QtGui, QtCore, uic
	90	from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT
[1618]	91	import shutil
	92	import subprocess as sub
	93	import sys
[1611]	94	import time
	95
	96	# Determine PALM directories
[3567]	97	# Check if configuration files exists and quit otherwise
	98	input_config = ".wd.config"
	99	for i in range(1,len(sys.argv)):
	100	input_config = str(sys.argv[i])
	101
	102	config = configparser.RawConfigParser(allow_no_value=True)
[1611]	103
[3567]	104	if ( os.path.isfile(input_config) == False ):
	105	print ("Error. No configuration file " + input_config + " found.")
	106	raise SystemExit
	107	else:
	108	print(os.path.isfile(input_config))
[1611]	109
[3567]	110	config.read(input_config)
[1618]	111
	112
	113	description = []
	114	hostname = []
	115	username = []
[3567]	116	pycall = []
[1618]	117
	118	for i in range(0,len(config.sections())):
	119
	120	description_tmp = config.sections()[i]
	121
	122	if ( description_tmp != 'Settings' ):
[1751]	123
	124	if ( config.get(description_tmp, 'enabled') == 'true' ):
	125	description.append(description_tmp)
	126	hostname.append(config.get(description_tmp, 'hostname'))
	127	username.append(config.get(description_tmp, 'username'))
[3567]	128	pycall.append(config.get(description_tmp, 'pycall'))
[1618]	129	else:
	130	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	131
	132	# Check if .wd.olddata and .wd.newdata exist. Otherwise create the files
	133	if ( os.path.exists(palm_dir + '/.wd.olddata') == False ):
	134	print "No .wd.olddata found. Creating..."
	135	file1 = open(palm_dir + '/.wd.olddata', 'a')
	136	file1.close()
	137
	138
	139	if ( os.path.exists(palm_dir + '/.wd.newdata') == False ):
	140	print "No .wd.newdata found. Creating..."
	141	file1 = open(palm_dir + '/.wd.newdata', 'a')
	142	file1.close()
	143
[1611]	144	# Dummy variables for the jobname and the progressbars
	145	jobname = ""
	146	timestamp = ""
	147	pbars = []
	148	job_data_list = []
	149
[1751]	150	# Message box for showing RUN_CONTROL output
[1613]	151	class MessageBoxScroll(QtGui.QMessageBox):
	152	def __init__(self):
	153	QtGui.QMessageBox.__init__(self)
	154	self.setSizeGripEnabled(True)
	155
	156	def event(self, e):
	157	result = QtGui.QMessageBox.event(self, e)
	158
	159	self.setMinimumHeight(100)
	160	self.setMaximumHeight(16777215)
	161	self.setMinimumWidth(400)
	162	self.setMaximumWidth(16777215)
	163	self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	164
	165	textEdit = self.findChild(QtGui.QTextEdit)
	166	if textEdit != None :
	167	textEdit.setMinimumHeight(800)
	168	textEdit.setMaximumHeight(16777215)
	169	textEdit.setMinimumWidth(1000)
	170	textEdit.setMaximumWidth(16777215)
	171	textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	172	f = QtGui.QFont('not_a_font') #
	173	f.setStyleHint(f.TypeWriter, f.PreferDefault)
	174	textEdit.setFont(f)
	175
	176	return result
	177
	178
[1751]	179	# Message box for showing RUN_CONTROL output
	180	class OptionBox(QtGui.QDialog):
	181	def __init__(self):
	182
	183	super(OptionBox, self).__init__()
	184
	185	uic.loadUi(palm_bin + '/palm_wd_files/wdoptions.ui', self)
	186
	187	hostname = []
	188	self.checkbox = []
	189	j = -1
	190	ypos = 0
	191
	192	self.vbox = QtGui.QVBoxLayout(self)
	193	self.vbox.setSpacing(0)
	194	self.vbox.setMargin(0)
	195
	196	config.read(palm_dir + '/.wd.config')
	197
	198	for i in range(0,len(config.sections())):
	199
	200	description_tmp = config.sections()[i]
	201
	202	if ( description_tmp != 'Settings' ):
	203	description.append(description_tmp)
	204	hostname.append(config.get(description_tmp, 'hostname'))
	205
	206	j = j + 1
	207	self.checkbox.append(j)
	208	self.checkbox[j] = QtGui.QCheckBox(description_tmp, self)
	209	ypos = ypos + 20
	210	self.checkbox[j].move(10,0)
	211	if ( config.get(description_tmp, 'enabled') == 'true' ):
	212	self.checkbox[j].toggle()
	213
	214	self.vbox.addWidget(self.checkbox[j])
	215	else:
	216	self.update_spin.setValue(int(config.get(description_tmp, 'update_frequency')))
	217
	218	self.hostBox.setLayout(self.vbox)
	219	self.hostBox.setGeometry(0, 0, 272, 50+ypos)
	220	self.settingBox.setGeometry(281, 0, 214, 50+ypos)
	221	self.ok.setGeometry(385, 50+ypos, 111, 24)
	222	self.cancel.setGeometry(270, 50+ypos, 111, 24)
	223
	224	self.setGeometry(0, 0, 500, 80+ypos)
	225	self.show()
	226
	227	return
	228
	229	# save updtes to file
	230	def SaveAndClose(self):
	231
	232	for i in range(0,len(self.checkbox)):
	233
	234	if ( self.checkbox[i].checkState() == 0 ):
	235	config.set(str(self.checkbox[i].text()),'enabled','false')
	236	else:
	237	config.set(str(self.checkbox[i].text()),'enabled','true')
	238
	239	config.set('Settings','update_frequency',self.update_spin.text())
	240
	241	with open(palm_dir + '/.wd.config', 'wb') as configfile:
	242	config.write(configfile)
	243
	244
	245	self.close()
	246
	247	return
	248
	249
	250
[1611]	251	# MainWindow class
	252	class Watchdog(QtGui.QMainWindow):
	253
	254	def __init__(self):
	255	super(Watchdog, self).__init__()
	256
	257	self.InitUi()
	258
	259
	260	# Initialized MainWindow UI
	261	def InitUi(self):
	262
	263	# Load predefined mainwindow
	264	uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self)
	265
	266	# Resize columns and set number of rows to zero. Column 6 is hidden and
	267	# contains the remote jobname (e.g. hannover.174610)
	268	self.table.setColumnWidth(0,230)
	269	self.table.setColumnWidth(1,80)
	270	self.table.setColumnWidth(2,50)
	271	self.table.setColumnWidth(3,80)
	272	self.table.setColumnWidth(4,180)
	273	self.table.setColumnWidth(5,115)
	274	self.table.setColumnHidden(6, True)
	275	self.table.setRowCount(0)
[1613]	276
[1611]	277	# Display MainWindow
	278	self.show()
	279	QtGui.QApplication.processEvents()
	280
	281	# Start refresh timer. On timeout perform update
	282	self.timer = QtCore.QTimer(self)
	283	self.timer.timeout.connect(self.Refresh)
	284	self.timer.setSingleShot(False)
	285	self.timer.start(update_frequency)
	286
	287	# The timetimer counts the time since last update
	288	self.timetimer= QtCore.QElapsedTimer()
	289	self.timetimer.start()
	290
	291	# The labeltimer induces the update of the remaining time in the UI
	292	self.labeltimer = QtCore.QTimer(self)
	293	self.labeltimer.timeout.connect(self.UpdateLabel)
	294	self.labeltimer.setSingleShot(False)
	295
	296	# Update in the UI will be performed at each 1/10 of the update interval
	297	self.labeltimer.start(update_frequency/10)
	298	self.label.setText("Next update in " + str(update_frequency/1000/60) + "min")
	299
	300	# Update the list now
	301	self.Refresh()
	302
	303
	304	# Add a custom context menu
	305	def OpenContextMenu(self, position):
	306	menu = QtGui.QMenu()
	307
	308	# "Show details" -> fetch details for the selected job
[1613]	309	detailsjobAction = menu.addAction('Show job details')
[1611]	310	detailsjobAction.setStatusTip('Display detailed info for job')
	311	detailsjobAction.triggered.connect(self.ShowDetails)
	312
[1613]	313	# "Show parameter file" -> show the contents of PARIN
	314	parinAction = menu.addAction('Show parameter file')
	315	parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)')
	316	parinAction.triggered.connect(self.ShowPARIN)
	317
	318	rcAction = menu.addAction('Show run control file')
	319	rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)')
	320	rcAction.triggered.connect(self.ShowRC)
	321
[1611]	322	# "Cancel job" -> remove job from queuing system
	323	canceljobAction = menu.addAction('Cancel job')
	324	canceljobAction.setStatusTip('Remove job from queueing system')
	325	canceljobAction.triggered.connect(self.CancelJob)
	326
[1613]	327	# "Force stop" -> force termination of job
[1611]	328	stopjobAction = menu.addAction('Force stop')
	329	stopjobAction.setStatusTip('Terminate job properly')
	330	stopjobAction.triggered.connect(self.DoStop)
	331
[1613]	332	# "Force restart" -> force rermination and restart of job
[1611]	333	restartjobAction = menu.addAction('Force restart')
	334	restartjobAction.setStatusTip('Terminate job properly and initiate restart')
	335	restartjobAction.triggered.connect(self.DoRestart)
	336
	337	# "Remove from list" -> remove a completed job from the list
	338	removefromlistAction = menu.addAction('Remove from list')
	339	removefromlistAction.setStatusTip('Remove finished job')
	340	removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1))
	341
	342
	343	# Activate/deactive contect menu items based on the status of the runs
	344	state = self.table.item(self.table.currentRow(),3).text()
	345	if (state == "Canceled" or state == "Completed" or state == "Terminated"):
	346	detailsjobAction.setEnabled(False)
	347	canceljobAction.setEnabled(False)
	348	removefromlistAction.setEnabled(True)
	349	else:
	350	detailsjobAction.setEnabled(True)
	351	canceljobAction.setEnabled(True)
	352	removefromlistAction.setEnabled(False)
	353
	354	if ( state == "Running" ):
	355	stopjobAction.setEnabled(True)
	356	restartjobAction.setEnabled(True)
[1613]	357	parinAction.setEnabled(True)
	358	rcAction.setEnabled(True)
[1611]	359	else:
	360	stopjobAction.setEnabled(False)
	361	restartjobAction.setEnabled(False)
[1613]	362	parinAction.setEnabled(False)
	363	rcAction.setEnabled(False)
	364
[1611]	365	# Show the context menu
	366	action = menu.exec_(self.table.mapToGlobal(position))
	367
	368
	369	# Collecting watchdog data and display in the UI
	370	def Refresh(self):
	371
	372	# Use global variables
	373	global pbars
[1751]	374	global update_frequency
[1611]	375
	376	# Deactivate timer processes until update is finished
	377	# QtGui.QApplication.processEvents() updates the UI whenever needed
	378	self.labeltimer.stop()
	379	self.label.setText("Updating...")
	380	QtGui.QApplication.processEvents()
	381	self.setEnabled(False)
	382	QtGui.QApplication.processEvents()
	383
	384	# Get current timestamp
	385	timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
	386
	387	# Open an empty file for the watchdog data
	388	file = open(palm_dir + '/.wd.newdata', 'w')
	389
	390
	391	# Set all required variables to their initial values. Sorting must be
	392	# disabled.
	393	self.table.setRowCount(0)
	394	self.table.setSortingEnabled(False)
	395	i = -1
	396	job_data_list = []
	397	pbars = []
	398
	399	# Scan all hosts in the variable hostname. For each host perform the
	400	# update loop
	401	for h in range(0,len(hostname)):
	402
	403	# Perform ssh command
	404	host = username[h] + "@" + hostname[h]
[3567]	405	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd queue " + username[h]],
[1611]	406	shell=False,
	407	stdout=sub.PIPE,
	408	stderr=sub.PIPE)
	409	result = ssh.stdout.readlines()
	410	result = filter(None, result)
	411
	412
	413	# Delete all job data
	414	job_data_tmp = []
	415	job_data = []
	416	job_data_n = []
	417	job_progress = 0
	418
	419	# In case of errors display error message
	420	if ( len(result) < 1 ):
	421	error = ssh.stderr.readlines()
	422	if ( error != [] ):
	423	notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error))
	424
	425	# No error -> save job data
	426	else:
	427
	428	# Successively write to job_data
	429	for j in range(0,len(result)):
	430
	431	job_data_tmp.append(j)
	432	job_data_tmp[j] = result[j].split(" ")
	433	job_data_tmp[j] = filter(None, job_data_tmp[j])
	434
	435	job_data.append(j)
	436	job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2],
	437	job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5],
	438	job_data_tmp[j][1], job_data_tmp[j][6].rstrip()]
	439	job_data_n.append(j)
	440	job_data_n[j] = job_data_tmp[j][1]
	441
	442	del(result)
	443
	444
	445
	446	# Now read the data from the last update from file. These data are
	447	# already in the prefered format
	448	file2 = open(palm_dir + '/.wd.olddata', 'r')
	449	result = file2.readlines()
	450	file2.close()
[1751]	451
	452	job_data_old = []
	453	job_data_old_n = []
	454	k = -1
[1611]	455	for j in range(0,len(result)):
	456	if ( result[j].split()[1] == description[h] ):
[1751]	457	k = k + 1
	458	job_data_old.append(k)
	459	job_data_old[k] = result[j].split(" ")
	460	job_data_old[k] = filter(None, job_data_old[k])
	461	job_data_old[k] = [line.rstrip('\n') for line in job_data_old[k]]
	462	job_data_old_n.append(k)
	463	job_data_old_n[k] = job_data_old[k][6]
[1611]	464
	465	# Merge the job_data and job_data_old to find completed, new and
	466	# still queued jobs
	467	if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ):
	468	jobs_full = list(set(job_data_old_n) \| set(job_data_n))
	469	jobs_known = list(set(job_data_old_n) & set(job_data_n))
	470	jobs_complete = set(job_data_old_n) - set(job_data_n)
	471	jobs_new = set(job_data_n) - set(job_data_old_n)
	472	elif ( len(job_data_n) > 0 ):
	473	jobs_full = job_data_n
	474	jobs_known = []
	475	jobs_new = job_data_n
	476	jobs_complete = []
	477	elif ( len(job_data_old_n) > 0 ):
	478	jobs_full = job_data_old_n
	479	jobs_known = []
	480	jobs_new = []
	481	jobs_complete = job_data_old_n
	482	else:
	483	jobs_full = []
	484	jobs_known = []
	485	jobs_new = []
	486	jobs_complete = []
	487
	488	# Display all known jobs
	489	to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)]
	490	for j in range(0,len(to_display)):
	491	i = i + 1
	492	self.table.insertRow(i)
	493	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	494	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	495	self.table.setItem(i, 0, item)
	496	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	497	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	498	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	499	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	500	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	501	self.table.setItem(i, 5, item)
	502	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	503	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	504	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	505	pbars.append(i)
	506	pbars[i] = QtGui.QProgressBar(self)
	507	pbars[i].setValue(int(job_data[to_display[j]][4]))
	508
	509	# Apply a color depending on the status of the job
	510	if ( job_data[to_display[j]][3] == "Running" ):
[1751]	511	self.table.setCellWidget(i,4,pbars[i])
[1611]	512	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	513	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	514	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	515	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	516	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	517	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	518	else:
	519	pbars[j].setEnabled(False)
[1751]	520	self.table.setCellWidget(i,4,pbars[i])
[1611]	521	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	522	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	523	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	524	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	525	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	526	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	527
	528	# Save the job data to file
	529	job_data_list.append(i)
	530	job_data_list[i] = job_data[to_display[j]][0]
	531
	532	printstring = str(job_data[to_display[j]][0]) + " " + \
	533	str(job_data[to_display[j]][1]) + " " + \
	534	str(job_data[to_display[j]][2]) + " " + \
	535	str(job_data[to_display[j]][3]) + " " + \
	536	str(job_data[to_display[j]][4]) + " " + \
	537	str(job_data[to_display[j]][5]) + " " + \
	538	str(job_data[to_display[j]][6]) + " " + \
	539	str(job_data[to_display[j]][7]) + "\n"
	540	file.write(printstring)
	541
	542
	543	# Display all new jobs
	544	to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)]
	545	for j in range(0,len(to_display)):
	546	i = i + 1
	547	self.table.insertRow(i)
	548	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	549	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	550	self.table.setItem(i, 0, item)
	551	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	552	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	553	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	554	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	555	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	556	self.table.setItem(i, 5, item)
	557	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	558	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	559	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	560
	561	pbars.append(i)
	562	pbars[i] = QtGui.QProgressBar(self)
	563	pbars[i].setValue(int(job_data[to_display[j]][4]))
	564
	565	# Apply a color depending on the status of the job
	566	if ( job_data[to_display[j]][3] == "Running" ):
	567	self.table.setCellWidget(i,4,pbars[i])
	568	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	569	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	570	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	571	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	572	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	573	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	574	else:
	575	pbars[j].setEnabled(False)
	576	self.table.setCellWidget(i,4,pbars[i])
	577	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	578	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	579	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	580	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	581	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	582	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	583
	584	# Save job data to file
	585	job_data_list.append(i)
	586	job_data_list[i] = job_data[to_display[j]][0]
	587
	588	printstring = str(job_data[to_display[j]][0]) + " " + \
	589	str(job_data[to_display[j]][1]) + " " + \
	590	str(job_data[to_display[j]][2]) + " " + \
	591	str(job_data[to_display[j]][3]) + " " + \
	592	str(job_data[to_display[j]][4]) + " " + \
	593	str(job_data[to_display[j]][5]) + " " + \
	594	str(job_data[to_display[j]][6]) + " " + \
	595	str(job_data[to_display[j]][7]) + "\n"
	596	file.write(printstring)
	597
	598
	599	# Display all completed/canceled/aborted jobs. The watchdog cannot
	600	# distinguish why the job has been finished
	601	to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)]
	602	for j in range(0,len(to_display)):
	603	i = i + 1
	604	self.table.insertRow(i)
	605	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0])
	606	item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6])
	607	self.table.setItem(i, 0, item)
	608	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1]))
	609	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2]))
	610	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3]))
	611	pbars.append(i)
	612	pbars[j] = QtGui.QProgressBar(self)
	613	pbars[j].setValue(int(job_data_old[to_display[j]][4]))
	614	pbars[j].setEnabled(False)
	615	self.table.setCellWidget(i,4,pbars[j])
	616	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5])
	617	item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7])
	618	self.table.setItem(i, 5, item)
	619	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6]))
	620	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	621	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	622	self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed"))
	623
	624	# Apply a color depending on the status of the job
	625	self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175))
	626	self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175))
	627	self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175))
	628	self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175))
	629	self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175))
	630	self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175))
	631
	632
	633	# Save job data to file
	634	job_data_list.append(i)
	635	job_data_list[i] = job_data_old[to_display[j]][0]
	636
	637	printstring = str(job_data_old[to_display[j]][0]) + " " + \
	638	str(job_data_old[to_display[j]][1]) + " " + \
	639	str(job_data_old[to_display[j]][2]) + " " + \
	640	str(job_data_old[to_display[j]][3]) + " " + \
	641	str(job_data_old[to_display[j]][4]) + " " + \
	642	str(job_data_old[to_display[j]][5]) + " " + \
	643	str(job_data_old[to_display[j]][6]) + " " + \
	644	str(job_data_old[to_display[j]][7]) + "\n"
	645	file.write(printstring)
	646
	647	del(jobs_complete)
	648	del(jobs_new)
	649	del(result)
[1751]	650	del(job_data_old)
	651	del(job_data_old_n)
	652
[1611]	653	file.close()
	654
	655
	656	# Update is complete, sorting can thus be re-enabled
	657	self.table.setSortingEnabled(True)
	658
	659	# Update the logfile
	660	if ( os.path.isfile(palm_dir + '/.wd.newdata') ):
	661	shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata')
	662
	663	# Re-enable timer processes
	664	self.setEnabled(True)
	665	self.label2.setText("Last update: " + timestamp)
	666	self.label.setText("Update complete.")
[1751]	667	self.timetimer.start()
	668	self.timer.start(update_frequency)
	669	self.labeltimer.start(update_frequency/10)
	670	self.UpdateLabel()
[1611]	671	QtGui.QApplication.processEvents()
	672
	673	# Canceling selected job from table
	674	def CancelJob(self):
	675
	676	# Return internal jobname
	677	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	678	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	679
	680	# Check description of job in order to login on the correct host
	681	descr = self.table.item(self.table.currentRow(),1).text()
[1751]	682
	683	querybox = QtGui.QMessageBox()
	684	querybox.setText("Attention!\nYou are trying to cancel a job. It will not be able to terminate properly.")
	685	querybox.setInformativeText("Do you really want to cancel " + jobname + " on host " + descr + "?")
	686	querybox.setStandardButtons(QtGui.QMessageBox.Yes \| QtGui.QMessageBox.No)
	687	querybox.setDefaultButton(QtGui.QMessageBox.No)
	688	returnvalue = querybox.exec_()
	689
	690	if ( returnvalue == QtGui.QMessageBox.Yes ):
	691
	692	for h in range(0,len(description)):
	693	if ( descr == description[h] ):
	694	host = username[h] + "@" + hostname[h]
[1611]	695
[3567]	696	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd cancel " + jobname],
[1751]	697	shell=False,
	698	stdout=sub.PIPE,
	699	stderr=sub.PIPE)
	700	result = ssh.stdout.readlines()
	701	result = filter(None, result)
[1611]	702
[1751]	703	# In case of error display a warning message
	704	if ( len(result) == 0 ):
	705	error = ssh.stderr.readlines()
	706	notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error))
[1611]	707
[1751]	708	# Otherwise inform the user and mark the job in the table
	709	else:
	710	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled"))
	711	notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result))
[1611]	712
	713
	714	# Show detailed information on job. For documentation see above
	715	def ShowDetails(self):
	716
[2416]	717	jobname = self.table.item(self.table.currentRow(),6).text()
[1611]	718	descr = self.table.item(self.table.currentRow(),1).text()
	719	for h in range(0,len(description)):
	720	if ( descr == description[h] ):
	721	host = username[h] + "@" + hostname[h]
	722
[3567]	723	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd check " + jobname],
[1611]	724	shell=False,
	725	stdout=sub.PIPE,
	726	stderr=sub.PIPE)
	727	result = ssh.stdout.readlines()
	728	result = filter(None, result)
	729
	730	if ( len(result) == 0 ):
	731	error = ssh.stderr.readlines()
	732	notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error))
	733	else:
	734	notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result))
	735
	736
	737	# Perform a forced stop on the job
	738	def DoStop(self):
	739
	740	# Return internal jobname
	741	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	742	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	743	# Check description of job in order to login on the correct host
	744	descr = self.table.item(self.table.currentRow(),1).text()
	745	for h in range(0,len(description)):
	746	if ( descr == description[h] ):
	747	host = username[h] + "@" + hostname[h]
	748	user = username[h]
	749
[3567]	750	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd stop " + jobrealname],
[1611]	751	shell=False,
	752	stdout=sub.PIPE,
	753	stderr=sub.PIPE)
	754	result = ssh.stdout.readlines()
	755	result = filter(None, result)
	756
	757	# In case of error display a warning message
	758	if ( len(result) == 0 ):
	759	error = ssh.stderr.readlines()
	760	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	761
	762	# Otherwise inform the user and mark the job in the table
	763	else:
	764	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
[1751]	765	notify = QtGui.QMessageBox.information(self,'Terminate job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
[1611]	766
	767
	768	# Perform a forced stop on the job
	769	def DoRestart(self):
	770
	771	# Return internal jobname
	772	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	773	jobrealname = self.table.item(self.table.currentRow(),0).text()
	774
[1611]	775	# Check description of job in order to login on the correct host
	776	descr = self.table.item(self.table.currentRow(),1).text()
	777	for h in range(0,len(description)):
	778	if ( descr == description[h] ):
	779	host = username[h] + "@" + hostname[h]
	780	user = username[h]
	781
[3567]	782	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd restart " + jobrealname],
[1611]	783	shell=False,
	784	stdout=sub.PIPE,
	785	stderr=sub.PIPE)
	786	result = ssh.stdout.readlines()
	787	result = filter(None, result)
	788
	789	# In case of error display a warning message
	790	if ( len(result) == 0 ):
	791	error = ssh.stderr.readlines()
	792	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	793
	794	# Otherwise inform the user and mark the job in the table
	795	else:
	796	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	797	notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	798
	799
[1613]	800	# Read the PARIN file of the job
	801	def ShowPARIN(self):
	802
	803	# Return internal jobname
	804	jobname = self.table.item(self.table.currentRow(),6).text()
	805	jobrealname = self.table.item(self.table.currentRow(),0).text()
	806
	807	# Check description of job in order to login on the correct host
	808	descr = self.table.item(self.table.currentRow(),1).text()
	809	for h in range(0,len(description)):
	810	if ( descr == description[h] ):
	811	host = username[h] + "@" + hostname[h]
	812	user = username[h]
	813
[3567]	814	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd parin " + jobrealname],
[1613]	815	shell=False,
	816	stdout=sub.PIPE,
	817	stderr=sub.PIPE)
	818	result = ssh.stdout.readlines()
	819	result = filter(None, result)
	820
	821	# In case of error display a warning message
	822	if ( len(result) == 0 ):
	823	error = ssh.stderr.readlines()
	824	notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error))
	825
	826	# Otherwise inform the user and mark the job in the table
	827	else:
	828	mb = MessageBoxScroll()
	829	mb.setText("Parameter file for job: " + jobrealname + "")
	830	mb.setDetailedText(''.join(result))
	831	mb.exec_()
	832
	833	# Read the PARIN file of the job
	834	def ShowRC(self):
	835
	836	# Return internal jobname and real job name
	837	jobname = self.table.item(self.table.currentRow(),6).text()
	838	jobrealname = self.table.item(self.table.currentRow(),0).text()
	839
	840	# Check description of job in order to login on the correct host
	841	descr = self.table.item(self.table.currentRow(),1).text()
	842	for h in range(0,len(description)):
	843	if ( descr == description[h] ):
	844	host = username[h] + "@" + hostname[h]
	845	user = username[h]
	846
[3567]	847	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd rc " + jobrealname],
[1613]	848	shell=False,
	849	stdout=sub.PIPE,
	850	stderr=sub.PIPE)
	851	result = ssh.stdout.readlines()
	852	result = filter(None, result)
	853
	854	# In case of error display a warning message
	855	if ( len(result) == 0 ):
	856	error = ssh.stderr.readlines()
	857	notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error))
	858
	859	# Otherwise inform the user and mark the job in the table
	860	else:
	861	mb = MessageBoxScroll()
	862	lastline = result[len(result)-2].split()[2]
	863	mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline)
	864	mb.setDetailedText(''.join(result))
	865	mb.exec_()
	866
[1611]	867	# Remove a job from list - removes the current row from the table and from
	868	# save file
	869	def RemoveFromList(self, row):
	870	if ( row == -1 ):
	871	row = self.table.currentRow()
	872
	873	# Read data from save file
	874	job_to_delete = self.table.item(row,6).text()
	875	self.table.removeRow(row)
	876	file = open(palm_dir + '/.wd.olddata', 'r')
	877	result = file.readlines()
	878	result = filter(None, result)
	879	file.close()
	880	file = open(palm_dir + '/.wd.olddata', 'w')
	881
	882	job_data_old = []
	883
	884	if ( len(result) == 0 ):
	885	notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.")
	886	else:
	887	# Save data in array job_data_old
	888	for j in range(0,len(result)):
	889	job_data_old.append(j)
	890	job_data_old[j] = result[j].split(" ")
	891	job_data_old[j] = filter(None, job_data_old[j])
	892	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	893
	894	# Check if line j is the selected job, if not -> save to file
	895	if ( job_data_old[j][6] != job_to_delete ):
	896	printstring = str(job_data_old[j][0]) + " " + \
	897	str(job_data_old[j][1]) + " " + \
	898	str(job_data_old[j][2]) + " " + \
	899	str(job_data_old[j][3]) + " " + \
	900	str(job_data_old[j][4]) + " " + \
	901	str(job_data_old[j][5]) + " " + \
	902	str(job_data_old[j][6]) + " " + \
	903	str(job_data_old[j][7]) + "\n"
	904	file.write(printstring)
	905
	906	file.close()
	907
	908	# Remove all completed jobs from list
	909	def ClearList(self):
	910
	911	num_of_lines = self.table.rowCount()
	912
	913	# Delete all lines with completed/canceled jobs from list. The counter
	914	# must decrease as the line numbers would be messed up otherwise
	915	for j in range(num_of_lines-1,-1,-1):
	916	state = self.table.item(j,3).text()
	917	if ( state == "Completed" or state == "Canceled" or state == "Terminated"):
	918	self.RemoveFromList(j)
	919
	920	# Update the label
	921	def UpdateLabel(self):
	922	remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60
	923	self.label.setText("Next update in " + str(remaining_time) + " min")
	924
	925
[1751]	926	# Enter Options menu
	927	def Options(self):
	928
	929	# disable mainwindow
	930	self.setEnabled(False)
	931
	932	# show Options Dialog
	933	opt = OptionBox()
	934	opt.exec_()
	935
	936	self.UpdateHosts()
	937	self.setEnabled(True)
	938
	939	# Update settings
	940	def UpdateHosts(self):
	941
	942	global update_frequency
[1753]	943	global description
	944	global hostname
	945	global username
	946
[1751]	947	description = []
	948	hostname = []
	949	username = []
	950
	951	for i in range(0,len(config.sections())):
	952
	953	description_tmp = config.sections()[i]
	954
	955	if ( description_tmp != 'Settings' ):
	956
	957	if ( config.get(description_tmp, 'enabled') == 'true' ):
	958	description.append(description_tmp)
	959	hostname.append(config.get(description_tmp, 'hostname'))
	960	username.append(config.get(description_tmp, 'username'))
	961	else:
	962	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	963
	964
[1611]	965	# Main loop
	966	def main():
	967
	968	app = QtGui.QApplication(sys.argv)
	969	res = Watchdog()
	970	sys.exit(app.exec_())
	971
	972
	973	if __name__ == '__main__':
[1613]	974	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |