Home

Context Navigation

source: palm/trunk/SCRIPTS/palm_wd @ 4085

Last change on this file since 4085 was 3943, checked in by maronga, 6 years ago
bugfixes in urban surface model; output of greenz roof transpiration added/corrected; minor formatting improvements
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 39.9 KB

Rev	Line
[2825]	1	#!/usr/bin/env python
[1611]	2	# -- coding: utf-8 --
	3	#--------------------------------------------------------------------------------#
[2696]	4	# This file is part of the PALM model system.
[1611]	5	#
	6	# PALM is free software: you can redistribute it and/or modify it under the terms
	7	# of the GNU General Public License as published by the Free Software Foundation,
	8	# either version 3 of the License, or (at your option) any later version.
	9	#
	10	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License along with
	15	# PALM. If not, see <http://www.gnu.org/licenses/>.
	16	#
[2718]	17	# Copyright 1997-2018 Leibniz Universitaet Hannover
[1611]	18	#--------------------------------------------------------------------------------#
	19	#
	20	# Current revisions:
	21	# -----------------
[1752]	22	#
[1754]	23	#
[1752]	24	# Former revisions:
	25	# -----------------
	26	# $Id: palm_wd 3943 2019-05-02 09:50:41Z knoop $
[3567]	27	# Adjustments and bugfixes
	28	#
	29	#
	30	# 2825 2018-02-20 21:48:27Z maronga
[2825]	31	# Modified header
	32	#
	33	# 2718 2018-01-02 08:49:38Z maronga
[2716]	34	# Corrected "Former revisions" section
	35	#
	36	# 2696 2017-12-14 17:12:51Z kanani
	37	# Change in file header (GPL part)
	38	#
	39	# 2416 2017-09-06 14:28:14Z maronga
[2416]	40	# Adapted for palmrun
	41	#
	42	# 1754 2016-02-22 13:50:22Z maronga
[1752]	43	#
[1754]	44	# 1753 2016-02-22 13:49:49Z maronga
	45	# Bugfix: use of global variables is required after updating configuration file
	46	#
[1752]	47	# 1751 2016-02-15 07:44:16Z maronga
[1751]	48	# Bugfixes: runs on multiple hosts caused crash of the watchdog. Progress bar
	49	# occasionally showed wrong progress.
	50	# New: Hosts can be switched on/off and update_frequency can be modified via
	51	# File->Options. Security check for cancelation of jobs.
[1611]	52	#
[1619]	53	# 1618 2015-07-13 06:52:15Z maronga
	54	# Added steering via configuration file .wd.config, to be place in the local
	55	# palm directory. Watchdog save files are automatically created upon first start.
	56	#
[1614]	57	# 1613 2015-07-08 14:53:29Z maronga
	58	# Bugfix: tooltip for queuing name did not show up on first update.
	59	# New: added contect menu for showing the parameter file and the run control
	60	# output
	61	#
[1612]	62	# 1611 2015-07-07 12:23:22Z maronga
	63	# Initial revision
	64	#
[1611]	65	#
	66	# Description:
	67	# ------------
	68	# PALM watchdog client for monitoring batch jobs on a variety of hosts specified
	69	# by the user
	70	#
	71	# Instructions:
	72	# -------------
	73	# 1) Modify the header section of palm_wd
	74	# 2) Move .wd.olddata and .wd.newdata to your palm directory
	75	# (e.g. /home/user/current_version/.wd.newdata etc.)
	76	# 3) Modify a copy of palm_wdd for each host to be monitored and move it to the
	77	# respective hosts
	78	# 4) Start the client either from mrungui or from shell by "nohup palm_wd&"
	79
	80	# To do:
	81	# ------
	82	# 1) Add "Options", "Help" and "Manual"
	83	# 2) Move user settings to a configuration file
	84	#------------------------------------------------------------------------------!
	85
[3943]	86	import ConfigParser as configparser
[1618]	87	import datetime
	88	import os
[1611]	89	from PyQt4 import QtGui, QtCore, uic
	90	from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT
[1618]	91	import shutil
	92	import subprocess as sub
	93	import sys
[1611]	94	import time
	95
	96	# Determine PALM directories
[3567]	97	# Check if configuration files exists and quit otherwise
	98	input_config = ".wd.config"
[3943]	99
	100	for i in range(0,len(sys.argv)):
	101	out = sub.check_output("echo $PALM_BIN", shell=True, stderr=sub.STDOUT)
[3567]	102	input_config = str(sys.argv[i])
[3943]	103	palm_bin = out.rstrip()
	104	palm_dir = out.split("palm")[0] + "palm/" + out.split("palm")[1].split("/")[1]
[3567]	105
	106	config = configparser.RawConfigParser(allow_no_value=True)
[1611]	107
[3567]	108	if ( os.path.isfile(input_config) == False ):
	109	print ("Error. No configuration file " + input_config + " found.")
	110	raise SystemExit
	111	else:
	112	print(os.path.isfile(input_config))
[1611]	113
[3943]	114	print(input_config)
	115
[3567]	116	config.read(input_config)
[1618]	117
[3943]	118	config = ConfigParser.RawConfigParser()
	119	config.read(palm_dir + '/.wd.config')
[1618]	120
	121	description = []
	122	hostname = []
	123	username = []
[3567]	124	pycall = []
[1618]	125
[3943]	126
[1618]	127	for i in range(0,len(config.sections())):
	128
	129	description_tmp = config.sections()[i]
	130
	131	if ( description_tmp != 'Settings' ):
[1751]	132
	133	if ( config.get(description_tmp, 'enabled') == 'true' ):
	134	description.append(description_tmp)
	135	hostname.append(config.get(description_tmp, 'hostname'))
	136	username.append(config.get(description_tmp, 'username'))
[3567]	137	pycall.append(config.get(description_tmp, 'pycall'))
[1618]	138	else:
	139	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	140
	141	# Check if .wd.olddata and .wd.newdata exist. Otherwise create the files
	142	if ( os.path.exists(palm_dir + '/.wd.olddata') == False ):
	143	print "No .wd.olddata found. Creating..."
	144	file1 = open(palm_dir + '/.wd.olddata', 'a')
	145	file1.close()
	146
	147
	148	if ( os.path.exists(palm_dir + '/.wd.newdata') == False ):
	149	print "No .wd.newdata found. Creating..."
	150	file1 = open(palm_dir + '/.wd.newdata', 'a')
	151	file1.close()
	152
[1611]	153	# Dummy variables for the jobname and the progressbars
	154	jobname = ""
	155	timestamp = ""
	156	pbars = []
	157	job_data_list = []
	158
[1751]	159	# Message box for showing RUN_CONTROL output
[1613]	160	class MessageBoxScroll(QtGui.QMessageBox):
	161	def __init__(self):
	162	QtGui.QMessageBox.__init__(self)
	163	self.setSizeGripEnabled(True)
	164
	165	def event(self, e):
	166	result = QtGui.QMessageBox.event(self, e)
	167
	168	self.setMinimumHeight(100)
	169	self.setMaximumHeight(16777215)
	170	self.setMinimumWidth(400)
	171	self.setMaximumWidth(16777215)
	172	self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	173
	174	textEdit = self.findChild(QtGui.QTextEdit)
	175	if textEdit != None :
	176	textEdit.setMinimumHeight(800)
	177	textEdit.setMaximumHeight(16777215)
	178	textEdit.setMinimumWidth(1000)
	179	textEdit.setMaximumWidth(16777215)
	180	textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	181	f = QtGui.QFont('not_a_font') #
	182	f.setStyleHint(f.TypeWriter, f.PreferDefault)
	183	textEdit.setFont(f)
	184
	185	return result
	186
	187
[1751]	188	# Message box for showing RUN_CONTROL output
	189	class OptionBox(QtGui.QDialog):
	190	def __init__(self):
	191
	192	super(OptionBox, self).__init__()
	193
	194	uic.loadUi(palm_bin + '/palm_wd_files/wdoptions.ui', self)
	195
	196	hostname = []
	197	self.checkbox = []
	198	j = -1
	199	ypos = 0
	200
	201	self.vbox = QtGui.QVBoxLayout(self)
	202	self.vbox.setSpacing(0)
	203	self.vbox.setMargin(0)
	204
	205	config.read(palm_dir + '/.wd.config')
	206
	207	for i in range(0,len(config.sections())):
	208
	209	description_tmp = config.sections()[i]
	210
	211	if ( description_tmp != 'Settings' ):
	212	description.append(description_tmp)
	213	hostname.append(config.get(description_tmp, 'hostname'))
	214
	215	j = j + 1
	216	self.checkbox.append(j)
	217	self.checkbox[j] = QtGui.QCheckBox(description_tmp, self)
	218	ypos = ypos + 20
	219	self.checkbox[j].move(10,0)
	220	if ( config.get(description_tmp, 'enabled') == 'true' ):
	221	self.checkbox[j].toggle()
	222
	223	self.vbox.addWidget(self.checkbox[j])
	224	else:
	225	self.update_spin.setValue(int(config.get(description_tmp, 'update_frequency')))
	226
	227	self.hostBox.setLayout(self.vbox)
	228	self.hostBox.setGeometry(0, 0, 272, 50+ypos)
	229	self.settingBox.setGeometry(281, 0, 214, 50+ypos)
	230	self.ok.setGeometry(385, 50+ypos, 111, 24)
	231	self.cancel.setGeometry(270, 50+ypos, 111, 24)
	232
	233	self.setGeometry(0, 0, 500, 80+ypos)
	234	self.show()
	235
	236	return
	237
	238	# save updtes to file
	239	def SaveAndClose(self):
	240
	241	for i in range(0,len(self.checkbox)):
	242
	243	if ( self.checkbox[i].checkState() == 0 ):
	244	config.set(str(self.checkbox[i].text()),'enabled','false')
	245	else:
	246	config.set(str(self.checkbox[i].text()),'enabled','true')
	247
	248	config.set('Settings','update_frequency',self.update_spin.text())
	249
	250	with open(palm_dir + '/.wd.config', 'wb') as configfile:
	251	config.write(configfile)
	252
	253
	254	self.close()
	255
	256	return
	257
	258
	259
[1611]	260	# MainWindow class
	261	class Watchdog(QtGui.QMainWindow):
	262
	263	def __init__(self):
	264	super(Watchdog, self).__init__()
	265
	266	self.InitUi()
	267
	268
	269	# Initialized MainWindow UI
	270	def InitUi(self):
	271
	272	# Load predefined mainwindow
	273	uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self)
	274
	275	# Resize columns and set number of rows to zero. Column 6 is hidden and
	276	# contains the remote jobname (e.g. hannover.174610)
	277	self.table.setColumnWidth(0,230)
	278	self.table.setColumnWidth(1,80)
	279	self.table.setColumnWidth(2,50)
	280	self.table.setColumnWidth(3,80)
	281	self.table.setColumnWidth(4,180)
	282	self.table.setColumnWidth(5,115)
	283	self.table.setColumnHidden(6, True)
	284	self.table.setRowCount(0)
[1613]	285
[1611]	286	# Display MainWindow
	287	self.show()
	288	QtGui.QApplication.processEvents()
	289
	290	# Start refresh timer. On timeout perform update
	291	self.timer = QtCore.QTimer(self)
	292	self.timer.timeout.connect(self.Refresh)
	293	self.timer.setSingleShot(False)
	294	self.timer.start(update_frequency)
	295
	296	# The timetimer counts the time since last update
	297	self.timetimer= QtCore.QElapsedTimer()
	298	self.timetimer.start()
	299
	300	# The labeltimer induces the update of the remaining time in the UI
	301	self.labeltimer = QtCore.QTimer(self)
	302	self.labeltimer.timeout.connect(self.UpdateLabel)
	303	self.labeltimer.setSingleShot(False)
	304
	305	# Update in the UI will be performed at each 1/10 of the update interval
	306	self.labeltimer.start(update_frequency/10)
	307	self.label.setText("Next update in " + str(update_frequency/1000/60) + "min")
	308
	309	# Update the list now
	310	self.Refresh()
	311
	312
	313	# Add a custom context menu
	314	def OpenContextMenu(self, position):
	315	menu = QtGui.QMenu()
	316
	317	# "Show details" -> fetch details for the selected job
[1613]	318	detailsjobAction = menu.addAction('Show job details')
[1611]	319	detailsjobAction.setStatusTip('Display detailed info for job')
	320	detailsjobAction.triggered.connect(self.ShowDetails)
	321
[1613]	322	# "Show parameter file" -> show the contents of PARIN
	323	parinAction = menu.addAction('Show parameter file')
	324	parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)')
	325	parinAction.triggered.connect(self.ShowPARIN)
	326
	327	rcAction = menu.addAction('Show run control file')
	328	rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)')
	329	rcAction.triggered.connect(self.ShowRC)
	330
[1611]	331	# "Cancel job" -> remove job from queuing system
	332	canceljobAction = menu.addAction('Cancel job')
	333	canceljobAction.setStatusTip('Remove job from queueing system')
	334	canceljobAction.triggered.connect(self.CancelJob)
	335
[1613]	336	# "Force stop" -> force termination of job
[1611]	337	stopjobAction = menu.addAction('Force stop')
	338	stopjobAction.setStatusTip('Terminate job properly')
	339	stopjobAction.triggered.connect(self.DoStop)
	340
[1613]	341	# "Force restart" -> force rermination and restart of job
[1611]	342	restartjobAction = menu.addAction('Force restart')
	343	restartjobAction.setStatusTip('Terminate job properly and initiate restart')
	344	restartjobAction.triggered.connect(self.DoRestart)
	345
	346	# "Remove from list" -> remove a completed job from the list
	347	removefromlistAction = menu.addAction('Remove from list')
	348	removefromlistAction.setStatusTip('Remove finished job')
	349	removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1))
	350
	351
	352	# Activate/deactive contect menu items based on the status of the runs
	353	state = self.table.item(self.table.currentRow(),3).text()
	354	if (state == "Canceled" or state == "Completed" or state == "Terminated"):
	355	detailsjobAction.setEnabled(False)
	356	canceljobAction.setEnabled(False)
	357	removefromlistAction.setEnabled(True)
	358	else:
	359	detailsjobAction.setEnabled(True)
	360	canceljobAction.setEnabled(True)
	361	removefromlistAction.setEnabled(False)
	362
	363	if ( state == "Running" ):
	364	stopjobAction.setEnabled(True)
	365	restartjobAction.setEnabled(True)
[1613]	366	parinAction.setEnabled(True)
	367	rcAction.setEnabled(True)
[1611]	368	else:
	369	stopjobAction.setEnabled(False)
	370	restartjobAction.setEnabled(False)
[1613]	371	parinAction.setEnabled(False)
	372	rcAction.setEnabled(False)
	373
[1611]	374	# Show the context menu
	375	action = menu.exec_(self.table.mapToGlobal(position))
	376
	377
	378	# Collecting watchdog data and display in the UI
	379	def Refresh(self):
	380
	381	# Use global variables
	382	global pbars
[1751]	383	global update_frequency
[1611]	384
	385	# Deactivate timer processes until update is finished
	386	# QtGui.QApplication.processEvents() updates the UI whenever needed
	387	self.labeltimer.stop()
	388	self.label.setText("Updating...")
	389	QtGui.QApplication.processEvents()
	390	self.setEnabled(False)
	391	QtGui.QApplication.processEvents()
	392
	393	# Get current timestamp
	394	timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
	395
	396	# Open an empty file for the watchdog data
	397	file = open(palm_dir + '/.wd.newdata', 'w')
	398
	399
	400	# Set all required variables to their initial values. Sorting must be
	401	# disabled.
	402	self.table.setRowCount(0)
	403	self.table.setSortingEnabled(False)
	404	i = -1
	405	job_data_list = []
	406	pbars = []
	407
	408	# Scan all hosts in the variable hostname. For each host perform the
	409	# update loop
	410	for h in range(0,len(hostname)):
	411
	412	# Perform ssh command
	413	host = username[h] + "@" + hostname[h]
[3567]	414	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd queue " + username[h]],
[1611]	415	shell=False,
	416	stdout=sub.PIPE,
	417	stderr=sub.PIPE)
	418	result = ssh.stdout.readlines()
	419	result = filter(None, result)
	420
	421
	422	# Delete all job data
	423	job_data_tmp = []
	424	job_data = []
	425	job_data_n = []
	426	job_progress = 0
	427
	428	# In case of errors display error message
	429	if ( len(result) < 1 ):
	430	error = ssh.stderr.readlines()
	431	if ( error != [] ):
	432	notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error))
	433
	434	# No error -> save job data
	435	else:
	436
	437	# Successively write to job_data
	438	for j in range(0,len(result)):
	439
	440	job_data_tmp.append(j)
	441	job_data_tmp[j] = result[j].split(" ")
	442	job_data_tmp[j] = filter(None, job_data_tmp[j])
	443
	444	job_data.append(j)
	445	job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2],
	446	job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5],
	447	job_data_tmp[j][1], job_data_tmp[j][6].rstrip()]
	448	job_data_n.append(j)
	449	job_data_n[j] = job_data_tmp[j][1]
	450
	451	del(result)
	452
	453
	454
	455	# Now read the data from the last update from file. These data are
	456	# already in the prefered format
	457	file2 = open(palm_dir + '/.wd.olddata', 'r')
	458	result = file2.readlines()
	459	file2.close()
[1751]	460
	461	job_data_old = []
	462	job_data_old_n = []
	463	k = -1
[1611]	464	for j in range(0,len(result)):
	465	if ( result[j].split()[1] == description[h] ):
[1751]	466	k = k + 1
	467	job_data_old.append(k)
	468	job_data_old[k] = result[j].split(" ")
	469	job_data_old[k] = filter(None, job_data_old[k])
	470	job_data_old[k] = [line.rstrip('\n') for line in job_data_old[k]]
	471	job_data_old_n.append(k)
	472	job_data_old_n[k] = job_data_old[k][6]
[1611]	473
	474	# Merge the job_data and job_data_old to find completed, new and
	475	# still queued jobs
	476	if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ):
	477	jobs_full = list(set(job_data_old_n) \| set(job_data_n))
	478	jobs_known = list(set(job_data_old_n) & set(job_data_n))
	479	jobs_complete = set(job_data_old_n) - set(job_data_n)
	480	jobs_new = set(job_data_n) - set(job_data_old_n)
	481	elif ( len(job_data_n) > 0 ):
	482	jobs_full = job_data_n
	483	jobs_known = []
	484	jobs_new = job_data_n
	485	jobs_complete = []
	486	elif ( len(job_data_old_n) > 0 ):
	487	jobs_full = job_data_old_n
	488	jobs_known = []
	489	jobs_new = []
	490	jobs_complete = job_data_old_n
	491	else:
	492	jobs_full = []
	493	jobs_known = []
	494	jobs_new = []
	495	jobs_complete = []
	496
	497	# Display all known jobs
	498	to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)]
	499	for j in range(0,len(to_display)):
	500	i = i + 1
	501	self.table.insertRow(i)
	502	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	503	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	504	self.table.setItem(i, 0, item)
	505	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	506	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	507	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	508	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	509	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	510	self.table.setItem(i, 5, item)
	511	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	512	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	513	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	514	pbars.append(i)
	515	pbars[i] = QtGui.QProgressBar(self)
	516	pbars[i].setValue(int(job_data[to_display[j]][4]))
	517
	518	# Apply a color depending on the status of the job
	519	if ( job_data[to_display[j]][3] == "Running" ):
[1751]	520	self.table.setCellWidget(i,4,pbars[i])
[1611]	521	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	522	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	523	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	524	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	525	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	526	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	527	else:
	528	pbars[j].setEnabled(False)
[1751]	529	self.table.setCellWidget(i,4,pbars[i])
[1611]	530	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	531	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	532	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	533	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	534	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	535	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	536
	537	# Save the job data to file
	538	job_data_list.append(i)
	539	job_data_list[i] = job_data[to_display[j]][0]
	540
	541	printstring = str(job_data[to_display[j]][0]) + " " + \
	542	str(job_data[to_display[j]][1]) + " " + \
	543	str(job_data[to_display[j]][2]) + " " + \
	544	str(job_data[to_display[j]][3]) + " " + \
	545	str(job_data[to_display[j]][4]) + " " + \
	546	str(job_data[to_display[j]][5]) + " " + \
	547	str(job_data[to_display[j]][6]) + " " + \
	548	str(job_data[to_display[j]][7]) + "\n"
	549	file.write(printstring)
	550
	551
	552	# Display all new jobs
	553	to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)]
	554	for j in range(0,len(to_display)):
	555	i = i + 1
	556	self.table.insertRow(i)
	557	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	558	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	559	self.table.setItem(i, 0, item)
	560	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	561	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	562	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	563	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	564	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	565	self.table.setItem(i, 5, item)
	566	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	567	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	568	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	569
	570	pbars.append(i)
	571	pbars[i] = QtGui.QProgressBar(self)
	572	pbars[i].setValue(int(job_data[to_display[j]][4]))
	573
	574	# Apply a color depending on the status of the job
	575	if ( job_data[to_display[j]][3] == "Running" ):
	576	self.table.setCellWidget(i,4,pbars[i])
	577	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	578	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	579	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	580	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	581	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	582	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	583	else:
	584	pbars[j].setEnabled(False)
	585	self.table.setCellWidget(i,4,pbars[i])
	586	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	587	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	588	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	589	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	590	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	591	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	592
	593	# Save job data to file
	594	job_data_list.append(i)
	595	job_data_list[i] = job_data[to_display[j]][0]
	596
	597	printstring = str(job_data[to_display[j]][0]) + " " + \
	598	str(job_data[to_display[j]][1]) + " " + \
	599	str(job_data[to_display[j]][2]) + " " + \
	600	str(job_data[to_display[j]][3]) + " " + \
	601	str(job_data[to_display[j]][4]) + " " + \
	602	str(job_data[to_display[j]][5]) + " " + \
	603	str(job_data[to_display[j]][6]) + " " + \
	604	str(job_data[to_display[j]][7]) + "\n"
	605	file.write(printstring)
	606
	607
	608	# Display all completed/canceled/aborted jobs. The watchdog cannot
	609	# distinguish why the job has been finished
	610	to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)]
	611	for j in range(0,len(to_display)):
	612	i = i + 1
	613	self.table.insertRow(i)
	614	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0])
	615	item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6])
	616	self.table.setItem(i, 0, item)
	617	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1]))
	618	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2]))
	619	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3]))
	620	pbars.append(i)
	621	pbars[j] = QtGui.QProgressBar(self)
	622	pbars[j].setValue(int(job_data_old[to_display[j]][4]))
	623	pbars[j].setEnabled(False)
	624	self.table.setCellWidget(i,4,pbars[j])
	625	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5])
	626	item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7])
	627	self.table.setItem(i, 5, item)
	628	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6]))
	629	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	630	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	631	self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed"))
	632
	633	# Apply a color depending on the status of the job
	634	self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175))
	635	self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175))
	636	self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175))
	637	self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175))
	638	self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175))
	639	self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175))
	640
	641
	642	# Save job data to file
	643	job_data_list.append(i)
	644	job_data_list[i] = job_data_old[to_display[j]][0]
	645
	646	printstring = str(job_data_old[to_display[j]][0]) + " " + \
	647	str(job_data_old[to_display[j]][1]) + " " + \
	648	str(job_data_old[to_display[j]][2]) + " " + \
	649	str(job_data_old[to_display[j]][3]) + " " + \
	650	str(job_data_old[to_display[j]][4]) + " " + \
	651	str(job_data_old[to_display[j]][5]) + " " + \
	652	str(job_data_old[to_display[j]][6]) + " " + \
	653	str(job_data_old[to_display[j]][7]) + "\n"
	654	file.write(printstring)
	655
	656	del(jobs_complete)
	657	del(jobs_new)
	658	del(result)
[1751]	659	del(job_data_old)
	660	del(job_data_old_n)
	661
[1611]	662	file.close()
	663
	664
	665	# Update is complete, sorting can thus be re-enabled
	666	self.table.setSortingEnabled(True)
	667
	668	# Update the logfile
	669	if ( os.path.isfile(palm_dir + '/.wd.newdata') ):
	670	shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata')
	671
	672	# Re-enable timer processes
	673	self.setEnabled(True)
	674	self.label2.setText("Last update: " + timestamp)
	675	self.label.setText("Update complete.")
[1751]	676	self.timetimer.start()
	677	self.timer.start(update_frequency)
	678	self.labeltimer.start(update_frequency/10)
	679	self.UpdateLabel()
[1611]	680	QtGui.QApplication.processEvents()
	681
	682	# Canceling selected job from table
	683	def CancelJob(self):
	684
	685	# Return internal jobname
	686	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	687	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	688
	689	# Check description of job in order to login on the correct host
	690	descr = self.table.item(self.table.currentRow(),1).text()
[1751]	691
	692	querybox = QtGui.QMessageBox()
	693	querybox.setText("Attention!\nYou are trying to cancel a job. It will not be able to terminate properly.")
	694	querybox.setInformativeText("Do you really want to cancel " + jobname + " on host " + descr + "?")
	695	querybox.setStandardButtons(QtGui.QMessageBox.Yes \| QtGui.QMessageBox.No)
	696	querybox.setDefaultButton(QtGui.QMessageBox.No)
	697	returnvalue = querybox.exec_()
	698
	699	if ( returnvalue == QtGui.QMessageBox.Yes ):
	700
	701	for h in range(0,len(description)):
	702	if ( descr == description[h] ):
	703	host = username[h] + "@" + hostname[h]
[1611]	704
[3567]	705	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd cancel " + jobname],
[1751]	706	shell=False,
	707	stdout=sub.PIPE,
	708	stderr=sub.PIPE)
	709	result = ssh.stdout.readlines()
	710	result = filter(None, result)
[1611]	711
[1751]	712	# In case of error display a warning message
	713	if ( len(result) == 0 ):
	714	error = ssh.stderr.readlines()
	715	notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error))
[1611]	716
[1751]	717	# Otherwise inform the user and mark the job in the table
	718	else:
	719	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled"))
	720	notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result))
[1611]	721
	722
	723	# Show detailed information on job. For documentation see above
	724	def ShowDetails(self):
	725
[2416]	726	jobname = self.table.item(self.table.currentRow(),6).text()
[1611]	727	descr = self.table.item(self.table.currentRow(),1).text()
	728	for h in range(0,len(description)):
	729	if ( descr == description[h] ):
	730	host = username[h] + "@" + hostname[h]
	731
[3567]	732	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd check " + jobname],
[1611]	733	shell=False,
	734	stdout=sub.PIPE,
	735	stderr=sub.PIPE)
	736	result = ssh.stdout.readlines()
	737	result = filter(None, result)
	738
	739	if ( len(result) == 0 ):
	740	error = ssh.stderr.readlines()
	741	notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error))
	742	else:
	743	notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result))
	744
	745
	746	# Perform a forced stop on the job
	747	def DoStop(self):
	748
	749	# Return internal jobname
	750	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	751	jobrealname = self.table.item(self.table.currentRow(),0).text()
[1611]	752	# Check description of job in order to login on the correct host
	753	descr = self.table.item(self.table.currentRow(),1).text()
	754	for h in range(0,len(description)):
	755	if ( descr == description[h] ):
	756	host = username[h] + "@" + hostname[h]
	757	user = username[h]
	758
[3567]	759	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd stop " + jobrealname],
[1611]	760	shell=False,
	761	stdout=sub.PIPE,
	762	stderr=sub.PIPE)
	763	result = ssh.stdout.readlines()
	764	result = filter(None, result)
	765
	766	# In case of error display a warning message
	767	if ( len(result) == 0 ):
	768	error = ssh.stderr.readlines()
	769	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	770
	771	# Otherwise inform the user and mark the job in the table
	772	else:
	773	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
[1751]	774	notify = QtGui.QMessageBox.information(self,'Terminate job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
[1611]	775
	776
	777	# Perform a forced stop on the job
	778	def DoRestart(self):
	779
	780	# Return internal jobname
	781	jobname = self.table.item(self.table.currentRow(),6).text()
[2416]	782	jobrealname = self.table.item(self.table.currentRow(),0).text()
	783
[1611]	784	# Check description of job in order to login on the correct host
	785	descr = self.table.item(self.table.currentRow(),1).text()
	786	for h in range(0,len(description)):
	787	if ( descr == description[h] ):
	788	host = username[h] + "@" + hostname[h]
	789	user = username[h]
	790
[3567]	791	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd restart " + jobrealname],
[1611]	792	shell=False,
	793	stdout=sub.PIPE,
	794	stderr=sub.PIPE)
	795	result = ssh.stdout.readlines()
	796	result = filter(None, result)
	797
	798	# In case of error display a warning message
	799	if ( len(result) == 0 ):
	800	error = ssh.stderr.readlines()
	801	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	802
	803	# Otherwise inform the user and mark the job in the table
	804	else:
	805	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	806	notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	807
	808
[1613]	809	# Read the PARIN file of the job
	810	def ShowPARIN(self):
	811
	812	# Return internal jobname
	813	jobname = self.table.item(self.table.currentRow(),6).text()
	814	jobrealname = self.table.item(self.table.currentRow(),0).text()
	815
	816	# Check description of job in order to login on the correct host
	817	descr = self.table.item(self.table.currentRow(),1).text()
	818	for h in range(0,len(description)):
	819	if ( descr == description[h] ):
	820	host = username[h] + "@" + hostname[h]
	821	user = username[h]
	822
[3567]	823	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd parin " + jobrealname],
[1613]	824	shell=False,
	825	stdout=sub.PIPE,
	826	stderr=sub.PIPE)
	827	result = ssh.stdout.readlines()
	828	result = filter(None, result)
	829
	830	# In case of error display a warning message
	831	if ( len(result) == 0 ):
	832	error = ssh.stderr.readlines()
	833	notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error))
	834
	835	# Otherwise inform the user and mark the job in the table
	836	else:
	837	mb = MessageBoxScroll()
	838	mb.setText("Parameter file for job: " + jobrealname + "")
	839	mb.setDetailedText(''.join(result))
	840	mb.exec_()
	841
	842	# Read the PARIN file of the job
	843	def ShowRC(self):
	844
	845	# Return internal jobname and real job name
	846	jobname = self.table.item(self.table.currentRow(),6).text()
	847	jobrealname = self.table.item(self.table.currentRow(),0).text()
	848
	849	# Check description of job in order to login on the correct host
	850	descr = self.table.item(self.table.currentRow(),1).text()
	851	for h in range(0,len(description)):
	852	if ( descr == description[h] ):
	853	host = username[h] + "@" + hostname[h]
	854	user = username[h]
	855
[3567]	856	ssh = sub.Popen(["ssh", "%s" % host, pycall[h] + " palm_wdd rc " + jobrealname],
[1613]	857	shell=False,
	858	stdout=sub.PIPE,
	859	stderr=sub.PIPE)
	860	result = ssh.stdout.readlines()
	861	result = filter(None, result)
	862
	863	# In case of error display a warning message
	864	if ( len(result) == 0 ):
	865	error = ssh.stderr.readlines()
	866	notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error))
	867
	868	# Otherwise inform the user and mark the job in the table
	869	else:
	870	mb = MessageBoxScroll()
	871	lastline = result[len(result)-2].split()[2]
	872	mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline)
	873	mb.setDetailedText(''.join(result))
	874	mb.exec_()
	875
[1611]	876	# Remove a job from list - removes the current row from the table and from
	877	# save file
	878	def RemoveFromList(self, row):
	879	if ( row == -1 ):
	880	row = self.table.currentRow()
	881
	882	# Read data from save file
	883	job_to_delete = self.table.item(row,6).text()
	884	self.table.removeRow(row)
	885	file = open(palm_dir + '/.wd.olddata', 'r')
	886	result = file.readlines()
	887	result = filter(None, result)
	888	file.close()
	889	file = open(palm_dir + '/.wd.olddata', 'w')
	890
	891	job_data_old = []
	892
	893	if ( len(result) == 0 ):
	894	notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.")
	895	else:
	896	# Save data in array job_data_old
	897	for j in range(0,len(result)):
	898	job_data_old.append(j)
	899	job_data_old[j] = result[j].split(" ")
	900	job_data_old[j] = filter(None, job_data_old[j])
	901	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	902
	903	# Check if line j is the selected job, if not -> save to file
	904	if ( job_data_old[j][6] != job_to_delete ):
	905	printstring = str(job_data_old[j][0]) + " " + \
	906	str(job_data_old[j][1]) + " " + \
	907	str(job_data_old[j][2]) + " " + \
	908	str(job_data_old[j][3]) + " " + \
	909	str(job_data_old[j][4]) + " " + \
	910	str(job_data_old[j][5]) + " " + \
	911	str(job_data_old[j][6]) + " " + \
	912	str(job_data_old[j][7]) + "\n"
	913	file.write(printstring)
	914
	915	file.close()
	916
	917	# Remove all completed jobs from list
	918	def ClearList(self):
	919
	920	num_of_lines = self.table.rowCount()
	921
	922	# Delete all lines with completed/canceled jobs from list. The counter
	923	# must decrease as the line numbers would be messed up otherwise
	924	for j in range(num_of_lines-1,-1,-1):
	925	state = self.table.item(j,3).text()
	926	if ( state == "Completed" or state == "Canceled" or state == "Terminated"):
	927	self.RemoveFromList(j)
	928
	929	# Update the label
	930	def UpdateLabel(self):
	931	remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60
	932	self.label.setText("Next update in " + str(remaining_time) + " min")
	933
	934
[1751]	935	# Enter Options menu
	936	def Options(self):
	937
	938	# disable mainwindow
	939	self.setEnabled(False)
	940
	941	# show Options Dialog
	942	opt = OptionBox()
	943	opt.exec_()
	944
	945	self.UpdateHosts()
	946	self.setEnabled(True)
	947
	948	# Update settings
	949	def UpdateHosts(self):
	950
	951	global update_frequency
[1753]	952	global description
	953	global hostname
	954	global username
	955
[1751]	956	description = []
	957	hostname = []
	958	username = []
	959
	960	for i in range(0,len(config.sections())):
	961
	962	description_tmp = config.sections()[i]
	963
	964	if ( description_tmp != 'Settings' ):
	965
	966	if ( config.get(description_tmp, 'enabled') == 'true' ):
	967	description.append(description_tmp)
	968	hostname.append(config.get(description_tmp, 'hostname'))
	969	username.append(config.get(description_tmp, 'username'))
	970	else:
	971	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	972
	973
[1611]	974	# Main loop
	975	def main():
	976
	977	app = QtGui.QApplication(sys.argv)
	978	res = Watchdog()
	979	sys.exit(app.exec_())
	980
	981
	982	if __name__ == '__main__':
[1613]	983	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |