Home

Context Navigation

source: palm/trunk/SCRIPTS/palm_wd @ 1679

Last change on this file since 1679 was 1619, checked in by maronga, 9 years ago
last commit documented
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 35.1 KB

Rev	Line
[1611]	1	#!/usr/bin/python
	2	# -- coding: utf-8 --
	3	#--------------------------------------------------------------------------------#
	4	# This file is part of PALM.
	5	#
	6	# PALM is free software: you can redistribute it and/or modify it under the terms
	7	# of the GNU General Public License as published by the Free Software Foundation,
	8	# either version 3 of the License, or (at your option) any later version.
	9	#
	10	# PALM is distributed in the hope that it will be useful, but WITHOUT ANY
	11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License along with
	15	# PALM. If not, see <http://www.gnu.org/licenses/>.
	16	#
	17	# Copyright 1997-2015 Leibniz Universitaet Hannover
	18	#--------------------------------------------------------------------------------#
	19	#
	20	# Current revisions:
	21	# -----------------
	22	#
[1619]	23	#
[1611]	24	# Former revisions:
	25	# -----------------
	26	# $Id: palm_wd 1619 2015-07-13 06:53:19Z boeske $
	27	#
[1619]	28	# 1618 2015-07-13 06:52:15Z maronga
	29	# Added steering via configuration file .wd.config, to be place in the local
	30	# palm directory. Watchdog save files are automatically created upon first start.
	31	#
[1614]	32	# 1613 2015-07-08 14:53:29Z maronga
	33	# Bugfix: tooltip for queuing name did not show up on first update.
	34	# New: added contect menu for showing the parameter file and the run control
	35	# output
	36	#
[1612]	37	# 1611 2015-07-07 12:23:22Z maronga
	38	# Initial revision
	39	#
[1611]	40	#
	41	# Description:
	42	# ------------
	43	# PALM watchdog client for monitoring batch jobs on a variety of hosts specified
	44	# by the user
	45	#
	46	# Instructions:
	47	# -------------
	48	# 1) Modify the header section of palm_wd
	49	# 2) Move .wd.olddata and .wd.newdata to your palm directory
	50	# (e.g. /home/user/current_version/.wd.newdata etc.)
	51	# 3) Modify a copy of palm_wdd for each host to be monitored and move it to the
	52	# respective hosts
	53	# 4) Start the client either from mrungui or from shell by "nohup palm_wd&"
	54
	55	# To do:
	56	# ------
	57	# 1) Add "Options", "Help" and "Manual"
	58	# 2) Move user settings to a configuration file
	59	#------------------------------------------------------------------------------!
	60
[1618]	61	import ConfigParser
	62	import datetime
	63	import os
[1611]	64	from PyQt4 import QtGui, QtCore, uic
	65	from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT
[1618]	66	import shutil
	67	import subprocess as sub
	68	import sys
[1611]	69	import time
	70
	71	# Determine PALM directories
	72	try:
	73	devnull = open(os.devnull, 'w')
	74	out = sub.check_output("echo $PALM_BIN", shell=True, stderr=sub.STDOUT)
	75	palm_bin = out.rstrip()
	76	palm_dir = out.split("palm")[0] + "palm/" + out.split("palm")[1].split("/")[1]
	77	out = None
	78	except:
	79	print "Error. $PALM_BIN is not set."
	80	raise SystemExit
	81
	82
[1618]	83	# Read configuration file
	84	# First check if the configuration file exists
	85	if ( os.path.exists(palm_dir + '/.wd.config') == False ):
	86	print "Error. No configuration file .wd.config found in " + palm_dir
	87	raise SystemExit
	88
	89	config = ConfigParser.RawConfigParser()
	90	config.read(palm_dir + '/.wd.config')
	91
	92	description = []
	93	hostname = []
	94	username = []
	95
	96	for i in range(0,len(config.sections())):
	97
	98	description_tmp = config.sections()[i]
	99
	100	if ( description_tmp != 'Settings' ):
	101	description.append(description_tmp)
	102	hostname.append(config.get(description_tmp, 'hostname'))
	103	username.append(config.get(description_tmp, 'username'))
	104	else:
	105	update_frequency = int(config.get(description_tmp, 'update_frequency'))*60000
	106
	107	# Check if .wd.olddata and .wd.newdata exist. Otherwise create the files
	108	if ( os.path.exists(palm_dir + '/.wd.olddata') == False ):
	109	print "No .wd.olddata found. Creating..."
	110	file1 = open(palm_dir + '/.wd.olddata', 'a')
	111	file1.close()
	112
	113
	114	if ( os.path.exists(palm_dir + '/.wd.newdata') == False ):
	115	print "No .wd.newdata found. Creating..."
	116	file1 = open(palm_dir + '/.wd.newdata', 'a')
	117	file1.close()
	118
[1611]	119	# Dummy variables for the jobname and the progressbars
	120	jobname = ""
	121	timestamp = ""
	122	pbars = []
	123	job_data_list = []
	124
[1613]	125	class MessageBoxScroll(QtGui.QMessageBox):
	126	def __init__(self):
	127	QtGui.QMessageBox.__init__(self)
	128	self.setSizeGripEnabled(True)
	129
	130	def event(self, e):
	131	result = QtGui.QMessageBox.event(self, e)
	132
	133	self.setMinimumHeight(100)
	134	self.setMaximumHeight(16777215)
	135	self.setMinimumWidth(400)
	136	self.setMaximumWidth(16777215)
	137	self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	138
	139	textEdit = self.findChild(QtGui.QTextEdit)
	140	if textEdit != None :
	141	textEdit.setMinimumHeight(800)
	142	textEdit.setMaximumHeight(16777215)
	143	textEdit.setMinimumWidth(1000)
	144	textEdit.setMaximumWidth(16777215)
	145	textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
	146	f = QtGui.QFont('not_a_font') #
	147	f.setStyleHint(f.TypeWriter, f.PreferDefault)
	148	textEdit.setFont(f)
	149
	150	return result
	151
	152
[1611]	153	# MainWindow class
	154	class Watchdog(QtGui.QMainWindow):
	155
	156	def __init__(self):
	157	super(Watchdog, self).__init__()
	158
	159	self.InitUi()
	160
	161
	162	# Initialized MainWindow UI
	163	def InitUi(self):
	164
	165	# Load predefined mainwindow
	166	uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self)
	167
	168	# Resize columns and set number of rows to zero. Column 6 is hidden and
	169	# contains the remote jobname (e.g. hannover.174610)
	170	self.table.setColumnWidth(0,230)
	171	self.table.setColumnWidth(1,80)
	172	self.table.setColumnWidth(2,50)
	173	self.table.setColumnWidth(3,80)
	174	self.table.setColumnWidth(4,180)
	175	self.table.setColumnWidth(5,115)
	176	self.table.setColumnHidden(6, True)
	177	self.table.setRowCount(0)
[1613]	178
[1611]	179	# Display MainWindow
	180	self.show()
	181	QtGui.QApplication.processEvents()
	182
	183	# Start refresh timer. On timeout perform update
	184	self.timer = QtCore.QTimer(self)
	185	self.timer.timeout.connect(self.Refresh)
	186	self.timer.setSingleShot(False)
	187	self.timer.start(update_frequency)
	188
	189	# The timetimer counts the time since last update
	190	self.timetimer= QtCore.QElapsedTimer()
	191	self.timetimer.start()
	192
	193	# The labeltimer induces the update of the remaining time in the UI
	194	self.labeltimer = QtCore.QTimer(self)
	195	self.labeltimer.timeout.connect(self.UpdateLabel)
	196	self.labeltimer.setSingleShot(False)
	197
	198	# Update in the UI will be performed at each 1/10 of the update interval
	199	self.labeltimer.start(update_frequency/10)
	200	self.label.setText("Next update in " + str(update_frequency/1000/60) + "min")
	201
	202	# Update the list now
	203	self.Refresh()
	204
	205
	206	# Add a custom context menu
	207	def OpenContextMenu(self, position):
	208	menu = QtGui.QMenu()
	209
	210	# "Show details" -> fetch details for the selected job
[1613]	211	detailsjobAction = menu.addAction('Show job details')
[1611]	212	detailsjobAction.setStatusTip('Display detailed info for job')
	213	detailsjobAction.triggered.connect(self.ShowDetails)
	214
[1613]	215	# "Show parameter file" -> show the contents of PARIN
	216	parinAction = menu.addAction('Show parameter file')
	217	parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)')
	218	parinAction.triggered.connect(self.ShowPARIN)
	219
	220	rcAction = menu.addAction('Show run control file')
	221	rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)')
	222	rcAction.triggered.connect(self.ShowRC)
	223
[1611]	224	# "Cancel job" -> remove job from queuing system
	225	canceljobAction = menu.addAction('Cancel job')
	226	canceljobAction.setStatusTip('Remove job from queueing system')
	227	canceljobAction.triggered.connect(self.CancelJob)
	228
[1613]	229	# "Force stop" -> force termination of job
[1611]	230	stopjobAction = menu.addAction('Force stop')
	231	stopjobAction.setStatusTip('Terminate job properly')
	232	stopjobAction.triggered.connect(self.DoStop)
	233
[1613]	234	# "Force restart" -> force rermination and restart of job
[1611]	235	restartjobAction = menu.addAction('Force restart')
	236	restartjobAction.setStatusTip('Terminate job properly and initiate restart')
	237	restartjobAction.triggered.connect(self.DoRestart)
	238
	239	# "Remove from list" -> remove a completed job from the list
	240	removefromlistAction = menu.addAction('Remove from list')
	241	removefromlistAction.setStatusTip('Remove finished job')
	242	removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1))
	243
	244
	245	# Activate/deactive contect menu items based on the status of the runs
	246	state = self.table.item(self.table.currentRow(),3).text()
	247	if (state == "Canceled" or state == "Completed" or state == "Terminated"):
	248	detailsjobAction.setEnabled(False)
	249	canceljobAction.setEnabled(False)
	250	removefromlistAction.setEnabled(True)
	251	else:
	252	detailsjobAction.setEnabled(True)
	253	canceljobAction.setEnabled(True)
	254	removefromlistAction.setEnabled(False)
	255
	256	if ( state == "Running" ):
	257	stopjobAction.setEnabled(True)
	258	restartjobAction.setEnabled(True)
[1613]	259	parinAction.setEnabled(True)
	260	rcAction.setEnabled(True)
[1611]	261	else:
	262	stopjobAction.setEnabled(False)
	263	restartjobAction.setEnabled(False)
[1613]	264	parinAction.setEnabled(False)
	265	rcAction.setEnabled(False)
	266
[1611]	267	# Show the context menu
	268	action = menu.exec_(self.table.mapToGlobal(position))
	269
	270
	271	# Collecting watchdog data and display in the UI
	272	def Refresh(self):
	273
	274	# Use global variables
	275	global jobs_save
	276	global pbars
	277
	278	# Deactivate timer processes until update is finished
	279	# QtGui.QApplication.processEvents() updates the UI whenever needed
	280	self.labeltimer.stop()
	281	self.label.setText("Updating...")
	282	QtGui.QApplication.processEvents()
	283	self.setEnabled(False)
	284	QtGui.QApplication.processEvents()
	285
	286	# Get current timestamp
	287	timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M")
	288
	289	# Open an empty file for the watchdog data
	290	file = open(palm_dir + '/.wd.newdata', 'w')
	291
	292
	293	# Set all required variables to their initial values. Sorting must be
	294	# disabled.
	295	self.table.setRowCount(0)
	296	self.table.setSortingEnabled(False)
	297	i = -1
	298	job_data_list = []
	299	pbars = []
	300
	301	# Scan all hosts in the variable hostname. For each host perform the
	302	# update loop
	303	for h in range(0,len(hostname)):
	304
	305	# Perform ssh command
	306	host = username[h] + "@" + hostname[h]
	307	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd queue " + username[h]],
	308	shell=False,
	309	stdout=sub.PIPE,
	310	stderr=sub.PIPE)
	311	result = ssh.stdout.readlines()
	312	result = filter(None, result)
	313
	314
	315	# Delete all job data
	316	job_data_tmp = []
	317	job_data = []
	318	job_data_n = []
	319	job_data_old = []
	320	job_data_old_n = []
	321	job_progress = 0
	322
	323	# In case of errors display error message
	324	if ( len(result) < 1 ):
	325	error = ssh.stderr.readlines()
	326	if ( error != [] ):
	327	notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error))
	328
	329	# No error -> save job data
	330	else:
	331
	332	# Successively write to job_data
	333	for j in range(0,len(result)):
	334
	335	job_data_tmp.append(j)
	336	job_data_tmp[j] = result[j].split(" ")
	337	job_data_tmp[j] = filter(None, job_data_tmp[j])
	338
	339	job_data.append(j)
	340	job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2],
	341	job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5],
	342	job_data_tmp[j][1], job_data_tmp[j][6].rstrip()]
	343	job_data_n.append(j)
	344	job_data_n[j] = job_data_tmp[j][1]
	345
	346	del(result)
	347
	348
	349
	350	# Now read the data from the last update from file. These data are
	351	# already in the prefered format
	352	file2 = open(palm_dir + '/.wd.olddata', 'r')
	353	result = file2.readlines()
	354	file2.close()
	355
	356	for j in range(0,len(result)):
	357	if ( result[j].split()[1] == description[h] ):
	358	job_data_old.append(j)
	359	job_data_old[j] = result[j].split(" ")
	360	job_data_old[j] = filter(None, job_data_old[j])
	361	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	362
	363	job_data_old_n.append(j)
	364	job_data_old_n[j] = job_data_old[j][6]
	365
	366	# Merge the job_data and job_data_old to find completed, new and
	367	# still queued jobs
	368	if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ):
	369	jobs_full = list(set(job_data_old_n) \| set(job_data_n))
	370	jobs_known = list(set(job_data_old_n) & set(job_data_n))
	371	jobs_complete = set(job_data_old_n) - set(job_data_n)
	372	jobs_new = set(job_data_n) - set(job_data_old_n)
	373	elif ( len(job_data_n) > 0 ):
	374	jobs_full = job_data_n
	375	jobs_known = []
	376	jobs_new = job_data_n
	377	jobs_complete = []
	378	elif ( len(job_data_old_n) > 0 ):
	379	jobs_full = job_data_old_n
	380	jobs_known = []
	381	jobs_new = []
	382	jobs_complete = job_data_old_n
	383	else:
	384	jobs_full = []
	385	jobs_known = []
	386	jobs_new = []
	387	jobs_complete = []
	388
	389	# Display all known jobs
	390	to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)]
	391	for j in range(0,len(to_display)):
	392	i = i + 1
	393	self.table.insertRow(i)
	394	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	395	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	396	self.table.setItem(i, 0, item)
	397	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	398	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	399	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	400	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	401	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	402	self.table.setItem(i, 5, item)
	403	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	404	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	405	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	406	pbars.append(i)
	407	pbars[i] = QtGui.QProgressBar(self)
	408	pbars[i].setValue(int(job_data[to_display[j]][4]))
	409
	410	# Apply a color depending on the status of the job
	411	if ( job_data[to_display[j]][3] == "Running" ):
	412	self.table.setCellWidget(i,4,pbars[j])
	413	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	414	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	415	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	416	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	417	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	418	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	419	else:
	420	pbars[j].setEnabled(False)
	421	self.table.setCellWidget(i,4,pbars[j])
	422	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	423	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	424	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	425	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	426	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	427	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	428
	429	# Save the job data to file
	430	job_data_list.append(i)
	431	job_data_list[i] = job_data[to_display[j]][0]
	432
	433	printstring = str(job_data[to_display[j]][0]) + " " + \
	434	str(job_data[to_display[j]][1]) + " " + \
	435	str(job_data[to_display[j]][2]) + " " + \
	436	str(job_data[to_display[j]][3]) + " " + \
	437	str(job_data[to_display[j]][4]) + " " + \
	438	str(job_data[to_display[j]][5]) + " " + \
	439	str(job_data[to_display[j]][6]) + " " + \
	440	str(job_data[to_display[j]][7]) + "\n"
	441	file.write(printstring)
	442
	443
	444	# Display all new jobs
	445	to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)]
	446	for j in range(0,len(to_display)):
	447	i = i + 1
	448	self.table.insertRow(i)
	449	item = QtGui.QTableWidgetItem(job_data[to_display[j]][0])
	450	item.setToolTip("Queuing name: " + job_data[to_display[j]][6])
	451	self.table.setItem(i, 0, item)
	452	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1]))
	453	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2]))
	454	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3]))
	455	item = QtGui.QTableWidgetItem(job_data[to_display[j]][5])
	456	item.setToolTip("Estimated job start: " + job_data[to_display[j]][7])
	457	self.table.setItem(i, 5, item)
	458	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6]))
	459	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	460	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	461
	462	pbars.append(i)
	463	pbars[i] = QtGui.QProgressBar(self)
	464	pbars[i].setValue(int(job_data[to_display[j]][4]))
	465
	466	# Apply a color depending on the status of the job
	467	if ( job_data[to_display[j]][3] == "Running" ):
	468	self.table.setCellWidget(i,4,pbars[i])
	469	self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168))
	470	self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168))
	471	self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168))
	472	self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168))
	473	self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168))
	474	self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168))
	475	else:
	476	pbars[j].setEnabled(False)
	477	self.table.setCellWidget(i,4,pbars[i])
	478	self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118))
	479	self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118))
	480	self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118))
	481	self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118))
	482	self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118))
	483	self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118))
	484
	485	# Save job data to file
	486	job_data_list.append(i)
	487	job_data_list[i] = job_data[to_display[j]][0]
	488
	489	printstring = str(job_data[to_display[j]][0]) + " " + \
	490	str(job_data[to_display[j]][1]) + " " + \
	491	str(job_data[to_display[j]][2]) + " " + \
	492	str(job_data[to_display[j]][3]) + " " + \
	493	str(job_data[to_display[j]][4]) + " " + \
	494	str(job_data[to_display[j]][5]) + " " + \
	495	str(job_data[to_display[j]][6]) + " " + \
	496	str(job_data[to_display[j]][7]) + "\n"
	497	file.write(printstring)
	498
	499
	500	# Display all completed/canceled/aborted jobs. The watchdog cannot
	501	# distinguish why the job has been finished
	502	to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)]
	503	for j in range(0,len(to_display)):
	504	i = i + 1
	505	self.table.insertRow(i)
	506	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0])
	507	item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6])
	508	self.table.setItem(i, 0, item)
	509	self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1]))
	510	self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2]))
	511	self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3]))
	512	pbars.append(i)
	513	pbars[j] = QtGui.QProgressBar(self)
	514	pbars[j].setValue(int(job_data_old[to_display[j]][4]))
	515	pbars[j].setEnabled(False)
	516	self.table.setCellWidget(i,4,pbars[j])
	517	item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5])
	518	item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7])
	519	self.table.setItem(i, 5, item)
	520	self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6]))
	521	self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight)
	522	self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight)
	523	self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed"))
	524
	525	# Apply a color depending on the status of the job
	526	self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175))
	527	self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175))
	528	self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175))
	529	self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175))
	530	self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175))
	531	self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175))
	532
	533
	534	# Save job data to file
	535	job_data_list.append(i)
	536	job_data_list[i] = job_data_old[to_display[j]][0]
	537
	538	printstring = str(job_data_old[to_display[j]][0]) + " " + \
	539	str(job_data_old[to_display[j]][1]) + " " + \
	540	str(job_data_old[to_display[j]][2]) + " " + \
	541	str(job_data_old[to_display[j]][3]) + " " + \
	542	str(job_data_old[to_display[j]][4]) + " " + \
	543	str(job_data_old[to_display[j]][5]) + " " + \
	544	str(job_data_old[to_display[j]][6]) + " " + \
	545	str(job_data_old[to_display[j]][7]) + "\n"
	546	file.write(printstring)
	547
	548	del(jobs_complete)
	549	del(jobs_new)
	550	del(result)
	551
	552	file.close()
	553
	554
	555	# Update is complete, sorting can thus be re-enabled
	556	self.table.setSortingEnabled(True)
	557
	558	# Update the logfile
	559	if ( os.path.isfile(palm_dir + '/.wd.newdata') ):
	560	shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata')
	561
	562	# Re-enable timer processes
	563	self.setEnabled(True)
	564	self.label2.setText("Last update: " + timestamp)
	565	self.label.setText("Update complete.")
	566	self.timetimer.restart()
	567	self.timer.start()
	568	self.labeltimer.start()
	569	QtGui.QApplication.processEvents()
	570
	571	# Canceling selected job from table
	572	def CancelJob(self):
	573
	574	# Return internal jobname
	575	jobname = self.table.item(self.table.currentRow(),6).text()
	576
	577	# Check description of job in order to login on the correct host
	578	descr = self.table.item(self.table.currentRow(),1).text()
	579	for h in range(0,len(description)):
	580	if ( descr == description[h] ):
	581	host = username[h] + "@" + hostname[h]
	582
	583	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd cancel " + jobname],
	584	shell=False,
	585	stdout=sub.PIPE,
	586	stderr=sub.PIPE)
	587	result = ssh.stdout.readlines()
	588	result = filter(None, result)
	589
	590	# In case of error display a warning message
	591	if ( len(result) == 0 ):
	592	error = ssh.stderr.readlines()
	593	notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error))
	594
	595	# Otherwise inform the user and mark the job in the table
	596	else:
	597	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled"))
	598	notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result))
	599
	600
	601	# Show detailed information on job. For documentation see above
	602	def ShowDetails(self):
	603
	604	jobname = self.table.item(self.table.currentRow(),6).text()
	605	descr = self.table.item(self.table.currentRow(),1).text()
	606	for h in range(0,len(description)):
	607	if ( descr == description[h] ):
	608	host = username[h] + "@" + hostname[h]
	609
	610	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd check " + jobname],
	611	shell=False,
	612	stdout=sub.PIPE,
	613	stderr=sub.PIPE)
	614	result = ssh.stdout.readlines()
	615	result = filter(None, result)
	616
	617	if ( len(result) == 0 ):
	618	error = ssh.stderr.readlines()
	619	notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error))
	620	else:
	621	notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result))
	622
	623
	624	# Perform a forced stop on the job
	625	def DoStop(self):
	626
	627	# Return internal jobname
	628	jobname = self.table.item(self.table.currentRow(),6).text()
	629
	630	# Check description of job in order to login on the correct host
	631	descr = self.table.item(self.table.currentRow(),1).text()
	632	for h in range(0,len(description)):
	633	if ( descr == description[h] ):
	634	host = username[h] + "@" + hostname[h]
	635	user = username[h]
	636
	637	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd stop " + user + " " + jobname],
	638	shell=False,
	639	stdout=sub.PIPE,
	640	stderr=sub.PIPE)
	641	result = ssh.stdout.readlines()
	642	result = filter(None, result)
	643
	644	# In case of error display a warning message
	645	if ( len(result) == 0 ):
	646	error = ssh.stderr.readlines()
	647	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	648
	649	# Otherwise inform the user and mark the job in the table
	650	else:
	651	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	652	notify = QtGui.QMessageBox.information(self,'Terminatie job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	653
	654
	655	# Perform a forced stop on the job
	656	def DoRestart(self):
	657
	658	# Return internal jobname
	659	jobname = self.table.item(self.table.currentRow(),6).text()
	660
	661	# Check description of job in order to login on the correct host
	662	descr = self.table.item(self.table.currentRow(),1).text()
	663	for h in range(0,len(description)):
	664	if ( descr == description[h] ):
	665	host = username[h] + "@" + hostname[h]
	666	user = username[h]
	667
	668	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd restart " + user + " " + jobname],
	669	shell=False,
	670	stdout=sub.PIPE,
	671	stderr=sub.PIPE)
	672	result = ssh.stdout.readlines()
	673	result = filter(None, result)
	674
	675	# In case of error display a warning message
	676	if ( len(result) == 0 ):
	677	error = ssh.stderr.readlines()
	678	notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error))
	679
	680	# Otherwise inform the user and mark the job in the table
	681	else:
	682	self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated"))
	683	notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result))
	684
	685
[1613]	686	# Read the PARIN file of the job
	687	def ShowPARIN(self):
	688
	689	# Return internal jobname
	690	jobname = self.table.item(self.table.currentRow(),6).text()
	691	jobrealname = self.table.item(self.table.currentRow(),0).text()
	692
	693	# Check description of job in order to login on the correct host
	694	descr = self.table.item(self.table.currentRow(),1).text()
	695	for h in range(0,len(description)):
	696	if ( descr == description[h] ):
	697	host = username[h] + "@" + hostname[h]
	698	user = username[h]
	699
	700	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd parin " + user + " " + jobname],
	701	shell=False,
	702	stdout=sub.PIPE,
	703	stderr=sub.PIPE)
	704	result = ssh.stdout.readlines()
	705	result = filter(None, result)
	706
	707	# In case of error display a warning message
	708	if ( len(result) == 0 ):
	709	error = ssh.stderr.readlines()
	710	notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error))
	711
	712	# Otherwise inform the user and mark the job in the table
	713	else:
	714	mb = MessageBoxScroll()
	715	mb.setText("Parameter file for job: " + jobrealname + "")
	716	mb.setDetailedText(''.join(result))
	717	mb.exec_()
	718
	719	# Read the PARIN file of the job
	720	def ShowRC(self):
	721
	722	# Return internal jobname and real job name
	723	jobname = self.table.item(self.table.currentRow(),6).text()
	724	jobrealname = self.table.item(self.table.currentRow(),0).text()
	725
	726	# Check description of job in order to login on the correct host
	727	descr = self.table.item(self.table.currentRow(),1).text()
	728	for h in range(0,len(description)):
	729	if ( descr == description[h] ):
	730	host = username[h] + "@" + hostname[h]
	731	user = username[h]
	732
	733	ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd rc " + user + " " + jobname],
	734	shell=False,
	735	stdout=sub.PIPE,
	736	stderr=sub.PIPE)
	737	result = ssh.stdout.readlines()
	738	result = filter(None, result)
	739
	740	# In case of error display a warning message
	741	if ( len(result) == 0 ):
	742	error = ssh.stderr.readlines()
	743	notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error))
	744
	745	# Otherwise inform the user and mark the job in the table
	746	else:
	747	mb = MessageBoxScroll()
	748	lastline = result[len(result)-2].split()[2]
	749	mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline)
	750	mb.setDetailedText(''.join(result))
	751	mb.exec_()
	752
[1611]	753	# Remove a job from list - removes the current row from the table and from
	754	# save file
	755	def RemoveFromList(self, row):
	756	if ( row == -1 ):
	757	row = self.table.currentRow()
	758
	759	# Read data from save file
	760	job_to_delete = self.table.item(row,6).text()
	761	self.table.removeRow(row)
	762	file = open(palm_dir + '/.wd.olddata', 'r')
	763	result = file.readlines()
	764	result = filter(None, result)
	765	file.close()
	766	file = open(palm_dir + '/.wd.olddata', 'w')
	767
	768	job_data_old = []
	769
	770	if ( len(result) == 0 ):
	771	notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.")
	772	else:
	773	# Save data in array job_data_old
	774	for j in range(0,len(result)):
	775	job_data_old.append(j)
	776	job_data_old[j] = result[j].split(" ")
	777	job_data_old[j] = filter(None, job_data_old[j])
	778	job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]]
	779
	780	# Check if line j is the selected job, if not -> save to file
	781	if ( job_data_old[j][6] != job_to_delete ):
	782	printstring = str(job_data_old[j][0]) + " " + \
	783	str(job_data_old[j][1]) + " " + \
	784	str(job_data_old[j][2]) + " " + \
	785	str(job_data_old[j][3]) + " " + \
	786	str(job_data_old[j][4]) + " " + \
	787	str(job_data_old[j][5]) + " " + \
	788	str(job_data_old[j][6]) + " " + \
	789	str(job_data_old[j][7]) + "\n"
	790	file.write(printstring)
	791
	792	file.close()
	793
	794	# Remove all completed jobs from list
	795	def ClearList(self):
	796
	797	num_of_lines = self.table.rowCount()
	798
	799	# Delete all lines with completed/canceled jobs from list. The counter
	800	# must decrease as the line numbers would be messed up otherwise
	801	for j in range(num_of_lines-1,-1,-1):
	802	state = self.table.item(j,3).text()
	803	if ( state == "Completed" or state == "Canceled" or state == "Terminated"):
	804	self.RemoveFromList(j)
	805
	806	# Update the label
	807	def UpdateLabel(self):
	808	remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60
	809	self.label.setText("Next update in " + str(remaining_time) + " min")
	810
	811
	812	# Main loop
	813	def main():
	814
	815	app = QtGui.QApplication(sys.argv)
	816	res = Watchdog()
	817	sys.exit(app.exec_())
	818
	819
	820	if __name__ == '__main__':
[1613]	821	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |