1 | #!/usr/bin/python |
---|
2 | # -*- coding: utf-8 -*- |
---|
3 | #--------------------------------------------------------------------------------# |
---|
4 | # This file is part of PALM. |
---|
5 | # |
---|
6 | # PALM is free software: you can redistribute it and/or modify it under the terms |
---|
7 | # of the GNU General Public License as published by the Free Software Foundation, |
---|
8 | # either version 3 of the License, or (at your option) any later version. |
---|
9 | # |
---|
10 | # PALM is distributed in the hope that it will be useful, but WITHOUT ANY |
---|
11 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
---|
12 | # A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
---|
13 | # |
---|
14 | # You should have received a copy of the GNU General Public License along with |
---|
15 | # PALM. If not, see <http://www.gnu.org/licenses/>. |
---|
16 | # |
---|
17 | # Copyright 1997-2015 Leibniz Universitaet Hannover |
---|
18 | #--------------------------------------------------------------------------------# |
---|
19 | # |
---|
20 | # Current revisions: |
---|
21 | # ----------------- |
---|
22 | # Bugfix: tooltip for queuing name did not show up on first update. |
---|
23 | # New: added contect menu for showing the parameter file and the run control |
---|
24 | # output |
---|
25 | # |
---|
26 | # Former revisions: |
---|
27 | # ----------------- |
---|
28 | # $Id: palm_wd 1613 2015-07-08 14:53:29Z maronga $ |
---|
29 | # |
---|
30 | # 1611 2015-07-07 12:23:22Z maronga |
---|
31 | # Initial revision |
---|
32 | # |
---|
33 | # |
---|
34 | # Description: |
---|
35 | # ------------ |
---|
36 | # PALM watchdog client for monitoring batch jobs on a variety of hosts specified |
---|
37 | # by the user |
---|
38 | # |
---|
39 | # Instructions: |
---|
40 | # ------------- |
---|
41 | # 1) Modify the header section of palm_wd |
---|
42 | # 2) Move .wd.olddata and .wd.newdata to your palm directory |
---|
43 | # (e.g. /home/user/current_version/.wd.newdata etc.) |
---|
44 | # 3) Modify a copy of palm_wdd for each host to be monitored and move it to the |
---|
45 | # respective hosts |
---|
46 | # 4) Start the client either from mrungui or from shell by "nohup palm_wd&" |
---|
47 | |
---|
48 | # To do: |
---|
49 | # ------ |
---|
50 | # 1) Add "Options", "Help" and "Manual" |
---|
51 | # 2) Move user settings to a configuration file |
---|
52 | #------------------------------------------------------------------------------! |
---|
53 | |
---|
54 | import sys |
---|
55 | import subprocess as sub |
---|
56 | from PyQt4 import QtGui, QtCore, uic |
---|
57 | from PyQt4.QtCore import pyqtSlot,SIGNAL,SLOT |
---|
58 | import time |
---|
59 | import datetime |
---|
60 | import shutil |
---|
61 | import os |
---|
62 | |
---|
63 | # START OF HEADER |
---|
64 | # Each list must contain the login host, the username and a description |
---|
65 | hostname = ["hlogin.hlrn.de", "blogin.hlrn.de"] |
---|
66 | username = ["nikmaron" , "nikmaron" ] |
---|
67 | description = ["Hannover" , "Berlin" ] |
---|
68 | |
---|
69 | update_frequency = 600*1000 |
---|
70 | |
---|
71 | # END OF HEADER |
---|
72 | |
---|
73 | # Determine PALM directories |
---|
74 | try: |
---|
75 | devnull = open(os.devnull, 'w') |
---|
76 | out = sub.check_output("echo $PALM_BIN", shell=True, stderr=sub.STDOUT) |
---|
77 | palm_bin = out.rstrip() |
---|
78 | palm_dir = out.split("palm")[0] + "palm/" + out.split("palm")[1].split("/")[1] |
---|
79 | out = None |
---|
80 | except: |
---|
81 | print "Error. $PALM_BIN is not set." |
---|
82 | raise SystemExit |
---|
83 | |
---|
84 | |
---|
85 | # Dummy variables for the jobname and the progressbars |
---|
86 | jobname = "" |
---|
87 | timestamp = "" |
---|
88 | pbars = [] |
---|
89 | job_data_list = [] |
---|
90 | |
---|
91 | class MessageBoxScroll(QtGui.QMessageBox): |
---|
92 | def __init__(self): |
---|
93 | QtGui.QMessageBox.__init__(self) |
---|
94 | self.setSizeGripEnabled(True) |
---|
95 | |
---|
96 | def event(self, e): |
---|
97 | result = QtGui.QMessageBox.event(self, e) |
---|
98 | |
---|
99 | self.setMinimumHeight(100) |
---|
100 | self.setMaximumHeight(16777215) |
---|
101 | self.setMinimumWidth(400) |
---|
102 | self.setMaximumWidth(16777215) |
---|
103 | self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding) |
---|
104 | |
---|
105 | textEdit = self.findChild(QtGui.QTextEdit) |
---|
106 | if textEdit != None : |
---|
107 | textEdit.setMinimumHeight(800) |
---|
108 | textEdit.setMaximumHeight(16777215) |
---|
109 | textEdit.setMinimumWidth(1000) |
---|
110 | textEdit.setMaximumWidth(16777215) |
---|
111 | textEdit.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding) |
---|
112 | f = QtGui.QFont('not_a_font') # |
---|
113 | f.setStyleHint(f.TypeWriter, f.PreferDefault) |
---|
114 | textEdit.setFont(f) |
---|
115 | |
---|
116 | return result |
---|
117 | |
---|
118 | |
---|
119 | # MainWindow class |
---|
120 | class Watchdog(QtGui.QMainWindow): |
---|
121 | |
---|
122 | def __init__(self): |
---|
123 | super(Watchdog, self).__init__() |
---|
124 | |
---|
125 | self.InitUi() |
---|
126 | |
---|
127 | |
---|
128 | # Initialized MainWindow UI |
---|
129 | def InitUi(self): |
---|
130 | |
---|
131 | # Load predefined mainwindow |
---|
132 | uic.loadUi(palm_bin + '/palm_wd_files/wd.ui', self) |
---|
133 | |
---|
134 | # Resize columns and set number of rows to zero. Column 6 is hidden and |
---|
135 | # contains the remote jobname (e.g. hannover.174610) |
---|
136 | self.table.setColumnWidth(0,230) |
---|
137 | self.table.setColumnWidth(1,80) |
---|
138 | self.table.setColumnWidth(2,50) |
---|
139 | self.table.setColumnWidth(3,80) |
---|
140 | self.table.setColumnWidth(4,180) |
---|
141 | self.table.setColumnWidth(5,115) |
---|
142 | self.table.setColumnHidden(6, True) |
---|
143 | self.table.setRowCount(0) |
---|
144 | |
---|
145 | # Display MainWindow |
---|
146 | self.show() |
---|
147 | QtGui.QApplication.processEvents() |
---|
148 | |
---|
149 | # Start refresh timer. On timeout perform update |
---|
150 | self.timer = QtCore.QTimer(self) |
---|
151 | self.timer.timeout.connect(self.Refresh) |
---|
152 | self.timer.setSingleShot(False) |
---|
153 | self.timer.start(update_frequency) |
---|
154 | |
---|
155 | # The timetimer counts the time since last update |
---|
156 | self.timetimer= QtCore.QElapsedTimer() |
---|
157 | self.timetimer.start() |
---|
158 | |
---|
159 | # The labeltimer induces the update of the remaining time in the UI |
---|
160 | self.labeltimer = QtCore.QTimer(self) |
---|
161 | self.labeltimer.timeout.connect(self.UpdateLabel) |
---|
162 | self.labeltimer.setSingleShot(False) |
---|
163 | |
---|
164 | # Update in the UI will be performed at each 1/10 of the update interval |
---|
165 | self.labeltimer.start(update_frequency/10) |
---|
166 | self.label.setText("Next update in " + str(update_frequency/1000/60) + "min") |
---|
167 | |
---|
168 | # Update the list now |
---|
169 | self.Refresh() |
---|
170 | |
---|
171 | |
---|
172 | # Add a custom context menu |
---|
173 | def OpenContextMenu(self, position): |
---|
174 | menu = QtGui.QMenu() |
---|
175 | |
---|
176 | # "Show details" -> fetch details for the selected job |
---|
177 | detailsjobAction = menu.addAction('Show job details') |
---|
178 | detailsjobAction.setStatusTip('Display detailed info for job') |
---|
179 | detailsjobAction.triggered.connect(self.ShowDetails) |
---|
180 | |
---|
181 | # "Show parameter file" -> show the contents of PARIN |
---|
182 | parinAction = menu.addAction('Show parameter file') |
---|
183 | parinAction.setStatusTip('Display the parameter file of the job (e.g. PARIN / _p3d)') |
---|
184 | parinAction.triggered.connect(self.ShowPARIN) |
---|
185 | |
---|
186 | rcAction = menu.addAction('Show run control file') |
---|
187 | rcAction.setStatusTip('Display the current run control file (e.g. RUN_CONTROL / _rc)') |
---|
188 | rcAction.triggered.connect(self.ShowRC) |
---|
189 | |
---|
190 | # "Cancel job" -> remove job from queuing system |
---|
191 | canceljobAction = menu.addAction('Cancel job') |
---|
192 | canceljobAction.setStatusTip('Remove job from queueing system') |
---|
193 | canceljobAction.triggered.connect(self.CancelJob) |
---|
194 | |
---|
195 | # "Force stop" -> force termination of job |
---|
196 | stopjobAction = menu.addAction('Force stop') |
---|
197 | stopjobAction.setStatusTip('Terminate job properly') |
---|
198 | stopjobAction.triggered.connect(self.DoStop) |
---|
199 | |
---|
200 | # "Force restart" -> force rermination and restart of job |
---|
201 | restartjobAction = menu.addAction('Force restart') |
---|
202 | restartjobAction.setStatusTip('Terminate job properly and initiate restart') |
---|
203 | restartjobAction.triggered.connect(self.DoRestart) |
---|
204 | |
---|
205 | # "Remove from list" -> remove a completed job from the list |
---|
206 | removefromlistAction = menu.addAction('Remove from list') |
---|
207 | removefromlistAction.setStatusTip('Remove finished job') |
---|
208 | removefromlistAction.triggered.connect(lambda: self.RemoveFromList(-1)) |
---|
209 | |
---|
210 | |
---|
211 | # Activate/deactive contect menu items based on the status of the runs |
---|
212 | state = self.table.item(self.table.currentRow(),3).text() |
---|
213 | if (state == "Canceled" or state == "Completed" or state == "Terminated"): |
---|
214 | detailsjobAction.setEnabled(False) |
---|
215 | canceljobAction.setEnabled(False) |
---|
216 | removefromlistAction.setEnabled(True) |
---|
217 | else: |
---|
218 | detailsjobAction.setEnabled(True) |
---|
219 | canceljobAction.setEnabled(True) |
---|
220 | removefromlistAction.setEnabled(False) |
---|
221 | |
---|
222 | if ( state == "Running" ): |
---|
223 | stopjobAction.setEnabled(True) |
---|
224 | restartjobAction.setEnabled(True) |
---|
225 | parinAction.setEnabled(True) |
---|
226 | rcAction.setEnabled(True) |
---|
227 | else: |
---|
228 | stopjobAction.setEnabled(False) |
---|
229 | restartjobAction.setEnabled(False) |
---|
230 | parinAction.setEnabled(False) |
---|
231 | rcAction.setEnabled(False) |
---|
232 | |
---|
233 | # Show the context menu |
---|
234 | action = menu.exec_(self.table.mapToGlobal(position)) |
---|
235 | |
---|
236 | |
---|
237 | # Collecting watchdog data and display in the UI |
---|
238 | def Refresh(self): |
---|
239 | |
---|
240 | # Use global variables |
---|
241 | global jobs_save |
---|
242 | global pbars |
---|
243 | |
---|
244 | # Deactivate timer processes until update is finished |
---|
245 | # QtGui.QApplication.processEvents() updates the UI whenever needed |
---|
246 | self.labeltimer.stop() |
---|
247 | self.label.setText("Updating...") |
---|
248 | QtGui.QApplication.processEvents() |
---|
249 | self.setEnabled(False) |
---|
250 | QtGui.QApplication.processEvents() |
---|
251 | |
---|
252 | # Get current timestamp |
---|
253 | timestamp = datetime.datetime.now().strftime("%d/%m/%Y %H:%M") |
---|
254 | |
---|
255 | # Open an empty file for the watchdog data |
---|
256 | file = open(palm_dir + '/.wd.newdata', 'w') |
---|
257 | |
---|
258 | |
---|
259 | # Set all required variables to their initial values. Sorting must be |
---|
260 | # disabled. |
---|
261 | self.table.setRowCount(0) |
---|
262 | self.table.setSortingEnabled(False) |
---|
263 | i = -1 |
---|
264 | job_data_list = [] |
---|
265 | pbars = [] |
---|
266 | |
---|
267 | # Scan all hosts in the variable hostname. For each host perform the |
---|
268 | # update loop |
---|
269 | for h in range(0,len(hostname)): |
---|
270 | |
---|
271 | # Perform ssh command |
---|
272 | host = username[h] + "@" + hostname[h] |
---|
273 | ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd queue " + username[h]], |
---|
274 | shell=False, |
---|
275 | stdout=sub.PIPE, |
---|
276 | stderr=sub.PIPE) |
---|
277 | result = ssh.stdout.readlines() |
---|
278 | result = filter(None, result) |
---|
279 | |
---|
280 | |
---|
281 | # Delete all job data |
---|
282 | job_data_tmp = [] |
---|
283 | job_data = [] |
---|
284 | job_data_n = [] |
---|
285 | job_data_old = [] |
---|
286 | job_data_old_n = [] |
---|
287 | job_progress = 0 |
---|
288 | |
---|
289 | # In case of errors display error message |
---|
290 | if ( len(result) < 1 ): |
---|
291 | error = ssh.stderr.readlines() |
---|
292 | if ( error != [] ): |
---|
293 | notify = QtGui.QMessageBox.warning(self,'Collect data',"Error. An error occured during read of job data for user " + username[h] +" for host " + description[h] + ".\n\nError message:\n" + ''.join(error)) |
---|
294 | |
---|
295 | # No error -> save job data |
---|
296 | else: |
---|
297 | |
---|
298 | # Successively write to job_data |
---|
299 | for j in range(0,len(result)): |
---|
300 | |
---|
301 | job_data_tmp.append(j) |
---|
302 | job_data_tmp[j] = result[j].split(" ") |
---|
303 | job_data_tmp[j] = filter(None, job_data_tmp[j]) |
---|
304 | |
---|
305 | job_data.append(j) |
---|
306 | job_data[j] = [job_data_tmp[j][0], description[h], job_data_tmp[j][2], |
---|
307 | job_data_tmp[j][3], int(float(job_data_tmp[j][4])*100), job_data_tmp[j][5], |
---|
308 | job_data_tmp[j][1], job_data_tmp[j][6].rstrip()] |
---|
309 | job_data_n.append(j) |
---|
310 | job_data_n[j] = job_data_tmp[j][1] |
---|
311 | |
---|
312 | del(result) |
---|
313 | |
---|
314 | |
---|
315 | |
---|
316 | # Now read the data from the last update from file. These data are |
---|
317 | # already in the prefered format |
---|
318 | file2 = open(palm_dir + '/.wd.olddata', 'r') |
---|
319 | result = file2.readlines() |
---|
320 | file2.close() |
---|
321 | |
---|
322 | for j in range(0,len(result)): |
---|
323 | if ( result[j].split()[1] == description[h] ): |
---|
324 | job_data_old.append(j) |
---|
325 | job_data_old[j] = result[j].split(" ") |
---|
326 | job_data_old[j] = filter(None, job_data_old[j]) |
---|
327 | job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]] |
---|
328 | |
---|
329 | job_data_old_n.append(j) |
---|
330 | job_data_old_n[j] = job_data_old[j][6] |
---|
331 | |
---|
332 | # Merge the job_data and job_data_old to find completed, new and |
---|
333 | # still queued jobs |
---|
334 | if ( len(job_data_n) > 0 and len(job_data_old_n) > 0 ): |
---|
335 | jobs_full = list(set(job_data_old_n) | set(job_data_n)) |
---|
336 | jobs_known = list(set(job_data_old_n) & set(job_data_n)) |
---|
337 | jobs_complete = set(job_data_old_n) - set(job_data_n) |
---|
338 | jobs_new = set(job_data_n) - set(job_data_old_n) |
---|
339 | elif ( len(job_data_n) > 0 ): |
---|
340 | jobs_full = job_data_n |
---|
341 | jobs_known = [] |
---|
342 | jobs_new = job_data_n |
---|
343 | jobs_complete = [] |
---|
344 | elif ( len(job_data_old_n) > 0 ): |
---|
345 | jobs_full = job_data_old_n |
---|
346 | jobs_known = [] |
---|
347 | jobs_new = [] |
---|
348 | jobs_complete = job_data_old_n |
---|
349 | else: |
---|
350 | jobs_full = [] |
---|
351 | jobs_known = [] |
---|
352 | jobs_new = [] |
---|
353 | jobs_complete = [] |
---|
354 | |
---|
355 | # Display all known jobs |
---|
356 | to_display = [list(job_data_n).index(item) for item in list(jobs_known) if list(jobs_known) and list(job_data_n)] |
---|
357 | for j in range(0,len(to_display)): |
---|
358 | i = i + 1 |
---|
359 | self.table.insertRow(i) |
---|
360 | item = QtGui.QTableWidgetItem(job_data[to_display[j]][0]) |
---|
361 | item.setToolTip("Queuing name: " + job_data[to_display[j]][6]) |
---|
362 | self.table.setItem(i, 0, item) |
---|
363 | self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1])) |
---|
364 | self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2])) |
---|
365 | self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3])) |
---|
366 | item = QtGui.QTableWidgetItem(job_data[to_display[j]][5]) |
---|
367 | item.setToolTip("Estimated job start: " + job_data[to_display[j]][7]) |
---|
368 | self.table.setItem(i, 5, item) |
---|
369 | self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6])) |
---|
370 | self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight) |
---|
371 | self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight) |
---|
372 | pbars.append(i) |
---|
373 | pbars[i] = QtGui.QProgressBar(self) |
---|
374 | pbars[i].setValue(int(job_data[to_display[j]][4])) |
---|
375 | |
---|
376 | # Apply a color depending on the status of the job |
---|
377 | if ( job_data[to_display[j]][3] == "Running" ): |
---|
378 | self.table.setCellWidget(i,4,pbars[j]) |
---|
379 | self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168)) |
---|
380 | self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168)) |
---|
381 | self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168)) |
---|
382 | self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168)) |
---|
383 | self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168)) |
---|
384 | self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168)) |
---|
385 | else: |
---|
386 | pbars[j].setEnabled(False) |
---|
387 | self.table.setCellWidget(i,4,pbars[j]) |
---|
388 | self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118)) |
---|
389 | self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118)) |
---|
390 | self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118)) |
---|
391 | self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118)) |
---|
392 | self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118)) |
---|
393 | self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118)) |
---|
394 | |
---|
395 | # Save the job data to file |
---|
396 | job_data_list.append(i) |
---|
397 | job_data_list[i] = job_data[to_display[j]][0] |
---|
398 | |
---|
399 | printstring = str(job_data[to_display[j]][0]) + " " + \ |
---|
400 | str(job_data[to_display[j]][1]) + " " + \ |
---|
401 | str(job_data[to_display[j]][2]) + " " + \ |
---|
402 | str(job_data[to_display[j]][3]) + " " + \ |
---|
403 | str(job_data[to_display[j]][4]) + " " + \ |
---|
404 | str(job_data[to_display[j]][5]) + " " + \ |
---|
405 | str(job_data[to_display[j]][6]) + " " + \ |
---|
406 | str(job_data[to_display[j]][7]) + "\n" |
---|
407 | file.write(printstring) |
---|
408 | |
---|
409 | |
---|
410 | # Display all new jobs |
---|
411 | to_display = [list(job_data_n).index(item) for item in list(jobs_new) if list(jobs_new) and list(job_data_n)] |
---|
412 | for j in range(0,len(to_display)): |
---|
413 | i = i + 1 |
---|
414 | self.table.insertRow(i) |
---|
415 | item = QtGui.QTableWidgetItem(job_data[to_display[j]][0]) |
---|
416 | item.setToolTip("Queuing name: " + job_data[to_display[j]][6]) |
---|
417 | self.table.setItem(i, 0, item) |
---|
418 | self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data[to_display[j]][1])) |
---|
419 | self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data[to_display[j]][2])) |
---|
420 | self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data[to_display[j]][3])) |
---|
421 | item = QtGui.QTableWidgetItem(job_data[to_display[j]][5]) |
---|
422 | item.setToolTip("Estimated job start: " + job_data[to_display[j]][7]) |
---|
423 | self.table.setItem(i, 5, item) |
---|
424 | self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data[to_display[j]][6])) |
---|
425 | self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight) |
---|
426 | self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight) |
---|
427 | |
---|
428 | pbars.append(i) |
---|
429 | pbars[i] = QtGui.QProgressBar(self) |
---|
430 | pbars[i].setValue(int(job_data[to_display[j]][4])) |
---|
431 | |
---|
432 | # Apply a color depending on the status of the job |
---|
433 | if ( job_data[to_display[j]][3] == "Running" ): |
---|
434 | self.table.setCellWidget(i,4,pbars[i]) |
---|
435 | self.table.item(i,0).setBackground(QtGui.QColor(255, 251, 168)) |
---|
436 | self.table.item(i,1).setBackground(QtGui.QColor(255, 251, 168)) |
---|
437 | self.table.item(i,2).setBackground(QtGui.QColor(255, 251, 168)) |
---|
438 | self.table.item(i,3).setBackground(QtGui.QColor(255, 251, 168)) |
---|
439 | self.table.item(i,5).setBackground(QtGui.QColor(255, 251, 168)) |
---|
440 | self.table.item(i,6).setBackground(QtGui.QColor(255, 251, 168)) |
---|
441 | else: |
---|
442 | pbars[j].setEnabled(False) |
---|
443 | self.table.setCellWidget(i,4,pbars[i]) |
---|
444 | self.table.item(i,0).setBackground(QtGui.QColor(255, 112, 118)) |
---|
445 | self.table.item(i,1).setBackground(QtGui.QColor(255, 112, 118)) |
---|
446 | self.table.item(i,2).setBackground(QtGui.QColor(255, 112, 118)) |
---|
447 | self.table.item(i,3).setBackground(QtGui.QColor(255, 112, 118)) |
---|
448 | self.table.item(i,5).setBackground(QtGui.QColor(255, 112, 118)) |
---|
449 | self.table.item(i,6).setBackground(QtGui.QColor(255, 112, 118)) |
---|
450 | |
---|
451 | # Save job data to file |
---|
452 | job_data_list.append(i) |
---|
453 | job_data_list[i] = job_data[to_display[j]][0] |
---|
454 | |
---|
455 | printstring = str(job_data[to_display[j]][0]) + " " + \ |
---|
456 | str(job_data[to_display[j]][1]) + " " + \ |
---|
457 | str(job_data[to_display[j]][2]) + " " + \ |
---|
458 | str(job_data[to_display[j]][3]) + " " + \ |
---|
459 | str(job_data[to_display[j]][4]) + " " + \ |
---|
460 | str(job_data[to_display[j]][5]) + " " + \ |
---|
461 | str(job_data[to_display[j]][6]) + " " + \ |
---|
462 | str(job_data[to_display[j]][7]) + "\n" |
---|
463 | file.write(printstring) |
---|
464 | |
---|
465 | |
---|
466 | # Display all completed/canceled/aborted jobs. The watchdog cannot |
---|
467 | # distinguish why the job has been finished |
---|
468 | to_display = [list(job_data_old_n).index(item) for item in list(jobs_complete) if list(jobs_complete) and list(job_data_old_n)] |
---|
469 | for j in range(0,len(to_display)): |
---|
470 | i = i + 1 |
---|
471 | self.table.insertRow(i) |
---|
472 | item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][0]) |
---|
473 | item.setToolTip("Queuing name: " + job_data_old[to_display[j]][6]) |
---|
474 | self.table.setItem(i, 0, item) |
---|
475 | self.table.setItem(i, 1, QtGui.QTableWidgetItem(job_data_old[to_display[j]][1])) |
---|
476 | self.table.setItem(i, 2, QtGui.QTableWidgetItem(job_data_old[to_display[j]][2])) |
---|
477 | self.table.setItem(i, 3, QtGui.QTableWidgetItem(job_data_old[to_display[j]][3])) |
---|
478 | pbars.append(i) |
---|
479 | pbars[j] = QtGui.QProgressBar(self) |
---|
480 | pbars[j].setValue(int(job_data_old[to_display[j]][4])) |
---|
481 | pbars[j].setEnabled(False) |
---|
482 | self.table.setCellWidget(i,4,pbars[j]) |
---|
483 | item = QtGui.QTableWidgetItem(job_data_old[to_display[j]][5]) |
---|
484 | item.setToolTip("Estimated job start: " + job_data_old[to_display[j]][7]) |
---|
485 | self.table.setItem(i, 5, item) |
---|
486 | self.table.setItem(i, 6, QtGui.QTableWidgetItem(job_data_old[to_display[j]][6])) |
---|
487 | self.table.item(i,2).setTextAlignment(QtCore.Qt.AlignRight) |
---|
488 | self.table.item(i,5).setTextAlignment(QtCore.Qt.AlignRight) |
---|
489 | self.table.setItem(i, 3, QtGui.QTableWidgetItem("Completed")) |
---|
490 | |
---|
491 | # Apply a color depending on the status of the job |
---|
492 | self.table.item(i,0).setBackground(QtGui.QColor(172, 252, 175)) |
---|
493 | self.table.item(i,1).setBackground(QtGui.QColor(172, 252, 175)) |
---|
494 | self.table.item(i,2).setBackground(QtGui.QColor(172, 252, 175)) |
---|
495 | self.table.item(i,3).setBackground(QtGui.QColor(172, 252, 175)) |
---|
496 | self.table.item(i,5).setBackground(QtGui.QColor(172, 252, 175)) |
---|
497 | self.table.item(i,6).setBackground(QtGui.QColor(172, 252, 175)) |
---|
498 | |
---|
499 | |
---|
500 | # Save job data to file |
---|
501 | job_data_list.append(i) |
---|
502 | job_data_list[i] = job_data_old[to_display[j]][0] |
---|
503 | |
---|
504 | printstring = str(job_data_old[to_display[j]][0]) + " " + \ |
---|
505 | str(job_data_old[to_display[j]][1]) + " " + \ |
---|
506 | str(job_data_old[to_display[j]][2]) + " " + \ |
---|
507 | str(job_data_old[to_display[j]][3]) + " " + \ |
---|
508 | str(job_data_old[to_display[j]][4]) + " " + \ |
---|
509 | str(job_data_old[to_display[j]][5]) + " " + \ |
---|
510 | str(job_data_old[to_display[j]][6]) + " " + \ |
---|
511 | str(job_data_old[to_display[j]][7]) + "\n" |
---|
512 | file.write(printstring) |
---|
513 | |
---|
514 | del(jobs_complete) |
---|
515 | del(jobs_new) |
---|
516 | del(result) |
---|
517 | |
---|
518 | file.close() |
---|
519 | |
---|
520 | |
---|
521 | # Update is complete, sorting can thus be re-enabled |
---|
522 | self.table.setSortingEnabled(True) |
---|
523 | |
---|
524 | # Update the logfile |
---|
525 | if ( os.path.isfile(palm_dir + '/.wd.newdata') ): |
---|
526 | shutil.copy(palm_dir + '/.wd.newdata',palm_dir + '/.wd.olddata') |
---|
527 | |
---|
528 | # Re-enable timer processes |
---|
529 | self.setEnabled(True) |
---|
530 | self.label2.setText("Last update: " + timestamp) |
---|
531 | self.label.setText("Update complete.") |
---|
532 | self.timetimer.restart() |
---|
533 | self.timer.start() |
---|
534 | self.labeltimer.start() |
---|
535 | QtGui.QApplication.processEvents() |
---|
536 | |
---|
537 | # Canceling selected job from table |
---|
538 | def CancelJob(self): |
---|
539 | |
---|
540 | # Return internal jobname |
---|
541 | jobname = self.table.item(self.table.currentRow(),6).text() |
---|
542 | |
---|
543 | # Check description of job in order to login on the correct host |
---|
544 | descr = self.table.item(self.table.currentRow(),1).text() |
---|
545 | for h in range(0,len(description)): |
---|
546 | if ( descr == description[h] ): |
---|
547 | host = username[h] + "@" + hostname[h] |
---|
548 | |
---|
549 | ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd cancel " + jobname], |
---|
550 | shell=False, |
---|
551 | stdout=sub.PIPE, |
---|
552 | stderr=sub.PIPE) |
---|
553 | result = ssh.stdout.readlines() |
---|
554 | result = filter(None, result) |
---|
555 | |
---|
556 | # In case of error display a warning message |
---|
557 | if ( len(result) == 0 ): |
---|
558 | error = ssh.stderr.readlines() |
---|
559 | notify = QtGui.QMessageBox.warning(self,'Cancel job',"Error. Could not cancel job " + jobname + ".\n\nError message:\n" + ''.join(error)) |
---|
560 | |
---|
561 | # Otherwise inform the user and mark the job in the table |
---|
562 | else: |
---|
563 | self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Canceled")) |
---|
564 | notify = QtGui.QMessageBox.information(self,'Cancel job',"Job" + jobname + " canceled!\n\nServer message:\n" + ''.join(result)) |
---|
565 | |
---|
566 | |
---|
567 | # Show detailed information on job. For documentation see above |
---|
568 | def ShowDetails(self): |
---|
569 | |
---|
570 | jobname = self.table.item(self.table.currentRow(),6).text() |
---|
571 | descr = self.table.item(self.table.currentRow(),1).text() |
---|
572 | for h in range(0,len(description)): |
---|
573 | if ( descr == description[h] ): |
---|
574 | host = username[h] + "@" + hostname[h] |
---|
575 | |
---|
576 | ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd check " + jobname], |
---|
577 | shell=False, |
---|
578 | stdout=sub.PIPE, |
---|
579 | stderr=sub.PIPE) |
---|
580 | result = ssh.stdout.readlines() |
---|
581 | result = filter(None, result) |
---|
582 | |
---|
583 | if ( len(result) == 0 ): |
---|
584 | error = ssh.stderr.readlines() |
---|
585 | notify = QtGui.QMessageBox.warning(self,'Show job details',"Error. Could not fetch job details for " + jobname + ".\n\nError message:\n" + ''.join(error)) |
---|
586 | else: |
---|
587 | notify = QtGui.QMessageBox.information(self,'Job details',"Details for job " + jobname + ":\n\n" + ''.join(result)) |
---|
588 | |
---|
589 | |
---|
590 | # Perform a forced stop on the job |
---|
591 | def DoStop(self): |
---|
592 | |
---|
593 | # Return internal jobname |
---|
594 | jobname = self.table.item(self.table.currentRow(),6).text() |
---|
595 | |
---|
596 | # Check description of job in order to login on the correct host |
---|
597 | descr = self.table.item(self.table.currentRow(),1).text() |
---|
598 | for h in range(0,len(description)): |
---|
599 | if ( descr == description[h] ): |
---|
600 | host = username[h] + "@" + hostname[h] |
---|
601 | user = username[h] |
---|
602 | |
---|
603 | ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd stop " + user + " " + jobname], |
---|
604 | shell=False, |
---|
605 | stdout=sub.PIPE, |
---|
606 | stderr=sub.PIPE) |
---|
607 | result = ssh.stdout.readlines() |
---|
608 | result = filter(None, result) |
---|
609 | |
---|
610 | # In case of error display a warning message |
---|
611 | if ( len(result) == 0 ): |
---|
612 | error = ssh.stderr.readlines() |
---|
613 | notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error)) |
---|
614 | |
---|
615 | # Otherwise inform the user and mark the job in the table |
---|
616 | else: |
---|
617 | self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated")) |
---|
618 | notify = QtGui.QMessageBox.information(self,'Terminatie job',"Termination of job " + jobname + " was initiated!\n\nServer message:\n" + ''.join(result)) |
---|
619 | |
---|
620 | |
---|
621 | # Perform a forced stop on the job |
---|
622 | def DoRestart(self): |
---|
623 | |
---|
624 | # Return internal jobname |
---|
625 | jobname = self.table.item(self.table.currentRow(),6).text() |
---|
626 | |
---|
627 | # Check description of job in order to login on the correct host |
---|
628 | descr = self.table.item(self.table.currentRow(),1).text() |
---|
629 | for h in range(0,len(description)): |
---|
630 | if ( descr == description[h] ): |
---|
631 | host = username[h] + "@" + hostname[h] |
---|
632 | user = username[h] |
---|
633 | |
---|
634 | ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd restart " + user + " " + jobname], |
---|
635 | shell=False, |
---|
636 | stdout=sub.PIPE, |
---|
637 | stderr=sub.PIPE) |
---|
638 | result = ssh.stdout.readlines() |
---|
639 | result = filter(None, result) |
---|
640 | |
---|
641 | # In case of error display a warning message |
---|
642 | if ( len(result) == 0 ): |
---|
643 | error = ssh.stderr.readlines() |
---|
644 | notify = QtGui.QMessageBox.warning(self,'Proper termination of job',"Error. Could not stop job " + jobname + ".\n\nError message:\n" + ''.join(error)) |
---|
645 | |
---|
646 | # Otherwise inform the user and mark the job in the table |
---|
647 | else: |
---|
648 | self.table.setItem(self.table.currentRow(),3,QtGui.QTableWidgetItem("Terminated")) |
---|
649 | notify = QtGui.QMessageBox.information(self,'Terminate job for restart',"Restart for job" + jobname + " was initiated!\n\nServer message:\n" + ''.join(result)) |
---|
650 | |
---|
651 | |
---|
652 | # Read the PARIN file of the job |
---|
653 | def ShowPARIN(self): |
---|
654 | |
---|
655 | # Return internal jobname |
---|
656 | jobname = self.table.item(self.table.currentRow(),6).text() |
---|
657 | jobrealname = self.table.item(self.table.currentRow(),0).text() |
---|
658 | |
---|
659 | # Check description of job in order to login on the correct host |
---|
660 | descr = self.table.item(self.table.currentRow(),1).text() |
---|
661 | for h in range(0,len(description)): |
---|
662 | if ( descr == description[h] ): |
---|
663 | host = username[h] + "@" + hostname[h] |
---|
664 | user = username[h] |
---|
665 | |
---|
666 | ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd parin " + user + " " + jobname], |
---|
667 | shell=False, |
---|
668 | stdout=sub.PIPE, |
---|
669 | stderr=sub.PIPE) |
---|
670 | result = ssh.stdout.readlines() |
---|
671 | result = filter(None, result) |
---|
672 | |
---|
673 | # In case of error display a warning message |
---|
674 | if ( len(result) == 0 ): |
---|
675 | error = ssh.stderr.readlines() |
---|
676 | notify = QtGui.QMessageBox.warning(self,'Showing parameter file',"Error. Could not fetch parameter file for job " + jobrealname + " (" + jobname + ").\n\nError message:\n" + ''.join(error)) |
---|
677 | |
---|
678 | # Otherwise inform the user and mark the job in the table |
---|
679 | else: |
---|
680 | mb = MessageBoxScroll() |
---|
681 | mb.setText("Parameter file for job: " + jobrealname + "") |
---|
682 | mb.setDetailedText(''.join(result)) |
---|
683 | mb.exec_() |
---|
684 | |
---|
685 | # Read the PARIN file of the job |
---|
686 | def ShowRC(self): |
---|
687 | |
---|
688 | # Return internal jobname and real job name |
---|
689 | jobname = self.table.item(self.table.currentRow(),6).text() |
---|
690 | jobrealname = self.table.item(self.table.currentRow(),0).text() |
---|
691 | |
---|
692 | # Check description of job in order to login on the correct host |
---|
693 | descr = self.table.item(self.table.currentRow(),1).text() |
---|
694 | for h in range(0,len(description)): |
---|
695 | if ( descr == description[h] ): |
---|
696 | host = username[h] + "@" + hostname[h] |
---|
697 | user = username[h] |
---|
698 | |
---|
699 | ssh = sub.Popen(["ssh", "%s" % host, "/sw/tools/python/2.7.6/generic/bin/python palm_wdd rc " + user + " " + jobname], |
---|
700 | shell=False, |
---|
701 | stdout=sub.PIPE, |
---|
702 | stderr=sub.PIPE) |
---|
703 | result = ssh.stdout.readlines() |
---|
704 | result = filter(None, result) |
---|
705 | |
---|
706 | # In case of error display a warning message |
---|
707 | if ( len(result) == 0 ): |
---|
708 | error = ssh.stderr.readlines() |
---|
709 | notify = QtGui.QMessageBox.warning(self,'Showing run control',"Error. Could not fetch run control file for job " + jobrealname + "(" + jobname + ").\n\nError message:\n" + ''.join(error)) |
---|
710 | |
---|
711 | # Otherwise inform the user and mark the job in the table |
---|
712 | else: |
---|
713 | mb = MessageBoxScroll() |
---|
714 | lastline = result[len(result)-2].split()[2] |
---|
715 | mb.setText("Simulated time for job " + jobrealname + " is currently: " + lastline) |
---|
716 | mb.setDetailedText(''.join(result)) |
---|
717 | mb.exec_() |
---|
718 | |
---|
719 | # Remove a job from list - removes the current row from the table and from |
---|
720 | # save file |
---|
721 | def RemoveFromList(self, row): |
---|
722 | if ( row == -1 ): |
---|
723 | row = self.table.currentRow() |
---|
724 | |
---|
725 | # Read data from save file |
---|
726 | job_to_delete = self.table.item(row,6).text() |
---|
727 | self.table.removeRow(row) |
---|
728 | file = open(palm_dir + '/.wd.olddata', 'r') |
---|
729 | result = file.readlines() |
---|
730 | result = filter(None, result) |
---|
731 | file.close() |
---|
732 | file = open(palm_dir + '/.wd.olddata', 'w') |
---|
733 | |
---|
734 | job_data_old = [] |
---|
735 | |
---|
736 | if ( len(result) == 0 ): |
---|
737 | notify = QtGui.QMessageBox.warning(self,'Read from .wd.olddata',"Error message:\n\nCould not read from file. I/O error.") |
---|
738 | else: |
---|
739 | # Save data in array job_data_old |
---|
740 | for j in range(0,len(result)): |
---|
741 | job_data_old.append(j) |
---|
742 | job_data_old[j] = result[j].split(" ") |
---|
743 | job_data_old[j] = filter(None, job_data_old[j]) |
---|
744 | job_data_old[j] = [line.rstrip('\n') for line in job_data_old[j]] |
---|
745 | |
---|
746 | # Check if line j is the selected job, if not -> save to file |
---|
747 | if ( job_data_old[j][6] != job_to_delete ): |
---|
748 | printstring = str(job_data_old[j][0]) + " " + \ |
---|
749 | str(job_data_old[j][1]) + " " + \ |
---|
750 | str(job_data_old[j][2]) + " " + \ |
---|
751 | str(job_data_old[j][3]) + " " + \ |
---|
752 | str(job_data_old[j][4]) + " " + \ |
---|
753 | str(job_data_old[j][5]) + " " + \ |
---|
754 | str(job_data_old[j][6]) + " " + \ |
---|
755 | str(job_data_old[j][7]) + "\n" |
---|
756 | file.write(printstring) |
---|
757 | |
---|
758 | file.close() |
---|
759 | |
---|
760 | # Remove all completed jobs from list |
---|
761 | def ClearList(self): |
---|
762 | |
---|
763 | num_of_lines = self.table.rowCount() |
---|
764 | |
---|
765 | # Delete all lines with completed/canceled jobs from list. The counter |
---|
766 | # must decrease as the line numbers would be messed up otherwise |
---|
767 | for j in range(num_of_lines-1,-1,-1): |
---|
768 | state = self.table.item(j,3).text() |
---|
769 | if ( state == "Completed" or state == "Canceled" or state == "Terminated"): |
---|
770 | self.RemoveFromList(j) |
---|
771 | |
---|
772 | # Update the label |
---|
773 | def UpdateLabel(self): |
---|
774 | remaining_time = (update_frequency - self.timetimer.elapsed()) / 1000 / 60 |
---|
775 | self.label.setText("Next update in " + str(remaining_time) + " min") |
---|
776 | |
---|
777 | |
---|
778 | # Main loop |
---|
779 | def main(): |
---|
780 | |
---|
781 | app = QtGui.QApplication(sys.argv) |
---|
782 | res = Watchdog() |
---|
783 | sys.exit(app.exec_()) |
---|
784 | |
---|
785 | |
---|
786 | if __name__ == '__main__': |
---|
787 | main() |
---|