Home

Context Navigation

source: palm/trunk/SCRIPTS/palmtest @ 4796

Last change on this file since 4796 was 4751, checked in by knoop, 4 years ago
New feature for palmtest to configure number of OpenMP threads per testcase
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 58.3 KB

Line
1	#!/usr/bin/env python3
2	# PYTHON_ARGCOMPLETE_OK
3
4	import os
5	import sys
6	import shutil
7	from datetime import datetime
8	import subprocess
9	import multiprocessing
10	import socket
11	import getpass
12	import math
13	import re
14	import threading
15	import queue
16	from contextlib import ContextDecorator
17
18	try:
19	from argparse import ArgumentParser
20	from argparse import RawTextHelpFormatter
21	except ImportError:
22	sys.exit(
23	'ERROR: You need argparse!\n' +
24	' install it from http://pypi.python.org/pypi/argparse\n' +
25	' or run \"pip install argparse\".'
26	)
27
28	try:
29	import numpy as np
30	except ImportError:
31	sys.exit(
32	'ERROR: You need numpy!\n' +
33	' install it from http://pypi.python.org/pypi/numpy\n' +
34	' or run \"python3 -m pip install numpy\".'
35	)
36
37	try:
38	import netCDF4
39	except ImportError:
40	sys.exit(
41	'ERROR: You need netCDF4!\n' +
42	' install it from http://pypi.python.org/pypi/netCDF4\n' +
43	' or run \"python3 -m pip install netCDF4\".'
44	)
45
46	try:
47	import yaml
48	except ImportError:
49	sys.exit(
50	'ERROR: You need PyYAML!\n' +
51	' install it from http://pypi.python.org/pypi/PyYAML\n' +
52	' or run \"python3 -m pip install PyYAML\".'
53	)
54
55	try:
56	import argcomplete
57	except ImportError:
58	print(
59	'INFO: To use Tab-completion you need argcomplete!\n' +
60	' install it from http://pypi.python.org/pypi/argcomplete\n' +
61	' or run \"python3 -m pip install argcomplete\".'
62	)
63	has_argcomplete = False
64	else:
65	has_argcomplete = True
66
67	try:
68	from termcolor import colored as tcolored
69	except ImportError:
70	def tcolored(string, color):
71	return string
72
73	disable_colored_output = False
74
75
76	def colored(string, color):
77	if not disable_colored_output:
78	return tcolored(string, color)
79	else:
80	return string
81
82
83	def disable_color():
84	global disable_colored_output
85	disable_colored_output = True
86
87
88	version = '1.0.1'
89
90
91	class Environment:
92
93	scripts_dir = os.path.dirname(os.path.realpath(__file__))
94	trunk_dir = os.path.realpath(os.path.join(scripts_dir, '..'))
95	workspace_dir = os.path.realpath(os.path.join(trunk_dir, '..'))
96
97	trunk_tests_dir = os.path.join(trunk_dir, 'TESTS')
98	trunk_tests_cases_dir = os.path.join(trunk_tests_dir, 'cases')
99	trunk_tests_builds_dir = os.path.join(trunk_tests_dir, 'builds')
100
101	tests_dir = os.path.join(workspace_dir, 'tests')
102
103
104	class LogFormatter:
105
106	terminal_columns, terminal_lines = shutil.get_terminal_size()
107	hline = '#' * min(terminal_columns, 300) + '\n'
108	table_width_intro = 12
109	table_width_builds = len(max([s for s in next(os.walk(Environment.trunk_tests_builds_dir))[1] if not s[0] == '.'], key=len)) + len('_debug')
110	table_width_cases = len(max([s for s in next(os.walk(Environment.trunk_tests_cases_dir))[1] if not s[0] == '.'], key=len))
111	table_width_cores = 7
112	table_width_total = table_width_intro + table_width_builds + table_width_cases + table_width_cores + 3
113
114	intro_table_line_template = \
115	'{:' + str(table_width_intro) + '} '
116
117	task_table_line_template = \
118	'{:' + str(table_width_intro) + '} ' + \
119	'{:' + str(table_width_cases) + '} ' + \
120	'{:' + str(table_width_builds) + '} ' + \
121	'{:' + str(table_width_cores) + '} '
122
123	config_table_line_template = \
124	'{:' + str(table_width_intro) + '} ' + \
125	'{:' + str(max(table_width_builds, table_width_cases)) + '} ' + \
126	'{:8} '
127
128	file_table_line_template = \
129	'{:' + str(table_width_intro) + '} ' + \
130	'{:' + str(table_width_cases + 13) + '} '
131
132
133	class SignificantDigitsRounder:
134
135	@staticmethod
136	def _round(value, digits=10):
137	if value == 0.0:
138	return value
139	negative = value < 0.0
140	value = -value if negative else value
141	rounded_value = round(value, -int(math.floor(math.log10(value))) + (digits - 1))
142	rounded_value = -rounded_value if negative else rounded_value
143	return rounded_value
144
145
146	vectorized_round = np.vectorize(_round)
147
148	_vectorized_round = np.vectorize(round)
149
150
151	@classmethod
152	def around(cls, array, digits=10):
153	# TODO: divide both arrays and check decimal point
154	sign_mask = np.ma.masked_where(array >= 0.0, array).mask
155	pos_array = np.where(sign_mask, array, -array)
156	non_zero_maks = np.ma.masked_where(pos_array == 0.0, pos_array).mask
157	non_zero_array = np.where(non_zero_maks, 1.0, pos_array)
158	i1 = -np.floor(np.log10(non_zero_array)).astype(int) + (digits - 1)
159	rounded_non_zero_array = cls._vectorized_round(non_zero_array, i1)
160	rounded_pos_array = np.where(non_zero_maks, 0.0, rounded_non_zero_array)
161	return np.where(sign_mask, rounded_pos_array, -rounded_pos_array)
162
163
164
165	class Logger(ContextDecorator):
166
167	def __init__(self, logfile_dir, logfile_name='palmtest.log', logfile_mode='a', verbose=False):
168	self.logfile_path = os.path.join(logfile_dir, logfile_name)
169	self.logfile_mode = logfile_mode
170	self.verbose = verbose
171
172	def __enter__(self):
173	self._file = open(self.logfile_path, self.logfile_mode)
174	return self
175
176	def to_file(self, message):
177	self._file.write(message)
178	self._file.flush()
179
180	def to_log(self, message):
181	if self.verbose:
182	sys.stdout.write(message)
183	sys.stdout.flush()
184	self._file.write(message)
185	self._file.flush()
186
187	def to_all(self, message):
188	sys.stdout.write(message)
189	sys.stdout.flush()
190	self._file.write(message)
191	self._file.flush()
192
193	def __exit__(self, *exc):
194	self._file.close()
195	return False
196
197
198	class Executor:
199
200	@staticmethod
201	def _enqueue_output(out, queue):
202	for line in iter(out.readline, b''):
203	queue.put(line)
204	out.close()
205
206	@staticmethod
207	def execute(cmd, cwd='.', verbose=True, dry_run=False):
208	assert isinstance(cmd, list)
209	if dry_run:
210	cmd = ['echo'] + cmd
211	cmd_str = ' '.join(cmd)
212	p = subprocess.Popen(cmd_str, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, bufsize=1)
213	q = queue.Queue()
214	t = threading.Thread(target=Executor._enqueue_output, args=(p.stdout, q))
215	t.daemon = True # thread dies with the program
216	t.start()
217
218	with Logger(cwd, verbose=verbose) as logger:
219	# read line without blocking
220	logger.to_log(LogFormatter.hline)
221	logger.to_log('CMD: ' + cmd_str + '\n')
222	logger.to_log(LogFormatter.hline)
223	while t.is_alive():
224	try:
225	line = q.get_nowait() # or q.get(timeout=.1)
226	except queue.Empty:
227	pass # print('no output yet')
228	else: # got line
229	logger.to_log(line.decode("utf-8"))
230	line = True
231	while line:
232	try:
233	line = q.get_nowait() # or q.get(timeout=.1)
234	except queue.Empty:
235	line = False
236	else: # got line
237	logger.to_log(line.decode("utf-8"))
238	logger.to_log(LogFormatter.hline)
239
240	rc = p.poll()
241	failed = rc != 0
242	return failed
243
244
245	class NetCDFInterface:
246
247	def __init__(self, filename):
248	self.filename = filename
249
250	def is_healthy(self):
251	try:
252	self.get_run_name()
253	except:
254	return False
255	else:
256	return True
257
258	def get_run_name(self):
259	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
260	l = getattr(netcdf, 'title').split()
261	i = l.index('run:')
262	return l[i+1]
263
264	def get_var_list(self):
265	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
266	var_list = list(netcdf.variables.keys())
267	var_list = filter(None, var_list)
268	return sorted(var_list)
269
270	def show_content(self):
271	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
272	for name in netcdf.ncattrs():
273	print("Global attr", name, "=", getattr(netcdf, name))
274	print(netcdf)
275	for v in netcdf.variables:
276	print(v)
277
278	def get_times_list(self):
279	attributes, times = self.read_var('time')
280	times = [str(time) for time in times]
281	times = list(filter(None, times))
282	return times
283
284	def contains(self, variable):
285	return variable in self.get_var_list()
286
287	def read_var(self, variable):
288	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
289	values = netcdf.variables[variable][:] # extract values
290	attributes = dict()
291	try:
292	attributes['long_name'] = netcdf.variables[variable].name
293	except:
294	attributes['long_name'] = ''
295	try:
296	attributes['unit'] = netcdf.variables[variable].units
297	except:
298	attributes['unit'] = ''
299	return attributes, values
300
301
302	class FileComparator:
303
304	@staticmethod
305	def compare_ascii(file_path1, file_path2, start_string=None):
306	try:
307	with open(file_path1, 'r') as file1:
308	content1 = file1.readlines()
309	except OSError:
310	return True, colored('[reference file not found]', 'red')
311	try:
312	with open(file_path2, 'r') as file2:
313	content2 = file2.readlines()
314	except OSError:
315	return True, colored('[output file not found]', 'red')
316	if start_string:
317	index1 = content1.index(start_string)
318	index2 = content2.index(start_string)
319	comparable_content1 = content1[index1:]
320	comparable_content2 = content2[index2:]
321	ln = index2 + 1
322	else:
323	comparable_content1 = content1
324	comparable_content2 = content2
325	ln = 1
326	if len(comparable_content1) != len(comparable_content2):
327	return True, colored('[mismatch in total number of lines]', 'red')
328	for line1, line2 in zip(comparable_content1, comparable_content2):
329	if not line1 == line2:
330	return True, colored('[mismatch in content starting line ' + str(ln) + ']', 'red')
331	ln += 1
332	return False, colored('[file ok]', 'green')
333
334	@staticmethod
335	def compare_netcdf(file_path1, file_path2, digits=None):
336	nci1 = NetCDFInterface(file_path1)
337	nci2 = NetCDFInterface(file_path2)
338	if not nci1.is_healthy():
339	return True, colored('[reference file not found]', 'red')
340	if not nci2.is_healthy():
341	return True, colored('[output file not found]', 'red')
342	times_list1 = nci1.get_times_list()
343	times_list2 = nci2.get_times_list()
344	if not times_list1 == times_list2:
345	return True, colored('[wrong time dimension]', 'red')
346	else:
347	time_list = times_list1
348	var_list1 = nci1.get_var_list()
349	var_list2 = nci2.get_var_list()
350	if not var_list1 == var_list2:
351	return True, colored('[wrong set of variables]', 'red')
352	else:
353	var_list = var_list1
354	content1 = dict()
355	content2 = dict()
356	for var in var_list:
357	attributes1, values1 = nci1.read_var(var)
358	attributes2, values2 = nci2.read_var(var)
359	if sorted(attributes1.keys()) != sorted(attributes2.keys()):
360	return True, colored('[wrong set of attributes in variable \"'+var+'\"]', 'red')
361	if isinstance(digits, int):
362	values1 = SignificantDigitsRounder.around(values1, digits=digits)
363	values2 = SignificantDigitsRounder.around(values2, digits=digits)
364	content1[var] = values1
365	content2[var] = values2
366	#for decimals in
367	for i, time in enumerate(time_list):
368	for var in var_list:
369	t_content1 = content1[var][i]
370	t_content2 = content2[var][i]
371	if not (t_content1==t_content2).all():
372	if isinstance(digits, int):
373	return True, colored('[1st mismatch within ' + str(digits) + ' digits at time index ' + str(i) + ' in variable \"' + var + '\"]', 'red')
374	else:
375	return True, colored('[1st mismatch at time index ' + str(i) + ' in variable \"' + var + '\"]', 'red')
376	return False, colored('[file ok]', 'green')
377
378
379	class OutputChecker:
380
381	def __init__(self, test_dir, setup_name, build_name, cores, significant_digits=None, verbose=True, dry_run=False):
382	self.test_dir = test_dir
383	self.setup_name = setup_name
384	self.build_name = build_name
385	self.cores = cores
386	self.significant_digits = significant_digits
387	self.verbose = verbose
388	self.dry_run = dry_run
389	self.job_name = self.setup_name + '__' + build_name + '__' + str(self.cores)
390	self.job_dir = os.path.join(self.test_dir, 'JOBS', self.job_name)
391	self.ref_monitoring_dir = os.path.join(Environment.trunk_tests_cases_dir, self.setup_name, 'MONITORING')
392	self.ref_output_dir = os.path.join(Environment.trunk_tests_cases_dir, self.setup_name, 'OUTPUT')
393	self.res_monitoring_dir = os.path.join(self.job_dir, 'MONITORING')
394	self.res_output_dir = os.path.join(self.job_dir, 'OUTPUT')
395	self.failed = None
396
397	def get_checkable_file_dicts(self):
398	if os.path.isdir(self.ref_monitoring_dir):
399	file_names_monitoring = [s for s in next(os.walk(self.ref_monitoring_dir))[2]]
400	else:
401	file_names_monitoring = []
402	file_paths_monitoring = []
403	for file_name in file_names_monitoring:
404	file_specific_ending = file_name[len(self.setup_name):]
405	file_specific_ending_split = file_specific_ending.split('.')
406	postfix = file_specific_ending_split[0]
407	if len(file_specific_ending_split) > 1:
408	extension = file_specific_ending_split[-1]
409	else:
410	extension = ''
411	if len(file_specific_ending_split) > 2:
412	cycle_info = file_specific_ending_split[1: -1]
413	else:
414	cycle_info = []
415	file_paths_monitoring.append(
416	dict(
417	postfix=postfix,
418	cycle_info=cycle_info,
419	extension=extension,
420	ref_path=self.ref_monitoring_dir,
421	res_path=self.res_monitoring_dir,
422	)
423	)
424	if os.path.isdir(self.ref_output_dir):
425	file_names_output = [s for s in next(os.walk(self.ref_output_dir))[2]]
426	else:
427	file_names_output = []
428	file_paths_output = []
429	for file_name in file_names_output:
430	file_specific_ending = file_name[len(self.setup_name):]
431	file_specific_ending_split = file_specific_ending.split('.')
432	postfix = file_specific_ending_split[0]
433	if len(file_specific_ending_split) > 1:
434	extension = file_specific_ending_split[-1]
435	else:
436	extension = ''
437	if len(file_specific_ending_split) > 2:
438	cycle_info = file_specific_ending_split[1: -1]
439	else:
440	cycle_info = []
441	file_paths_output.append(
442	dict(
443	postfix=postfix,
444	cycle_info=cycle_info,
445	extension=extension,
446	ref_path=self.ref_output_dir,
447	res_path=self.res_output_dir,
448	)
449	)
450	return file_paths_monitoring + file_paths_output
451
452	def check(self):
453	with Logger(self.test_dir, verbose=self.verbose) as logger:
454	logger.to_log('Checking output files:')
455	logger.to_all('\n')
456	failed = False
457	for file_dict in self.get_checkable_file_dicts():
458	file_failed = False
459	ext_list = [file_dict['extension']] if file_dict['extension'] else []
460	file_specific_ending = '.'.join([file_dict['postfix']] + file_dict['cycle_info'] + ext_list )
461	logger.to_all(LogFormatter.file_table_line_template.format('Checking:', self.setup_name + file_specific_ending))
462	ref_file_path = os.path.join(file_dict['ref_path'], self.setup_name + file_specific_ending)
463	res_file_path = os.path.join(file_dict['res_path'], self.job_name + file_specific_ending)
464	if re.match('_rc', file_dict['postfix']) and re.match('[0-9]{3}', file_dict['extension']):
465	file_failed, message = FileComparator.compare_ascii(ref_file_path, res_file_path, start_string='Run-control output:\n')
466	elif re.match('nc', file_dict['extension']):
467	if self.significant_digits is not None:
468	if re.match('_ts', file_dict['postfix']) and 'timeseries' in self.significant_digits:
469	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
470	digits=self.significant_digits['timeseries'])
471	elif re.match('_pr', file_dict['postfix']) and 'profiles' in self.significant_digits:
472	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
473	digits=self.significant_digits['profiles'])
474	else:
475	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
476	digits=self.significant_digits['other'])
477	else:
478	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path)
479	else:
480	message = colored('[ignored]', 'blue')
481	if file_failed:
482	failed = True
483	logger.to_all(message + '\n')
484	if self.dry_run:
485	failed = False
486	return failed
487
488
489	class PALMJob:
490	"""The PALM job class deals with the execution of a single PALM job"""
491
492	@staticmethod
493	def get_job_name(setup_name, build_name, cores):
494	return setup_name + '__' + build_name + '__' + str(cores)
495
496	def __init__(self, test_dir, test_case, build_name, cores, verbose=False, dry_run=False):
497	self.test_dir = test_dir
498	self.test_case = test_case
499	self.build_name = build_name
500	self.cores = cores
501	self.verbose = verbose
502	self.dry_run = dry_run
503
504	self.attempted_debug = False
505	self.failed_debug = None
506	self.attempted_non_debug = False
507	self.failed_non_debug = None
508
509	def _link_restart_files(self, build_name):
510	if self.dry_run:
511	return True, colored('[restart data dry]', 'blue')
512	name = self.get_job_name(self.test_case.name, build_name, self.cores)
513	source_name = self.get_job_name(self.test_case.use_binary_files_from, build_name, self.cores)
514	source_restart_dir = os.path.join(self.test_dir, 'JOBS', source_name, 'RESTART')
515	try:
516	source_data_dirs_grp = [d for r, d, f in os.walk(source_restart_dir)]
517	except:
518	source_data_dirs_grp = []
519	if len(source_data_dirs_grp) == 0:
520	source_data_dirs = []
521	else:
522	source_data_dirs = source_data_dirs_grp[0]
523	if len(source_data_dirs) == 0 and re.match('.+_debug', build_name):
524	source_build_name = build_name[:-len('_debug')]
525	source_name = self.get_job_name(self.test_case.use_binary_files_from, source_build_name, self.cores)
526	source_restart_dir = os.path.join(self.test_dir, 'JOBS', source_name, 'RESTART')
527	try:
528	source_data_dirs_grp = [d for r, d, f in os.walk(source_restart_dir)]
529	except:
530	source_data_dirs_grp = []
531	if len(source_data_dirs_grp) == 0:
532	source_data_dirs = []
533	else:
534	source_data_dirs = source_data_dirs_grp[0]
535	if len(source_data_dirs) == 0:
536	source_data_dir = 'no_restart_data'
537	else:
538	source_data_dir = sorted(source_data_dirs)[-1]
539	source_data_dir_path = os.path.join(source_restart_dir, source_data_dir)
540	if os.path.isdir(source_data_dir_path) and re.match('.+_d3d.*', source_data_dir):
541	job_restart_dir = os.path.join(self.test_dir, 'JOBS', name, 'RESTART')
542	os.makedirs(job_restart_dir, exist_ok=False)
543	job_data_dir_path = os.path.join(job_restart_dir, name + '_d3d')
544	os.symlink(source_data_dir_path, job_data_dir_path, target_is_directory=True)
545	return False, colored('[linked restart data from: ' + source_data_dir_path + ']', 'green')
546	else:
547	return True, colored('[no restart data found]', 'red')
548
549	def _execute(self, name, build_name):
550	cmd_list = [
551	os.path.join(self.test_dir, 'trunk', 'SCRIPTS', 'palmrun'),
552	'-c', '\"' + build_name + '\"',
553	'-r', name,
554	'-a', '\"' + ' '.join(self.test_case.activation_strings) + '\"',
555	'-X', str(self.cores),
556	]
557	if self.test_case.omp_num_threads_found:
558	cmd_list.extend(
559	[
560	'-O', str(self.test_case.omp_num_threads),
561	]
562	)
563	cmd_list.extend(
564	[
565	'-B',
566	'-v',
567	'-z',
568	]
569	)
570	execution_failed = Executor.execute(
571	cmd_list,
572	cwd=self.test_dir,
573	verbose=self.verbose,
574	dry_run=self.dry_run,
575	)
576
577	if self.dry_run:
578	return False, colored('[execution dry]', 'blue')
579	elif execution_failed:
580	return True, colored('[execution failed]', 'red')
581	else:
582	return False, colored('[execution ok]', 'green')
583
584	def _check(self, build_name):
585	checker = OutputChecker(
586	self.test_dir,
587	self.test_case.name,
588	build_name,
589	self.cores,
590	significant_digits=self.test_case.significant_digits,
591	verbose=self.verbose,
592	dry_run=self.dry_run,
593	)
594	check_failed = checker.check()
595
596	if self.dry_run:
597	return False, colored('[checks dry]', 'blue')
598	if check_failed:
599	return True, colored('[checks failed]', 'red')
600	else:
601	return False, colored('[checks ok]', 'green')
602
603	def execute(self, debug=False):
604	if debug:
605	attempted = self.attempted_debug
606	build_name = self.build_name + '_debug'
607	failed = self.failed_debug
608	else:
609	attempted = self.attempted_non_debug
610	build_name = self.build_name
611	failed = self.failed_non_debug
612
613	if not attempted:
614	with Logger(self.test_dir, verbose=self.verbose) as logger:
615	status_prefix = LogFormatter.task_table_line_template.format('Testing:', self.test_case.name, build_name, self.cores)
616	logger.to_all(status_prefix)
617	logger.to_log('[started]' + '\n')
618	attempted = True
619
620	name = self.get_job_name(self.test_case.name, build_name, self.cores)
621
622	input_dir = os.path.join(self.test_dir, 'JOBS', name, 'INPUT')
623	os.makedirs(input_dir, exist_ok=False)
624
625	# copying needs to be done per file, because input files need to be renamed
626	for input_file in self.test_case.input_file_names:
627	postfix = input_file[len(self.test_case.name):]
628	src = os.path.join(self.test_case.input_dir, input_file)
629	dst = os.path.join(input_dir, name + postfix)
630	shutil.copy(src, dst)
631
632	# copying the entire directory is ok, because source files do not need to be renamed
633	user_code_dir = os.path.join(self.test_dir, 'JOBS', name, 'USER_CODE')
634	if os.path.isdir(self.test_case.user_code_dir):
635	shutil.copytree(self.test_case.user_code_dir, user_code_dir, copy_function=shutil.copy)
636
637	if self.test_case.requires_binary_files:
638	link_restart_files_failed, message = self._link_restart_files(build_name)
639	logger.to_log(status_prefix)
640	logger.to_log(message + ' ')
641	logger.to_log('\n')
642
643	failed, message = self._execute(name, build_name)
644	logger.to_log(status_prefix)
645	logger.to_all(message + ' ')
646	logger.to_log('\n')
647
648	failed, message = self._check(build_name)
649	logger.to_log(status_prefix)
650	logger.to_log(message + ' ')
651
652	logger.to_all('\n')
653
654	if debug:
655	self.attempted_debug = attempted
656	self.failed_debug = failed
657	else:
658	self.attempted_non_debug = attempted
659	self.failed_non_debug = failed
660
661	return failed
662
663	def status(self):
664	return dict(
665	attempted=self.attempted_non_debug or self.attempted_debug,
666	failed=self.failed_non_debug and self.failed_debug,
667	debugged=self.attempted_debug,
668	non_debug_failed=self.failed_non_debug,
669	)
670
671
672	class PALMBuild:
673	"""The PALM build class deals with configuration and execution of all required PALM builds"""
674
675	def __init__(self, test_dir, build_name, verbose=False, dry_run=False):
676	self.test_dir = test_dir
677	self.build_name = build_name
678	self.verbose = verbose
679	self.dry_run = dry_run
680	self.configured = False
681	self.executed = False
682	self.available = False
683	self.requires_mpi = False
684	self.requires_netcdf = False
685	self.requires_fftw = False
686	self.requires_rrtmg = False
687	self.attempted_non_debug = False
688	self.attempted_debug = False
689	self.failed_non_debug = None
690	self.failed_debug = None
691
692	def configure(self):
693	try:
694	with open(os.path.join(Environment.trunk_tests_builds_dir, self.build_name, 'build_config.yml'), 'r') as f:
695	build_config = yaml.load(f, Loader=yaml.SafeLoader)
696	except:
697	return True, colored('[build not found]', 'red')
698
699	if 'compiler' in build_config:
700	self.compiler = build_config['compiler']
701	else:
702	return True, colored('[missing \"compiler\" keyword]', 'red')
703
704	if not isinstance(self.compiler, dict):
705	return True, colored('[\"compiler\" keyword must be dict]', 'red')
706
707	if 'linker' in build_config:
708	self.linker = build_config['linker']
709	else:
710	return True, colored('[missing \"linker\" keyword]', 'red')
711
712	if not isinstance(self.linker, dict):
713	return True, colored('[\"linker\" keyword must be dict]', 'red')
714
715	if 'mpi_wrapper' in self.compiler:
716	if 'mpi_wrapper}}' in self.compiler['mpi_wrapper']:
717	self.requires_mpi = True
718	else:
719	return True, colored('[missing \"mpi_wrapper\" keyword]', 'red')
720
721	if 'includes' in self.compiler:
722	for include in self.compiler['includes']:
723	if 'include.netcdf}}' in include:
724	self.requires_netcdf = True
725	if 'include.fftw}}' in include:
726	self.requires_fftw = True
727	if 'include.rrtmg}}' in include:
728	self.requires_rrtmg = True
729	else:
730	return True, colored('[missing \"includes\" keyword in compiler]', 'red')
731
732	if 'options' in self.linker:
733	for lib in self.linker['options']:
734	if 'lib.netcdf}}' in lib:
735	self.requires_netcdf = True
736	if 'lib.fftw}}' in lib:
737	self.requires_fftw = True
738	if 'lib.rrtmg}}' in lib:
739	self.requires_rrtmg = True
740	else:
741	return True, colored('[missing \"options\" keyword in linker]', 'red')
742
743	library_names = []
744	if self.requires_netcdf:
745	library_names.append('netcdf')
746	if self.requires_fftw:
747	library_names.append('fftw')
748	if self.requires_rrtmg:
749	library_names.append('rrtmg')
750
751	if not 'executable' in self.compiler:
752	return True, colored('[missing \"executable\" keyword in compiler]', 'red')
753
754	if not 'definitions' in self.compiler:
755	return True, colored('[missing \"definitions\" keyword in compiler]', 'red')
756
757	if not 'options' in self.compiler:
758	return True, colored('[missing \"options\" keyword in compiler]', 'red')
759
760	if not 'default' in self.compiler['options']:
761	return True, colored('[missing \"default\" keyword in compiler.options]', 'red')
762
763	if not 'debug' in self.compiler['options']:
764	return True, colored('[missing \"debug\" keyword in compiler.options]', 'red')
765
766	try:
767	with open(os.path.join(Environment.workspace_dir, 'palmtest.yml'), 'r') as f:
768	palmtest_config = yaml.load(f, Loader=yaml.SafeLoader)
769	except:
770	return True, colored('[palmtest.yml not found]', 'red')
771
772	if 'palm_config_template' in palmtest_config:
773	if isinstance(palmtest_config['palm_config_template'], str):
774	custom_template = palmtest_config['palm_config_template']
775	try:
776	with open(os.path.join(custom_template), 'r') as palm_config_template_file:
777	template = palm_config_template_file.read()
778	except:
779	try:
780	with open(os.path.join(Environment.scripts_dir, '.palm.config.default.in'), 'r') as palm_config_template_file:
781	template = palm_config_template_file.read()
782	except:
783	return True, colored('[trunk/SCRIPTS/.palm.config.default.in not found]', 'red')
784
785	template = template.replace('@CMAKE_INSTALL_PREFIX@', self.test_dir)
786	template = template.replace('@PALM_HOSTNAME@', socket.gethostname())
787	template = template.replace('@CMAKE_USERNAME@', getpass.getuser())
788	template = template.replace('@MPI_Fortran_COMPILER@', self.compiler['mpi_wrapper'])
789	template = template.replace('@CMAKE_Fortran_COMPILER@', self.compiler['executable'])
790	cpp_options_str = ['-D' + s for s in self.compiler['definitions']]
791	template = template.replace('@PALM_CPP_OPTIONS_STR@', ' '.join(cpp_options_str))
792	template = template.replace('@PALM_CORES@', str(multiprocessing.cpu_count()))
793	template = template.replace('@PALM_COMPILER_OPTIONS@', '{{palmtest.compiler.options}} ' + ' '.join(self.compiler['includes']))
794	template = template.replace('@PALM_LINKER_OPTIONS@', ' '.join(self.linker['options']))
795
796	if 'environments' in palmtest_config:
797	available_environments = palmtest_config['environments']
798	else:
799	return True, colored('[missing \"environments\" keyword in palmtest.yml]', 'red')
800
801	if 'id' in self.compiler:
802	c_id = self.compiler['id']
803	else:
804	return True, colored('[missing \"id\" keyword in compiler]', 'red')
805
806	if c_id in available_environments:
807	self.available = True
808
809	environment = available_environments[c_id]
810
811	if 'mpi_execution_command' in environment:
812	template = template.replace('@PALM_EXECUTE_COMMAND@', environment['mpi_execution_command'])
813	else:
814	template = template.replace('@PALM_EXECUTE_COMMAND@', 'mpirun -n {{mpi_tasks}}')
815
816	if 'executable' not in environment:
817	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"executable\"]', 'red')
818	value = environment['executable']
819	if isinstance(value, str):
820	template = template.replace('{{' + '.'.join([c_id, 'executable']) + '}}', value)
821	if self.requires_mpi:
822	if 'mpi_wrapper' not in environment:
823	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"mpi_wrapper\"]', 'red')
824	value = environment['mpi_wrapper']
825	if isinstance(value, str):
826	template = template.replace('{{' + '.'.join([c_id, 'mpi_wrapper']) + '}}', value)
827	if 'include' not in environment:
828	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"include\"]', 'red')
829	if 'lib' not in environment:
830	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"lib\"]', 'red')
831	for lib in library_names:
832	if lib not in environment['include']:
833	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"include.'+lib+'\"]', 'red')
834	value = environment['include'][lib]
835	if isinstance(value, str):
836	template = template.replace('{{' + '.'.join([c_id, 'include', lib]) + '}}', value)
837	if lib not in environment['lib']:
838	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"lib.'+lib+'\"]', 'red')
839	value = environment['lib'][lib]
840	if isinstance(value, str):
841	template = template.replace('{{' + '.'.join([c_id, 'lib', lib]) + '}}', value)
842
843	with open(os.path.join(self.test_dir, '.palm.config.' + self.build_name), 'w') as palm_config_file:
844	palm_config_file.write(
845	template.replace(
846	'{{palmtest.compiler.options}}',
847	' '.join(self.compiler['options']['default']),
848	)
849	)
850	with open(os.path.join(self.test_dir, '.palm.config.' + self.build_name + '_debug'), 'w') as palm_config_file:
851	palm_config_file.write(
852	template.replace(
853	'{{palmtest.compiler.options}}',
854	' '.join(self.compiler['options']['debug']),
855	)
856	)
857	self.configured = True
858	return False, colored('[configuration ok]', 'green')
859
860	else:
861	return True, colored('[palmtest.yml environment \"' + c_id + '\" not found]', 'red')
862
863	def _execute(self, build_name):
864	self.attempted = True
865	build_failed = Executor.execute(
866	[
867	os.path.join(self.test_dir, 'trunk', 'SCRIPTS', 'palmbuild'),
868	'-c', '\"' + build_name + '\"',
869	'-v',
870	],
871	cwd=self.test_dir,
872	verbose=self.verbose,
873	dry_run=self.dry_run,
874	)
875
876	if self.dry_run:
877	return False, colored('[build dry]', 'blue')
878	if build_failed:
879	return True, colored('[build failed]', 'red')
880	else:
881	return False, colored('[build ok]', 'green')
882
883	def build(self, debug=False):
884	if debug:
885	attempted = self.attempted_debug
886	build_name = self.build_name + '_debug'
887	failed = self.failed_debug
888	else:
889	attempted = self.attempted_non_debug
890	build_name = self.build_name
891	failed = self.failed_non_debug
892
893	if not attempted:
894	with Logger(self.test_dir, verbose=self.verbose) as logger:
895	status_prefix = LogFormatter.task_table_line_template.format('Building:', '', build_name, '')
896	logger.to_all(status_prefix)
897	logger.to_log('[started]' + '\n')
898	attempted = True
899
900	failed, message = self._execute(build_name)
901	logger.to_log(status_prefix)
902	logger.to_all(message + ' ')
903	logger.to_all('\n')
904
905	if debug:
906	self.attempted_debug = attempted
907	self.failed_debug = failed
908	else:
909	self.attempted_non_debug = attempted
910	self.failed_non_debug = failed
911
912	return failed
913
914	def report(self):
915	return dict(
916	failed_debug=self.failed_debug,
917	failed_non_debug=self.failed_non_debug,
918	)
919
920
921	class PALMTestCase:
922	"""The PALM test case class deals with the configuration and execution of all PALM test cases"""
923
924	def __init__(self,test_dir, name, verbose=False, dry_run=False):
925	self.test_dir = test_dir
926	self.name = name
927	self.verbose = verbose
928	self.dry_run = dry_run
929	self.user_code_dir = os.path.join(Environment.trunk_tests_cases_dir, self.name, 'USER_CODE')
930	self.input_dir = os.path.join(Environment.trunk_tests_cases_dir, self.name, 'INPUT')
931	self.number_of_cores = []
932	self.build_names = []
933	self.input_file_names = []
934	self.configured = False
935
936	def configure(self, requested_build_names, requested_cores):
937	f_name = os.path.join(Environment.trunk_tests_cases_dir, self.name, 'case_config.yml')
938	try:
939	with open(f_name, 'r') as f:
940	config = yaml.load(f, Loader=yaml.SafeLoader)
941	except:
942	return True, colored('[Case \"' + self.name + '\" could not be found.]', 'red')
943	try:
944	self.use_binary_files_from = config['use_binary_files_from']
945	except:
946	self.use_binary_files_from = None
947	self.requires_binary_files = bool(self.use_binary_files_from)
948
949	if 'allowed_builds' not in config:
950	return True, colored('[missing \"allowed_builds\" keyword]', 'red')
951	self.allowed_build_names = config['allowed_builds']
952
953	if 'allowed_number_of_cores' not in config:
954	return True, colored('[missing \"allowed_number_of_cores\" keyword]', 'red')
955	self.allowed_number_of_cores = config['allowed_number_of_cores']
956
957	if 'omp_num_threads' in config:
958	self.omp_num_threads_found = True
959	self.omp_num_threads = config['omp_num_threads']
960	else:
961	self.omp_num_threads_found = False
962
963	if 'activation_strings' not in config:
964	return True, colored('[missing \"activation_strings\" keyword]', 'red')
965	self.activation_strings = config['activation_strings']
966
967	if 'significant_digits_for_netcdf_checks' not in config:
968	return True, colored('[missing \"significant_digits_for_netcdf_checks\" keyword]', 'red')
969	self.significant_digits = config['significant_digits_for_netcdf_checks']
970
971	if 'timeseries' not in config['significant_digits_for_netcdf_checks']:
972	return True, colored('[missing \"timeseries\" keyword in significant_digits_for_netcdf_checks]', 'red')
973
974	if 'profiles' not in config['significant_digits_for_netcdf_checks']:
975	return True, colored('[missing \"profiles\" keyword in significant_digits_for_netcdf_checks]', 'red')
976
977	if 'other' not in config['significant_digits_for_netcdf_checks']:
978	return True, colored('[missing \"other\" keyword in significant_digits_for_netcdf_checks]', 'red')
979
980	self.number_of_cores = sorted(set(requested_cores).intersection(self.allowed_number_of_cores))
981	self.build_names = sorted(set(requested_build_names).intersection(self.allowed_build_names))
982	self.input_file_names = [s for s in next(os.walk(self.input_dir))[2]]
983	self.configured = True
984	if len(self.number_of_cores) == 0 :
985	return True, colored('[no allowed cores requested]', 'blue')
986	if len(self.build_names) == 0:
987	return True, colored('[no allowed builds requested]', 'blue')
988	if len(self.input_file_names) == 0:
989	return True, colored('[no input files found]', 'red')
990	return False, colored('[configuration ok]', 'green')
991
992
993
994	class PALMTest:
995
996	def __init__(self, args):
997	self.verbose = args.verbose
998	self.no_auto_debug = args.no_auto_debug
999	self.force_debug = args.force_debug
1000	self.fail_on_debug = args.fail_on_debug
1001	self.dry_run = args.dry_run
1002	self.no_color = args.no_color
1003	self.test_id = args.test_id
1004	self.test_case_names = args.cases
1005	self.requested_build_names = args.builds
1006	self.requested_cores = args.cores
1007	self.test_case_queue = []
1008	self.build_database = dict()
1009
1010	def prepare(self):
1011	if self.no_color:
1012	disable_color()
1013	self.test_dir = os.path.join(Environment.tests_dir, self.test_id)
1014	try:
1015	os.makedirs(self.test_dir, exist_ok=False)
1016	except:
1017	print('ERROR: Found existing test directory: ' + self.test_dir)
1018	exit(1)
1019	with Logger(self.test_dir, verbose=self.verbose) as logger:
1020	logger.to_all(LogFormatter.hline)
1021	logger.to_all('This is the PALM tester (version: ' + version + ')' + '\n')
1022	logger.to_all(LogFormatter.hline)
1023	try:
1024	with open(os.path.join(Environment.workspace_dir, 'palmtest.yml'), 'r') as f:
1025	pass
1026	except:
1027	logger.to_all('ERROR: No palmtest.yml file was found in your working directory!\n')
1028	logger.to_all('INFO: A template for this file can be found at: trunk/TESTS/palmtest.yml\n')
1029	logger.to_all(' Please copy the template to your working directory and adjust it to your system!\n')
1030	exit(1)
1031
1032	self.execution_trunk_dir = os.path.join(self.test_dir, 'trunk')
1033	os.symlink(Environment.trunk_dir, self.execution_trunk_dir)
1034	self.execution_jobs_dir = os.path.join(self.test_dir, 'JOBS')
1035	os.makedirs(self.execution_jobs_dir, exist_ok=False)
1036
1037	try:
1038	with open(os.path.join(Environment.scripts_dir, '.palm.iofiles'), 'r') as iofiles_template_file:
1039	iofiles_template = iofiles_template_file.read()
1040	with open(os.path.join(self.test_dir, '.palm.iofiles'), 'w') as iofiles_file:
1041	iofiles_file.write(iofiles_template.replace('$fast_io_catalog', '$base_data'))
1042	except:
1043	logger.to_all('ERROR: No .palm.iofiles file was found in trunk/SCRIPTS/')
1044	exit(1)
1045
1046	available_cores = multiprocessing.cpu_count()
1047	final_cores_list = list(filter(lambda x: x <= available_cores, self.requested_cores))
1048
1049	logger.to_all(LogFormatter.config_table_line_template.format('Object:', 'Name:', 'Action:') + 'Status:\n')
1050	logger.to_all(LogFormatter.hline)
1051
1052	if 'all' in self.requested_build_names:
1053	self.requested_build_names = [name for name in next(os.walk(Environment.trunk_tests_builds_dir))[1] if not name[0] == '.']
1054	found_build_names = []
1055	for build_name in self.requested_build_names:
1056	build = PALMBuild(self.test_dir, build_name, verbose=self.verbose, dry_run=self.dry_run)
1057	configuration_failed, message = build.configure()
1058	if not configuration_failed:
1059	self.build_database[build_name] = build
1060	found_build_names.append(build_name)
1061	logger.to_all(LogFormatter.config_table_line_template.format('Build', build_name, 'approved'))
1062	logger.to_all(message + '\n')
1063	else:
1064	logger.to_all(LogFormatter.config_table_line_template.format('Build', build_name, 'rejected'))
1065	logger.to_all(message + '\n')
1066	final_build_list = found_build_names
1067
1068	if 'all' in self.test_case_names:
1069	self.test_case_names = sorted([name for name in next(os.walk(Environment.trunk_tests_cases_dir))[1] if not name[0] == '.'])
1070
1071	additional_initial_runs_2 = [self.test_case_names]
1072	while len(additional_initial_runs_2[-1]) > 0:
1073	additional_initial_runs_1 = []
1074	for test_case_name in additional_initial_runs_2[-1]:
1075	test_case = PALMTestCase(self.test_dir, test_case_name, verbose=self.verbose)
1076	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
1077	if not test_case_configuration_failed:
1078	if test_case.requires_binary_files:
1079	additional_initial_runs_1.append(test_case.use_binary_files_from)
1080	additional_initial_runs_2.append(sorted(set(additional_initial_runs_1)))
1081
1082	test_case_order = []
1083	for i in range(len(additional_initial_runs_2)-1):
1084	# low and high refer to priority
1085	low = additional_initial_runs_2[i]
1086	high = additional_initial_runs_2[i+1]
1087	for item in high:
1088	while item in low:
1089	low.remove(item)
1090	test_case_order.append(low)
1091
1092	test_case_order_no_dublicates = []
1093	for test_cases in test_case_order:
1094	seen = set()
1095	seen_add = seen.add
1096	test_case_order_no_dublicates.append( [x for x in test_cases if not (x in seen or seen_add(x))] )
1097
1098	approved_test_case_order = [[]] + list(reversed(test_case_order_no_dublicates))
1099	for i, test_cases in enumerate(list(approved_test_case_order)):
1100	info = 'Case (dep)' if i < len(approved_test_case_order)-1 else 'Case'
1101	for test_case_name in list(test_cases):
1102	sys.stdout.flush()
1103	test_case = PALMTestCase(self.test_dir, test_case_name, verbose=self.verbose)
1104	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
1105	if test_case_configuration_failed:
1106	# removing as configuration failed should only apply to added dependencies
1107	approved_test_case_order[i].remove(test_case_name)
1108	logger.to_all(LogFormatter.config_table_line_template.format(info, test_case_name, 'rejected'))
1109	logger.to_all(message + '\n')
1110	elif test_case.requires_binary_files:
1111	if test_case.use_binary_files_from not in approved_test_case_order[i-1]:
1112	# removing as dependency is already removed
1113	approved_test_case_order[i].remove(test_case_name)
1114	logger.to_all(LogFormatter.config_table_line_template.format(info, test_case_name, 'disabled'))
1115	logger.to_all(colored('[requires dependency \"' + test_case.use_binary_files_from + '\"]', 'red') + '\n')
1116	else:
1117	logger.to_all(LogFormatter.config_table_line_template.format(info, test_case_name, 'approved'))
1118	logger.to_all(message + '\n')
1119	else:
1120	logger.to_all(LogFormatter.config_table_line_template.format(info, test_case_name, 'approved'))
1121	logger.to_all(message + '\n')
1122
1123	final_case_list = []
1124	for cases in approved_test_case_order:
1125	for case in cases:
1126	if case not in final_case_list:
1127	final_case_list.append(case)
1128
1129	for build_name in final_build_list:
1130	build = PALMBuild(
1131	self.test_dir,
1132	build_name,
1133	verbose=self.verbose,
1134	dry_run=self.dry_run,
1135	)
1136	configuration_failed, message = build.configure()
1137	if not configuration_failed:
1138	self.build_database[build_name] = build
1139	else:
1140	logger.to_all(message + '\n')
1141
1142	for case_name in final_case_list:
1143	test_case = PALMTestCase(
1144	self.test_dir,
1145	case_name,
1146	verbose=self.verbose,
1147	dry_run=self.dry_run,
1148	)
1149	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
1150	if not test_case_configuration_failed:
1151	self.test_case_queue.append(test_case)
1152	logger.to_all(LogFormatter.hline)
1153
1154	logger.to_all(LogFormatter.intro_table_line_template.format('Test ID:') +
1155	self.test_id + '\n')
1156	logger.to_all(LogFormatter.intro_table_line_template.format('Builds:') +
1157	str('\n' + LogFormatter.intro_table_line_template.format('')).join(sorted(self.build_database.keys())) + '\n')
1158	logger.to_all(LogFormatter.intro_table_line_template.format('Cases:') +
1159	str('\n' + LogFormatter.intro_table_line_template.format('')).join([c.name for c in self.test_case_queue]) + '\n')
1160	logger.to_all(LogFormatter.intro_table_line_template.format('Cores:') +
1161	' '.join([str(i) for i in final_cores_list]) + '\n')
1162
1163	def _execute(self, test_case, build_name, cores):
1164	job = PALMJob(
1165	self.test_dir,
1166	test_case,
1167	build_name,
1168	cores,
1169	verbose=self.verbose,
1170	dry_run=self.dry_run
1171	)
1172	if self.force_debug:
1173	build_failed_non_debug = True
1174	job_failed_non_debug = True
1175	build_failed_debug = self.build_database[build_name].build(debug=True)
1176	if build_failed_debug:
1177	job_failed_debug = True
1178	else:
1179	job_failed_debug = job.execute(debug=True)
1180	elif self.no_auto_debug:
1181	build_failed_non_debug = self.build_database[build_name].build(debug=False)
1182	if build_failed_non_debug:
1183	job_failed_non_debug = True
1184	else:
1185	job_failed_non_debug = job.execute(debug=False)
1186	build_failed_debug = None
1187	job_failed_debug = None
1188	else:
1189	build_failed_non_debug = self.build_database[build_name].build(debug=False)
1190	if build_failed_non_debug:
1191	job_failed_non_debug = True
1192	build_failed_debug = self.build_database[build_name].build(debug=True)
1193	if build_failed_debug:
1194	job_failed_debug = False
1195	else:
1196	job_failed_debug = job.execute(debug=True)
1197	else:
1198	job_failed_non_debug = job.execute(debug=False)
1199	if job_failed_non_debug:
1200	build_failed_debug = self.build_database[build_name].build(debug=True)
1201	if build_failed_debug:
1202	job_failed_debug = True
1203	else:
1204	job_failed_debug = job.execute(debug=True)
1205	else:
1206	build_failed_debug = None
1207	job_failed_debug = None
1208	return dict(
1209	build_failed_non_debug=build_failed_non_debug,
1210	job_failed_non_debug=job_failed_non_debug,
1211	build_failed_debug=build_failed_debug,
1212	job_failed_debug=job_failed_debug,
1213	)
1214
1215	def execute(self):
1216	with Logger(self.test_dir, verbose=self.verbose) as logger:
1217	logger.to_all(LogFormatter.hline)
1218	logger.to_all(LogFormatter.task_table_line_template.format('Task:', 'Case:', 'Build:', 'Cores:') + 'Status:\n')
1219	logger.to_all(LogFormatter.hline)
1220	self.test_report = dict()
1221	for test_case in self.test_case_queue:
1222	logger.to_log(LogFormatter.hline)
1223	logger.to_file(LogFormatter.hline)
1224	logger.to_file(LogFormatter.hline)
1225	status_dict = dict()
1226	for build_name in test_case.build_names:
1227	status_dict[build_name] = dict()
1228	for cores in test_case.number_of_cores:
1229	status_dict[build_name][cores] = self._execute(test_case, build_name, cores)
1230	self.test_report[test_case.name] = status_dict
1231	logger.to_log(LogFormatter.hline)
1232	logger.to_file('\n' * 10)
1233
1234	def report(self):
1235	with Logger(self.test_dir, verbose=self.verbose) as logger:
1236	logger.to_all(LogFormatter.hline)
1237	r = '{:10}' + ' total: ' + '{:<3d}' + \
1238	' ok: ' + colored('{:<3d}', 'green') + \
1239	' debugged: ' + colored('{:<3d}', 'yellow') + \
1240	' failed: ' + colored('{:<3d}', 'red')
1241	n_all = 0
1242	n_ok = 0
1243	n_debugged = 0
1244	n_failed = 0
1245	for build_name, build in self.build_database.items():
1246	status = build.report()
1247	b = status['failed_non_debug']
1248	bd = status['failed_debug']
1249	n_all += 1
1250	if not b and b is not None:
1251	n_ok += 1
1252	if bd is not None:
1253	n_debugged += 1
1254	if b and (bd or bd is None):
1255	n_failed += 1
1256	logger.to_all(r.format('Builds:', n_all, n_ok, n_debugged, n_failed) + '\n')
1257	total_failed = n_failed
1258	total_debugged = n_debugged
1259	n_all = 0
1260	n_ok = 0
1261	n_debugged = 0
1262	n_failed = 0
1263	# {'case_name': {'build_name': {4: {'build_failed_debug': None,
1264	# 'build_failed_non_debug': False,
1265	# 'job_failed_debug': None,
1266	# 'job_failed_non_debug': False}}},
1267	for case_name, case in self.test_report.items():
1268	for build_name, build in case.items():
1269	for cores, results in build.items():
1270	n_all += 1
1271	b = results['build_failed_non_debug']
1272	bd = results['build_failed_debug']
1273	j = results['job_failed_non_debug']
1274	jd = results['job_failed_debug']
1275	if not j:
1276	n_ok += 1
1277	if jd is not None:
1278	n_debugged += 1
1279	if j and (jd or jd is None):
1280	n_failed += 1
1281	logger.to_all(r.format('Tests:', n_all, n_ok, n_debugged, n_failed) + '\n')
1282	total_failed += n_failed
1283	total_debugged += n_debugged
1284	if self.fail_on_debug:
1285	return (total_failed + total_debugged) > 0
1286	else:
1287	return total_failed > 0
1288
1289
1290	class CustomCompleter:
1291
1292	def __init__(self):
1293	pass
1294
1295	def __call__(self, prefix, parsed_args, **kwargs):
1296	return (i for i in self.get_items() if i.startswith(prefix))
1297
1298	def get_items(self):
1299	return []
1300
1301
1302	class CaseCompleter(CustomCompleter):
1303
1304	def get_items(self):
1305	case_names = [name for name in next(os.walk(Environment.trunk_tests_cases_dir))[1] if not name[0] == '.']
1306	return case_names + ['all']
1307
1308
1309	class BuildCompleter(CustomCompleter):
1310
1311	def get_items(self):
1312	build_names = [name for name in next(os.walk(Environment.trunk_tests_builds_dir))[1] if not name[0] == '.']
1313	return build_names + ['all']
1314
1315
1316	class PALMTestArgumentParser(ArgumentParser):
1317
1318	def __init__(self):
1319	super().__init__(
1320	description='This is the PALM tester\n' +
1321	'Developer Support: knoop@muk.uni-hannover.de',
1322	formatter_class=RawTextHelpFormatter,
1323	add_help=True,
1324	)
1325	self.add_argument(
1326	'--version',
1327	action='version',
1328	version=version,
1329	)
1330	self.add_argument(
1331	'--verbose',
1332	action='store_true',
1333	dest='verbose',
1334	help='Increase verbosity of terminal output.',
1335	required=False,
1336	)
1337	self.add_argument(
1338	'--no-auto-debug',
1339	action='store_true',
1340	dest='no_auto_debug',
1341	help='Disable automatic debugging in case of test failure.',
1342	required=False,
1343	)
1344	self.add_argument(
1345	'--force-debug',
1346	action='store_true',
1347	dest='force_debug',
1348	help='Force debugging regardless of test failure (ignores --no-auto-debug).',
1349	required=False,
1350	)
1351	self.add_argument(
1352	'--fail-on-debug',
1353	action='store_true',
1354	dest='fail_on_debug',
1355	help='Return a non-zero exit status in case debugging was required.',
1356	required=False,
1357	)
1358	self.add_argument(
1359	'--dry-run',
1360	action='store_true',
1361	dest='dry_run',
1362	help='Prepare and process all requested tests without actually building or executing PALM.',
1363	required=False,
1364	)
1365	self.add_argument(
1366	'--no-color',
1367	action='store_true',
1368	dest='no_color',
1369	help='Disable colored terminal output.',
1370	required=False,
1371	)
1372	self.add_argument(
1373	'--cases',
1374	action='store',
1375	dest='cases',
1376	default=['all'],
1377	help='A list of test cases to be executed. (default: %(default)s)',
1378	nargs='+',
1379	required=False,
1380	type=str,
1381	metavar='STR',
1382	).completer = CaseCompleter()
1383	self.add_argument(
1384	'--builds',
1385	action='store',
1386	dest='builds',
1387	default=['all'],
1388	help='A list of builds to be executed. (default: %(default)s)',
1389	nargs='+',
1390	required=False,
1391	type=str,
1392	metavar='STR',
1393	).completer = BuildCompleter()
1394	self.add_argument(
1395	'--cores',
1396	action='store',
1397	dest='cores',
1398	default=[i for i in range(1, multiprocessing.cpu_count()+1)],
1399	choices=[i for i in range(1, multiprocessing.cpu_count()+1)],
1400	help='The number of cores tests are supposed to be executed on. (default: %(default)s)',
1401	nargs='+',
1402	required=False,
1403	type=int,
1404	metavar='INT',
1405	)
1406	self.add_argument(
1407	'--test-id',
1408	action='store',
1409	dest='test_id',
1410	default=datetime.now().strftime('%Y-%m-%d_%H:%M:%S.%f'),
1411	help='An individual test id. (default: current timestamp)',
1412	required=False,
1413	type=str,
1414	metavar='STR',
1415	)
1416
1417
1418	if __name__ == '__main__':
1419	parser = PALMTestArgumentParser()
1420	if has_argcomplete:
1421	argcomplete.autocomplete(parser)
1422	args = parser.parse_args()
1423	palm_test = PALMTest(args)
1424	palm_test.prepare()
1425	palm_test.execute()
1426	failed = palm_test.report()
1427	exit(1 if failed else 0)

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |