Home

Context Navigation

← Previous Revision
Next Revision →
Normal
Revision Log

palmtest

Last change on this file was 4838, checked in by raasch, 3 years ago
hybrid MPI/openmp testcase added
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 58.8 KB

Rev	Line
[3353]	1	#!/usr/bin/env python3
	2	# PYTHON_ARGCOMPLETE_OK
[2497]	3
[3353]	4	import os
	5	import sys
	6	import shutil
	7	from datetime import datetime
	8	import subprocess
	9	import multiprocessing
	10	import socket
	11	import getpass
	12	import math
	13	import re
	14	import threading
	15	import queue
	16	from contextlib import ContextDecorator
[2497]	17
[3353]	18	try:
	19	from argparse import ArgumentParser
	20	from argparse import RawTextHelpFormatter
	21	except ImportError:
	22	sys.exit(
	23	'ERROR: You need argparse!\n' +
	24	' install it from http://pypi.python.org/pypi/argparse\n' +
	25	' or run \"pip install argparse\".'
	26	)
[2497]	27
[3353]	28	try:
	29	import numpy as np
	30	except ImportError:
	31	sys.exit(
	32	'ERROR: You need numpy!\n' +
	33	' install it from http://pypi.python.org/pypi/numpy\n' +
	34	' or run \"python3 -m pip install numpy\".'
	35	)
[2497]	36
[3353]	37	try:
	38	import netCDF4
	39	except ImportError:
	40	sys.exit(
	41	'ERROR: You need netCDF4!\n' +
	42	' install it from http://pypi.python.org/pypi/netCDF4\n' +
	43	' or run \"python3 -m pip install netCDF4\".'
	44	)
[2497]	45
[3353]	46	try:
	47	import yaml
	48	except ImportError:
	49	sys.exit(
	50	'ERROR: You need PyYAML!\n' +
	51	' install it from http://pypi.python.org/pypi/PyYAML\n' +
	52	' or run \"python3 -m pip install PyYAML\".'
	53	)
[2497]	54
[3353]	55	try:
	56	import argcomplete
	57	except ImportError:
	58	print(
	59	'INFO: To use Tab-completion you need argcomplete!\n' +
	60	' install it from http://pypi.python.org/pypi/argcomplete\n' +
	61	' or run \"python3 -m pip install argcomplete\".'
	62	)
	63	has_argcomplete = False
	64	else:
	65	has_argcomplete = True
[2497]	66
[3353]	67	try:
[3354]	68	from termcolor import colored as tcolored
[3353]	69	except ImportError:
[3354]	70	def tcolored(string, color):
[3353]	71	return string
[2497]	72
[3354]	73	disable_colored_output = False
[2497]	74
	75
[3354]	76	def colored(string, color):
	77	if not disable_colored_output:
	78	return tcolored(string, color)
	79	else:
	80	return string
	81
	82
[4091]	83	def disable_color():
	84	global disable_colored_output
	85	disable_colored_output = True
	86
	87
[3354]	88	version = '1.0.1'
	89
[2497]	90
[4091]	91	class Environment:
[2497]	92
[4091]	93	scripts_dir = os.path.dirname(os.path.realpath(__file__))
	94	trunk_dir = os.path.realpath(os.path.join(scripts_dir, '..'))
	95	workspace_dir = os.path.realpath(os.path.join(trunk_dir, '..'))
[2497]	96
[4091]	97	trunk_tests_dir = os.path.join(trunk_dir, 'TESTS')
	98	trunk_tests_cases_dir = os.path.join(trunk_tests_dir, 'cases')
	99	trunk_tests_builds_dir = os.path.join(trunk_tests_dir, 'builds')
[2497]	100
[4091]	101	tests_dir = os.path.join(workspace_dir, 'tests')
[2579]	102
	103
[4091]	104	class LogFormatter:
[3353]	105
[4091]	106	terminal_columns, terminal_lines = shutil.get_terminal_size()
	107	hline = '#' * min(terminal_columns, 300) + '\n'
	108	table_width_intro = 12
	109	table_width_builds = len(max([s for s in next(os.walk(Environment.trunk_tests_builds_dir))[1] if not s[0] == '.'], key=len)) + len('_debug')
	110	table_width_cases = len(max([s for s in next(os.walk(Environment.trunk_tests_cases_dir))[1] if not s[0] == '.'], key=len))
	111	table_width_cores = 7
	112	table_width_total = table_width_intro + table_width_builds + table_width_cases + table_width_cores + 3
[3353]	113
[4091]	114	intro_table_line_template = \
	115	'{:' + str(table_width_intro) + '} '
[3353]	116
[4091]	117	task_table_line_template = \
	118	'{:' + str(table_width_intro) + '} ' + \
	119	'{:' + str(table_width_cases) + '} ' + \
	120	'{:' + str(table_width_builds) + '} ' + \
	121	'{:' + str(table_width_cores) + '} '
	122
	123	config_table_line_template = \
	124	'{:' + str(table_width_intro) + '} ' + \
	125	'{:' + str(max(table_width_builds, table_width_cases)) + '} ' + \
	126	'{:8} '
	127
	128	file_table_line_template = \
	129	'{:' + str(table_width_intro) + '} ' + \
	130	'{:' + str(table_width_cases + 13) + '} '
	131
	132
[3353]	133	class SignificantDigitsRounder:
	134
	135	@staticmethod
	136	def _round(value, digits=10):
	137	if value == 0.0:
	138	return value
	139	negative = value < 0.0
	140	value = -value if negative else value
	141	rounded_value = round(value, -int(math.floor(math.log10(value))) + (digits - 1))
	142	rounded_value = -rounded_value if negative else rounded_value
	143	return rounded_value
	144
	145
	146	vectorized_round = np.vectorize(_round)
	147
	148	_vectorized_round = np.vectorize(round)
	149
	150
	151	@classmethod
	152	def around(cls, array, digits=10):
	153	# TODO: divide both arrays and check decimal point
	154	sign_mask = np.ma.masked_where(array >= 0.0, array).mask
	155	pos_array = np.where(sign_mask, array, -array)
	156	non_zero_maks = np.ma.masked_where(pos_array == 0.0, pos_array).mask
	157	non_zero_array = np.where(non_zero_maks, 1.0, pos_array)
	158	i1 = -np.floor(np.log10(non_zero_array)).astype(int) + (digits - 1)
	159	rounded_non_zero_array = cls._vectorized_round(non_zero_array, i1)
	160	rounded_pos_array = np.where(non_zero_maks, 0.0, rounded_non_zero_array)
	161	return np.where(sign_mask, rounded_pos_array, -rounded_pos_array)
	162
	163
	164
	165	class Logger(ContextDecorator):
	166
	167	def __init__(self, logfile_dir, logfile_name='palmtest.log', logfile_mode='a', verbose=False):
	168	self.logfile_path = os.path.join(logfile_dir, logfile_name)
	169	self.logfile_mode = logfile_mode
	170	self.verbose = verbose
	171
	172	def __enter__(self):
	173	self._file = open(self.logfile_path, self.logfile_mode)
	174	return self
	175
	176	def to_file(self, message):
	177	self._file.write(message)
	178	self._file.flush()
	179
	180	def to_log(self, message):
	181	if self.verbose:
	182	sys.stdout.write(message)
	183	sys.stdout.flush()
	184	self._file.write(message)
	185	self._file.flush()
	186
	187	def to_all(self, message):
	188	sys.stdout.write(message)
	189	sys.stdout.flush()
	190	self._file.write(message)
	191	self._file.flush()
	192
	193	def __exit__(self, *exc):
	194	self._file.close()
	195	return False
	196
	197
	198	class Executor:
	199
	200	@staticmethod
	201	def _enqueue_output(out, queue):
	202	for line in iter(out.readline, b''):
	203	queue.put(line)
	204	out.close()
	205
	206	@staticmethod
	207	def execute(cmd, cwd='.', verbose=True, dry_run=False):
	208	assert isinstance(cmd, list)
	209	if dry_run:
	210	cmd = ['echo'] + cmd
	211	cmd_str = ' '.join(cmd)
	212	p = subprocess.Popen(cmd_str, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, bufsize=1)
	213	q = queue.Queue()
	214	t = threading.Thread(target=Executor._enqueue_output, args=(p.stdout, q))
	215	t.daemon = True # thread dies with the program
	216	t.start()
	217
	218	with Logger(cwd, verbose=verbose) as logger:
	219	# read line without blocking
[4091]	220	logger.to_log(LogFormatter.hline)
[3353]	221	logger.to_log('CMD: ' + cmd_str + '\n')
[4091]	222	logger.to_log(LogFormatter.hline)
[3353]	223	while t.is_alive():
	224	try:
	225	line = q.get_nowait() # or q.get(timeout=.1)
	226	except queue.Empty:
	227	pass # print('no output yet')
	228	else: # got line
	229	logger.to_log(line.decode("utf-8"))
	230	line = True
	231	while line:
	232	try:
	233	line = q.get_nowait() # or q.get(timeout=.1)
	234	except queue.Empty:
	235	line = False
	236	else: # got line
	237	logger.to_log(line.decode("utf-8"))
[4091]	238	logger.to_log(LogFormatter.hline)
[3353]	239
	240	rc = p.poll()
	241	failed = rc != 0
	242	return failed
	243
	244
	245	class NetCDFInterface:
	246
	247	def __init__(self, filename):
	248	self.filename = filename
	249
	250	def is_healthy(self):
	251	try:
	252	self.get_run_name()
	253	except:
	254	return False
	255	else:
	256	return True
	257
	258	def get_run_name(self):
	259	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
	260	l = getattr(netcdf, 'title').split()
	261	i = l.index('run:')
	262	return l[i+1]
	263
	264	def get_var_list(self):
	265	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
[3561]	266	var_list = list(netcdf.variables.keys())
[3353]	267	var_list = filter(None, var_list)
	268	return sorted(var_list)
	269
	270	def show_content(self):
	271	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
	272	for name in netcdf.ncattrs():
	273	print("Global attr", name, "=", getattr(netcdf, name))
	274	print(netcdf)
	275	for v in netcdf.variables:
	276	print(v)
	277
	278	def get_times_list(self):
	279	attributes, times = self.read_var('time')
	280	times = [str(time) for time in times]
	281	times = list(filter(None, times))
	282	return times
	283
	284	def contains(self, variable):
	285	return variable in self.get_var_list()
	286
	287	def read_var(self, variable):
	288	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
	289	values = netcdf.variables[variable][:] # extract values
[3561]	290	attributes = dict()
	291	try:
	292	attributes['long_name'] = netcdf.variables[variable].name
	293	except:
	294	attributes['long_name'] = ''
	295	try:
	296	attributes['unit'] = netcdf.variables[variable].units
	297	except:
	298	attributes['unit'] = ''
[3353]	299	return attributes, values
	300
	301
	302	class FileComparator:
	303
	304	@staticmethod
	305	def compare_ascii(file_path1, file_path2, start_string=None):
	306	try:
	307	with open(file_path1, 'r') as file1:
	308	content1 = file1.readlines()
	309	except OSError:
	310	return True, colored('[reference file not found]', 'red')
	311	try:
	312	with open(file_path2, 'r') as file2:
	313	content2 = file2.readlines()
	314	except OSError:
	315	return True, colored('[output file not found]', 'red')
	316	if start_string:
	317	index1 = content1.index(start_string)
	318	index2 = content2.index(start_string)
	319	comparable_content1 = content1[index1:]
	320	comparable_content2 = content2[index2:]
	321	ln = index2 + 1
	322	else:
	323	comparable_content1 = content1
	324	comparable_content2 = content2
	325	ln = 1
	326	if len(comparable_content1) != len(comparable_content2):
	327	return True, colored('[mismatch in total number of lines]', 'red')
	328	for line1, line2 in zip(comparable_content1, comparable_content2):
	329	if not line1 == line2:
	330	return True, colored('[mismatch in content starting line ' + str(ln) + ']', 'red')
	331	ln += 1
	332	return False, colored('[file ok]', 'green')
	333
	334	@staticmethod
	335	def compare_netcdf(file_path1, file_path2, digits=None):
	336	nci1 = NetCDFInterface(file_path1)
	337	nci2 = NetCDFInterface(file_path2)
	338	if not nci1.is_healthy():
	339	return True, colored('[reference file not found]', 'red')
	340	if not nci2.is_healthy():
	341	return True, colored('[output file not found]', 'red')
	342	times_list1 = nci1.get_times_list()
	343	times_list2 = nci2.get_times_list()
	344	if not times_list1 == times_list2:
	345	return True, colored('[wrong time dimension]', 'red')
	346	else:
	347	time_list = times_list1
	348	var_list1 = nci1.get_var_list()
	349	var_list2 = nci2.get_var_list()
	350	if not var_list1 == var_list2:
	351	return True, colored('[wrong set of variables]', 'red')
	352	else:
	353	var_list = var_list1
	354	content1 = dict()
	355	content2 = dict()
	356	for var in var_list:
	357	attributes1, values1 = nci1.read_var(var)
	358	attributes2, values2 = nci2.read_var(var)
	359	if sorted(attributes1.keys()) != sorted(attributes2.keys()):
	360	return True, colored('[wrong set of attributes in variable \"'+var+'\"]', 'red')
	361	if isinstance(digits, int):
	362	values1 = SignificantDigitsRounder.around(values1, digits=digits)
	363	values2 = SignificantDigitsRounder.around(values2, digits=digits)
	364	content1[var] = values1
	365	content2[var] = values2
	366	#for decimals in
	367	for i, time in enumerate(time_list):
	368	for var in var_list:
	369	t_content1 = content1[var][i]
	370	t_content2 = content2[var][i]
	371	if not (t_content1==t_content2).all():
	372	if isinstance(digits, int):
	373	return True, colored('[1st mismatch within ' + str(digits) + ' digits at time index ' + str(i) + ' in variable \"' + var + '\"]', 'red')
	374	else:
	375	return True, colored('[1st mismatch at time index ' + str(i) + ' in variable \"' + var + '\"]', 'red')
	376	return False, colored('[file ok]', 'green')
	377
	378
	379	class OutputChecker:
	380
	381	def __init__(self, test_dir, setup_name, build_name, cores, significant_digits=None, verbose=True, dry_run=False):
	382	self.test_dir = test_dir
	383	self.setup_name = setup_name
	384	self.build_name = build_name
	385	self.cores = cores
	386	self.significant_digits = significant_digits
	387	self.verbose = verbose
	388	self.dry_run = dry_run
	389	self.job_name = self.setup_name + '__' + build_name + '__' + str(self.cores)
	390	self.job_dir = os.path.join(self.test_dir, 'JOBS', self.job_name)
[4091]	391	self.ref_monitoring_dir = os.path.join(Environment.trunk_tests_cases_dir, self.setup_name, 'MONITORING')
	392	self.ref_output_dir = os.path.join(Environment.trunk_tests_cases_dir, self.setup_name, 'OUTPUT')
[3353]	393	self.res_monitoring_dir = os.path.join(self.job_dir, 'MONITORING')
	394	self.res_output_dir = os.path.join(self.job_dir, 'OUTPUT')
	395	self.failed = None
	396
	397	def get_checkable_file_dicts(self):
	398	if os.path.isdir(self.ref_monitoring_dir):
	399	file_names_monitoring = [s for s in next(os.walk(self.ref_monitoring_dir))[2]]
	400	else:
	401	file_names_monitoring = []
	402	file_paths_monitoring = []
	403	for file_name in file_names_monitoring:
	404	file_specific_ending = file_name[len(self.setup_name):]
	405	file_specific_ending_split = file_specific_ending.split('.')
	406	postfix = file_specific_ending_split[0]
	407	if len(file_specific_ending_split) > 1:
	408	extension = file_specific_ending_split[-1]
	409	else:
	410	extension = ''
	411	if len(file_specific_ending_split) > 2:
	412	cycle_info = file_specific_ending_split[1: -1]
	413	else:
	414	cycle_info = []
	415	file_paths_monitoring.append(
	416	dict(
	417	postfix=postfix,
	418	cycle_info=cycle_info,
	419	extension=extension,
	420	ref_path=self.ref_monitoring_dir,
	421	res_path=self.res_monitoring_dir,
	422	)
	423	)
	424	if os.path.isdir(self.ref_output_dir):
	425	file_names_output = [s for s in next(os.walk(self.ref_output_dir))[2]]
	426	else:
	427	file_names_output = []
	428	file_paths_output = []
	429	for file_name in file_names_output:
	430	file_specific_ending = file_name[len(self.setup_name):]
	431	file_specific_ending_split = file_specific_ending.split('.')
	432	postfix = file_specific_ending_split[0]
	433	if len(file_specific_ending_split) > 1:
	434	extension = file_specific_ending_split[-1]
	435	else:
	436	extension = ''
	437	if len(file_specific_ending_split) > 2:
	438	cycle_info = file_specific_ending_split[1: -1]
	439	else:
	440	cycle_info = []
	441	file_paths_output.append(
	442	dict(
	443	postfix=postfix,
	444	cycle_info=cycle_info,
	445	extension=extension,
	446	ref_path=self.ref_output_dir,
	447	res_path=self.res_output_dir,
	448	)
	449	)
	450	return file_paths_monitoring + file_paths_output
	451
	452	def check(self):
	453	with Logger(self.test_dir, verbose=self.verbose) as logger:
	454	logger.to_log('Checking output files:')
	455	logger.to_all('\n')
	456	failed = False
	457	for file_dict in self.get_checkable_file_dicts():
	458	file_failed = False
	459	ext_list = [file_dict['extension']] if file_dict['extension'] else []
	460	file_specific_ending = '.'.join([file_dict['postfix']] + file_dict['cycle_info'] + ext_list )
[4091]	461	logger.to_all(LogFormatter.file_table_line_template.format('Checking:', self.setup_name + file_specific_ending))
[3353]	462	ref_file_path = os.path.join(file_dict['ref_path'], self.setup_name + file_specific_ending)
	463	res_file_path = os.path.join(file_dict['res_path'], self.job_name + file_specific_ending)
[4456]	464	if re.match('_rc', file_dict['postfix']) and re.match('[0-9]{3}', file_dict['extension']):
[3353]	465	file_failed, message = FileComparator.compare_ascii(ref_file_path, res_file_path, start_string='Run-control output:\n')
	466	elif re.match('nc', file_dict['extension']):
	467	if self.significant_digits is not None:
	468	if re.match('_ts', file_dict['postfix']) and 'timeseries' in self.significant_digits:
	469	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
	470	digits=self.significant_digits['timeseries'])
	471	elif re.match('_pr', file_dict['postfix']) and 'profiles' in self.significant_digits:
	472	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
	473	digits=self.significant_digits['profiles'])
	474	else:
	475	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
	476	digits=self.significant_digits['other'])
	477	else:
	478	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path)
	479	else:
	480	message = colored('[ignored]', 'blue')
	481	if file_failed:
	482	failed = True
	483	logger.to_all(message + '\n')
	484	if self.dry_run:
	485	failed = False
	486	return failed
	487
	488
	489	class PALMJob:
	490	"""The PALM job class deals with the execution of a single PALM job"""
	491
	492	@staticmethod
	493	def get_job_name(setup_name, build_name, cores):
	494	return setup_name + '__' + build_name + '__' + str(cores)
	495
	496	def __init__(self, test_dir, test_case, build_name, cores, verbose=False, dry_run=False):
	497	self.test_dir = test_dir
	498	self.test_case = test_case
	499	self.build_name = build_name
	500	self.cores = cores
	501	self.verbose = verbose
	502	self.dry_run = dry_run
	503
	504	self.attempted_debug = False
	505	self.failed_debug = None
	506	self.attempted_non_debug = False
	507	self.failed_non_debug = None
	508
	509	def _link_restart_files(self, build_name):
	510	if self.dry_run:
	511	return True, colored('[restart data dry]', 'blue')
	512	name = self.get_job_name(self.test_case.name, build_name, self.cores)
	513	source_name = self.get_job_name(self.test_case.use_binary_files_from, build_name, self.cores)
	514	source_restart_dir = os.path.join(self.test_dir, 'JOBS', source_name, 'RESTART')
	515	try:
	516	source_data_dirs_grp = [d for r, d, f in os.walk(source_restart_dir)]
	517	except:
	518	source_data_dirs_grp = []
	519	if len(source_data_dirs_grp) == 0:
	520	source_data_dirs = []
	521	else:
	522	source_data_dirs = source_data_dirs_grp[0]
	523	if len(source_data_dirs) == 0 and re.match('.+_debug', build_name):
	524	source_build_name = build_name[:-len('_debug')]
	525	source_name = self.get_job_name(self.test_case.use_binary_files_from, source_build_name, self.cores)
	526	source_restart_dir = os.path.join(self.test_dir, 'JOBS', source_name, 'RESTART')
	527	try:
	528	source_data_dirs_grp = [d for r, d, f in os.walk(source_restart_dir)]
	529	except:
	530	source_data_dirs_grp = []
	531	if len(source_data_dirs_grp) == 0:
	532	source_data_dirs = []
	533	else:
	534	source_data_dirs = source_data_dirs_grp[0]
	535	if len(source_data_dirs) == 0:
	536	source_data_dir = 'no_restart_data'
	537	else:
	538	source_data_dir = sorted(source_data_dirs)[-1]
	539	source_data_dir_path = os.path.join(source_restart_dir, source_data_dir)
	540	if os.path.isdir(source_data_dir_path) and re.match('.+_d3d.*', source_data_dir):
	541	job_restart_dir = os.path.join(self.test_dir, 'JOBS', name, 'RESTART')
[4091]	542	os.makedirs(job_restart_dir, exist_ok=False)
[3353]	543	job_data_dir_path = os.path.join(job_restart_dir, name + '_d3d')
	544	os.symlink(source_data_dir_path, job_data_dir_path, target_is_directory=True)
	545	return False, colored('[linked restart data from: ' + source_data_dir_path + ']', 'green')
	546	else:
	547	return True, colored('[no restart data found]', 'red')
	548
	549	def _execute(self, name, build_name):
[4751]	550	cmd_list = [
	551	os.path.join(self.test_dir, 'trunk', 'SCRIPTS', 'palmrun'),
	552	'-c', '\"' + build_name + '\"',
	553	'-r', name,
	554	'-a', '\"' + ' '.join(self.test_case.activation_strings) + '\"',
	555	'-X', str(self.cores),
	556	]
	557	if self.test_case.omp_num_threads_found:
	558	cmd_list.extend(
	559	[
	560	'-O', str(self.test_case.omp_num_threads),
	561	]
	562	)
[4838]	563	if self.test_case.tasks_per_node_found:
	564	cmd_list.extend(
	565	[
	566	'-T', str(self.test_case.tasks_per_node),
	567	]
	568	)
[4751]	569	cmd_list.extend(
[3353]	570	[
	571	'-B',
	572	'-v',
	573	'-z',
[4751]	574	]
	575	)
	576	execution_failed = Executor.execute(
	577	cmd_list,
[3353]	578	cwd=self.test_dir,
	579	verbose=self.verbose,
	580	dry_run=self.dry_run,
	581	)
	582
	583	if self.dry_run:
	584	return False, colored('[execution dry]', 'blue')
	585	elif execution_failed:
	586	return True, colored('[execution failed]', 'red')
	587	else:
	588	return False, colored('[execution ok]', 'green')
	589
	590	def _check(self, build_name):
	591	checker = OutputChecker(
	592	self.test_dir,
	593	self.test_case.name,
	594	build_name,
	595	self.cores,
	596	significant_digits=self.test_case.significant_digits,
	597	verbose=self.verbose,
	598	dry_run=self.dry_run,
	599	)
	600	check_failed = checker.check()
	601
	602	if self.dry_run:
	603	return False, colored('[checks dry]', 'blue')
	604	if check_failed:
	605	return True, colored('[checks failed]', 'red')
	606	else:
	607	return False, colored('[checks ok]', 'green')
	608
	609	def execute(self, debug=False):
	610	if debug:
	611	attempted = self.attempted_debug
	612	build_name = self.build_name + '_debug'
	613	failed = self.failed_debug
	614	else:
	615	attempted = self.attempted_non_debug
	616	build_name = self.build_name
	617	failed = self.failed_non_debug
	618
	619	if not attempted:
	620	with Logger(self.test_dir, verbose=self.verbose) as logger:
[4091]	621	status_prefix = LogFormatter.task_table_line_template.format('Testing:', self.test_case.name, build_name, self.cores)
[3353]	622	logger.to_all(status_prefix)
	623	logger.to_log('[started]' + '\n')
	624	attempted = True
	625
	626	name = self.get_job_name(self.test_case.name, build_name, self.cores)
	627
	628	input_dir = os.path.join(self.test_dir, 'JOBS', name, 'INPUT')
[4091]	629	os.makedirs(input_dir, exist_ok=False)
[3353]	630
	631	# copying needs to be done per file, because input files need to be renamed
	632	for input_file in self.test_case.input_file_names:
	633	postfix = input_file[len(self.test_case.name):]
	634	src = os.path.join(self.test_case.input_dir, input_file)
	635	dst = os.path.join(input_dir, name + postfix)
	636	shutil.copy(src, dst)
	637
	638	# copying the entire directory is ok, because source files do not need to be renamed
	639	user_code_dir = os.path.join(self.test_dir, 'JOBS', name, 'USER_CODE')
	640	if os.path.isdir(self.test_case.user_code_dir):
	641	shutil.copytree(self.test_case.user_code_dir, user_code_dir, copy_function=shutil.copy)
	642
	643	if self.test_case.requires_binary_files:
	644	link_restart_files_failed, message = self._link_restart_files(build_name)
	645	logger.to_log(status_prefix)
	646	logger.to_log(message + ' ')
	647	logger.to_log('\n')
	648
	649	failed, message = self._execute(name, build_name)
	650	logger.to_log(status_prefix)
	651	logger.to_all(message + ' ')
	652	logger.to_log('\n')
	653
	654	failed, message = self._check(build_name)
	655	logger.to_log(status_prefix)
	656	logger.to_log(message + ' ')
	657
	658	logger.to_all('\n')
	659
	660	if debug:
	661	self.attempted_debug = attempted
	662	self.failed_debug = failed
	663	else:
	664	self.attempted_non_debug = attempted
	665	self.failed_non_debug = failed
	666
	667	return failed
	668
	669	def status(self):
	670	return dict(
	671	attempted=self.attempted_non_debug or self.attempted_debug,
	672	failed=self.failed_non_debug and self.failed_debug,
	673	debugged=self.attempted_debug,
	674	non_debug_failed=self.failed_non_debug,
	675	)
	676
	677
	678	class PALMBuild:
	679	"""The PALM build class deals with configuration and execution of all required PALM builds"""
	680
	681	def __init__(self, test_dir, build_name, verbose=False, dry_run=False):
	682	self.test_dir = test_dir
	683	self.build_name = build_name
	684	self.verbose = verbose
	685	self.dry_run = dry_run
	686	self.configured = False
	687	self.executed = False
	688	self.available = False
	689	self.requires_mpi = False
	690	self.requires_netcdf = False
	691	self.requires_fftw = False
	692	self.requires_rrtmg = False
	693	self.attempted_non_debug = False
	694	self.attempted_debug = False
	695	self.failed_non_debug = None
	696	self.failed_debug = None
	697
	698	def configure(self):
	699	try:
[4091]	700	with open(os.path.join(Environment.trunk_tests_builds_dir, self.build_name, 'build_config.yml'), 'r') as f:
[4554]	701	build_config = yaml.load(f, Loader=yaml.SafeLoader)
[3353]	702	except:
	703	return True, colored('[build not found]', 'red')
	704
	705	if 'compiler' in build_config:
	706	self.compiler = build_config['compiler']
	707	else:
	708	return True, colored('[missing \"compiler\" keyword]', 'red')
	709
	710	if not isinstance(self.compiler, dict):
	711	return True, colored('[\"compiler\" keyword must be dict]', 'red')
	712
	713	if 'linker' in build_config:
	714	self.linker = build_config['linker']
	715	else:
	716	return True, colored('[missing \"linker\" keyword]', 'red')
	717
	718	if not isinstance(self.linker, dict):
	719	return True, colored('[\"linker\" keyword must be dict]', 'red')
	720
	721	if 'mpi_wrapper' in self.compiler:
	722	if 'mpi_wrapper}}' in self.compiler['mpi_wrapper']:
	723	self.requires_mpi = True
	724	else:
	725	return True, colored('[missing \"mpi_wrapper\" keyword]', 'red')
	726
	727	if 'includes' in self.compiler:
	728	for include in self.compiler['includes']:
	729	if 'include.netcdf}}' in include:
	730	self.requires_netcdf = True
	731	if 'include.fftw}}' in include:
	732	self.requires_fftw = True
	733	if 'include.rrtmg}}' in include:
	734	self.requires_rrtmg = True
	735	else:
	736	return True, colored('[missing \"includes\" keyword in compiler]', 'red')
	737
	738	if 'options' in self.linker:
	739	for lib in self.linker['options']:
	740	if 'lib.netcdf}}' in lib:
	741	self.requires_netcdf = True
	742	if 'lib.fftw}}' in lib:
	743	self.requires_fftw = True
	744	if 'lib.rrtmg}}' in lib:
	745	self.requires_rrtmg = True
	746	else:
	747	return True, colored('[missing \"options\" keyword in linker]', 'red')
	748
	749	library_names = []
	750	if self.requires_netcdf:
	751	library_names.append('netcdf')
	752	if self.requires_fftw:
	753	library_names.append('fftw')
	754	if self.requires_rrtmg:
	755	library_names.append('rrtmg')
	756
	757	if not 'executable' in self.compiler:
	758	return True, colored('[missing \"executable\" keyword in compiler]', 'red')
	759
	760	if not 'definitions' in self.compiler:
	761	return True, colored('[missing \"definitions\" keyword in compiler]', 'red')
	762
	763	if not 'options' in self.compiler:
	764	return True, colored('[missing \"options\" keyword in compiler]', 'red')
	765
	766	if not 'default' in self.compiler['options']:
	767	return True, colored('[missing \"default\" keyword in compiler.options]', 'red')
	768
	769	if not 'debug' in self.compiler['options']:
	770	return True, colored('[missing \"debug\" keyword in compiler.options]', 'red')
	771
	772	try:
[4091]	773	with open(os.path.join(Environment.workspace_dir, 'palmtest.yml'), 'r') as f:
[4554]	774	palmtest_config = yaml.load(f, Loader=yaml.SafeLoader)
[3353]	775	except:
	776	return True, colored('[palmtest.yml not found]', 'red')
	777
	778	if 'palm_config_template' in palmtest_config:
	779	if isinstance(palmtest_config['palm_config_template'], str):
	780	custom_template = palmtest_config['palm_config_template']
	781	try:
	782	with open(os.path.join(custom_template), 'r') as palm_config_template_file:
	783	template = palm_config_template_file.read()
	784	except:
	785	try:
[4091]	786	with open(os.path.join(Environment.scripts_dir, '.palm.config.default.in'), 'r') as palm_config_template_file:
[3353]	787	template = palm_config_template_file.read()
	788	except:
	789	return True, colored('[trunk/SCRIPTS/.palm.config.default.in not found]', 'red')
	790
	791	template = template.replace('@CMAKE_INSTALL_PREFIX@', self.test_dir)
	792	template = template.replace('@PALM_HOSTNAME@', socket.gethostname())
	793	template = template.replace('@CMAKE_USERNAME@', getpass.getuser())
	794	template = template.replace('@MPI_Fortran_COMPILER@', self.compiler['mpi_wrapper'])
	795	template = template.replace('@CMAKE_Fortran_COMPILER@', self.compiler['executable'])
	796	cpp_options_str = ['-D' + s for s in self.compiler['definitions']]
	797	template = template.replace('@PALM_CPP_OPTIONS_STR@', ' '.join(cpp_options_str))
[4091]	798	template = template.replace('@PALM_CORES@', str(multiprocessing.cpu_count()))
[3353]	799	template = template.replace('@PALM_COMPILER_OPTIONS@', '{{palmtest.compiler.options}} ' + ' '.join(self.compiler['includes']))
	800	template = template.replace('@PALM_LINKER_OPTIONS@', ' '.join(self.linker['options']))
	801
	802	if 'environments' in palmtest_config:
	803	available_environments = palmtest_config['environments']
	804	else:
	805	return True, colored('[missing \"environments\" keyword in palmtest.yml]', 'red')
	806
	807	if 'id' in self.compiler:
	808	c_id = self.compiler['id']
	809	else:
	810	return True, colored('[missing \"id\" keyword in compiler]', 'red')
	811
	812	if c_id in available_environments:
	813	self.available = True
	814
	815	environment = available_environments[c_id]
[3682]	816
	817	if 'mpi_execution_command' in environment:
	818	template = template.replace('@PALM_EXECUTE_COMMAND@', environment['mpi_execution_command'])
	819	else:
	820	template = template.replace('@PALM_EXECUTE_COMMAND@', 'mpirun -n {{mpi_tasks}}')
	821
[3353]	822	if 'executable' not in environment:
	823	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"executable\"]', 'red')
	824	value = environment['executable']
	825	if isinstance(value, str):
	826	template = template.replace('{{' + '.'.join([c_id, 'executable']) + '}}', value)
	827	if self.requires_mpi:
	828	if 'mpi_wrapper' not in environment:
	829	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"mpi_wrapper\"]', 'red')
	830	value = environment['mpi_wrapper']
	831	if isinstance(value, str):
	832	template = template.replace('{{' + '.'.join([c_id, 'mpi_wrapper']) + '}}', value)
	833	if 'include' not in environment:
	834	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"include\"]', 'red')
	835	if 'lib' not in environment:
	836	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"lib\"]', 'red')
	837	for lib in library_names:
	838	if lib not in environment['include']:
	839	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"include.'+lib+'\"]', 'red')
	840	value = environment['include'][lib]
	841	if isinstance(value, str):
	842	template = template.replace('{{' + '.'.join([c_id, 'include', lib]) + '}}', value)
	843	if lib not in environment['lib']:
	844	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"lib.'+lib+'\"]', 'red')
	845	value = environment['lib'][lib]
	846	if isinstance(value, str):
	847	template = template.replace('{{' + '.'.join([c_id, 'lib', lib]) + '}}', value)
	848
	849	with open(os.path.join(self.test_dir, '.palm.config.' + self.build_name), 'w') as palm_config_file:
	850	palm_config_file.write(
	851	template.replace(
	852	'{{palmtest.compiler.options}}',
	853	' '.join(self.compiler['options']['default']),
	854	)
	855	)
	856	with open(os.path.join(self.test_dir, '.palm.config.' + self.build_name + '_debug'), 'w') as palm_config_file:
	857	palm_config_file.write(
	858	template.replace(
	859	'{{palmtest.compiler.options}}',
	860	' '.join(self.compiler['options']['debug']),
	861	)
	862	)
	863	self.configured = True
	864	return False, colored('[configuration ok]', 'green')
	865
	866	else:
	867	return True, colored('[palmtest.yml environment \"' + c_id + '\" not found]', 'red')
	868
	869	def _execute(self, build_name):
	870	self.attempted = True
	871	build_failed = Executor.execute(
	872	[
	873	os.path.join(self.test_dir, 'trunk', 'SCRIPTS', 'palmbuild'),
[3457]	874	'-c', '\"' + build_name + '\"',
[3353]	875	'-v',
	876	],
	877	cwd=self.test_dir,
	878	verbose=self.verbose,
	879	dry_run=self.dry_run,
	880	)
	881
	882	if self.dry_run:
	883	return False, colored('[build dry]', 'blue')
	884	if build_failed:
	885	return True, colored('[build failed]', 'red')
	886	else:
	887	return False, colored('[build ok]', 'green')
	888
	889	def build(self, debug=False):
	890	if debug:
	891	attempted = self.attempted_debug
	892	build_name = self.build_name + '_debug'
	893	failed = self.failed_debug
	894	else:
	895	attempted = self.attempted_non_debug
	896	build_name = self.build_name
	897	failed = self.failed_non_debug
	898
	899	if not attempted:
	900	with Logger(self.test_dir, verbose=self.verbose) as logger:
[4091]	901	status_prefix = LogFormatter.task_table_line_template.format('Building:', '', build_name, '')
[3353]	902	logger.to_all(status_prefix)
	903	logger.to_log('[started]' + '\n')
	904	attempted = True
	905
	906	failed, message = self._execute(build_name)
	907	logger.to_log(status_prefix)
	908	logger.to_all(message + ' ')
	909	logger.to_all('\n')
	910
	911	if debug:
	912	self.attempted_debug = attempted
	913	self.failed_debug = failed
	914	else:
	915	self.attempted_non_debug = attempted
	916	self.failed_non_debug = failed
	917
	918	return failed
	919
	920	def report(self):
	921	return dict(
	922	failed_debug=self.failed_debug,
	923	failed_non_debug=self.failed_non_debug,
	924	)
	925
	926
	927	class PALMTestCase:
	928	"""The PALM test case class deals with the configuration and execution of all PALM test cases"""
	929
	930	def __init__(self,test_dir, name, verbose=False, dry_run=False):
	931	self.test_dir = test_dir
	932	self.name = name
	933	self.verbose = verbose
	934	self.dry_run = dry_run
[4091]	935	self.user_code_dir = os.path.join(Environment.trunk_tests_cases_dir, self.name, 'USER_CODE')
	936	self.input_dir = os.path.join(Environment.trunk_tests_cases_dir, self.name, 'INPUT')
[3353]	937	self.number_of_cores = []
	938	self.build_names = []
	939	self.input_file_names = []
	940	self.configured = False
	941
	942	def configure(self, requested_build_names, requested_cores):
[4091]	943	f_name = os.path.join(Environment.trunk_tests_cases_dir, self.name, 'case_config.yml')
[3353]	944	try:
	945	with open(f_name, 'r') as f:
[4554]	946	config = yaml.load(f, Loader=yaml.SafeLoader)
[3353]	947	except:
	948	return True, colored('[Case \"' + self.name + '\" could not be found.]', 'red')
	949	try:
	950	self.use_binary_files_from = config['use_binary_files_from']
	951	except:
	952	self.use_binary_files_from = None
	953	self.requires_binary_files = bool(self.use_binary_files_from)
	954
	955	if 'allowed_builds' not in config:
	956	return True, colored('[missing \"allowed_builds\" keyword]', 'red')
	957	self.allowed_build_names = config['allowed_builds']
	958
	959	if 'allowed_number_of_cores' not in config:
	960	return True, colored('[missing \"allowed_number_of_cores\" keyword]', 'red')
	961	self.allowed_number_of_cores = config['allowed_number_of_cores']
	962
[4751]	963	if 'omp_num_threads' in config:
	964	self.omp_num_threads_found = True
	965	self.omp_num_threads = config['omp_num_threads']
	966	else:
	967	self.omp_num_threads_found = False
	968
[4838]	969	if 'tasks_per_node' in config:
	970	self.tasks_per_node_found = True
	971	self.tasks_per_node = config['tasks_per_node']
	972	else:
	973	self.tasks_per_node_found = False
	974
[3353]	975	if 'activation_strings' not in config:
	976	return True, colored('[missing \"activation_strings\" keyword]', 'red')
	977	self.activation_strings = config['activation_strings']
	978
	979	if 'significant_digits_for_netcdf_checks' not in config:
	980	return True, colored('[missing \"significant_digits_for_netcdf_checks\" keyword]', 'red')
	981	self.significant_digits = config['significant_digits_for_netcdf_checks']
	982
	983	if 'timeseries' not in config['significant_digits_for_netcdf_checks']:
	984	return True, colored('[missing \"timeseries\" keyword in significant_digits_for_netcdf_checks]', 'red')
	985
	986	if 'profiles' not in config['significant_digits_for_netcdf_checks']:
	987	return True, colored('[missing \"profiles\" keyword in significant_digits_for_netcdf_checks]', 'red')
	988
	989	if 'other' not in config['significant_digits_for_netcdf_checks']:
	990	return True, colored('[missing \"other\" keyword in significant_digits_for_netcdf_checks]', 'red')
	991
	992	self.number_of_cores = sorted(set(requested_cores).intersection(self.allowed_number_of_cores))
	993	self.build_names = sorted(set(requested_build_names).intersection(self.allowed_build_names))
	994	self.input_file_names = [s for s in next(os.walk(self.input_dir))[2]]
	995	self.configured = True
	996	if len(self.number_of_cores) == 0 :
	997	return True, colored('[no allowed cores requested]', 'blue')
	998	if len(self.build_names) == 0:
	999	return True, colored('[no allowed builds requested]', 'blue')
	1000	if len(self.input_file_names) == 0:
	1001	return True, colored('[no input files found]', 'red')
	1002	return False, colored('[configuration ok]', 'green')
	1003
	1004
	1005
	1006	class PALMTest:
	1007
	1008	def __init__(self, args):
	1009	self.verbose = args.verbose
	1010	self.no_auto_debug = args.no_auto_debug
	1011	self.force_debug = args.force_debug
	1012	self.fail_on_debug = args.fail_on_debug
	1013	self.dry_run = args.dry_run
[3354]	1014	self.no_color = args.no_color
[3353]	1015	self.test_id = args.test_id
	1016	self.test_case_names = args.cases
	1017	self.requested_build_names = args.builds
	1018	self.requested_cores = args.cores
	1019	self.test_case_queue = []
	1020	self.build_database = dict()
	1021
	1022	def prepare(self):
[4091]	1023	if self.no_color:
	1024	disable_color()
	1025	self.test_dir = os.path.join(Environment.tests_dir, self.test_id)
[3353]	1026	try:
[4091]	1027	os.makedirs(self.test_dir, exist_ok=False)
[3353]	1028	except:
	1029	print('ERROR: Found existing test directory: ' + self.test_dir)
	1030	exit(1)
	1031	with Logger(self.test_dir, verbose=self.verbose) as logger:
[4091]	1032	logger.to_all(LogFormatter.hline)
[3353]	1033	logger.to_all('This is the PALM tester (version: ' + version + ')' + '\n')
[4091]	1034	logger.to_all(LogFormatter.hline)
[3353]	1035	try:
[4091]	1036	with open(os.path.join(Environment.workspace_dir, 'palmtest.yml'), 'r') as f:
[3353]	1037	pass
	1038	except:
	1039	logger.to_all('ERROR: No palmtest.yml file was found in your working directory!\n')
	1040	logger.to_all('INFO: A template for this file can be found at: trunk/TESTS/palmtest.yml\n')
	1041	logger.to_all(' Please copy the template to your working directory and adjust it to your system!\n')
	1042	exit(1)
	1043
	1044	self.execution_trunk_dir = os.path.join(self.test_dir, 'trunk')
[4091]	1045	os.symlink(Environment.trunk_dir, self.execution_trunk_dir)
[3353]	1046	self.execution_jobs_dir = os.path.join(self.test_dir, 'JOBS')
[4091]	1047	os.makedirs(self.execution_jobs_dir, exist_ok=False)
[3353]	1048
	1049	try:
[4091]	1050	with open(os.path.join(Environment.scripts_dir, '.palm.iofiles'), 'r') as iofiles_template_file:
[3353]	1051	iofiles_template = iofiles_template_file.read()
	1052	with open(os.path.join(self.test_dir, '.palm.iofiles'), 'w') as iofiles_file:
[4816]	1053	iofiles_file.write(iofiles_template.replace('$fast_io_catalog', '$base_data').replace('$restart_data_path', '$base_data').replace('$output_data_path', '$base_data'))
[3353]	1054	except:
	1055	logger.to_all('ERROR: No .palm.iofiles file was found in trunk/SCRIPTS/')
	1056	exit(1)
	1057
[4091]	1058	available_cores = multiprocessing.cpu_count()
[3353]	1059	final_cores_list = list(filter(lambda x: x <= available_cores, self.requested_cores))
	1060
[4091]	1061	logger.to_all(LogFormatter.config_table_line_template.format('Object:', 'Name:', 'Action:') + 'Status:\n')
	1062	logger.to_all(LogFormatter.hline)
[3353]	1063
	1064	if 'all' in self.requested_build_names:
[4091]	1065	self.requested_build_names = [name for name in next(os.walk(Environment.trunk_tests_builds_dir))[1] if not name[0] == '.']
[3353]	1066	found_build_names = []
	1067	for build_name in self.requested_build_names:
	1068	build = PALMBuild(self.test_dir, build_name, verbose=self.verbose, dry_run=self.dry_run)
	1069	configuration_failed, message = build.configure()
	1070	if not configuration_failed:
	1071	self.build_database[build_name] = build
	1072	found_build_names.append(build_name)
[4091]	1073	logger.to_all(LogFormatter.config_table_line_template.format('Build', build_name, 'approved'))
[3353]	1074	logger.to_all(message + '\n')
	1075	else:
[4091]	1076	logger.to_all(LogFormatter.config_table_line_template.format('Build', build_name, 'rejected'))
[3353]	1077	logger.to_all(message + '\n')
	1078	final_build_list = found_build_names
	1079
	1080	if 'all' in self.test_case_names:
[4091]	1081	self.test_case_names = sorted([name for name in next(os.walk(Environment.trunk_tests_cases_dir))[1] if not name[0] == '.'])
[3353]	1082
	1083	additional_initial_runs_2 = [self.test_case_names]
	1084	while len(additional_initial_runs_2[-1]) > 0:
	1085	additional_initial_runs_1 = []
	1086	for test_case_name in additional_initial_runs_2[-1]:
	1087	test_case = PALMTestCase(self.test_dir, test_case_name, verbose=self.verbose)
	1088	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
	1089	if not test_case_configuration_failed:
	1090	if test_case.requires_binary_files:
	1091	additional_initial_runs_1.append(test_case.use_binary_files_from)
	1092	additional_initial_runs_2.append(sorted(set(additional_initial_runs_1)))
	1093
	1094	test_case_order = []
	1095	for i in range(len(additional_initial_runs_2)-1):
	1096	# low and high refer to priority
	1097	low = additional_initial_runs_2[i]
	1098	high = additional_initial_runs_2[i+1]
	1099	for item in high:
	1100	while item in low:
	1101	low.remove(item)
	1102	test_case_order.append(low)
	1103
	1104	test_case_order_no_dublicates = []
	1105	for test_cases in test_case_order:
	1106	seen = set()
	1107	seen_add = seen.add
	1108	test_case_order_no_dublicates.append( [x for x in test_cases if not (x in seen or seen_add(x))] )
	1109
	1110	approved_test_case_order = [[]] + list(reversed(test_case_order_no_dublicates))
	1111	for i, test_cases in enumerate(list(approved_test_case_order)):
	1112	info = 'Case (dep)' if i < len(approved_test_case_order)-1 else 'Case'
	1113	for test_case_name in list(test_cases):
	1114	sys.stdout.flush()
	1115	test_case = PALMTestCase(self.test_dir, test_case_name, verbose=self.verbose)
	1116	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
	1117	if test_case_configuration_failed:
	1118	# removing as configuration failed should only apply to added dependencies
	1119	approved_test_case_order[i].remove(test_case_name)
[4091]	1120	logger.to_all(LogFormatter.config_table_line_template.format(info, test_case_name, 'rejected'))
[3353]	1121	logger.to_all(message + '\n')
	1122	elif test_case.requires_binary_files:
	1123	if test_case.use_binary_files_from not in approved_test_case_order[i-1]:
	1124	# removing as dependency is already removed
	1125	approved_test_case_order[i].remove(test_case_name)
[4091]	1126	logger.to_all(LogFormatter.config_table_line_template.format(info, test_case_name, 'disabled'))
[3353]	1127	logger.to_all(colored('[requires dependency \"' + test_case.use_binary_files_from + '\"]', 'red') + '\n')
	1128	else:
[4091]	1129	logger.to_all(LogFormatter.config_table_line_template.format(info, test_case_name, 'approved'))
[3353]	1130	logger.to_all(message + '\n')
	1131	else:
[4091]	1132	logger.to_all(LogFormatter.config_table_line_template.format(info, test_case_name, 'approved'))
[3353]	1133	logger.to_all(message + '\n')
	1134
	1135	final_case_list = []
	1136	for cases in approved_test_case_order:
	1137	for case in cases:
	1138	if case not in final_case_list:
	1139	final_case_list.append(case)
	1140
	1141	for build_name in final_build_list:
	1142	build = PALMBuild(
	1143	self.test_dir,
	1144	build_name,
	1145	verbose=self.verbose,
	1146	dry_run=self.dry_run,
	1147	)
	1148	configuration_failed, message = build.configure()
	1149	if not configuration_failed:
	1150	self.build_database[build_name] = build
	1151	else:
	1152	logger.to_all(message + '\n')
	1153
	1154	for case_name in final_case_list:
	1155	test_case = PALMTestCase(
	1156	self.test_dir,
	1157	case_name,
	1158	verbose=self.verbose,
	1159	dry_run=self.dry_run,
	1160	)
	1161	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
	1162	if not test_case_configuration_failed:
	1163	self.test_case_queue.append(test_case)
[4091]	1164	logger.to_all(LogFormatter.hline)
[3353]	1165
[4091]	1166	logger.to_all(LogFormatter.intro_table_line_template.format('Test ID:') +
	1167	self.test_id + '\n')
	1168	logger.to_all(LogFormatter.intro_table_line_template.format('Builds:') +
	1169	str('\n' + LogFormatter.intro_table_line_template.format('')).join(sorted(self.build_database.keys())) + '\n')
	1170	logger.to_all(LogFormatter.intro_table_line_template.format('Cases:') +
	1171	str('\n' + LogFormatter.intro_table_line_template.format('')).join([c.name for c in self.test_case_queue]) + '\n')
	1172	logger.to_all(LogFormatter.intro_table_line_template.format('Cores:') +
	1173	' '.join([str(i) for i in final_cores_list]) + '\n')
[3353]	1174
	1175	def _execute(self, test_case, build_name, cores):
	1176	job = PALMJob(
	1177	self.test_dir,
	1178	test_case,
	1179	build_name,
	1180	cores,
	1181	verbose=self.verbose,
	1182	dry_run=self.dry_run
	1183	)
	1184	if self.force_debug:
	1185	build_failed_non_debug = True
	1186	job_failed_non_debug = True
	1187	build_failed_debug = self.build_database[build_name].build(debug=True)
	1188	if build_failed_debug:
	1189	job_failed_debug = True
	1190	else:
	1191	job_failed_debug = job.execute(debug=True)
	1192	elif self.no_auto_debug:
	1193	build_failed_non_debug = self.build_database[build_name].build(debug=False)
	1194	if build_failed_non_debug:
	1195	job_failed_non_debug = True
	1196	else:
	1197	job_failed_non_debug = job.execute(debug=False)
	1198	build_failed_debug = None
	1199	job_failed_debug = None
	1200	else:
	1201	build_failed_non_debug = self.build_database[build_name].build(debug=False)
	1202	if build_failed_non_debug:
	1203	job_failed_non_debug = True
	1204	build_failed_debug = self.build_database[build_name].build(debug=True)
	1205	if build_failed_debug:
	1206	job_failed_debug = False
	1207	else:
	1208	job_failed_debug = job.execute(debug=True)
	1209	else:
	1210	job_failed_non_debug = job.execute(debug=False)
	1211	if job_failed_non_debug:
	1212	build_failed_debug = self.build_database[build_name].build(debug=True)
	1213	if build_failed_debug:
	1214	job_failed_debug = True
	1215	else:
	1216	job_failed_debug = job.execute(debug=True)
	1217	else:
	1218	build_failed_debug = None
	1219	job_failed_debug = None
	1220	return dict(
	1221	build_failed_non_debug=build_failed_non_debug,
	1222	job_failed_non_debug=job_failed_non_debug,
	1223	build_failed_debug=build_failed_debug,
	1224	job_failed_debug=job_failed_debug,
	1225	)
	1226
	1227	def execute(self):
	1228	with Logger(self.test_dir, verbose=self.verbose) as logger:
[4091]	1229	logger.to_all(LogFormatter.hline)
	1230	logger.to_all(LogFormatter.task_table_line_template.format('Task:', 'Case:', 'Build:', 'Cores:') + 'Status:\n')
	1231	logger.to_all(LogFormatter.hline)
[3353]	1232	self.test_report = dict()
	1233	for test_case in self.test_case_queue:
[4091]	1234	logger.to_log(LogFormatter.hline)
	1235	logger.to_file(LogFormatter.hline)
	1236	logger.to_file(LogFormatter.hline)
[3353]	1237	status_dict = dict()
	1238	for build_name in test_case.build_names:
	1239	status_dict[build_name] = dict()
	1240	for cores in test_case.number_of_cores:
	1241	status_dict[build_name][cores] = self._execute(test_case, build_name, cores)
	1242	self.test_report[test_case.name] = status_dict
[4091]	1243	logger.to_log(LogFormatter.hline)
[3353]	1244	logger.to_file('\n' * 10)
	1245
	1246	def report(self):
	1247	with Logger(self.test_dir, verbose=self.verbose) as logger:
[4091]	1248	logger.to_all(LogFormatter.hline)
[3353]	1249	r = '{:10}' + ' total: ' + '{:<3d}' + \
	1250	' ok: ' + colored('{:<3d}', 'green') + \
	1251	' debugged: ' + colored('{:<3d}', 'yellow') + \
	1252	' failed: ' + colored('{:<3d}', 'red')
	1253	n_all = 0
	1254	n_ok = 0
	1255	n_debugged = 0
	1256	n_failed = 0
	1257	for build_name, build in self.build_database.items():
	1258	status = build.report()
	1259	b = status['failed_non_debug']
	1260	bd = status['failed_debug']
	1261	n_all += 1
	1262	if not b and b is not None:
	1263	n_ok += 1
	1264	if bd is not None:
	1265	n_debugged += 1
	1266	if b and (bd or bd is None):
	1267	n_failed += 1
	1268	logger.to_all(r.format('Builds:', n_all, n_ok, n_debugged, n_failed) + '\n')
	1269	total_failed = n_failed
	1270	total_debugged = n_debugged
	1271	n_all = 0
	1272	n_ok = 0
	1273	n_debugged = 0
	1274	n_failed = 0
	1275	# {'case_name': {'build_name': {4: {'build_failed_debug': None,
	1276	# 'build_failed_non_debug': False,
	1277	# 'job_failed_debug': None,
	1278	# 'job_failed_non_debug': False}}},
	1279	for case_name, case in self.test_report.items():
	1280	for build_name, build in case.items():
	1281	for cores, results in build.items():
	1282	n_all += 1
	1283	b = results['build_failed_non_debug']
	1284	bd = results['build_failed_debug']
	1285	j = results['job_failed_non_debug']
	1286	jd = results['job_failed_debug']
	1287	if not j:
	1288	n_ok += 1
	1289	if jd is not None:
	1290	n_debugged += 1
	1291	if j and (jd or jd is None):
	1292	n_failed += 1
[3354]	1293	logger.to_all(r.format('Tests:', n_all, n_ok, n_debugged, n_failed) + '\n')
[3353]	1294	total_failed += n_failed
	1295	total_debugged += n_debugged
	1296	if self.fail_on_debug:
	1297	return (total_failed + total_debugged) > 0
	1298	else:
	1299	return total_failed > 0
	1300
	1301
	1302	class CustomCompleter:
	1303
	1304	def __init__(self):
	1305	pass
	1306
	1307	def __call__(self, prefix, parsed_args, **kwargs):
	1308	return (i for i in self.get_items() if i.startswith(prefix))
	1309
	1310	def get_items(self):
	1311	return []
	1312
	1313
	1314	class CaseCompleter(CustomCompleter):
	1315
	1316	def get_items(self):
[4091]	1317	case_names = [name for name in next(os.walk(Environment.trunk_tests_cases_dir))[1] if not name[0] == '.']
[3353]	1318	return case_names + ['all']
	1319
	1320
	1321	class BuildCompleter(CustomCompleter):
	1322
	1323	def get_items(self):
[4091]	1324	build_names = [name for name in next(os.walk(Environment.trunk_tests_builds_dir))[1] if not name[0] == '.']
[3353]	1325	return build_names + ['all']
	1326
	1327
	1328	class PALMTestArgumentParser(ArgumentParser):
	1329
	1330	def __init__(self):
	1331	super().__init__(
	1332	description='This is the PALM tester\n' +
	1333	'Developer Support: knoop@muk.uni-hannover.de',
	1334	formatter_class=RawTextHelpFormatter,
	1335	add_help=True,
	1336	)
	1337	self.add_argument(
	1338	'--version',
	1339	action='version',
	1340	version=version,
	1341	)
	1342	self.add_argument(
	1343	'--verbose',
	1344	action='store_true',
	1345	dest='verbose',
	1346	help='Increase verbosity of terminal output.',
	1347	required=False,
	1348	)
	1349	self.add_argument(
	1350	'--no-auto-debug',
	1351	action='store_true',
	1352	dest='no_auto_debug',
	1353	help='Disable automatic debugging in case of test failure.',
	1354	required=False,
	1355	)
	1356	self.add_argument(
	1357	'--force-debug',
	1358	action='store_true',
	1359	dest='force_debug',
	1360	help='Force debugging regardless of test failure (ignores --no-auto-debug).',
	1361	required=False,
	1362	)
	1363	self.add_argument(
	1364	'--fail-on-debug',
	1365	action='store_true',
	1366	dest='fail_on_debug',
	1367	help='Return a non-zero exit status in case debugging was required.',
	1368	required=False,
	1369	)
	1370	self.add_argument(
	1371	'--dry-run',
	1372	action='store_true',
	1373	dest='dry_run',
	1374	help='Prepare and process all requested tests without actually building or executing PALM.',
	1375	required=False,
	1376	)
	1377	self.add_argument(
[3354]	1378	'--no-color',
	1379	action='store_true',
	1380	dest='no_color',
	1381	help='Disable colored terminal output.',
	1382	required=False,
	1383	)
	1384	self.add_argument(
[3353]	1385	'--cases',
	1386	action='store',
	1387	dest='cases',
	1388	default=['all'],
	1389	help='A list of test cases to be executed. (default: %(default)s)',
	1390	nargs='+',
	1391	required=False,
	1392	type=str,
	1393	metavar='STR',
	1394	).completer = CaseCompleter()
	1395	self.add_argument(
	1396	'--builds',
	1397	action='store',
	1398	dest='builds',
	1399	default=['all'],
	1400	help='A list of builds to be executed. (default: %(default)s)',
	1401	nargs='+',
	1402	required=False,
	1403	type=str,
	1404	metavar='STR',
	1405	).completer = BuildCompleter()
	1406	self.add_argument(
	1407	'--cores',
	1408	action='store',
	1409	dest='cores',
[4091]	1410	default=[i for i in range(1, multiprocessing.cpu_count()+1)],
	1411	choices=[i for i in range(1, multiprocessing.cpu_count()+1)],
[3353]	1412	help='The number of cores tests are supposed to be executed on. (default: %(default)s)',
	1413	nargs='+',
	1414	required=False,
	1415	type=int,
	1416	metavar='INT',
	1417	)
	1418	self.add_argument(
	1419	'--test-id',
	1420	action='store',
	1421	dest='test_id',
	1422	default=datetime.now().strftime('%Y-%m-%d_%H:%M:%S.%f'),
	1423	help='An individual test id. (default: current timestamp)',
	1424	required=False,
	1425	type=str,
	1426	metavar='STR',
	1427	)
	1428
	1429
	1430	if __name__ == '__main__':
	1431	parser = PALMTestArgumentParser()
	1432	if has_argcomplete:
	1433	argcomplete.autocomplete(parser)
	1434	args = parser.parse_args()
	1435	palm_test = PALMTest(args)
	1436	palm_test.prepare()
	1437	palm_test.execute()
	1438	failed = palm_test.report()
	1439	exit(1 if failed else 0)

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: palm/trunk/SCRIPTS/palmtest

Download in other formats: