Home

Context Navigation

source: palm/trunk/SCRIPTS/palmtest @ 4064

Last change on this file since 4064 was 3682, checked in by knoop, 6 years ago
Extended palmtest to use custom MPI execution commands
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 56.8 KB

Rev	Line
[3353]	1	#!/usr/bin/env python3
	2	# PYTHON_ARGCOMPLETE_OK
[2497]	3
[3353]	4	import os
	5	import sys
	6	import shutil
	7	from datetime import datetime
	8	import subprocess
	9	import multiprocessing
	10	import socket
	11	import getpass
	12	import math
	13	import re
	14	import threading
	15	import queue
	16	from contextlib import ContextDecorator
[2497]	17
[3353]	18	try:
	19	from argparse import ArgumentParser
	20	from argparse import RawTextHelpFormatter
	21	except ImportError:
	22	sys.exit(
	23	'ERROR: You need argparse!\n' +
	24	' install it from http://pypi.python.org/pypi/argparse\n' +
	25	' or run \"pip install argparse\".'
	26	)
[2497]	27
[3353]	28	try:
	29	import numpy as np
	30	except ImportError:
	31	sys.exit(
	32	'ERROR: You need numpy!\n' +
	33	' install it from http://pypi.python.org/pypi/numpy\n' +
	34	' or run \"python3 -m pip install numpy\".'
	35	)
[2497]	36
[3353]	37	try:
	38	import netCDF4
	39	except ImportError:
	40	sys.exit(
	41	'ERROR: You need netCDF4!\n' +
	42	' install it from http://pypi.python.org/pypi/netCDF4\n' +
	43	' or run \"python3 -m pip install netCDF4\".'
	44	)
[2497]	45
[3353]	46	try:
	47	import yaml
	48	except ImportError:
	49	sys.exit(
	50	'ERROR: You need PyYAML!\n' +
	51	' install it from http://pypi.python.org/pypi/PyYAML\n' +
	52	' or run \"python3 -m pip install PyYAML\".'
	53	)
[2497]	54
[3353]	55	try:
	56	import argcomplete
	57	except ImportError:
	58	print(
	59	'INFO: To use Tab-completion you need argcomplete!\n' +
	60	' install it from http://pypi.python.org/pypi/argcomplete\n' +
	61	' or run \"python3 -m pip install argcomplete\".'
	62	)
	63	has_argcomplete = False
	64	else:
	65	has_argcomplete = True
[2497]	66
[3353]	67	try:
[3354]	68	from termcolor import colored as tcolored
[3353]	69	except ImportError:
[3354]	70	def tcolored(string, color):
[3353]	71	return string
[2497]	72
[3354]	73	disable_colored_output = False
[2497]	74
	75
[3354]	76	def colored(string, color):
	77	if not disable_colored_output:
	78	return tcolored(string, color)
	79	else:
	80	return string
	81
	82
	83	version = '1.0.1'
	84
[3353]	85	scripts_dir = os.path.dirname(os.path.realpath(__file__))
	86	trunk_dir = os.path.realpath(os.path.join(scripts_dir, '..'))
	87	workspace_dir = os.path.realpath(os.path.join(trunk_dir, '..'))
[2497]	88
[3353]	89	trunk_tests_dir = os.path.join(trunk_dir, 'TESTS')
	90	trunk_tests_cases_dir = os.path.join(trunk_tests_dir, 'cases')
	91	trunk_tests_builds_dir = os.path.join(trunk_tests_dir, 'builds')
[2497]	92
[3353]	93	tests_dir = os.path.join(workspace_dir, 'tests')
[2497]	94
[3353]	95	available_cores = multiprocessing.cpu_count()
	96	terminal_columns, terminal_lines = shutil.get_terminal_size()
	97	hline = '#' * min(terminal_columns, 300) + '\n'
	98	table_width_intro = 12
[3354]	99	table_width_builds = len(max([s for s in next(os.walk(trunk_tests_builds_dir))[1] if not s[0] == '.'], key=len)) + len('_debug')
	100	table_width_cases = len(max([s for s in next(os.walk(trunk_tests_cases_dir))[1] if not s[0] == '.'], key=len))
[3353]	101	table_width_cores = 7
	102	table_width_total = table_width_intro + table_width_builds + table_width_cases + table_width_cores + 3
[2497]	103
[3353]	104	task_table_line_template = \
	105	'{:' + str(table_width_intro) + '} ' + \
	106	'{:' + str(table_width_cases) + '} ' + \
	107	'{:' + str(table_width_builds) + '} ' + \
	108	'{:' + str(table_width_cores) + '} '
[2579]	109
[3353]	110	config_table_line_template = \
	111	'{:' + str(table_width_intro) + '} ' + \
	112	'{:' + str(max(table_width_builds, table_width_cases)) + '} ' + \
	113	'{:8} '
[2579]	114
[3353]	115	file_table_line_template = \
	116	'{:' + str(table_width_intro) + '} ' + \
	117	'{:' + str(table_width_cases + 13) + '} '
	118
	119	# for debugging
	120	exist_ok = False
	121
	122
	123	class SignificantDigitsRounder:
	124
	125	@staticmethod
	126	def _round(value, digits=10):
	127	if value == 0.0:
	128	return value
	129	negative = value < 0.0
	130	value = -value if negative else value
	131	rounded_value = round(value, -int(math.floor(math.log10(value))) + (digits - 1))
	132	rounded_value = -rounded_value if negative else rounded_value
	133	return rounded_value
	134
	135
	136	vectorized_round = np.vectorize(_round)
	137
	138	_vectorized_round = np.vectorize(round)
	139
	140
	141	@classmethod
	142	def around(cls, array, digits=10):
	143	# TODO: divide both arrays and check decimal point
	144	sign_mask = np.ma.masked_where(array >= 0.0, array).mask
	145	pos_array = np.where(sign_mask, array, -array)
	146	non_zero_maks = np.ma.masked_where(pos_array == 0.0, pos_array).mask
	147	non_zero_array = np.where(non_zero_maks, 1.0, pos_array)
	148	i1 = -np.floor(np.log10(non_zero_array)).astype(int) + (digits - 1)
	149	rounded_non_zero_array = cls._vectorized_round(non_zero_array, i1)
	150	rounded_pos_array = np.where(non_zero_maks, 0.0, rounded_non_zero_array)
	151	return np.where(sign_mask, rounded_pos_array, -rounded_pos_array)
	152
	153
	154
	155	class Logger(ContextDecorator):
	156
	157	def __init__(self, logfile_dir, logfile_name='palmtest.log', logfile_mode='a', verbose=False):
	158	self.logfile_path = os.path.join(logfile_dir, logfile_name)
	159	self.logfile_mode = logfile_mode
	160	self.verbose = verbose
	161
	162	def __enter__(self):
	163	self._file = open(self.logfile_path, self.logfile_mode)
	164	return self
	165
	166	def to_file(self, message):
	167	self._file.write(message)
	168	self._file.flush()
	169
	170	def to_log(self, message):
	171	if self.verbose:
	172	sys.stdout.write(message)
	173	sys.stdout.flush()
	174	self._file.write(message)
	175	self._file.flush()
	176
	177	def to_all(self, message):
	178	sys.stdout.write(message)
	179	sys.stdout.flush()
	180	self._file.write(message)
	181	self._file.flush()
	182
	183	def __exit__(self, *exc):
	184	self._file.close()
	185	return False
	186
	187
	188	class Executor:
	189
	190	@staticmethod
	191	def _enqueue_output(out, queue):
	192	for line in iter(out.readline, b''):
	193	queue.put(line)
	194	out.close()
	195
	196	@staticmethod
	197	def execute(cmd, cwd='.', verbose=True, dry_run=False):
	198	assert isinstance(cmd, list)
	199	if dry_run:
	200	cmd = ['echo'] + cmd
	201	cmd_str = ' '.join(cmd)
	202	p = subprocess.Popen(cmd_str, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, bufsize=1)
	203	q = queue.Queue()
	204	t = threading.Thread(target=Executor._enqueue_output, args=(p.stdout, q))
	205	t.daemon = True # thread dies with the program
	206	t.start()
	207
	208	with Logger(cwd, verbose=verbose) as logger:
	209	# read line without blocking
	210	logger.to_log(hline)
	211	logger.to_log('CMD: ' + cmd_str + '\n')
	212	logger.to_log(hline)
	213	while t.is_alive():
	214	try:
	215	line = q.get_nowait() # or q.get(timeout=.1)
	216	except queue.Empty:
	217	pass # print('no output yet')
	218	else: # got line
	219	logger.to_log(line.decode("utf-8"))
	220	line = True
	221	while line:
	222	try:
	223	line = q.get_nowait() # or q.get(timeout=.1)
	224	except queue.Empty:
	225	line = False
	226	else: # got line
	227	logger.to_log(line.decode("utf-8"))
	228	logger.to_log(hline)
	229
	230	rc = p.poll()
	231	failed = rc != 0
	232	return failed
	233
	234
	235	class NetCDFInterface:
	236
	237	def __init__(self, filename):
	238	self.filename = filename
	239
	240	def is_healthy(self):
	241	try:
	242	self.get_run_name()
	243	except:
	244	return False
	245	else:
	246	return True
	247
	248	def get_run_name(self):
	249	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
	250	l = getattr(netcdf, 'title').split()
	251	i = l.index('run:')
	252	return l[i+1]
	253
	254	def get_var_list(self):
	255	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
[3561]	256	var_list = list(netcdf.variables.keys())
[3353]	257	var_list = filter(None, var_list)
	258	return sorted(var_list)
	259
	260	def show_content(self):
	261	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
	262	for name in netcdf.ncattrs():
	263	print("Global attr", name, "=", getattr(netcdf, name))
	264	print(netcdf)
	265	for v in netcdf.variables:
	266	print(v)
	267
	268	def get_times_list(self):
	269	attributes, times = self.read_var('time')
	270	times = [str(time) for time in times]
	271	times = list(filter(None, times))
	272	return times
	273
	274	def contains(self, variable):
	275	return variable in self.get_var_list()
	276
	277	def read_var(self, variable):
	278	with netCDF4.Dataset(self.filename, mode='r') as netcdf:
	279	values = netcdf.variables[variable][:] # extract values
[3561]	280	attributes = dict()
	281	try:
	282	attributes['long_name'] = netcdf.variables[variable].name
	283	except:
	284	attributes['long_name'] = ''
	285	try:
	286	attributes['unit'] = netcdf.variables[variable].units
	287	except:
	288	attributes['unit'] = ''
[3353]	289	return attributes, values
	290
	291
	292	class FileComparator:
	293
	294	@staticmethod
	295	def compare_ascii(file_path1, file_path2, start_string=None):
	296	try:
	297	with open(file_path1, 'r') as file1:
	298	content1 = file1.readlines()
	299	except OSError:
	300	return True, colored('[reference file not found]', 'red')
	301	try:
	302	with open(file_path2, 'r') as file2:
	303	content2 = file2.readlines()
	304	except OSError:
	305	return True, colored('[output file not found]', 'red')
	306	if start_string:
	307	index1 = content1.index(start_string)
	308	index2 = content2.index(start_string)
	309	comparable_content1 = content1[index1:]
	310	comparable_content2 = content2[index2:]
	311	ln = index2 + 1
	312	else:
	313	comparable_content1 = content1
	314	comparable_content2 = content2
	315	ln = 1
	316	if len(comparable_content1) != len(comparable_content2):
	317	return True, colored('[mismatch in total number of lines]', 'red')
	318	for line1, line2 in zip(comparable_content1, comparable_content2):
	319	if not line1 == line2:
	320	return True, colored('[mismatch in content starting line ' + str(ln) + ']', 'red')
	321	ln += 1
	322	return False, colored('[file ok]', 'green')
	323
	324	@staticmethod
	325	def compare_netcdf(file_path1, file_path2, digits=None):
	326	nci1 = NetCDFInterface(file_path1)
	327	nci2 = NetCDFInterface(file_path2)
	328	if not nci1.is_healthy():
	329	return True, colored('[reference file not found]', 'red')
	330	if not nci2.is_healthy():
	331	return True, colored('[output file not found]', 'red')
	332	times_list1 = nci1.get_times_list()
	333	times_list2 = nci2.get_times_list()
	334	if not times_list1 == times_list2:
	335	return True, colored('[wrong time dimension]', 'red')
	336	else:
	337	time_list = times_list1
	338	var_list1 = nci1.get_var_list()
	339	var_list2 = nci2.get_var_list()
	340	if not var_list1 == var_list2:
	341	return True, colored('[wrong set of variables]', 'red')
	342	else:
	343	var_list = var_list1
	344	content1 = dict()
	345	content2 = dict()
	346	for var in var_list:
	347	attributes1, values1 = nci1.read_var(var)
	348	attributes2, values2 = nci2.read_var(var)
	349	if sorted(attributes1.keys()) != sorted(attributes2.keys()):
	350	return True, colored('[wrong set of attributes in variable \"'+var+'\"]', 'red')
	351	if isinstance(digits, int):
	352	values1 = SignificantDigitsRounder.around(values1, digits=digits)
	353	values2 = SignificantDigitsRounder.around(values2, digits=digits)
	354	content1[var] = values1
	355	content2[var] = values2
	356	#for decimals in
	357	for i, time in enumerate(time_list):
	358	for var in var_list:
	359	t_content1 = content1[var][i]
	360	t_content2 = content2[var][i]
	361	if not (t_content1==t_content2).all():
	362	if isinstance(digits, int):
	363	return True, colored('[1st mismatch within ' + str(digits) + ' digits at time index ' + str(i) + ' in variable \"' + var + '\"]', 'red')
	364	else:
	365	return True, colored('[1st mismatch at time index ' + str(i) + ' in variable \"' + var + '\"]', 'red')
	366	return False, colored('[file ok]', 'green')
	367
	368
	369	class OutputChecker:
	370
	371	def __init__(self, test_dir, setup_name, build_name, cores, significant_digits=None, verbose=True, dry_run=False):
	372	self.test_dir = test_dir
	373	self.setup_name = setup_name
	374	self.build_name = build_name
	375	self.cores = cores
	376	self.significant_digits = significant_digits
	377	self.verbose = verbose
	378	self.dry_run = dry_run
	379	self.job_name = self.setup_name + '__' + build_name + '__' + str(self.cores)
	380	self.job_dir = os.path.join(self.test_dir, 'JOBS', self.job_name)
	381	self.ref_monitoring_dir = os.path.join(trunk_tests_cases_dir, self.setup_name, 'MONITORING')
	382	self.ref_output_dir = os.path.join(trunk_tests_cases_dir, self.setup_name, 'OUTPUT')
	383	self.res_monitoring_dir = os.path.join(self.job_dir, 'MONITORING')
	384	self.res_output_dir = os.path.join(self.job_dir, 'OUTPUT')
	385	self.failed = None
	386
	387	def get_checkable_file_dicts(self):
	388	if os.path.isdir(self.ref_monitoring_dir):
	389	file_names_monitoring = [s for s in next(os.walk(self.ref_monitoring_dir))[2]]
	390	else:
	391	file_names_monitoring = []
	392	file_paths_monitoring = []
	393	for file_name in file_names_monitoring:
	394	file_specific_ending = file_name[len(self.setup_name):]
	395	file_specific_ending_split = file_specific_ending.split('.')
	396	postfix = file_specific_ending_split[0]
	397	if len(file_specific_ending_split) > 1:
	398	extension = file_specific_ending_split[-1]
	399	else:
	400	extension = ''
	401	if len(file_specific_ending_split) > 2:
	402	cycle_info = file_specific_ending_split[1: -1]
	403	else:
	404	cycle_info = []
	405	file_paths_monitoring.append(
	406	dict(
	407	postfix=postfix,
	408	cycle_info=cycle_info,
	409	extension=extension,
	410	ref_path=self.ref_monitoring_dir,
	411	res_path=self.res_monitoring_dir,
	412	)
	413	)
	414	if os.path.isdir(self.ref_output_dir):
	415	file_names_output = [s for s in next(os.walk(self.ref_output_dir))[2]]
	416	else:
	417	file_names_output = []
	418	file_paths_output = []
	419	for file_name in file_names_output:
	420	file_specific_ending = file_name[len(self.setup_name):]
	421	file_specific_ending_split = file_specific_ending.split('.')
	422	postfix = file_specific_ending_split[0]
	423	if len(file_specific_ending_split) > 1:
	424	extension = file_specific_ending_split[-1]
	425	else:
	426	extension = ''
	427	if len(file_specific_ending_split) > 2:
	428	cycle_info = file_specific_ending_split[1: -1]
	429	else:
	430	cycle_info = []
	431	file_paths_output.append(
	432	dict(
	433	postfix=postfix,
	434	cycle_info=cycle_info,
	435	extension=extension,
	436	ref_path=self.ref_output_dir,
	437	res_path=self.res_output_dir,
	438	)
	439	)
	440	return file_paths_monitoring + file_paths_output
	441
	442	def check(self):
	443	with Logger(self.test_dir, verbose=self.verbose) as logger:
	444	logger.to_log('Checking output files:')
	445	logger.to_all('\n')
	446	failed = False
	447	for file_dict in self.get_checkable_file_dicts():
	448	file_failed = False
	449	ext_list = [file_dict['extension']] if file_dict['extension'] else []
	450	file_specific_ending = '.'.join([file_dict['postfix']] + file_dict['cycle_info'] + ext_list )
	451	logger.to_all(file_table_line_template.format('Checking:', self.setup_name + file_specific_ending))
	452	ref_file_path = os.path.join(file_dict['ref_path'], self.setup_name + file_specific_ending)
	453	res_file_path = os.path.join(file_dict['res_path'], self.job_name + file_specific_ending)
	454	if re.match('_rc', file_dict['postfix']) and not file_dict['extension']:
	455	file_failed, message = FileComparator.compare_ascii(ref_file_path, res_file_path, start_string='Run-control output:\n')
	456	elif re.match('nc', file_dict['extension']):
	457	if self.significant_digits is not None:
	458	if re.match('_ts', file_dict['postfix']) and 'timeseries' in self.significant_digits:
	459	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
	460	digits=self.significant_digits['timeseries'])
	461	elif re.match('_pr', file_dict['postfix']) and 'profiles' in self.significant_digits:
	462	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
	463	digits=self.significant_digits['profiles'])
	464	else:
	465	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path,
	466	digits=self.significant_digits['other'])
	467	else:
	468	file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path)
	469	else:
	470	message = colored('[ignored]', 'blue')
	471	if file_failed:
	472	failed = True
	473	logger.to_all(message + '\n')
	474	if self.dry_run:
	475	failed = False
	476	return failed
	477
	478
	479	class PALMJob:
	480	"""The PALM job class deals with the execution of a single PALM job"""
	481
	482	@staticmethod
	483	def get_job_name(setup_name, build_name, cores):
	484	return setup_name + '__' + build_name + '__' + str(cores)
	485
	486	def __init__(self, test_dir, test_case, build_name, cores, verbose=False, dry_run=False):
	487	self.test_dir = test_dir
	488	self.test_case = test_case
	489	self.build_name = build_name
	490	self.cores = cores
	491	self.verbose = verbose
	492	self.dry_run = dry_run
	493
	494	self.attempted_debug = False
	495	self.failed_debug = None
	496	self.attempted_non_debug = False
	497	self.failed_non_debug = None
	498
	499	def _link_restart_files(self, build_name):
	500	if self.dry_run:
	501	return True, colored('[restart data dry]', 'blue')
	502	name = self.get_job_name(self.test_case.name, build_name, self.cores)
	503	source_name = self.get_job_name(self.test_case.use_binary_files_from, build_name, self.cores)
	504	source_restart_dir = os.path.join(self.test_dir, 'JOBS', source_name, 'RESTART')
	505	try:
	506	source_data_dirs_grp = [d for r, d, f in os.walk(source_restart_dir)]
	507	except:
	508	source_data_dirs_grp = []
	509	if len(source_data_dirs_grp) == 0:
	510	source_data_dirs = []
	511	else:
	512	source_data_dirs = source_data_dirs_grp[0]
	513	if len(source_data_dirs) == 0 and re.match('.+_debug', build_name):
	514	source_build_name = build_name[:-len('_debug')]
	515	source_name = self.get_job_name(self.test_case.use_binary_files_from, source_build_name, self.cores)
	516	source_restart_dir = os.path.join(self.test_dir, 'JOBS', source_name, 'RESTART')
	517	try:
	518	source_data_dirs_grp = [d for r, d, f in os.walk(source_restart_dir)]
	519	except:
	520	source_data_dirs_grp = []
	521	if len(source_data_dirs_grp) == 0:
	522	source_data_dirs = []
	523	else:
	524	source_data_dirs = source_data_dirs_grp[0]
	525	if len(source_data_dirs) == 0:
	526	source_data_dir = 'no_restart_data'
	527	else:
	528	source_data_dir = sorted(source_data_dirs)[-1]
	529	source_data_dir_path = os.path.join(source_restart_dir, source_data_dir)
	530	if os.path.isdir(source_data_dir_path) and re.match('.+_d3d.*', source_data_dir):
	531	job_restart_dir = os.path.join(self.test_dir, 'JOBS', name, 'RESTART')
	532	os.makedirs(job_restart_dir, exist_ok=exist_ok)
	533	job_data_dir_path = os.path.join(job_restart_dir, name + '_d3d')
	534	os.symlink(source_data_dir_path, job_data_dir_path, target_is_directory=True)
	535	return False, colored('[linked restart data from: ' + source_data_dir_path + ']', 'green')
	536	else:
	537	return True, colored('[no restart data found]', 'red')
	538
	539	def _execute(self, name, build_name):
	540	execution_failed = Executor.execute(
	541	[
	542	os.path.join(self.test_dir, 'trunk', 'SCRIPTS', 'palmrun'),
[3457]	543	'-c', '\"' + build_name + '\"',
	544	'-r', name,
[3353]	545	'-a', '\"' + ' '.join(self.test_case.activation_strings) + '\"',
	546	'-X', str(self.cores),
	547	'-T', str(self.cores),
	548	'-B',
	549	'-v',
	550	'-z',
	551	],
	552	cwd=self.test_dir,
	553	verbose=self.verbose,
	554	dry_run=self.dry_run,
	555	)
	556
	557	if self.dry_run:
	558	return False, colored('[execution dry]', 'blue')
	559	elif execution_failed:
	560	return True, colored('[execution failed]', 'red')
	561	else:
	562	return False, colored('[execution ok]', 'green')
	563
	564	def _check(self, build_name):
	565	checker = OutputChecker(
	566	self.test_dir,
	567	self.test_case.name,
	568	build_name,
	569	self.cores,
	570	significant_digits=self.test_case.significant_digits,
	571	verbose=self.verbose,
	572	dry_run=self.dry_run,
	573	)
	574	check_failed = checker.check()
	575
	576	if self.dry_run:
	577	return False, colored('[checks dry]', 'blue')
	578	if check_failed:
	579	return True, colored('[checks failed]', 'red')
	580	else:
	581	return False, colored('[checks ok]', 'green')
	582
	583	def execute(self, debug=False):
	584	if debug:
	585	attempted = self.attempted_debug
	586	build_name = self.build_name + '_debug'
	587	failed = self.failed_debug
	588	else:
	589	attempted = self.attempted_non_debug
	590	build_name = self.build_name
	591	failed = self.failed_non_debug
	592
	593	if not attempted:
	594	with Logger(self.test_dir, verbose=self.verbose) as logger:
	595	status_prefix = task_table_line_template.format('Testing:', self.test_case.name, build_name, self.cores)
	596	logger.to_all(status_prefix)
	597	logger.to_log('[started]' + '\n')
	598	attempted = True
	599
	600	name = self.get_job_name(self.test_case.name, build_name, self.cores)
	601
	602	input_dir = os.path.join(self.test_dir, 'JOBS', name, 'INPUT')
	603	os.makedirs(input_dir, exist_ok=exist_ok)
	604
	605	# copying needs to be done per file, because input files need to be renamed
	606	for input_file in self.test_case.input_file_names:
	607	postfix = input_file[len(self.test_case.name):]
	608	src = os.path.join(self.test_case.input_dir, input_file)
	609	dst = os.path.join(input_dir, name + postfix)
	610	shutil.copy(src, dst)
	611
	612	# copying the entire directory is ok, because source files do not need to be renamed
	613	user_code_dir = os.path.join(self.test_dir, 'JOBS', name, 'USER_CODE')
	614	if os.path.isdir(self.test_case.user_code_dir):
	615	shutil.copytree(self.test_case.user_code_dir, user_code_dir, copy_function=shutil.copy)
	616
	617	if self.test_case.requires_binary_files:
	618	link_restart_files_failed, message = self._link_restart_files(build_name)
	619	logger.to_log(status_prefix)
	620	logger.to_log(message + ' ')
	621	logger.to_log('\n')
	622
	623	failed, message = self._execute(name, build_name)
	624	logger.to_log(status_prefix)
	625	logger.to_all(message + ' ')
	626	logger.to_log('\n')
	627
	628	failed, message = self._check(build_name)
	629	logger.to_log(status_prefix)
	630	logger.to_log(message + ' ')
	631
	632	logger.to_all('\n')
	633
	634	if debug:
	635	self.attempted_debug = attempted
	636	self.failed_debug = failed
	637	else:
	638	self.attempted_non_debug = attempted
	639	self.failed_non_debug = failed
	640
	641	return failed
	642
	643	def status(self):
	644	return dict(
	645	attempted=self.attempted_non_debug or self.attempted_debug,
	646	failed=self.failed_non_debug and self.failed_debug,
	647	debugged=self.attempted_debug,
	648	non_debug_failed=self.failed_non_debug,
	649	)
	650
	651
	652	class PALMBuild:
	653	"""The PALM build class deals with configuration and execution of all required PALM builds"""
	654
	655	def __init__(self, test_dir, build_name, verbose=False, dry_run=False):
	656	self.test_dir = test_dir
	657	self.build_name = build_name
	658	self.verbose = verbose
	659	self.dry_run = dry_run
	660	self.configured = False
	661	self.executed = False
	662	self.available = False
	663	self.requires_mpi = False
	664	self.requires_netcdf = False
	665	self.requires_fftw = False
	666	self.requires_rrtmg = False
	667	self.attempted_non_debug = False
	668	self.attempted_debug = False
	669	self.failed_non_debug = None
	670	self.failed_debug = None
	671
	672	def configure(self):
	673	try:
	674	with open(os.path.join(trunk_tests_builds_dir, self.build_name, 'build_config.yml'), 'r') as f:
	675	build_config = yaml.load(f)
	676	except:
	677	return True, colored('[build not found]', 'red')
	678
	679	if 'compiler' in build_config:
	680	self.compiler = build_config['compiler']
	681	else:
	682	return True, colored('[missing \"compiler\" keyword]', 'red')
	683
	684	if not isinstance(self.compiler, dict):
	685	return True, colored('[\"compiler\" keyword must be dict]', 'red')
	686
	687	if 'linker' in build_config:
	688	self.linker = build_config['linker']
	689	else:
	690	return True, colored('[missing \"linker\" keyword]', 'red')
	691
	692	if not isinstance(self.linker, dict):
	693	return True, colored('[\"linker\" keyword must be dict]', 'red')
	694
	695	if 'mpi_wrapper' in self.compiler:
	696	if 'mpi_wrapper}}' in self.compiler['mpi_wrapper']:
	697	self.requires_mpi = True
	698	else:
	699	return True, colored('[missing \"mpi_wrapper\" keyword]', 'red')
	700
	701	if 'includes' in self.compiler:
	702	for include in self.compiler['includes']:
	703	if 'include.netcdf}}' in include:
	704	self.requires_netcdf = True
	705	if 'include.fftw}}' in include:
	706	self.requires_fftw = True
	707	if 'include.rrtmg}}' in include:
	708	self.requires_rrtmg = True
	709	else:
	710	return True, colored('[missing \"includes\" keyword in compiler]', 'red')
	711
	712	if 'options' in self.linker:
	713	for lib in self.linker['options']:
	714	if 'lib.netcdf}}' in lib:
	715	self.requires_netcdf = True
	716	if 'lib.fftw}}' in lib:
	717	self.requires_fftw = True
	718	if 'lib.rrtmg}}' in lib:
	719	self.requires_rrtmg = True
	720	else:
	721	return True, colored('[missing \"options\" keyword in linker]', 'red')
	722
	723	library_names = []
	724	if self.requires_netcdf:
	725	library_names.append('netcdf')
	726	if self.requires_fftw:
	727	library_names.append('fftw')
	728	if self.requires_rrtmg:
	729	library_names.append('rrtmg')
	730
	731	if not 'executable' in self.compiler:
	732	return True, colored('[missing \"executable\" keyword in compiler]', 'red')
	733
	734	if not 'definitions' in self.compiler:
	735	return True, colored('[missing \"definitions\" keyword in compiler]', 'red')
	736
	737	if not 'options' in self.compiler:
	738	return True, colored('[missing \"options\" keyword in compiler]', 'red')
	739
	740	if not 'default' in self.compiler['options']:
	741	return True, colored('[missing \"default\" keyword in compiler.options]', 'red')
	742
	743	if not 'debug' in self.compiler['options']:
	744	return True, colored('[missing \"debug\" keyword in compiler.options]', 'red')
	745
	746	try:
	747	with open(os.path.join(workspace_dir, 'palmtest.yml'), 'r') as f:
	748	palmtest_config = yaml.load(f)
	749	except:
	750	return True, colored('[palmtest.yml not found]', 'red')
	751
	752	if 'palm_config_template' in palmtest_config:
	753	if isinstance(palmtest_config['palm_config_template'], str):
	754	custom_template = palmtest_config['palm_config_template']
	755	try:
	756	with open(os.path.join(custom_template), 'r') as palm_config_template_file:
	757	template = palm_config_template_file.read()
	758	except:
	759	try:
	760	with open(os.path.join(scripts_dir, '.palm.config.default.in'), 'r') as palm_config_template_file:
	761	template = palm_config_template_file.read()
	762	except:
	763	return True, colored('[trunk/SCRIPTS/.palm.config.default.in not found]', 'red')
	764
	765	template = template.replace('@CMAKE_INSTALL_PREFIX@', self.test_dir)
	766	template = template.replace('@PALM_HOSTNAME@', socket.gethostname())
	767	template = template.replace('@CMAKE_USERNAME@', getpass.getuser())
	768	template = template.replace('@MPI_Fortran_COMPILER@', self.compiler['mpi_wrapper'])
	769	template = template.replace('@CMAKE_Fortran_COMPILER@', self.compiler['executable'])
	770	cpp_options_str = ['-D' + s for s in self.compiler['definitions']]
	771	template = template.replace('@PALM_CPP_OPTIONS_STR@', ' '.join(cpp_options_str))
	772	template = template.replace('@PALM_CORES@', str(available_cores))
	773	template = template.replace('@PALM_COMPILER_OPTIONS@', '{{palmtest.compiler.options}} ' + ' '.join(self.compiler['includes']))
	774	template = template.replace('@PALM_LINKER_OPTIONS@', ' '.join(self.linker['options']))
	775
	776	if 'environments' in palmtest_config:
	777	available_environments = palmtest_config['environments']
	778	else:
	779	return True, colored('[missing \"environments\" keyword in palmtest.yml]', 'red')
	780
	781	if 'id' in self.compiler:
	782	c_id = self.compiler['id']
	783	else:
	784	return True, colored('[missing \"id\" keyword in compiler]', 'red')
	785
	786	if c_id in available_environments:
	787	self.available = True
	788
	789	environment = available_environments[c_id]
[3682]	790
	791	if 'mpi_execution_command' in environment:
	792	template = template.replace('@PALM_EXECUTE_COMMAND@', environment['mpi_execution_command'])
	793	else:
	794	template = template.replace('@PALM_EXECUTE_COMMAND@', 'mpirun -n {{mpi_tasks}}')
	795
[3353]	796	if 'executable' not in environment:
	797	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"executable\"]', 'red')
	798	value = environment['executable']
	799	if isinstance(value, str):
	800	template = template.replace('{{' + '.'.join([c_id, 'executable']) + '}}', value)
	801	if self.requires_mpi:
	802	if 'mpi_wrapper' not in environment:
	803	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"mpi_wrapper\"]', 'red')
	804	value = environment['mpi_wrapper']
	805	if isinstance(value, str):
	806	template = template.replace('{{' + '.'.join([c_id, 'mpi_wrapper']) + '}}', value)
	807	if 'include' not in environment:
	808	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"include\"]', 'red')
	809	if 'lib' not in environment:
	810	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"lib\"]', 'red')
	811	for lib in library_names:
	812	if lib not in environment['include']:
	813	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"include.'+lib+'\"]', 'red')
	814	value = environment['include'][lib]
	815	if isinstance(value, str):
	816	template = template.replace('{{' + '.'.join([c_id, 'include', lib]) + '}}', value)
	817	if lib not in environment['lib']:
	818	return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"lib.'+lib+'\"]', 'red')
	819	value = environment['lib'][lib]
	820	if isinstance(value, str):
	821	template = template.replace('{{' + '.'.join([c_id, 'lib', lib]) + '}}', value)
	822
	823	with open(os.path.join(self.test_dir, '.palm.config.' + self.build_name), 'w') as palm_config_file:
	824	palm_config_file.write(
	825	template.replace(
	826	'{{palmtest.compiler.options}}',
	827	' '.join(self.compiler['options']['default']),
	828	)
	829	)
	830	with open(os.path.join(self.test_dir, '.palm.config.' + self.build_name + '_debug'), 'w') as palm_config_file:
	831	palm_config_file.write(
	832	template.replace(
	833	'{{palmtest.compiler.options}}',
	834	' '.join(self.compiler['options']['debug']),
	835	)
	836	)
	837	self.configured = True
	838	return False, colored('[configuration ok]', 'green')
	839
	840	else:
	841	return True, colored('[palmtest.yml environment \"' + c_id + '\" not found]', 'red')
	842
	843	def _execute(self, build_name):
	844	self.attempted = True
	845	build_failed = Executor.execute(
	846	[
	847	os.path.join(self.test_dir, 'trunk', 'SCRIPTS', 'palmbuild'),
[3457]	848	'-c', '\"' + build_name + '\"',
[3353]	849	'-v',
	850	],
	851	cwd=self.test_dir,
	852	verbose=self.verbose,
	853	dry_run=self.dry_run,
	854	)
	855
	856	if self.dry_run:
	857	return False, colored('[build dry]', 'blue')
	858	if build_failed:
	859	return True, colored('[build failed]', 'red')
	860	else:
	861	return False, colored('[build ok]', 'green')
	862
	863	def build(self, debug=False):
	864	if debug:
	865	attempted = self.attempted_debug
	866	build_name = self.build_name + '_debug'
	867	failed = self.failed_debug
	868	else:
	869	attempted = self.attempted_non_debug
	870	build_name = self.build_name
	871	failed = self.failed_non_debug
	872
	873	if not attempted:
	874	with Logger(self.test_dir, verbose=self.verbose) as logger:
	875	status_prefix = task_table_line_template.format('Building:', '', build_name, '')
	876	logger.to_all(status_prefix)
	877	logger.to_log('[started]' + '\n')
	878	attempted = True
	879
	880	failed, message = self._execute(build_name)
	881	logger.to_log(status_prefix)
	882	logger.to_all(message + ' ')
	883	logger.to_all('\n')
	884
	885	if debug:
	886	self.attempted_debug = attempted
	887	self.failed_debug = failed
	888	else:
	889	self.attempted_non_debug = attempted
	890	self.failed_non_debug = failed
	891
	892	return failed
	893
	894	def report(self):
	895	return dict(
	896	failed_debug=self.failed_debug,
	897	failed_non_debug=self.failed_non_debug,
	898	)
	899
	900
	901	class PALMTestCase:
	902	"""The PALM test case class deals with the configuration and execution of all PALM test cases"""
	903
	904	def __init__(self,test_dir, name, verbose=False, dry_run=False):
	905	self.test_dir = test_dir
	906	self.name = name
	907	self.verbose = verbose
	908	self.dry_run = dry_run
	909	self.user_code_dir = os.path.join(trunk_tests_cases_dir, self.name, 'USER_CODE')
	910	self.input_dir = os.path.join(trunk_tests_cases_dir, self.name, 'INPUT')
	911	self.number_of_cores = []
	912	self.build_names = []
	913	self.input_file_names = []
	914	self.configured = False
	915
	916	def configure(self, requested_build_names, requested_cores):
	917	f_name = os.path.join(trunk_tests_cases_dir, self.name, 'case_config.yml')
	918	try:
	919	with open(f_name, 'r') as f:
	920	config = yaml.load(f)
	921	except:
	922	return True, colored('[Case \"' + self.name + '\" could not be found.]', 'red')
	923	try:
	924	self.use_binary_files_from = config['use_binary_files_from']
	925	except:
	926	self.use_binary_files_from = None
	927	self.requires_binary_files = bool(self.use_binary_files_from)
	928
	929	if 'allowed_builds' not in config:
	930	return True, colored('[missing \"allowed_builds\" keyword]', 'red')
	931	self.allowed_build_names = config['allowed_builds']
	932
	933	if 'allowed_number_of_cores' not in config:
	934	return True, colored('[missing \"allowed_number_of_cores\" keyword]', 'red')
	935	self.allowed_number_of_cores = config['allowed_number_of_cores']
	936
	937	if 'activation_strings' not in config:
	938	return True, colored('[missing \"activation_strings\" keyword]', 'red')
	939	self.activation_strings = config['activation_strings']
	940
	941	if 'significant_digits_for_netcdf_checks' not in config:
	942	return True, colored('[missing \"significant_digits_for_netcdf_checks\" keyword]', 'red')
	943	self.significant_digits = config['significant_digits_for_netcdf_checks']
	944
	945	if 'timeseries' not in config['significant_digits_for_netcdf_checks']:
	946	return True, colored('[missing \"timeseries\" keyword in significant_digits_for_netcdf_checks]', 'red')
	947
	948	if 'profiles' not in config['significant_digits_for_netcdf_checks']:
	949	return True, colored('[missing \"profiles\" keyword in significant_digits_for_netcdf_checks]', 'red')
	950
	951	if 'other' not in config['significant_digits_for_netcdf_checks']:
	952	return True, colored('[missing \"other\" keyword in significant_digits_for_netcdf_checks]', 'red')
	953
	954	self.number_of_cores = sorted(set(requested_cores).intersection(self.allowed_number_of_cores))
	955	self.build_names = sorted(set(requested_build_names).intersection(self.allowed_build_names))
	956	self.input_file_names = [s for s in next(os.walk(self.input_dir))[2]]
	957	self.configured = True
	958	if len(self.number_of_cores) == 0 :
	959	return True, colored('[no allowed cores requested]', 'blue')
	960	if len(self.build_names) == 0:
	961	return True, colored('[no allowed builds requested]', 'blue')
	962	if len(self.input_file_names) == 0:
	963	return True, colored('[no input files found]', 'red')
	964	return False, colored('[configuration ok]', 'green')
	965
	966
	967
	968	class PALMTest:
	969
	970	def __init__(self, args):
	971	self.verbose = args.verbose
	972	self.no_auto_debug = args.no_auto_debug
	973	self.force_debug = args.force_debug
	974	self.fail_on_debug = args.fail_on_debug
	975	self.dry_run = args.dry_run
[3354]	976	self.no_color = args.no_color
[3353]	977	self.test_id = args.test_id
	978	self.test_case_names = args.cases
	979	self.requested_build_names = args.builds
	980	self.requested_cores = args.cores
	981	self.test_case_queue = []
	982	self.build_database = dict()
	983
	984	def prepare(self):
[3354]	985	global disable_colored_output
	986	disable_colored_output = self.no_color
[3353]	987	self.test_dir = os.path.join(tests_dir, self.test_id)
	988	try:
	989	os.makedirs(self.test_dir, exist_ok=exist_ok)
	990	except:
	991	print('ERROR: Found existing test directory: ' + self.test_dir)
	992	exit(1)
	993	with Logger(self.test_dir, verbose=self.verbose) as logger:
	994	logger.to_all(hline)
	995	logger.to_all('This is the PALM tester (version: ' + version + ')' + '\n')
	996	logger.to_all(hline)
	997	try:
	998	with open(os.path.join(workspace_dir, 'palmtest.yml'), 'r') as f:
	999	pass
	1000	except:
	1001	logger.to_all('ERROR: No palmtest.yml file was found in your working directory!\n')
	1002	logger.to_all('INFO: A template for this file can be found at: trunk/TESTS/palmtest.yml\n')
	1003	logger.to_all(' Please copy the template to your working directory and adjust it to your system!\n')
	1004	exit(1)
	1005
	1006	self.execution_trunk_dir = os.path.join(self.test_dir, 'trunk')
	1007	os.symlink(trunk_dir, self.execution_trunk_dir)
	1008	self.execution_jobs_dir = os.path.join(self.test_dir, 'JOBS')
	1009	os.makedirs(self.execution_jobs_dir, exist_ok=exist_ok)
	1010
	1011	try:
	1012	with open(os.path.join(scripts_dir, '.palm.iofiles'), 'r') as iofiles_template_file:
	1013	iofiles_template = iofiles_template_file.read()
	1014	with open(os.path.join(self.test_dir, '.palm.iofiles'), 'w') as iofiles_file:
	1015	iofiles_file.write(iofiles_template.replace('$fast_io_catalog', '$base_data'))
	1016	except:
	1017	logger.to_all('ERROR: No .palm.iofiles file was found in trunk/SCRIPTS/')
	1018	exit(1)
	1019
	1020	final_cores_list = list(filter(lambda x: x <= available_cores, self.requested_cores))
	1021
	1022	logger.to_all(config_table_line_template.format('Object:', 'Name:', 'Action:') + 'Status:\n')
	1023	logger.to_all(hline)
	1024
	1025	if 'all' in self.requested_build_names:
[3354]	1026	self.requested_build_names = [name for name in next(os.walk(trunk_tests_builds_dir))[1] if not name[0] == '.']
[3353]	1027	found_build_names = []
	1028	for build_name in self.requested_build_names:
	1029	build = PALMBuild(self.test_dir, build_name, verbose=self.verbose, dry_run=self.dry_run)
	1030	configuration_failed, message = build.configure()
	1031	if not configuration_failed:
	1032	self.build_database[build_name] = build
	1033	found_build_names.append(build_name)
	1034	logger.to_all(config_table_line_template.format('Build', build_name, 'approved'))
	1035	logger.to_all(message + '\n')
	1036	else:
	1037	logger.to_all(config_table_line_template.format('Build', build_name, 'rejected'))
	1038	logger.to_all(message + '\n')
	1039	final_build_list = found_build_names
	1040
	1041	if 'all' in self.test_case_names:
[3354]	1042	self.test_case_names = sorted([name for name in next(os.walk(trunk_tests_cases_dir))[1] if not name[0] == '.'])
[3353]	1043
	1044	additional_initial_runs_2 = [self.test_case_names]
	1045	while len(additional_initial_runs_2[-1]) > 0:
	1046	additional_initial_runs_1 = []
	1047	for test_case_name in additional_initial_runs_2[-1]:
	1048	test_case = PALMTestCase(self.test_dir, test_case_name, verbose=self.verbose)
	1049	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
	1050	if not test_case_configuration_failed:
	1051	if test_case.requires_binary_files:
	1052	additional_initial_runs_1.append(test_case.use_binary_files_from)
	1053	additional_initial_runs_2.append(sorted(set(additional_initial_runs_1)))
	1054
	1055	test_case_order = []
	1056	for i in range(len(additional_initial_runs_2)-1):
	1057	# low and high refer to priority
	1058	low = additional_initial_runs_2[i]
	1059	high = additional_initial_runs_2[i+1]
	1060	for item in high:
	1061	while item in low:
	1062	low.remove(item)
	1063	test_case_order.append(low)
	1064
	1065	test_case_order_no_dublicates = []
	1066	for test_cases in test_case_order:
	1067	seen = set()
	1068	seen_add = seen.add
	1069	test_case_order_no_dublicates.append( [x for x in test_cases if not (x in seen or seen_add(x))] )
	1070
	1071	approved_test_case_order = [[]] + list(reversed(test_case_order_no_dublicates))
	1072	for i, test_cases in enumerate(list(approved_test_case_order)):
	1073	info = 'Case (dep)' if i < len(approved_test_case_order)-1 else 'Case'
	1074	for test_case_name in list(test_cases):
	1075	sys.stdout.flush()
	1076	test_case = PALMTestCase(self.test_dir, test_case_name, verbose=self.verbose)
	1077	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
	1078	if test_case_configuration_failed:
	1079	# removing as configuration failed should only apply to added dependencies
	1080	approved_test_case_order[i].remove(test_case_name)
	1081	logger.to_all(config_table_line_template.format(info, test_case_name, 'rejected'))
	1082	logger.to_all(message + '\n')
	1083	elif test_case.requires_binary_files:
	1084	if test_case.use_binary_files_from not in approved_test_case_order[i-1]:
	1085	# removing as dependency is already removed
	1086	approved_test_case_order[i].remove(test_case_name)
	1087	logger.to_all(config_table_line_template.format(info, test_case_name, 'disabled'))
	1088	logger.to_all(colored('[requires dependency \"' + test_case.use_binary_files_from + '\"]', 'red') + '\n')
	1089	else:
	1090	logger.to_all(config_table_line_template.format(info, test_case_name, 'approved'))
	1091	logger.to_all(message + '\n')
	1092	else:
	1093	logger.to_all(config_table_line_template.format(info, test_case_name, 'approved'))
	1094	logger.to_all(message + '\n')
	1095
	1096	final_case_list = []
	1097	for cases in approved_test_case_order:
	1098	for case in cases:
	1099	if case not in final_case_list:
	1100	final_case_list.append(case)
	1101
	1102	for build_name in final_build_list:
	1103	build = PALMBuild(
	1104	self.test_dir,
	1105	build_name,
	1106	verbose=self.verbose,
	1107	dry_run=self.dry_run,
	1108	)
	1109	configuration_failed, message = build.configure()
	1110	if not configuration_failed:
	1111	self.build_database[build_name] = build
	1112	else:
	1113	logger.to_all(message + '\n')
	1114
	1115	for case_name in final_case_list:
	1116	test_case = PALMTestCase(
	1117	self.test_dir,
	1118	case_name,
	1119	verbose=self.verbose,
	1120	dry_run=self.dry_run,
	1121	)
	1122	test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list)
	1123	if not test_case_configuration_failed:
	1124	self.test_case_queue.append(test_case)
	1125	logger.to_all(hline)
	1126
	1127	r = '{:' + str(table_width_intro) + '} '
	1128
	1129	logger.to_all(r.format('Test ID:') + self.test_id + '\n')
	1130	logger.to_all(r.format('Builds:') + str('\n' + r.format('')).join(sorted(self.build_database.keys())) + '\n')
	1131	logger.to_all(r.format('Cases:') + str('\n' + r.format('')).join([c.name for c in self.test_case_queue]) + '\n')
	1132	logger.to_all(r.format('Cores:') + ' '.join([str(i) for i in final_cores_list]) + '\n')
	1133
	1134	def _execute(self, test_case, build_name, cores):
	1135	job = PALMJob(
	1136	self.test_dir,
	1137	test_case,
	1138	build_name,
	1139	cores,
	1140	verbose=self.verbose,
	1141	dry_run=self.dry_run
	1142	)
	1143	if self.force_debug:
	1144	build_failed_non_debug = True
	1145	job_failed_non_debug = True
	1146	build_failed_debug = self.build_database[build_name].build(debug=True)
	1147	if build_failed_debug:
	1148	job_failed_debug = True
	1149	else:
	1150	job_failed_debug = job.execute(debug=True)
	1151	elif self.no_auto_debug:
	1152	build_failed_non_debug = self.build_database[build_name].build(debug=False)
	1153	if build_failed_non_debug:
	1154	job_failed_non_debug = True
	1155	else:
	1156	job_failed_non_debug = job.execute(debug=False)
	1157	build_failed_debug = None
	1158	job_failed_debug = None
	1159	else:
	1160	build_failed_non_debug = self.build_database[build_name].build(debug=False)
	1161	if build_failed_non_debug:
	1162	job_failed_non_debug = True
	1163	build_failed_debug = self.build_database[build_name].build(debug=True)
	1164	if build_failed_debug:
	1165	job_failed_debug = False
	1166	else:
	1167	job_failed_debug = job.execute(debug=True)
	1168	else:
	1169	job_failed_non_debug = job.execute(debug=False)
	1170	if job_failed_non_debug:
	1171	build_failed_debug = self.build_database[build_name].build(debug=True)
	1172	if build_failed_debug:
	1173	job_failed_debug = True
	1174	else:
	1175	job_failed_debug = job.execute(debug=True)
	1176	else:
	1177	build_failed_debug = None
	1178	job_failed_debug = None
	1179	return dict(
	1180	build_failed_non_debug=build_failed_non_debug,
	1181	job_failed_non_debug=job_failed_non_debug,
	1182	build_failed_debug=build_failed_debug,
	1183	job_failed_debug=job_failed_debug,
	1184	)
	1185
	1186	def execute(self):
	1187	with Logger(self.test_dir, verbose=self.verbose) as logger:
	1188	logger.to_all(hline)
	1189	logger.to_all(task_table_line_template.format('Task:', 'Case:', 'Build:', 'Cores:') + 'Status:\n')
	1190	logger.to_all(hline)
	1191	self.test_report = dict()
	1192	for test_case in self.test_case_queue:
	1193	logger.to_log(hline)
	1194	logger.to_file(hline)
	1195	logger.to_file(hline)
	1196	status_dict = dict()
	1197	for build_name in test_case.build_names:
	1198	status_dict[build_name] = dict()
	1199	for cores in test_case.number_of_cores:
	1200	status_dict[build_name][cores] = self._execute(test_case, build_name, cores)
	1201	self.test_report[test_case.name] = status_dict
	1202	logger.to_log(hline)
	1203	logger.to_file('\n' * 10)
	1204
	1205	def report(self):
	1206	with Logger(self.test_dir, verbose=self.verbose) as logger:
	1207	logger.to_all(hline)
	1208	r = '{:10}' + ' total: ' + '{:<3d}' + \
	1209	' ok: ' + colored('{:<3d}', 'green') + \
	1210	' debugged: ' + colored('{:<3d}', 'yellow') + \
	1211	' failed: ' + colored('{:<3d}', 'red')
	1212	n_all = 0
	1213	n_ok = 0
	1214	n_debugged = 0
	1215	n_failed = 0
	1216	for build_name, build in self.build_database.items():
	1217	status = build.report()
	1218	b = status['failed_non_debug']
	1219	bd = status['failed_debug']
	1220	n_all += 1
	1221	if not b and b is not None:
	1222	n_ok += 1
	1223	if bd is not None:
	1224	n_debugged += 1
	1225	if b and (bd or bd is None):
	1226	n_failed += 1
	1227	logger.to_all(r.format('Builds:', n_all, n_ok, n_debugged, n_failed) + '\n')
	1228	total_failed = n_failed
	1229	total_debugged = n_debugged
	1230	n_all = 0
	1231	n_ok = 0
	1232	n_debugged = 0
	1233	n_failed = 0
	1234	# {'case_name': {'build_name': {4: {'build_failed_debug': None,
	1235	# 'build_failed_non_debug': False,
	1236	# 'job_failed_debug': None,
	1237	# 'job_failed_non_debug': False}}},
	1238	for case_name, case in self.test_report.items():
	1239	for build_name, build in case.items():
	1240	for cores, results in build.items():
	1241	n_all += 1
	1242	b = results['build_failed_non_debug']
	1243	bd = results['build_failed_debug']
	1244	j = results['job_failed_non_debug']
	1245	jd = results['job_failed_debug']
	1246	if not j:
	1247	n_ok += 1
	1248	if jd is not None:
	1249	n_debugged += 1
	1250	if j and (jd or jd is None):
	1251	n_failed += 1
[3354]	1252	logger.to_all(r.format('Tests:', n_all, n_ok, n_debugged, n_failed) + '\n')
[3353]	1253	total_failed += n_failed
	1254	total_debugged += n_debugged
	1255	if self.fail_on_debug:
	1256	return (total_failed + total_debugged) > 0
	1257	else:
	1258	return total_failed > 0
	1259
	1260
	1261	class CustomCompleter:
	1262
	1263	def __init__(self):
	1264	pass
	1265
	1266	def __call__(self, prefix, parsed_args, **kwargs):
	1267	return (i for i in self.get_items() if i.startswith(prefix))
	1268
	1269	def get_items(self):
	1270	return []
	1271
	1272
	1273	class CaseCompleter(CustomCompleter):
	1274
	1275	def get_items(self):
[3354]	1276	case_names = [name for name in next(os.walk(trunk_tests_cases_dir))[1] if not name[0] == '.']
[3353]	1277	return case_names + ['all']
	1278
	1279
	1280	class BuildCompleter(CustomCompleter):
	1281
	1282	def get_items(self):
[3354]	1283	build_names = [name for name in next(os.walk(trunk_tests_builds_dir))[1] if not name[0] == '.']
[3353]	1284	return build_names + ['all']
	1285
	1286
	1287	class PALMTestArgumentParser(ArgumentParser):
	1288
	1289	def __init__(self):
	1290	super().__init__(
	1291	description='This is the PALM tester\n' +
	1292	'Developer Support: knoop@muk.uni-hannover.de',
	1293	formatter_class=RawTextHelpFormatter,
	1294	add_help=True,
	1295	)
	1296	self.add_argument(
	1297	'--version',
	1298	action='version',
	1299	version=version,
	1300	)
	1301	self.add_argument(
	1302	'--verbose',
	1303	action='store_true',
	1304	dest='verbose',
	1305	help='Increase verbosity of terminal output.',
	1306	required=False,
	1307	)
	1308	self.add_argument(
	1309	'--no-auto-debug',
	1310	action='store_true',
	1311	dest='no_auto_debug',
	1312	help='Disable automatic debugging in case of test failure.',
	1313	required=False,
	1314	)
	1315	self.add_argument(
	1316	'--force-debug',
	1317	action='store_true',
	1318	dest='force_debug',
	1319	help='Force debugging regardless of test failure (ignores --no-auto-debug).',
	1320	required=False,
	1321	)
	1322	self.add_argument(
	1323	'--fail-on-debug',
	1324	action='store_true',
	1325	dest='fail_on_debug',
	1326	help='Return a non-zero exit status in case debugging was required.',
	1327	required=False,
	1328	)
	1329	self.add_argument(
	1330	'--dry-run',
	1331	action='store_true',
	1332	dest='dry_run',
	1333	help='Prepare and process all requested tests without actually building or executing PALM.',
	1334	required=False,
	1335	)
	1336	self.add_argument(
[3354]	1337	'--no-color',
	1338	action='store_true',
	1339	dest='no_color',
	1340	help='Disable colored terminal output.',
	1341	required=False,
	1342	)
	1343	self.add_argument(
[3353]	1344	'--cases',
	1345	action='store',
	1346	dest='cases',
	1347	default=['all'],
	1348	help='A list of test cases to be executed. (default: %(default)s)',
	1349	nargs='+',
	1350	required=False,
	1351	type=str,
	1352	metavar='STR',
	1353	).completer = CaseCompleter()
	1354	self.add_argument(
	1355	'--builds',
	1356	action='store',
	1357	dest='builds',
	1358	default=['all'],
	1359	help='A list of builds to be executed. (default: %(default)s)',
	1360	nargs='+',
	1361	required=False,
	1362	type=str,
	1363	metavar='STR',
	1364	).completer = BuildCompleter()
	1365	self.add_argument(
	1366	'--cores',
	1367	action='store',
	1368	dest='cores',
	1369	default=[i for i in range(1, available_cores+1)],
	1370	choices=[i for i in range(1, available_cores+1)],
	1371	help='The number of cores tests are supposed to be executed on. (default: %(default)s)',
	1372	nargs='+',
	1373	required=False,
	1374	type=int,
	1375	metavar='INT',
	1376	)
	1377	self.add_argument(
	1378	'--test-id',
	1379	action='store',
	1380	dest='test_id',
	1381	default=datetime.now().strftime('%Y-%m-%d_%H:%M:%S.%f'),
	1382	help='An individual test id. (default: current timestamp)',
	1383	required=False,
	1384	type=str,
	1385	metavar='STR',
	1386	)
	1387
	1388
	1389	if __name__ == '__main__':
	1390	parser = PALMTestArgumentParser()
	1391	if has_argcomplete:
	1392	argcomplete.autocomplete(parser)
	1393	args = parser.parse_args()
	1394	palm_test = PALMTest(args)
	1395	palm_test.prepare()
	1396	palm_test.execute()
	1397	failed = palm_test.report()
	1398	exit(1 if failed else 0)

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

| Impressum | ©Leibniz Universität Hannover |