[3353] | 1 | #!/usr/bin/env python3 |
---|
| 2 | # PYTHON_ARGCOMPLETE_OK |
---|
[2497] | 3 | |
---|
[3353] | 4 | import os |
---|
| 5 | import sys |
---|
| 6 | import shutil |
---|
| 7 | from datetime import datetime |
---|
| 8 | import subprocess |
---|
| 9 | import multiprocessing |
---|
| 10 | import socket |
---|
| 11 | import getpass |
---|
| 12 | import math |
---|
| 13 | import re |
---|
| 14 | import threading |
---|
| 15 | import queue |
---|
| 16 | from contextlib import ContextDecorator |
---|
[2497] | 17 | |
---|
[3353] | 18 | try: |
---|
| 19 | from argparse import ArgumentParser |
---|
| 20 | from argparse import RawTextHelpFormatter |
---|
| 21 | except ImportError: |
---|
| 22 | sys.exit( |
---|
| 23 | 'ERROR: You need argparse!\n' + |
---|
| 24 | ' install it from http://pypi.python.org/pypi/argparse\n' + |
---|
| 25 | ' or run \"pip install argparse\".' |
---|
| 26 | ) |
---|
[2497] | 27 | |
---|
[3353] | 28 | try: |
---|
| 29 | import numpy as np |
---|
| 30 | except ImportError: |
---|
| 31 | sys.exit( |
---|
| 32 | 'ERROR: You need numpy!\n' + |
---|
| 33 | ' install it from http://pypi.python.org/pypi/numpy\n' + |
---|
| 34 | ' or run \"python3 -m pip install numpy\".' |
---|
| 35 | ) |
---|
[2497] | 36 | |
---|
[3353] | 37 | try: |
---|
| 38 | import netCDF4 |
---|
| 39 | except ImportError: |
---|
| 40 | sys.exit( |
---|
| 41 | 'ERROR: You need netCDF4!\n' + |
---|
| 42 | ' install it from http://pypi.python.org/pypi/netCDF4\n' + |
---|
| 43 | ' or run \"python3 -m pip install netCDF4\".' |
---|
| 44 | ) |
---|
[2497] | 45 | |
---|
[3353] | 46 | try: |
---|
| 47 | import yaml |
---|
| 48 | except ImportError: |
---|
| 49 | sys.exit( |
---|
| 50 | 'ERROR: You need PyYAML!\n' + |
---|
| 51 | ' install it from http://pypi.python.org/pypi/PyYAML\n' + |
---|
| 52 | ' or run \"python3 -m pip install PyYAML\".' |
---|
| 53 | ) |
---|
[2497] | 54 | |
---|
[3353] | 55 | try: |
---|
| 56 | import argcomplete |
---|
| 57 | except ImportError: |
---|
| 58 | print( |
---|
| 59 | 'INFO: To use Tab-completion you need argcomplete!\n' + |
---|
| 60 | ' install it from http://pypi.python.org/pypi/argcomplete\n' + |
---|
| 61 | ' or run \"python3 -m pip install argcomplete\".' |
---|
| 62 | ) |
---|
| 63 | has_argcomplete = False |
---|
| 64 | else: |
---|
| 65 | has_argcomplete = True |
---|
[2497] | 66 | |
---|
[3353] | 67 | try: |
---|
[3354] | 68 | from termcolor import colored as tcolored |
---|
[3353] | 69 | except ImportError: |
---|
[3354] | 70 | def tcolored(string, color): |
---|
[3353] | 71 | return string |
---|
[2497] | 72 | |
---|
[3354] | 73 | disable_colored_output = False |
---|
[2497] | 74 | |
---|
| 75 | |
---|
[3354] | 76 | def colored(string, color): |
---|
| 77 | if not disable_colored_output: |
---|
| 78 | return tcolored(string, color) |
---|
| 79 | else: |
---|
| 80 | return string |
---|
| 81 | |
---|
| 82 | |
---|
| 83 | version = '1.0.1' |
---|
| 84 | |
---|
[3353] | 85 | scripts_dir = os.path.dirname(os.path.realpath(__file__)) |
---|
| 86 | trunk_dir = os.path.realpath(os.path.join(scripts_dir, '..')) |
---|
| 87 | workspace_dir = os.path.realpath(os.path.join(trunk_dir, '..')) |
---|
[2497] | 88 | |
---|
[3353] | 89 | trunk_tests_dir = os.path.join(trunk_dir, 'TESTS') |
---|
| 90 | trunk_tests_cases_dir = os.path.join(trunk_tests_dir, 'cases') |
---|
| 91 | trunk_tests_builds_dir = os.path.join(trunk_tests_dir, 'builds') |
---|
[2497] | 92 | |
---|
[3353] | 93 | tests_dir = os.path.join(workspace_dir, 'tests') |
---|
[2497] | 94 | |
---|
[3353] | 95 | available_cores = multiprocessing.cpu_count() |
---|
| 96 | terminal_columns, terminal_lines = shutil.get_terminal_size() |
---|
| 97 | hline = '#' * min(terminal_columns, 300) + '\n' |
---|
| 98 | table_width_intro = 12 |
---|
[3354] | 99 | table_width_builds = len(max([s for s in next(os.walk(trunk_tests_builds_dir))[1] if not s[0] == '.'], key=len)) + len('_debug') |
---|
| 100 | table_width_cases = len(max([s for s in next(os.walk(trunk_tests_cases_dir))[1] if not s[0] == '.'], key=len)) |
---|
[3353] | 101 | table_width_cores = 7 |
---|
| 102 | table_width_total = table_width_intro + table_width_builds + table_width_cases + table_width_cores + 3 |
---|
[2497] | 103 | |
---|
[3353] | 104 | task_table_line_template = \ |
---|
| 105 | '{:' + str(table_width_intro) + '} ' + \ |
---|
| 106 | '{:' + str(table_width_cases) + '} ' + \ |
---|
| 107 | '{:' + str(table_width_builds) + '} ' + \ |
---|
| 108 | '{:' + str(table_width_cores) + '} ' |
---|
[2579] | 109 | |
---|
[3353] | 110 | config_table_line_template = \ |
---|
| 111 | '{:' + str(table_width_intro) + '} ' + \ |
---|
| 112 | '{:' + str(max(table_width_builds, table_width_cases)) + '} ' + \ |
---|
| 113 | '{:8} ' |
---|
[2579] | 114 | |
---|
[3353] | 115 | file_table_line_template = \ |
---|
| 116 | '{:' + str(table_width_intro) + '} ' + \ |
---|
| 117 | '{:' + str(table_width_cases + 13) + '} ' |
---|
| 118 | |
---|
| 119 | # for debugging |
---|
| 120 | exist_ok = False |
---|
| 121 | |
---|
| 122 | |
---|
| 123 | class SignificantDigitsRounder: |
---|
| 124 | |
---|
| 125 | @staticmethod |
---|
| 126 | def _round(value, digits=10): |
---|
| 127 | if value == 0.0: |
---|
| 128 | return value |
---|
| 129 | negative = value < 0.0 |
---|
| 130 | value = -value if negative else value |
---|
| 131 | rounded_value = round(value, -int(math.floor(math.log10(value))) + (digits - 1)) |
---|
| 132 | rounded_value = -rounded_value if negative else rounded_value |
---|
| 133 | return rounded_value |
---|
| 134 | |
---|
| 135 | |
---|
| 136 | vectorized_round = np.vectorize(_round) |
---|
| 137 | |
---|
| 138 | _vectorized_round = np.vectorize(round) |
---|
| 139 | |
---|
| 140 | |
---|
| 141 | @classmethod |
---|
| 142 | def around(cls, array, digits=10): |
---|
| 143 | # TODO: divide both arrays and check decimal point |
---|
| 144 | sign_mask = np.ma.masked_where(array >= 0.0, array).mask |
---|
| 145 | pos_array = np.where(sign_mask, array, -array) |
---|
| 146 | non_zero_maks = np.ma.masked_where(pos_array == 0.0, pos_array).mask |
---|
| 147 | non_zero_array = np.where(non_zero_maks, 1.0, pos_array) |
---|
| 148 | i1 = -np.floor(np.log10(non_zero_array)).astype(int) + (digits - 1) |
---|
| 149 | rounded_non_zero_array = cls._vectorized_round(non_zero_array, i1) |
---|
| 150 | rounded_pos_array = np.where(non_zero_maks, 0.0, rounded_non_zero_array) |
---|
| 151 | return np.where(sign_mask, rounded_pos_array, -rounded_pos_array) |
---|
| 152 | |
---|
| 153 | |
---|
| 154 | |
---|
| 155 | class Logger(ContextDecorator): |
---|
| 156 | |
---|
| 157 | def __init__(self, logfile_dir, logfile_name='palmtest.log', logfile_mode='a', verbose=False): |
---|
| 158 | self.logfile_path = os.path.join(logfile_dir, logfile_name) |
---|
| 159 | self.logfile_mode = logfile_mode |
---|
| 160 | self.verbose = verbose |
---|
| 161 | |
---|
| 162 | def __enter__(self): |
---|
| 163 | self._file = open(self.logfile_path, self.logfile_mode) |
---|
| 164 | return self |
---|
| 165 | |
---|
| 166 | def to_file(self, message): |
---|
| 167 | self._file.write(message) |
---|
| 168 | self._file.flush() |
---|
| 169 | |
---|
| 170 | def to_log(self, message): |
---|
| 171 | if self.verbose: |
---|
| 172 | sys.stdout.write(message) |
---|
| 173 | sys.stdout.flush() |
---|
| 174 | self._file.write(message) |
---|
| 175 | self._file.flush() |
---|
| 176 | |
---|
| 177 | def to_all(self, message): |
---|
| 178 | sys.stdout.write(message) |
---|
| 179 | sys.stdout.flush() |
---|
| 180 | self._file.write(message) |
---|
| 181 | self._file.flush() |
---|
| 182 | |
---|
| 183 | def __exit__(self, *exc): |
---|
| 184 | self._file.close() |
---|
| 185 | return False |
---|
| 186 | |
---|
| 187 | |
---|
| 188 | class Executor: |
---|
| 189 | |
---|
| 190 | @staticmethod |
---|
| 191 | def _enqueue_output(out, queue): |
---|
| 192 | for line in iter(out.readline, b''): |
---|
| 193 | queue.put(line) |
---|
| 194 | out.close() |
---|
| 195 | |
---|
| 196 | @staticmethod |
---|
| 197 | def execute(cmd, cwd='.', verbose=True, dry_run=False): |
---|
| 198 | assert isinstance(cmd, list) |
---|
| 199 | if dry_run: |
---|
| 200 | cmd = ['echo'] + cmd |
---|
| 201 | cmd_str = ' '.join(cmd) |
---|
| 202 | p = subprocess.Popen(cmd_str, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, bufsize=1) |
---|
| 203 | q = queue.Queue() |
---|
| 204 | t = threading.Thread(target=Executor._enqueue_output, args=(p.stdout, q)) |
---|
| 205 | t.daemon = True # thread dies with the program |
---|
| 206 | t.start() |
---|
| 207 | |
---|
| 208 | with Logger(cwd, verbose=verbose) as logger: |
---|
| 209 | # read line without blocking |
---|
| 210 | logger.to_log(hline) |
---|
| 211 | logger.to_log('CMD: ' + cmd_str + '\n') |
---|
| 212 | logger.to_log(hline) |
---|
| 213 | while t.is_alive(): |
---|
| 214 | try: |
---|
| 215 | line = q.get_nowait() # or q.get(timeout=.1) |
---|
| 216 | except queue.Empty: |
---|
| 217 | pass # print('no output yet') |
---|
| 218 | else: # got line |
---|
| 219 | logger.to_log(line.decode("utf-8")) |
---|
| 220 | line = True |
---|
| 221 | while line: |
---|
| 222 | try: |
---|
| 223 | line = q.get_nowait() # or q.get(timeout=.1) |
---|
| 224 | except queue.Empty: |
---|
| 225 | line = False |
---|
| 226 | else: # got line |
---|
| 227 | logger.to_log(line.decode("utf-8")) |
---|
| 228 | logger.to_log(hline) |
---|
| 229 | |
---|
| 230 | rc = p.poll() |
---|
| 231 | failed = rc != 0 |
---|
| 232 | return failed |
---|
| 233 | |
---|
| 234 | |
---|
| 235 | class NetCDFInterface: |
---|
| 236 | |
---|
| 237 | def __init__(self, filename): |
---|
| 238 | self.filename = filename |
---|
| 239 | |
---|
| 240 | def is_healthy(self): |
---|
| 241 | try: |
---|
| 242 | self.get_run_name() |
---|
| 243 | except: |
---|
| 244 | return False |
---|
| 245 | else: |
---|
| 246 | return True |
---|
| 247 | |
---|
| 248 | def get_run_name(self): |
---|
| 249 | with netCDF4.Dataset(self.filename, mode='r') as netcdf: |
---|
| 250 | l = getattr(netcdf, 'title').split() |
---|
| 251 | i = l.index('run:') |
---|
| 252 | return l[i+1] |
---|
| 253 | |
---|
| 254 | def get_var_list(self): |
---|
| 255 | with netCDF4.Dataset(self.filename, mode='r') as netcdf: |
---|
[3561] | 256 | var_list = list(netcdf.variables.keys()) |
---|
[3353] | 257 | var_list = filter(None, var_list) |
---|
| 258 | return sorted(var_list) |
---|
| 259 | |
---|
| 260 | def show_content(self): |
---|
| 261 | with netCDF4.Dataset(self.filename, mode='r') as netcdf: |
---|
| 262 | for name in netcdf.ncattrs(): |
---|
| 263 | print("Global attr", name, "=", getattr(netcdf, name)) |
---|
| 264 | print(netcdf) |
---|
| 265 | for v in netcdf.variables: |
---|
| 266 | print(v) |
---|
| 267 | |
---|
| 268 | def get_times_list(self): |
---|
| 269 | attributes, times = self.read_var('time') |
---|
| 270 | times = [str(time) for time in times] |
---|
| 271 | times = list(filter(None, times)) |
---|
| 272 | return times |
---|
| 273 | |
---|
| 274 | def contains(self, variable): |
---|
| 275 | return variable in self.get_var_list() |
---|
| 276 | |
---|
| 277 | def read_var(self, variable): |
---|
| 278 | with netCDF4.Dataset(self.filename, mode='r') as netcdf: |
---|
| 279 | values = netcdf.variables[variable][:] # extract values |
---|
[3561] | 280 | attributes = dict() |
---|
| 281 | try: |
---|
| 282 | attributes['long_name'] = netcdf.variables[variable].name |
---|
| 283 | except: |
---|
| 284 | attributes['long_name'] = '' |
---|
| 285 | try: |
---|
| 286 | attributes['unit'] = netcdf.variables[variable].units |
---|
| 287 | except: |
---|
| 288 | attributes['unit'] = '' |
---|
[3353] | 289 | return attributes, values |
---|
| 290 | |
---|
| 291 | |
---|
| 292 | class FileComparator: |
---|
| 293 | |
---|
| 294 | @staticmethod |
---|
| 295 | def compare_ascii(file_path1, file_path2, start_string=None): |
---|
| 296 | try: |
---|
| 297 | with open(file_path1, 'r') as file1: |
---|
| 298 | content1 = file1.readlines() |
---|
| 299 | except OSError: |
---|
| 300 | return True, colored('[reference file not found]', 'red') |
---|
| 301 | try: |
---|
| 302 | with open(file_path2, 'r') as file2: |
---|
| 303 | content2 = file2.readlines() |
---|
| 304 | except OSError: |
---|
| 305 | return True, colored('[output file not found]', 'red') |
---|
| 306 | if start_string: |
---|
| 307 | index1 = content1.index(start_string) |
---|
| 308 | index2 = content2.index(start_string) |
---|
| 309 | comparable_content1 = content1[index1:] |
---|
| 310 | comparable_content2 = content2[index2:] |
---|
| 311 | ln = index2 + 1 |
---|
| 312 | else: |
---|
| 313 | comparable_content1 = content1 |
---|
| 314 | comparable_content2 = content2 |
---|
| 315 | ln = 1 |
---|
| 316 | if len(comparable_content1) != len(comparable_content2): |
---|
| 317 | return True, colored('[mismatch in total number of lines]', 'red') |
---|
| 318 | for line1, line2 in zip(comparable_content1, comparable_content2): |
---|
| 319 | if not line1 == line2: |
---|
| 320 | return True, colored('[mismatch in content starting line ' + str(ln) + ']', 'red') |
---|
| 321 | ln += 1 |
---|
| 322 | return False, colored('[file ok]', 'green') |
---|
| 323 | |
---|
| 324 | @staticmethod |
---|
| 325 | def compare_netcdf(file_path1, file_path2, digits=None): |
---|
| 326 | nci1 = NetCDFInterface(file_path1) |
---|
| 327 | nci2 = NetCDFInterface(file_path2) |
---|
| 328 | if not nci1.is_healthy(): |
---|
| 329 | return True, colored('[reference file not found]', 'red') |
---|
| 330 | if not nci2.is_healthy(): |
---|
| 331 | return True, colored('[output file not found]', 'red') |
---|
| 332 | times_list1 = nci1.get_times_list() |
---|
| 333 | times_list2 = nci2.get_times_list() |
---|
| 334 | if not times_list1 == times_list2: |
---|
| 335 | return True, colored('[wrong time dimension]', 'red') |
---|
| 336 | else: |
---|
| 337 | time_list = times_list1 |
---|
| 338 | var_list1 = nci1.get_var_list() |
---|
| 339 | var_list2 = nci2.get_var_list() |
---|
| 340 | if not var_list1 == var_list2: |
---|
| 341 | return True, colored('[wrong set of variables]', 'red') |
---|
| 342 | else: |
---|
| 343 | var_list = var_list1 |
---|
| 344 | content1 = dict() |
---|
| 345 | content2 = dict() |
---|
| 346 | for var in var_list: |
---|
| 347 | attributes1, values1 = nci1.read_var(var) |
---|
| 348 | attributes2, values2 = nci2.read_var(var) |
---|
| 349 | if sorted(attributes1.keys()) != sorted(attributes2.keys()): |
---|
| 350 | return True, colored('[wrong set of attributes in variable \"'+var+'\"]', 'red') |
---|
| 351 | if isinstance(digits, int): |
---|
| 352 | values1 = SignificantDigitsRounder.around(values1, digits=digits) |
---|
| 353 | values2 = SignificantDigitsRounder.around(values2, digits=digits) |
---|
| 354 | content1[var] = values1 |
---|
| 355 | content2[var] = values2 |
---|
| 356 | #for decimals in |
---|
| 357 | for i, time in enumerate(time_list): |
---|
| 358 | for var in var_list: |
---|
| 359 | t_content1 = content1[var][i] |
---|
| 360 | t_content2 = content2[var][i] |
---|
| 361 | if not (t_content1==t_content2).all(): |
---|
| 362 | if isinstance(digits, int): |
---|
| 363 | return True, colored('[1st mismatch within ' + str(digits) + ' digits at time index ' + str(i) + ' in variable \"' + var + '\"]', 'red') |
---|
| 364 | else: |
---|
| 365 | return True, colored('[1st mismatch at time index ' + str(i) + ' in variable \"' + var + '\"]', 'red') |
---|
| 366 | return False, colored('[file ok]', 'green') |
---|
| 367 | |
---|
| 368 | |
---|
| 369 | class OutputChecker: |
---|
| 370 | |
---|
| 371 | def __init__(self, test_dir, setup_name, build_name, cores, significant_digits=None, verbose=True, dry_run=False): |
---|
| 372 | self.test_dir = test_dir |
---|
| 373 | self.setup_name = setup_name |
---|
| 374 | self.build_name = build_name |
---|
| 375 | self.cores = cores |
---|
| 376 | self.significant_digits = significant_digits |
---|
| 377 | self.verbose = verbose |
---|
| 378 | self.dry_run = dry_run |
---|
| 379 | self.job_name = self.setup_name + '__' + build_name + '__' + str(self.cores) |
---|
| 380 | self.job_dir = os.path.join(self.test_dir, 'JOBS', self.job_name) |
---|
| 381 | self.ref_monitoring_dir = os.path.join(trunk_tests_cases_dir, self.setup_name, 'MONITORING') |
---|
| 382 | self.ref_output_dir = os.path.join(trunk_tests_cases_dir, self.setup_name, 'OUTPUT') |
---|
| 383 | self.res_monitoring_dir = os.path.join(self.job_dir, 'MONITORING') |
---|
| 384 | self.res_output_dir = os.path.join(self.job_dir, 'OUTPUT') |
---|
| 385 | self.failed = None |
---|
| 386 | |
---|
| 387 | def get_checkable_file_dicts(self): |
---|
| 388 | if os.path.isdir(self.ref_monitoring_dir): |
---|
| 389 | file_names_monitoring = [s for s in next(os.walk(self.ref_monitoring_dir))[2]] |
---|
| 390 | else: |
---|
| 391 | file_names_monitoring = [] |
---|
| 392 | file_paths_monitoring = [] |
---|
| 393 | for file_name in file_names_monitoring: |
---|
| 394 | file_specific_ending = file_name[len(self.setup_name):] |
---|
| 395 | file_specific_ending_split = file_specific_ending.split('.') |
---|
| 396 | postfix = file_specific_ending_split[0] |
---|
| 397 | if len(file_specific_ending_split) > 1: |
---|
| 398 | extension = file_specific_ending_split[-1] |
---|
| 399 | else: |
---|
| 400 | extension = '' |
---|
| 401 | if len(file_specific_ending_split) > 2: |
---|
| 402 | cycle_info = file_specific_ending_split[1: -1] |
---|
| 403 | else: |
---|
| 404 | cycle_info = [] |
---|
| 405 | file_paths_monitoring.append( |
---|
| 406 | dict( |
---|
| 407 | postfix=postfix, |
---|
| 408 | cycle_info=cycle_info, |
---|
| 409 | extension=extension, |
---|
| 410 | ref_path=self.ref_monitoring_dir, |
---|
| 411 | res_path=self.res_monitoring_dir, |
---|
| 412 | ) |
---|
| 413 | ) |
---|
| 414 | if os.path.isdir(self.ref_output_dir): |
---|
| 415 | file_names_output = [s for s in next(os.walk(self.ref_output_dir))[2]] |
---|
| 416 | else: |
---|
| 417 | file_names_output = [] |
---|
| 418 | file_paths_output = [] |
---|
| 419 | for file_name in file_names_output: |
---|
| 420 | file_specific_ending = file_name[len(self.setup_name):] |
---|
| 421 | file_specific_ending_split = file_specific_ending.split('.') |
---|
| 422 | postfix = file_specific_ending_split[0] |
---|
| 423 | if len(file_specific_ending_split) > 1: |
---|
| 424 | extension = file_specific_ending_split[-1] |
---|
| 425 | else: |
---|
| 426 | extension = '' |
---|
| 427 | if len(file_specific_ending_split) > 2: |
---|
| 428 | cycle_info = file_specific_ending_split[1: -1] |
---|
| 429 | else: |
---|
| 430 | cycle_info = [] |
---|
| 431 | file_paths_output.append( |
---|
| 432 | dict( |
---|
| 433 | postfix=postfix, |
---|
| 434 | cycle_info=cycle_info, |
---|
| 435 | extension=extension, |
---|
| 436 | ref_path=self.ref_output_dir, |
---|
| 437 | res_path=self.res_output_dir, |
---|
| 438 | ) |
---|
| 439 | ) |
---|
| 440 | return file_paths_monitoring + file_paths_output |
---|
| 441 | |
---|
| 442 | def check(self): |
---|
| 443 | with Logger(self.test_dir, verbose=self.verbose) as logger: |
---|
| 444 | logger.to_log('Checking output files:') |
---|
| 445 | logger.to_all('\n') |
---|
| 446 | failed = False |
---|
| 447 | for file_dict in self.get_checkable_file_dicts(): |
---|
| 448 | file_failed = False |
---|
| 449 | ext_list = [file_dict['extension']] if file_dict['extension'] else [] |
---|
| 450 | file_specific_ending = '.'.join([file_dict['postfix']] + file_dict['cycle_info'] + ext_list ) |
---|
| 451 | logger.to_all(file_table_line_template.format('Checking:', self.setup_name + file_specific_ending)) |
---|
| 452 | ref_file_path = os.path.join(file_dict['ref_path'], self.setup_name + file_specific_ending) |
---|
| 453 | res_file_path = os.path.join(file_dict['res_path'], self.job_name + file_specific_ending) |
---|
| 454 | if re.match('_rc', file_dict['postfix']) and not file_dict['extension']: |
---|
| 455 | file_failed, message = FileComparator.compare_ascii(ref_file_path, res_file_path, start_string='Run-control output:\n') |
---|
| 456 | elif re.match('nc', file_dict['extension']): |
---|
| 457 | if self.significant_digits is not None: |
---|
| 458 | if re.match('_ts', file_dict['postfix']) and 'timeseries' in self.significant_digits: |
---|
| 459 | file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path, |
---|
| 460 | digits=self.significant_digits['timeseries']) |
---|
| 461 | elif re.match('_pr', file_dict['postfix']) and 'profiles' in self.significant_digits: |
---|
| 462 | file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path, |
---|
| 463 | digits=self.significant_digits['profiles']) |
---|
| 464 | else: |
---|
| 465 | file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path, |
---|
| 466 | digits=self.significant_digits['other']) |
---|
| 467 | else: |
---|
| 468 | file_failed, message = FileComparator.compare_netcdf(ref_file_path, res_file_path) |
---|
| 469 | else: |
---|
| 470 | message = colored('[ignored]', 'blue') |
---|
| 471 | if file_failed: |
---|
| 472 | failed = True |
---|
| 473 | logger.to_all(message + '\n') |
---|
| 474 | if self.dry_run: |
---|
| 475 | failed = False |
---|
| 476 | return failed |
---|
| 477 | |
---|
| 478 | |
---|
| 479 | class PALMJob: |
---|
| 480 | """The PALM job class deals with the execution of a single PALM job""" |
---|
| 481 | |
---|
| 482 | @staticmethod |
---|
| 483 | def get_job_name(setup_name, build_name, cores): |
---|
| 484 | return setup_name + '__' + build_name + '__' + str(cores) |
---|
| 485 | |
---|
| 486 | def __init__(self, test_dir, test_case, build_name, cores, verbose=False, dry_run=False): |
---|
| 487 | self.test_dir = test_dir |
---|
| 488 | self.test_case = test_case |
---|
| 489 | self.build_name = build_name |
---|
| 490 | self.cores = cores |
---|
| 491 | self.verbose = verbose |
---|
| 492 | self.dry_run = dry_run |
---|
| 493 | |
---|
| 494 | self.attempted_debug = False |
---|
| 495 | self.failed_debug = None |
---|
| 496 | self.attempted_non_debug = False |
---|
| 497 | self.failed_non_debug = None |
---|
| 498 | |
---|
| 499 | def _link_restart_files(self, build_name): |
---|
| 500 | if self.dry_run: |
---|
| 501 | return True, colored('[restart data dry]', 'blue') |
---|
| 502 | name = self.get_job_name(self.test_case.name, build_name, self.cores) |
---|
| 503 | source_name = self.get_job_name(self.test_case.use_binary_files_from, build_name, self.cores) |
---|
| 504 | source_restart_dir = os.path.join(self.test_dir, 'JOBS', source_name, 'RESTART') |
---|
| 505 | try: |
---|
| 506 | source_data_dirs_grp = [d for r, d, f in os.walk(source_restart_dir)] |
---|
| 507 | except: |
---|
| 508 | source_data_dirs_grp = [] |
---|
| 509 | if len(source_data_dirs_grp) == 0: |
---|
| 510 | source_data_dirs = [] |
---|
| 511 | else: |
---|
| 512 | source_data_dirs = source_data_dirs_grp[0] |
---|
| 513 | if len(source_data_dirs) == 0 and re.match('.+_debug', build_name): |
---|
| 514 | source_build_name = build_name[:-len('_debug')] |
---|
| 515 | source_name = self.get_job_name(self.test_case.use_binary_files_from, source_build_name, self.cores) |
---|
| 516 | source_restart_dir = os.path.join(self.test_dir, 'JOBS', source_name, 'RESTART') |
---|
| 517 | try: |
---|
| 518 | source_data_dirs_grp = [d for r, d, f in os.walk(source_restart_dir)] |
---|
| 519 | except: |
---|
| 520 | source_data_dirs_grp = [] |
---|
| 521 | if len(source_data_dirs_grp) == 0: |
---|
| 522 | source_data_dirs = [] |
---|
| 523 | else: |
---|
| 524 | source_data_dirs = source_data_dirs_grp[0] |
---|
| 525 | if len(source_data_dirs) == 0: |
---|
| 526 | source_data_dir = 'no_restart_data' |
---|
| 527 | else: |
---|
| 528 | source_data_dir = sorted(source_data_dirs)[-1] |
---|
| 529 | source_data_dir_path = os.path.join(source_restart_dir, source_data_dir) |
---|
| 530 | if os.path.isdir(source_data_dir_path) and re.match('.+_d3d.*', source_data_dir): |
---|
| 531 | job_restart_dir = os.path.join(self.test_dir, 'JOBS', name, 'RESTART') |
---|
| 532 | os.makedirs(job_restart_dir, exist_ok=exist_ok) |
---|
| 533 | job_data_dir_path = os.path.join(job_restart_dir, name + '_d3d') |
---|
| 534 | os.symlink(source_data_dir_path, job_data_dir_path, target_is_directory=True) |
---|
| 535 | return False, colored('[linked restart data from: ' + source_data_dir_path + ']', 'green') |
---|
| 536 | else: |
---|
| 537 | return True, colored('[no restart data found]', 'red') |
---|
| 538 | |
---|
| 539 | def _execute(self, name, build_name): |
---|
| 540 | execution_failed = Executor.execute( |
---|
| 541 | [ |
---|
| 542 | os.path.join(self.test_dir, 'trunk', 'SCRIPTS', 'palmrun'), |
---|
[3457] | 543 | '-c', '\"' + build_name + '\"', |
---|
| 544 | '-r', name, |
---|
[3353] | 545 | '-a', '\"' + ' '.join(self.test_case.activation_strings) + '\"', |
---|
| 546 | '-X', str(self.cores), |
---|
| 547 | '-T', str(self.cores), |
---|
| 548 | '-B', |
---|
| 549 | '-v', |
---|
| 550 | '-z', |
---|
| 551 | ], |
---|
| 552 | cwd=self.test_dir, |
---|
| 553 | verbose=self.verbose, |
---|
| 554 | dry_run=self.dry_run, |
---|
| 555 | ) |
---|
| 556 | |
---|
| 557 | if self.dry_run: |
---|
| 558 | return False, colored('[execution dry]', 'blue') |
---|
| 559 | elif execution_failed: |
---|
| 560 | return True, colored('[execution failed]', 'red') |
---|
| 561 | else: |
---|
| 562 | return False, colored('[execution ok]', 'green') |
---|
| 563 | |
---|
| 564 | def _check(self, build_name): |
---|
| 565 | checker = OutputChecker( |
---|
| 566 | self.test_dir, |
---|
| 567 | self.test_case.name, |
---|
| 568 | build_name, |
---|
| 569 | self.cores, |
---|
| 570 | significant_digits=self.test_case.significant_digits, |
---|
| 571 | verbose=self.verbose, |
---|
| 572 | dry_run=self.dry_run, |
---|
| 573 | ) |
---|
| 574 | check_failed = checker.check() |
---|
| 575 | |
---|
| 576 | if self.dry_run: |
---|
| 577 | return False, colored('[checks dry]', 'blue') |
---|
| 578 | if check_failed: |
---|
| 579 | return True, colored('[checks failed]', 'red') |
---|
| 580 | else: |
---|
| 581 | return False, colored('[checks ok]', 'green') |
---|
| 582 | |
---|
| 583 | def execute(self, debug=False): |
---|
| 584 | if debug: |
---|
| 585 | attempted = self.attempted_debug |
---|
| 586 | build_name = self.build_name + '_debug' |
---|
| 587 | failed = self.failed_debug |
---|
| 588 | else: |
---|
| 589 | attempted = self.attempted_non_debug |
---|
| 590 | build_name = self.build_name |
---|
| 591 | failed = self.failed_non_debug |
---|
| 592 | |
---|
| 593 | if not attempted: |
---|
| 594 | with Logger(self.test_dir, verbose=self.verbose) as logger: |
---|
| 595 | status_prefix = task_table_line_template.format('Testing:', self.test_case.name, build_name, self.cores) |
---|
| 596 | logger.to_all(status_prefix) |
---|
| 597 | logger.to_log('[started]' + '\n') |
---|
| 598 | attempted = True |
---|
| 599 | |
---|
| 600 | name = self.get_job_name(self.test_case.name, build_name, self.cores) |
---|
| 601 | |
---|
| 602 | input_dir = os.path.join(self.test_dir, 'JOBS', name, 'INPUT') |
---|
| 603 | os.makedirs(input_dir, exist_ok=exist_ok) |
---|
| 604 | |
---|
| 605 | # copying needs to be done per file, because input files need to be renamed |
---|
| 606 | for input_file in self.test_case.input_file_names: |
---|
| 607 | postfix = input_file[len(self.test_case.name):] |
---|
| 608 | src = os.path.join(self.test_case.input_dir, input_file) |
---|
| 609 | dst = os.path.join(input_dir, name + postfix) |
---|
| 610 | shutil.copy(src, dst) |
---|
| 611 | |
---|
| 612 | # copying the entire directory is ok, because source files do not need to be renamed |
---|
| 613 | user_code_dir = os.path.join(self.test_dir, 'JOBS', name, 'USER_CODE') |
---|
| 614 | if os.path.isdir(self.test_case.user_code_dir): |
---|
| 615 | shutil.copytree(self.test_case.user_code_dir, user_code_dir, copy_function=shutil.copy) |
---|
| 616 | |
---|
| 617 | if self.test_case.requires_binary_files: |
---|
| 618 | link_restart_files_failed, message = self._link_restart_files(build_name) |
---|
| 619 | logger.to_log(status_prefix) |
---|
| 620 | logger.to_log(message + ' ') |
---|
| 621 | logger.to_log('\n') |
---|
| 622 | |
---|
| 623 | failed, message = self._execute(name, build_name) |
---|
| 624 | logger.to_log(status_prefix) |
---|
| 625 | logger.to_all(message + ' ') |
---|
| 626 | logger.to_log('\n') |
---|
| 627 | |
---|
| 628 | failed, message = self._check(build_name) |
---|
| 629 | logger.to_log(status_prefix) |
---|
| 630 | logger.to_log(message + ' ') |
---|
| 631 | |
---|
| 632 | logger.to_all('\n') |
---|
| 633 | |
---|
| 634 | if debug: |
---|
| 635 | self.attempted_debug = attempted |
---|
| 636 | self.failed_debug = failed |
---|
| 637 | else: |
---|
| 638 | self.attempted_non_debug = attempted |
---|
| 639 | self.failed_non_debug = failed |
---|
| 640 | |
---|
| 641 | return failed |
---|
| 642 | |
---|
| 643 | def status(self): |
---|
| 644 | return dict( |
---|
| 645 | attempted=self.attempted_non_debug or self.attempted_debug, |
---|
| 646 | failed=self.failed_non_debug and self.failed_debug, |
---|
| 647 | debugged=self.attempted_debug, |
---|
| 648 | non_debug_failed=self.failed_non_debug, |
---|
| 649 | ) |
---|
| 650 | |
---|
| 651 | |
---|
| 652 | class PALMBuild: |
---|
| 653 | """The PALM build class deals with configuration and execution of all required PALM builds""" |
---|
| 654 | |
---|
| 655 | def __init__(self, test_dir, build_name, verbose=False, dry_run=False): |
---|
| 656 | self.test_dir = test_dir |
---|
| 657 | self.build_name = build_name |
---|
| 658 | self.verbose = verbose |
---|
| 659 | self.dry_run = dry_run |
---|
| 660 | self.configured = False |
---|
| 661 | self.executed = False |
---|
| 662 | self.available = False |
---|
| 663 | self.requires_mpi = False |
---|
| 664 | self.requires_netcdf = False |
---|
| 665 | self.requires_fftw = False |
---|
| 666 | self.requires_rrtmg = False |
---|
| 667 | self.attempted_non_debug = False |
---|
| 668 | self.attempted_debug = False |
---|
| 669 | self.failed_non_debug = None |
---|
| 670 | self.failed_debug = None |
---|
| 671 | |
---|
| 672 | def configure(self): |
---|
| 673 | try: |
---|
| 674 | with open(os.path.join(trunk_tests_builds_dir, self.build_name, 'build_config.yml'), 'r') as f: |
---|
| 675 | build_config = yaml.load(f) |
---|
| 676 | except: |
---|
| 677 | return True, colored('[build not found]', 'red') |
---|
| 678 | |
---|
| 679 | if 'compiler' in build_config: |
---|
| 680 | self.compiler = build_config['compiler'] |
---|
| 681 | else: |
---|
| 682 | return True, colored('[missing \"compiler\" keyword]', 'red') |
---|
| 683 | |
---|
| 684 | if not isinstance(self.compiler, dict): |
---|
| 685 | return True, colored('[\"compiler\" keyword must be dict]', 'red') |
---|
| 686 | |
---|
| 687 | if 'linker' in build_config: |
---|
| 688 | self.linker = build_config['linker'] |
---|
| 689 | else: |
---|
| 690 | return True, colored('[missing \"linker\" keyword]', 'red') |
---|
| 691 | |
---|
| 692 | if not isinstance(self.linker, dict): |
---|
| 693 | return True, colored('[\"linker\" keyword must be dict]', 'red') |
---|
| 694 | |
---|
| 695 | if 'mpi_wrapper' in self.compiler: |
---|
| 696 | if 'mpi_wrapper}}' in self.compiler['mpi_wrapper']: |
---|
| 697 | self.requires_mpi = True |
---|
| 698 | else: |
---|
| 699 | return True, colored('[missing \"mpi_wrapper\" keyword]', 'red') |
---|
| 700 | |
---|
| 701 | if 'includes' in self.compiler: |
---|
| 702 | for include in self.compiler['includes']: |
---|
| 703 | if 'include.netcdf}}' in include: |
---|
| 704 | self.requires_netcdf = True |
---|
| 705 | if 'include.fftw}}' in include: |
---|
| 706 | self.requires_fftw = True |
---|
| 707 | if 'include.rrtmg}}' in include: |
---|
| 708 | self.requires_rrtmg = True |
---|
| 709 | else: |
---|
| 710 | return True, colored('[missing \"includes\" keyword in compiler]', 'red') |
---|
| 711 | |
---|
| 712 | if 'options' in self.linker: |
---|
| 713 | for lib in self.linker['options']: |
---|
| 714 | if 'lib.netcdf}}' in lib: |
---|
| 715 | self.requires_netcdf = True |
---|
| 716 | if 'lib.fftw}}' in lib: |
---|
| 717 | self.requires_fftw = True |
---|
| 718 | if 'lib.rrtmg}}' in lib: |
---|
| 719 | self.requires_rrtmg = True |
---|
| 720 | else: |
---|
| 721 | return True, colored('[missing \"options\" keyword in linker]', 'red') |
---|
| 722 | |
---|
| 723 | library_names = [] |
---|
| 724 | if self.requires_netcdf: |
---|
| 725 | library_names.append('netcdf') |
---|
| 726 | if self.requires_fftw: |
---|
| 727 | library_names.append('fftw') |
---|
| 728 | if self.requires_rrtmg: |
---|
| 729 | library_names.append('rrtmg') |
---|
| 730 | |
---|
| 731 | if not 'executable' in self.compiler: |
---|
| 732 | return True, colored('[missing \"executable\" keyword in compiler]', 'red') |
---|
| 733 | |
---|
| 734 | if not 'definitions' in self.compiler: |
---|
| 735 | return True, colored('[missing \"definitions\" keyword in compiler]', 'red') |
---|
| 736 | |
---|
| 737 | if not 'options' in self.compiler: |
---|
| 738 | return True, colored('[missing \"options\" keyword in compiler]', 'red') |
---|
| 739 | |
---|
| 740 | if not 'default' in self.compiler['options']: |
---|
| 741 | return True, colored('[missing \"default\" keyword in compiler.options]', 'red') |
---|
| 742 | |
---|
| 743 | if not 'debug' in self.compiler['options']: |
---|
| 744 | return True, colored('[missing \"debug\" keyword in compiler.options]', 'red') |
---|
| 745 | |
---|
| 746 | try: |
---|
| 747 | with open(os.path.join(workspace_dir, 'palmtest.yml'), 'r') as f: |
---|
| 748 | palmtest_config = yaml.load(f) |
---|
| 749 | except: |
---|
| 750 | return True, colored('[palmtest.yml not found]', 'red') |
---|
| 751 | |
---|
| 752 | if 'palm_config_template' in palmtest_config: |
---|
| 753 | if isinstance(palmtest_config['palm_config_template'], str): |
---|
| 754 | custom_template = palmtest_config['palm_config_template'] |
---|
| 755 | try: |
---|
| 756 | with open(os.path.join(custom_template), 'r') as palm_config_template_file: |
---|
| 757 | template = palm_config_template_file.read() |
---|
| 758 | except: |
---|
| 759 | try: |
---|
| 760 | with open(os.path.join(scripts_dir, '.palm.config.default.in'), 'r') as palm_config_template_file: |
---|
| 761 | template = palm_config_template_file.read() |
---|
| 762 | except: |
---|
| 763 | return True, colored('[trunk/SCRIPTS/.palm.config.default.in not found]', 'red') |
---|
| 764 | |
---|
| 765 | template = template.replace('@CMAKE_INSTALL_PREFIX@', self.test_dir) |
---|
| 766 | template = template.replace('@PALM_HOSTNAME@', socket.gethostname()) |
---|
| 767 | template = template.replace('@CMAKE_USERNAME@', getpass.getuser()) |
---|
| 768 | template = template.replace('@MPI_Fortran_COMPILER@', self.compiler['mpi_wrapper']) |
---|
| 769 | template = template.replace('@CMAKE_Fortran_COMPILER@', self.compiler['executable']) |
---|
| 770 | cpp_options_str = ['-D' + s for s in self.compiler['definitions']] |
---|
| 771 | template = template.replace('@PALM_CPP_OPTIONS_STR@', ' '.join(cpp_options_str)) |
---|
| 772 | template = template.replace('@PALM_CORES@', str(available_cores)) |
---|
| 773 | template = template.replace('@PALM_COMPILER_OPTIONS@', '{{palmtest.compiler.options}} ' + ' '.join(self.compiler['includes'])) |
---|
| 774 | template = template.replace('@PALM_LINKER_OPTIONS@', ' '.join(self.linker['options'])) |
---|
| 775 | |
---|
| 776 | if 'environments' in palmtest_config: |
---|
| 777 | available_environments = palmtest_config['environments'] |
---|
| 778 | else: |
---|
| 779 | return True, colored('[missing \"environments\" keyword in palmtest.yml]', 'red') |
---|
| 780 | |
---|
| 781 | if 'id' in self.compiler: |
---|
| 782 | c_id = self.compiler['id'] |
---|
| 783 | else: |
---|
| 784 | return True, colored('[missing \"id\" keyword in compiler]', 'red') |
---|
| 785 | |
---|
| 786 | if c_id in available_environments: |
---|
| 787 | self.available = True |
---|
| 788 | |
---|
| 789 | environment = available_environments[c_id] |
---|
[3682] | 790 | |
---|
| 791 | if 'mpi_execution_command' in environment: |
---|
| 792 | template = template.replace('@PALM_EXECUTE_COMMAND@', environment['mpi_execution_command']) |
---|
| 793 | else: |
---|
| 794 | template = template.replace('@PALM_EXECUTE_COMMAND@', 'mpirun -n {{mpi_tasks}}') |
---|
| 795 | |
---|
[3353] | 796 | if 'executable' not in environment: |
---|
| 797 | return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"executable\"]', 'red') |
---|
| 798 | value = environment['executable'] |
---|
| 799 | if isinstance(value, str): |
---|
| 800 | template = template.replace('{{' + '.'.join([c_id, 'executable']) + '}}', value) |
---|
| 801 | if self.requires_mpi: |
---|
| 802 | if 'mpi_wrapper' not in environment: |
---|
| 803 | return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"mpi_wrapper\"]', 'red') |
---|
| 804 | value = environment['mpi_wrapper'] |
---|
| 805 | if isinstance(value, str): |
---|
| 806 | template = template.replace('{{' + '.'.join([c_id, 'mpi_wrapper']) + '}}', value) |
---|
| 807 | if 'include' not in environment: |
---|
| 808 | return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"include\"]', 'red') |
---|
| 809 | if 'lib' not in environment: |
---|
| 810 | return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"lib\"]', 'red') |
---|
| 811 | for lib in library_names: |
---|
| 812 | if lib not in environment['include']: |
---|
| 813 | return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"include.'+lib+'\"]', 'red') |
---|
| 814 | value = environment['include'][lib] |
---|
| 815 | if isinstance(value, str): |
---|
| 816 | template = template.replace('{{' + '.'.join([c_id, 'include', lib]) + '}}', value) |
---|
| 817 | if lib not in environment['lib']: |
---|
| 818 | return True, colored('[palmtest.yml environment \"' + c_id + '\" has no \"lib.'+lib+'\"]', 'red') |
---|
| 819 | value = environment['lib'][lib] |
---|
| 820 | if isinstance(value, str): |
---|
| 821 | template = template.replace('{{' + '.'.join([c_id, 'lib', lib]) + '}}', value) |
---|
| 822 | |
---|
| 823 | with open(os.path.join(self.test_dir, '.palm.config.' + self.build_name), 'w') as palm_config_file: |
---|
| 824 | palm_config_file.write( |
---|
| 825 | template.replace( |
---|
| 826 | '{{palmtest.compiler.options}}', |
---|
| 827 | ' '.join(self.compiler['options']['default']), |
---|
| 828 | ) |
---|
| 829 | ) |
---|
| 830 | with open(os.path.join(self.test_dir, '.palm.config.' + self.build_name + '_debug'), 'w') as palm_config_file: |
---|
| 831 | palm_config_file.write( |
---|
| 832 | template.replace( |
---|
| 833 | '{{palmtest.compiler.options}}', |
---|
| 834 | ' '.join(self.compiler['options']['debug']), |
---|
| 835 | ) |
---|
| 836 | ) |
---|
| 837 | self.configured = True |
---|
| 838 | return False, colored('[configuration ok]', 'green') |
---|
| 839 | |
---|
| 840 | else: |
---|
| 841 | return True, colored('[palmtest.yml environment \"' + c_id + '\" not found]', 'red') |
---|
| 842 | |
---|
| 843 | def _execute(self, build_name): |
---|
| 844 | self.attempted = True |
---|
| 845 | build_failed = Executor.execute( |
---|
| 846 | [ |
---|
| 847 | os.path.join(self.test_dir, 'trunk', 'SCRIPTS', 'palmbuild'), |
---|
[3457] | 848 | '-c', '\"' + build_name + '\"', |
---|
[3353] | 849 | '-v', |
---|
| 850 | ], |
---|
| 851 | cwd=self.test_dir, |
---|
| 852 | verbose=self.verbose, |
---|
| 853 | dry_run=self.dry_run, |
---|
| 854 | ) |
---|
| 855 | |
---|
| 856 | if self.dry_run: |
---|
| 857 | return False, colored('[build dry]', 'blue') |
---|
| 858 | if build_failed: |
---|
| 859 | return True, colored('[build failed]', 'red') |
---|
| 860 | else: |
---|
| 861 | return False, colored('[build ok]', 'green') |
---|
| 862 | |
---|
| 863 | def build(self, debug=False): |
---|
| 864 | if debug: |
---|
| 865 | attempted = self.attempted_debug |
---|
| 866 | build_name = self.build_name + '_debug' |
---|
| 867 | failed = self.failed_debug |
---|
| 868 | else: |
---|
| 869 | attempted = self.attempted_non_debug |
---|
| 870 | build_name = self.build_name |
---|
| 871 | failed = self.failed_non_debug |
---|
| 872 | |
---|
| 873 | if not attempted: |
---|
| 874 | with Logger(self.test_dir, verbose=self.verbose) as logger: |
---|
| 875 | status_prefix = task_table_line_template.format('Building:', '', build_name, '') |
---|
| 876 | logger.to_all(status_prefix) |
---|
| 877 | logger.to_log('[started]' + '\n') |
---|
| 878 | attempted = True |
---|
| 879 | |
---|
| 880 | failed, message = self._execute(build_name) |
---|
| 881 | logger.to_log(status_prefix) |
---|
| 882 | logger.to_all(message + ' ') |
---|
| 883 | logger.to_all('\n') |
---|
| 884 | |
---|
| 885 | if debug: |
---|
| 886 | self.attempted_debug = attempted |
---|
| 887 | self.failed_debug = failed |
---|
| 888 | else: |
---|
| 889 | self.attempted_non_debug = attempted |
---|
| 890 | self.failed_non_debug = failed |
---|
| 891 | |
---|
| 892 | return failed |
---|
| 893 | |
---|
| 894 | def report(self): |
---|
| 895 | return dict( |
---|
| 896 | failed_debug=self.failed_debug, |
---|
| 897 | failed_non_debug=self.failed_non_debug, |
---|
| 898 | ) |
---|
| 899 | |
---|
| 900 | |
---|
| 901 | class PALMTestCase: |
---|
| 902 | """The PALM test case class deals with the configuration and execution of all PALM test cases""" |
---|
| 903 | |
---|
| 904 | def __init__(self,test_dir, name, verbose=False, dry_run=False): |
---|
| 905 | self.test_dir = test_dir |
---|
| 906 | self.name = name |
---|
| 907 | self.verbose = verbose |
---|
| 908 | self.dry_run = dry_run |
---|
| 909 | self.user_code_dir = os.path.join(trunk_tests_cases_dir, self.name, 'USER_CODE') |
---|
| 910 | self.input_dir = os.path.join(trunk_tests_cases_dir, self.name, 'INPUT') |
---|
| 911 | self.number_of_cores = [] |
---|
| 912 | self.build_names = [] |
---|
| 913 | self.input_file_names = [] |
---|
| 914 | self.configured = False |
---|
| 915 | |
---|
| 916 | def configure(self, requested_build_names, requested_cores): |
---|
| 917 | f_name = os.path.join(trunk_tests_cases_dir, self.name, 'case_config.yml') |
---|
| 918 | try: |
---|
| 919 | with open(f_name, 'r') as f: |
---|
| 920 | config = yaml.load(f) |
---|
| 921 | except: |
---|
| 922 | return True, colored('[Case \"' + self.name + '\" could not be found.]', 'red') |
---|
| 923 | try: |
---|
| 924 | self.use_binary_files_from = config['use_binary_files_from'] |
---|
| 925 | except: |
---|
| 926 | self.use_binary_files_from = None |
---|
| 927 | self.requires_binary_files = bool(self.use_binary_files_from) |
---|
| 928 | |
---|
| 929 | if 'allowed_builds' not in config: |
---|
| 930 | return True, colored('[missing \"allowed_builds\" keyword]', 'red') |
---|
| 931 | self.allowed_build_names = config['allowed_builds'] |
---|
| 932 | |
---|
| 933 | if 'allowed_number_of_cores' not in config: |
---|
| 934 | return True, colored('[missing \"allowed_number_of_cores\" keyword]', 'red') |
---|
| 935 | self.allowed_number_of_cores = config['allowed_number_of_cores'] |
---|
| 936 | |
---|
| 937 | if 'activation_strings' not in config: |
---|
| 938 | return True, colored('[missing \"activation_strings\" keyword]', 'red') |
---|
| 939 | self.activation_strings = config['activation_strings'] |
---|
| 940 | |
---|
| 941 | if 'significant_digits_for_netcdf_checks' not in config: |
---|
| 942 | return True, colored('[missing \"significant_digits_for_netcdf_checks\" keyword]', 'red') |
---|
| 943 | self.significant_digits = config['significant_digits_for_netcdf_checks'] |
---|
| 944 | |
---|
| 945 | if 'timeseries' not in config['significant_digits_for_netcdf_checks']: |
---|
| 946 | return True, colored('[missing \"timeseries\" keyword in significant_digits_for_netcdf_checks]', 'red') |
---|
| 947 | |
---|
| 948 | if 'profiles' not in config['significant_digits_for_netcdf_checks']: |
---|
| 949 | return True, colored('[missing \"profiles\" keyword in significant_digits_for_netcdf_checks]', 'red') |
---|
| 950 | |
---|
| 951 | if 'other' not in config['significant_digits_for_netcdf_checks']: |
---|
| 952 | return True, colored('[missing \"other\" keyword in significant_digits_for_netcdf_checks]', 'red') |
---|
| 953 | |
---|
| 954 | self.number_of_cores = sorted(set(requested_cores).intersection(self.allowed_number_of_cores)) |
---|
| 955 | self.build_names = sorted(set(requested_build_names).intersection(self.allowed_build_names)) |
---|
| 956 | self.input_file_names = [s for s in next(os.walk(self.input_dir))[2]] |
---|
| 957 | self.configured = True |
---|
| 958 | if len(self.number_of_cores) == 0 : |
---|
| 959 | return True, colored('[no allowed cores requested]', 'blue') |
---|
| 960 | if len(self.build_names) == 0: |
---|
| 961 | return True, colored('[no allowed builds requested]', 'blue') |
---|
| 962 | if len(self.input_file_names) == 0: |
---|
| 963 | return True, colored('[no input files found]', 'red') |
---|
| 964 | return False, colored('[configuration ok]', 'green') |
---|
| 965 | |
---|
| 966 | |
---|
| 967 | |
---|
| 968 | class PALMTest: |
---|
| 969 | |
---|
| 970 | def __init__(self, args): |
---|
| 971 | self.verbose = args.verbose |
---|
| 972 | self.no_auto_debug = args.no_auto_debug |
---|
| 973 | self.force_debug = args.force_debug |
---|
| 974 | self.fail_on_debug = args.fail_on_debug |
---|
| 975 | self.dry_run = args.dry_run |
---|
[3354] | 976 | self.no_color = args.no_color |
---|
[3353] | 977 | self.test_id = args.test_id |
---|
| 978 | self.test_case_names = args.cases |
---|
| 979 | self.requested_build_names = args.builds |
---|
| 980 | self.requested_cores = args.cores |
---|
| 981 | self.test_case_queue = [] |
---|
| 982 | self.build_database = dict() |
---|
| 983 | |
---|
| 984 | def prepare(self): |
---|
[3354] | 985 | global disable_colored_output |
---|
| 986 | disable_colored_output = self.no_color |
---|
[3353] | 987 | self.test_dir = os.path.join(tests_dir, self.test_id) |
---|
| 988 | try: |
---|
| 989 | os.makedirs(self.test_dir, exist_ok=exist_ok) |
---|
| 990 | except: |
---|
| 991 | print('ERROR: Found existing test directory: ' + self.test_dir) |
---|
| 992 | exit(1) |
---|
| 993 | with Logger(self.test_dir, verbose=self.verbose) as logger: |
---|
| 994 | logger.to_all(hline) |
---|
| 995 | logger.to_all('This is the PALM tester (version: ' + version + ')' + '\n') |
---|
| 996 | logger.to_all(hline) |
---|
| 997 | try: |
---|
| 998 | with open(os.path.join(workspace_dir, 'palmtest.yml'), 'r') as f: |
---|
| 999 | pass |
---|
| 1000 | except: |
---|
| 1001 | logger.to_all('ERROR: No palmtest.yml file was found in your working directory!\n') |
---|
| 1002 | logger.to_all('INFO: A template for this file can be found at: trunk/TESTS/palmtest.yml\n') |
---|
| 1003 | logger.to_all(' Please copy the template to your working directory and adjust it to your system!\n') |
---|
| 1004 | exit(1) |
---|
| 1005 | |
---|
| 1006 | self.execution_trunk_dir = os.path.join(self.test_dir, 'trunk') |
---|
| 1007 | os.symlink(trunk_dir, self.execution_trunk_dir) |
---|
| 1008 | self.execution_jobs_dir = os.path.join(self.test_dir, 'JOBS') |
---|
| 1009 | os.makedirs(self.execution_jobs_dir, exist_ok=exist_ok) |
---|
| 1010 | |
---|
| 1011 | try: |
---|
| 1012 | with open(os.path.join(scripts_dir, '.palm.iofiles'), 'r') as iofiles_template_file: |
---|
| 1013 | iofiles_template = iofiles_template_file.read() |
---|
| 1014 | with open(os.path.join(self.test_dir, '.palm.iofiles'), 'w') as iofiles_file: |
---|
| 1015 | iofiles_file.write(iofiles_template.replace('$fast_io_catalog', '$base_data')) |
---|
| 1016 | except: |
---|
| 1017 | logger.to_all('ERROR: No .palm.iofiles file was found in trunk/SCRIPTS/') |
---|
| 1018 | exit(1) |
---|
| 1019 | |
---|
| 1020 | final_cores_list = list(filter(lambda x: x <= available_cores, self.requested_cores)) |
---|
| 1021 | |
---|
| 1022 | logger.to_all(config_table_line_template.format('Object:', 'Name:', 'Action:') + 'Status:\n') |
---|
| 1023 | logger.to_all(hline) |
---|
| 1024 | |
---|
| 1025 | if 'all' in self.requested_build_names: |
---|
[3354] | 1026 | self.requested_build_names = [name for name in next(os.walk(trunk_tests_builds_dir))[1] if not name[0] == '.'] |
---|
[3353] | 1027 | found_build_names = [] |
---|
| 1028 | for build_name in self.requested_build_names: |
---|
| 1029 | build = PALMBuild(self.test_dir, build_name, verbose=self.verbose, dry_run=self.dry_run) |
---|
| 1030 | configuration_failed, message = build.configure() |
---|
| 1031 | if not configuration_failed: |
---|
| 1032 | self.build_database[build_name] = build |
---|
| 1033 | found_build_names.append(build_name) |
---|
| 1034 | logger.to_all(config_table_line_template.format('Build', build_name, 'approved')) |
---|
| 1035 | logger.to_all(message + '\n') |
---|
| 1036 | else: |
---|
| 1037 | logger.to_all(config_table_line_template.format('Build', build_name, 'rejected')) |
---|
| 1038 | logger.to_all(message + '\n') |
---|
| 1039 | final_build_list = found_build_names |
---|
| 1040 | |
---|
| 1041 | if 'all' in self.test_case_names: |
---|
[3354] | 1042 | self.test_case_names = sorted([name for name in next(os.walk(trunk_tests_cases_dir))[1] if not name[0] == '.']) |
---|
[3353] | 1043 | |
---|
| 1044 | additional_initial_runs_2 = [self.test_case_names] |
---|
| 1045 | while len(additional_initial_runs_2[-1]) > 0: |
---|
| 1046 | additional_initial_runs_1 = [] |
---|
| 1047 | for test_case_name in additional_initial_runs_2[-1]: |
---|
| 1048 | test_case = PALMTestCase(self.test_dir, test_case_name, verbose=self.verbose) |
---|
| 1049 | test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list) |
---|
| 1050 | if not test_case_configuration_failed: |
---|
| 1051 | if test_case.requires_binary_files: |
---|
| 1052 | additional_initial_runs_1.append(test_case.use_binary_files_from) |
---|
| 1053 | additional_initial_runs_2.append(sorted(set(additional_initial_runs_1))) |
---|
| 1054 | |
---|
| 1055 | test_case_order = [] |
---|
| 1056 | for i in range(len(additional_initial_runs_2)-1): |
---|
| 1057 | # low and high refer to priority |
---|
| 1058 | low = additional_initial_runs_2[i] |
---|
| 1059 | high = additional_initial_runs_2[i+1] |
---|
| 1060 | for item in high: |
---|
| 1061 | while item in low: |
---|
| 1062 | low.remove(item) |
---|
| 1063 | test_case_order.append(low) |
---|
| 1064 | |
---|
| 1065 | test_case_order_no_dublicates = [] |
---|
| 1066 | for test_cases in test_case_order: |
---|
| 1067 | seen = set() |
---|
| 1068 | seen_add = seen.add |
---|
| 1069 | test_case_order_no_dublicates.append( [x for x in test_cases if not (x in seen or seen_add(x))] ) |
---|
| 1070 | |
---|
| 1071 | approved_test_case_order = [[]] + list(reversed(test_case_order_no_dublicates)) |
---|
| 1072 | for i, test_cases in enumerate(list(approved_test_case_order)): |
---|
| 1073 | info = 'Case (dep)' if i < len(approved_test_case_order)-1 else 'Case' |
---|
| 1074 | for test_case_name in list(test_cases): |
---|
| 1075 | sys.stdout.flush() |
---|
| 1076 | test_case = PALMTestCase(self.test_dir, test_case_name, verbose=self.verbose) |
---|
| 1077 | test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list) |
---|
| 1078 | if test_case_configuration_failed: |
---|
| 1079 | # removing as configuration failed should only apply to added dependencies |
---|
| 1080 | approved_test_case_order[i].remove(test_case_name) |
---|
| 1081 | logger.to_all(config_table_line_template.format(info, test_case_name, 'rejected')) |
---|
| 1082 | logger.to_all(message + '\n') |
---|
| 1083 | elif test_case.requires_binary_files: |
---|
| 1084 | if test_case.use_binary_files_from not in approved_test_case_order[i-1]: |
---|
| 1085 | # removing as dependency is already removed |
---|
| 1086 | approved_test_case_order[i].remove(test_case_name) |
---|
| 1087 | logger.to_all(config_table_line_template.format(info, test_case_name, 'disabled')) |
---|
| 1088 | logger.to_all(colored('[requires dependency \"' + test_case.use_binary_files_from + '\"]', 'red') + '\n') |
---|
| 1089 | else: |
---|
| 1090 | logger.to_all(config_table_line_template.format(info, test_case_name, 'approved')) |
---|
| 1091 | logger.to_all(message + '\n') |
---|
| 1092 | else: |
---|
| 1093 | logger.to_all(config_table_line_template.format(info, test_case_name, 'approved')) |
---|
| 1094 | logger.to_all(message + '\n') |
---|
| 1095 | |
---|
| 1096 | final_case_list = [] |
---|
| 1097 | for cases in approved_test_case_order: |
---|
| 1098 | for case in cases: |
---|
| 1099 | if case not in final_case_list: |
---|
| 1100 | final_case_list.append(case) |
---|
| 1101 | |
---|
| 1102 | for build_name in final_build_list: |
---|
| 1103 | build = PALMBuild( |
---|
| 1104 | self.test_dir, |
---|
| 1105 | build_name, |
---|
| 1106 | verbose=self.verbose, |
---|
| 1107 | dry_run=self.dry_run, |
---|
| 1108 | ) |
---|
| 1109 | configuration_failed, message = build.configure() |
---|
| 1110 | if not configuration_failed: |
---|
| 1111 | self.build_database[build_name] = build |
---|
| 1112 | else: |
---|
| 1113 | logger.to_all(message + '\n') |
---|
| 1114 | |
---|
| 1115 | for case_name in final_case_list: |
---|
| 1116 | test_case = PALMTestCase( |
---|
| 1117 | self.test_dir, |
---|
| 1118 | case_name, |
---|
| 1119 | verbose=self.verbose, |
---|
| 1120 | dry_run=self.dry_run, |
---|
| 1121 | ) |
---|
| 1122 | test_case_configuration_failed, message = test_case.configure(final_build_list, final_cores_list) |
---|
| 1123 | if not test_case_configuration_failed: |
---|
| 1124 | self.test_case_queue.append(test_case) |
---|
| 1125 | logger.to_all(hline) |
---|
| 1126 | |
---|
| 1127 | r = '{:' + str(table_width_intro) + '} ' |
---|
| 1128 | |
---|
| 1129 | logger.to_all(r.format('Test ID:') + self.test_id + '\n') |
---|
| 1130 | logger.to_all(r.format('Builds:') + str('\n' + r.format('')).join(sorted(self.build_database.keys())) + '\n') |
---|
| 1131 | logger.to_all(r.format('Cases:') + str('\n' + r.format('')).join([c.name for c in self.test_case_queue]) + '\n') |
---|
| 1132 | logger.to_all(r.format('Cores:') + ' '.join([str(i) for i in final_cores_list]) + '\n') |
---|
| 1133 | |
---|
| 1134 | def _execute(self, test_case, build_name, cores): |
---|
| 1135 | job = PALMJob( |
---|
| 1136 | self.test_dir, |
---|
| 1137 | test_case, |
---|
| 1138 | build_name, |
---|
| 1139 | cores, |
---|
| 1140 | verbose=self.verbose, |
---|
| 1141 | dry_run=self.dry_run |
---|
| 1142 | ) |
---|
| 1143 | if self.force_debug: |
---|
| 1144 | build_failed_non_debug = True |
---|
| 1145 | job_failed_non_debug = True |
---|
| 1146 | build_failed_debug = self.build_database[build_name].build(debug=True) |
---|
| 1147 | if build_failed_debug: |
---|
| 1148 | job_failed_debug = True |
---|
| 1149 | else: |
---|
| 1150 | job_failed_debug = job.execute(debug=True) |
---|
| 1151 | elif self.no_auto_debug: |
---|
| 1152 | build_failed_non_debug = self.build_database[build_name].build(debug=False) |
---|
| 1153 | if build_failed_non_debug: |
---|
| 1154 | job_failed_non_debug = True |
---|
| 1155 | else: |
---|
| 1156 | job_failed_non_debug = job.execute(debug=False) |
---|
| 1157 | build_failed_debug = None |
---|
| 1158 | job_failed_debug = None |
---|
| 1159 | else: |
---|
| 1160 | build_failed_non_debug = self.build_database[build_name].build(debug=False) |
---|
| 1161 | if build_failed_non_debug: |
---|
| 1162 | job_failed_non_debug = True |
---|
| 1163 | build_failed_debug = self.build_database[build_name].build(debug=True) |
---|
| 1164 | if build_failed_debug: |
---|
| 1165 | job_failed_debug = False |
---|
| 1166 | else: |
---|
| 1167 | job_failed_debug = job.execute(debug=True) |
---|
| 1168 | else: |
---|
| 1169 | job_failed_non_debug = job.execute(debug=False) |
---|
| 1170 | if job_failed_non_debug: |
---|
| 1171 | build_failed_debug = self.build_database[build_name].build(debug=True) |
---|
| 1172 | if build_failed_debug: |
---|
| 1173 | job_failed_debug = True |
---|
| 1174 | else: |
---|
| 1175 | job_failed_debug = job.execute(debug=True) |
---|
| 1176 | else: |
---|
| 1177 | build_failed_debug = None |
---|
| 1178 | job_failed_debug = None |
---|
| 1179 | return dict( |
---|
| 1180 | build_failed_non_debug=build_failed_non_debug, |
---|
| 1181 | job_failed_non_debug=job_failed_non_debug, |
---|
| 1182 | build_failed_debug=build_failed_debug, |
---|
| 1183 | job_failed_debug=job_failed_debug, |
---|
| 1184 | ) |
---|
| 1185 | |
---|
| 1186 | def execute(self): |
---|
| 1187 | with Logger(self.test_dir, verbose=self.verbose) as logger: |
---|
| 1188 | logger.to_all(hline) |
---|
| 1189 | logger.to_all(task_table_line_template.format('Task:', 'Case:', 'Build:', 'Cores:') + 'Status:\n') |
---|
| 1190 | logger.to_all(hline) |
---|
| 1191 | self.test_report = dict() |
---|
| 1192 | for test_case in self.test_case_queue: |
---|
| 1193 | logger.to_log(hline) |
---|
| 1194 | logger.to_file(hline) |
---|
| 1195 | logger.to_file(hline) |
---|
| 1196 | status_dict = dict() |
---|
| 1197 | for build_name in test_case.build_names: |
---|
| 1198 | status_dict[build_name] = dict() |
---|
| 1199 | for cores in test_case.number_of_cores: |
---|
| 1200 | status_dict[build_name][cores] = self._execute(test_case, build_name, cores) |
---|
| 1201 | self.test_report[test_case.name] = status_dict |
---|
| 1202 | logger.to_log(hline) |
---|
| 1203 | logger.to_file('\n' * 10) |
---|
| 1204 | |
---|
| 1205 | def report(self): |
---|
| 1206 | with Logger(self.test_dir, verbose=self.verbose) as logger: |
---|
| 1207 | logger.to_all(hline) |
---|
| 1208 | r = '{:10}' + ' total: ' + '{:<3d}' + \ |
---|
| 1209 | ' ok: ' + colored('{:<3d}', 'green') + \ |
---|
| 1210 | ' debugged: ' + colored('{:<3d}', 'yellow') + \ |
---|
| 1211 | ' failed: ' + colored('{:<3d}', 'red') |
---|
| 1212 | n_all = 0 |
---|
| 1213 | n_ok = 0 |
---|
| 1214 | n_debugged = 0 |
---|
| 1215 | n_failed = 0 |
---|
| 1216 | for build_name, build in self.build_database.items(): |
---|
| 1217 | status = build.report() |
---|
| 1218 | b = status['failed_non_debug'] |
---|
| 1219 | bd = status['failed_debug'] |
---|
| 1220 | n_all += 1 |
---|
| 1221 | if not b and b is not None: |
---|
| 1222 | n_ok += 1 |
---|
| 1223 | if bd is not None: |
---|
| 1224 | n_debugged += 1 |
---|
| 1225 | if b and (bd or bd is None): |
---|
| 1226 | n_failed += 1 |
---|
| 1227 | logger.to_all(r.format('Builds:', n_all, n_ok, n_debugged, n_failed) + '\n') |
---|
| 1228 | total_failed = n_failed |
---|
| 1229 | total_debugged = n_debugged |
---|
| 1230 | n_all = 0 |
---|
| 1231 | n_ok = 0 |
---|
| 1232 | n_debugged = 0 |
---|
| 1233 | n_failed = 0 |
---|
| 1234 | # {'case_name': {'build_name': {4: {'build_failed_debug': None, |
---|
| 1235 | # 'build_failed_non_debug': False, |
---|
| 1236 | # 'job_failed_debug': None, |
---|
| 1237 | # 'job_failed_non_debug': False}}}, |
---|
| 1238 | for case_name, case in self.test_report.items(): |
---|
| 1239 | for build_name, build in case.items(): |
---|
| 1240 | for cores, results in build.items(): |
---|
| 1241 | n_all += 1 |
---|
| 1242 | b = results['build_failed_non_debug'] |
---|
| 1243 | bd = results['build_failed_debug'] |
---|
| 1244 | j = results['job_failed_non_debug'] |
---|
| 1245 | jd = results['job_failed_debug'] |
---|
| 1246 | if not j: |
---|
| 1247 | n_ok += 1 |
---|
| 1248 | if jd is not None: |
---|
| 1249 | n_debugged += 1 |
---|
| 1250 | if j and (jd or jd is None): |
---|
| 1251 | n_failed += 1 |
---|
[3354] | 1252 | logger.to_all(r.format('Tests:', n_all, n_ok, n_debugged, n_failed) + '\n') |
---|
[3353] | 1253 | total_failed += n_failed |
---|
| 1254 | total_debugged += n_debugged |
---|
| 1255 | if self.fail_on_debug: |
---|
| 1256 | return (total_failed + total_debugged) > 0 |
---|
| 1257 | else: |
---|
| 1258 | return total_failed > 0 |
---|
| 1259 | |
---|
| 1260 | |
---|
| 1261 | class CustomCompleter: |
---|
| 1262 | |
---|
| 1263 | def __init__(self): |
---|
| 1264 | pass |
---|
| 1265 | |
---|
| 1266 | def __call__(self, prefix, parsed_args, **kwargs): |
---|
| 1267 | return (i for i in self.get_items() if i.startswith(prefix)) |
---|
| 1268 | |
---|
| 1269 | def get_items(self): |
---|
| 1270 | return [] |
---|
| 1271 | |
---|
| 1272 | |
---|
| 1273 | class CaseCompleter(CustomCompleter): |
---|
| 1274 | |
---|
| 1275 | def get_items(self): |
---|
[3354] | 1276 | case_names = [name for name in next(os.walk(trunk_tests_cases_dir))[1] if not name[0] == '.'] |
---|
[3353] | 1277 | return case_names + ['all'] |
---|
| 1278 | |
---|
| 1279 | |
---|
| 1280 | class BuildCompleter(CustomCompleter): |
---|
| 1281 | |
---|
| 1282 | def get_items(self): |
---|
[3354] | 1283 | build_names = [name for name in next(os.walk(trunk_tests_builds_dir))[1] if not name[0] == '.'] |
---|
[3353] | 1284 | return build_names + ['all'] |
---|
| 1285 | |
---|
| 1286 | |
---|
| 1287 | class PALMTestArgumentParser(ArgumentParser): |
---|
| 1288 | |
---|
| 1289 | def __init__(self): |
---|
| 1290 | super().__init__( |
---|
| 1291 | description='This is the PALM tester\n' + |
---|
| 1292 | 'Developer Support: knoop@muk.uni-hannover.de', |
---|
| 1293 | formatter_class=RawTextHelpFormatter, |
---|
| 1294 | add_help=True, |
---|
| 1295 | ) |
---|
| 1296 | self.add_argument( |
---|
| 1297 | '--version', |
---|
| 1298 | action='version', |
---|
| 1299 | version=version, |
---|
| 1300 | ) |
---|
| 1301 | self.add_argument( |
---|
| 1302 | '--verbose', |
---|
| 1303 | action='store_true', |
---|
| 1304 | dest='verbose', |
---|
| 1305 | help='Increase verbosity of terminal output.', |
---|
| 1306 | required=False, |
---|
| 1307 | ) |
---|
| 1308 | self.add_argument( |
---|
| 1309 | '--no-auto-debug', |
---|
| 1310 | action='store_true', |
---|
| 1311 | dest='no_auto_debug', |
---|
| 1312 | help='Disable automatic debugging in case of test failure.', |
---|
| 1313 | required=False, |
---|
| 1314 | ) |
---|
| 1315 | self.add_argument( |
---|
| 1316 | '--force-debug', |
---|
| 1317 | action='store_true', |
---|
| 1318 | dest='force_debug', |
---|
| 1319 | help='Force debugging regardless of test failure (ignores --no-auto-debug).', |
---|
| 1320 | required=False, |
---|
| 1321 | ) |
---|
| 1322 | self.add_argument( |
---|
| 1323 | '--fail-on-debug', |
---|
| 1324 | action='store_true', |
---|
| 1325 | dest='fail_on_debug', |
---|
| 1326 | help='Return a non-zero exit status in case debugging was required.', |
---|
| 1327 | required=False, |
---|
| 1328 | ) |
---|
| 1329 | self.add_argument( |
---|
| 1330 | '--dry-run', |
---|
| 1331 | action='store_true', |
---|
| 1332 | dest='dry_run', |
---|
| 1333 | help='Prepare and process all requested tests without actually building or executing PALM.', |
---|
| 1334 | required=False, |
---|
| 1335 | ) |
---|
| 1336 | self.add_argument( |
---|
[3354] | 1337 | '--no-color', |
---|
| 1338 | action='store_true', |
---|
| 1339 | dest='no_color', |
---|
| 1340 | help='Disable colored terminal output.', |
---|
| 1341 | required=False, |
---|
| 1342 | ) |
---|
| 1343 | self.add_argument( |
---|
[3353] | 1344 | '--cases', |
---|
| 1345 | action='store', |
---|
| 1346 | dest='cases', |
---|
| 1347 | default=['all'], |
---|
| 1348 | help='A list of test cases to be executed. (default: %(default)s)', |
---|
| 1349 | nargs='+', |
---|
| 1350 | required=False, |
---|
| 1351 | type=str, |
---|
| 1352 | metavar='STR', |
---|
| 1353 | ).completer = CaseCompleter() |
---|
| 1354 | self.add_argument( |
---|
| 1355 | '--builds', |
---|
| 1356 | action='store', |
---|
| 1357 | dest='builds', |
---|
| 1358 | default=['all'], |
---|
| 1359 | help='A list of builds to be executed. (default: %(default)s)', |
---|
| 1360 | nargs='+', |
---|
| 1361 | required=False, |
---|
| 1362 | type=str, |
---|
| 1363 | metavar='STR', |
---|
| 1364 | ).completer = BuildCompleter() |
---|
| 1365 | self.add_argument( |
---|
| 1366 | '--cores', |
---|
| 1367 | action='store', |
---|
| 1368 | dest='cores', |
---|
| 1369 | default=[i for i in range(1, available_cores+1)], |
---|
| 1370 | choices=[i for i in range(1, available_cores+1)], |
---|
| 1371 | help='The number of cores tests are supposed to be executed on. (default: %(default)s)', |
---|
| 1372 | nargs='+', |
---|
| 1373 | required=False, |
---|
| 1374 | type=int, |
---|
| 1375 | metavar='INT', |
---|
| 1376 | ) |
---|
| 1377 | self.add_argument( |
---|
| 1378 | '--test-id', |
---|
| 1379 | action='store', |
---|
| 1380 | dest='test_id', |
---|
| 1381 | default=datetime.now().strftime('%Y-%m-%d_%H:%M:%S.%f'), |
---|
| 1382 | help='An individual test id. (default: current timestamp)', |
---|
| 1383 | required=False, |
---|
| 1384 | type=str, |
---|
| 1385 | metavar='STR', |
---|
| 1386 | ) |
---|
| 1387 | |
---|
| 1388 | |
---|
| 1389 | if __name__ == '__main__': |
---|
| 1390 | parser = PALMTestArgumentParser() |
---|
| 1391 | if has_argcomplete: |
---|
| 1392 | argcomplete.autocomplete(parser) |
---|
| 1393 | args = parser.parse_args() |
---|
| 1394 | palm_test = PALMTest(args) |
---|
| 1395 | palm_test.prepare() |
---|
| 1396 | palm_test.execute() |
---|
| 1397 | failed = palm_test.report() |
---|
| 1398 | exit(1 if failed else 0) |
---|