#!/usr/bin/python # -*- coding: utf-8 -*- # --------------------------------------------------------------------------------------------------------- # Author : Ulrich Carmesin # Source : gitea.ucarmesin.de # --------------------------------------------------------------------------------------------------------- # Template Batchrahmen # #import sys, getopt import argparse import copy import yaml import os from datetime import datetime import basic.constants as B import basic.message import basic.message import basic.componentHandling import utils.date_tool import utils.path_tool import utils.file_tool jobdef = { "unit": { "pardef": "", "pfilesource": "", "pfiletarget": "envparfile", "basedir": "envbase", "dirname": "envdir", "logdir": "{job.par.envdir}/{log}/log_{time}.txt" }, "check_environment": { "pardef": "", "pfilesource": "", "pfiletarget": "envparfile", "basedir": "envbase", "dirname": "envdir", "logdir": "{job.par.envdir}/{log}/log_{time}.txt" }, "test_executer": { "pardef": "", "pfilesource": "tsparfile", "pfiletarget": "tsparfile", "basedir": "tsbase", "dirname": "tsdir", "logdir": "{job.par.tsdir}/{log}/log_{time}.txt" }, "init_testsuite": { "pardef": "tsdir", # ,tdtyp,tdsrc,tdname", "pfilesource": "envparfile", "pfiletarget": "tsparfile", "basedir": "tsbase", "dirname": "tsdir", "logdir": "{job.par.tsdir}/{log}/log_{tstime}.txt" }, "init_testcase": { "pardef": "tcdir", # ",tdtyp,tdsrc,tdname", "pfilesource": "envparfile", "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" }, "execute_testcase": { "pardef": "tcdir", # ",tdtyp,tdsrc,tdname", "pfilesource": "tcparfile", "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" }, "collect_testcase": { "pardef": "tcdir", # ",tdtyp,tdsrc,tdname", "pfilesource": "tcparfile", "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" }, "compare_testcase": { "pardef": "tcdir", # ",tdtyp,tdsrc,tdname", "pfilesource": "tcparfile", "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" }, "test_system": { "pardef": "tcdir,tdtyp,tdsrc,tdname", "pfilesource": "tsparfile", "pfiletarget": "tsparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"}, "finish_testcase": { "pardef": "tcdir", "pfilesource": "tcparfile", "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"}, "finish_testsuite": { "pardef": "tsdir", "pfilesource": "tsparfile", "pfiletarget": "tssarfile", "basedir": "tsbase", "dirname": "tsdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"} } def setGlobal(): pass EXCP_CANT_POP = "cant pop this job from the instances" class Job: __instance = None instances = [] def __init__ (self, program): print ("################# init Job ## " + program + " #################") self.program = program Job.__instance = self Job.pushInstance(self) par = Parameter(program) self.par = par print("prog-42 " + str(self.par.basedir)) conf = Configuration(program) self.conf = conf print("prog-45 " + str(self.par.basedir)) dirpath = self.par.getDirParameter() setGlobal() if dirpath is not None: utils.path_tool.extractPath(dirpath[0], dirpath[1]) if program == "unit": # no job will be started self.start = datetime.now() logTime = self.start.strftime("%Y%m%d_%H%M%S") self.m = basic.message.Message(basic.message.LIMIT_DEBUG, logTime, None) print("prog-50 " + str(self.par.basedir)) @staticmethod def popInstance(pjob=None): """ pop and stop a subjob """ if pjob is not None and Job.__instance is not None and Job.__instance != pjob: raise Exception(B.EXP_CANT_POP) # for guarantee that both the stack and the original variable is empty if one is empty if len(Job.instances) < 1: Job.__instance = None return None if Job.__instance is None: while len(Job.instances) > 0: Job.instances.pop() return None if len(Job.instances) == 1: job = Job.__instance job.stopJob(1) Job.instances.pop() Job.__instance = None return None job = Job.instances.pop() if pjob is not None and job is not None and job != pjob: Job.instances.append(job) Job.__instance = job raise Exception(B.EXP_CANT_POP) if len(Job.instances) > 0: topjob = Job.instances.pop() Job.instances.append(topjob) Job.__instance = topjob else: Job.__instance = None return job @staticmethod def pushInstance(pjob): """ push a new created subjob """ if len(Job.instances) > 0: job = Job.instances.pop() if pjob is not None and job is not None and job != pjob: Job.instances.append(job) Job.instances.append(pjob) Job.__instance = pjob return pjob @staticmethod def resetInstance(program): job = Job.getInstance() if job is not None: job.stopJob(1) Job.__instance = None Job(program) return Job.__instance def setProgram(self, program): self.program = program basedir = jobdef[program]["basedir"] self.basedir = basedir if (self.par is not None): setattr(self.par, "program", program) setattr(self.par, "basedir", basedir) parstring = getattr(self.par, "parstring") parstring = parstring[parstring.find("--"):] parstring = "python "+program+" "+parstring setattr(self.par, "parstring", parstring) if not hasattr(self.par, jobdef[program]["dirname"]): setattr(self.par, jobdef[program]["dirname"], utils.path_tool.composePattern("{"+basedir+"}", None)) self.par.setParameterLoaded() @staticmethod def getInstance(): if (Job.__instance is not None): return Job.__instance else: return None def startJob(self): self.start = datetime.now() print("prog-68 " + str(self.par.basedir)) logTime = self.start.strftime("%Y%m%d_%H%M%S") self.m = basic.message.Message(basic.message.LIMIT_DEBUG, logTime, None) print("prog-68 " + str(self.m.rc)) self.par.setParameterLoaded() self.m.logInfo("# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") self.m.debug(basic.message.LIMIT_INFO, "# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") self.par.checkParameter() def stopJob(self, reboot=0): self.ende = datetime.now() self.dumpParameter() print("stopJob " + str(self.m.messages) + ", " + str(self.m.debugfile)) self.m.logInfo("# # " + self.m.topmessage + " # # # ") self.m.logInfo("# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") self.m.debug(basic.message.LIMIT_INFO, "# # " + self.m.topmessage + " # # # ") self.m.debug(basic.message.LIMIT_INFO, "# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # RC: " + str(self.m.getFinalRc())) self.m.closeMessage() rc = self.m.getFinalRc() print ("rc " + str(rc)) if reboot == 0: exit(rc) def dumpParameter(self): parpath = utils.path_tool.composePath(jobdef[self.program]["pfiletarget"], None) output = {} cconf = basic.componentHandling.getComponentDict() output["par"] = self.par.__dict__ if len(cconf) < 1: utils.file_tool.writeFileDict(self.m, parpath, output) return output[B.SUBJECT_COMPS] = {} for c in cconf: output[B.SUBJECT_COMPS][c] = {} for x in ["function", "conn"]: output[B.SUBJECT_COMPS][c][x] = cconf[c][x] if x == B.SUBJECT_CONN and "passwd" in cconf[c][x]: cconf[B.SUBJECT_COMPS][c][x]["passwd"] = "xxxxx" utils.file_tool.writeFileDict(self.m, parpath, output) def loadParameter(self): output = {} if len(str(jobdef[self.program]["pfilesource"])) < 2: return None parpath = utils.path_tool.composePath(jobdef[self.program]["pfilesource"], None) if not os.path.join(parpath): return None doc = utils.file_tool.readFileDict(parpath, self.m) for k in doc.keys(): output[k] = copy.deepcopy(doc[k]) return output def getParameter(self, parameter): if hasattr(self.par, parameter): return getattr(self.par, parameter) elif "xxxtime" in parameter: neu = utils.date_tool.getActdate(utils.date_tool.F_DIR) # setattr(self.par, parameter, neu) return neu def hasElement(self, parameter, elem): if hasattr(self.par, parameter): print (parameter + " in Parameter") if getattr(self.par, parameter).find(elem) >= 0: return True return False return True def hascomponente(self, komp): return self.hasElement("componente", komp) def hasFunction(self, fct): return self.hasElement("function", fct) def hasTool(self, tool): return self.hasElement("tool", tool) def getMessageLevel(self, errtyp, elem): if (not hasattr(self, "m")) or (self.m is None): return basic.message.LIMIT_DEBUG elif elem.find("tool") > 1: if not hasattr(self.par, "tool") or getattr(self.par, "tool").find(elem) <= 0: return int(self.m.CONST_ERRTYP[errtyp]) -1 else: return int(self.m.CONST_ERRTYP[errtyp]) else: return int(self.m.CONST_ERRTYP[errtyp]) def getInfoLevel(self, elem): return self.getMessageLevel("info", elem) def getDebugLevel(self, elem): return self.getMessageLevel("debug", elem) def getTraceLevel(self, elem): return self.getMessageLevel("trace", elem) def debug(self, prio, text): #print("job.debug "+str(prio)+" "+text) if hasattr(self, "m"): self.m.debug(prio, text) else: print(text) # ------------------------------------------------------------------------------------------------------------------ class Parameter: print ("class Parameter") def __init__ (self, program): print ("# init Parameter for " + program) self.program = program self.basedir = "debugs" self.setBasedir(program) print (f"# Parameter initialisiert {self.program} mit basedir {self.basedir}") if (program != "unit"): self.setParameter() def setBasedir(self, program): if jobdef[program]: self.basedir = jobdef[program]["basedir"] if hasattr(self, jobdef[program]["dirname"]): utils.path_tool.extractPath(self.basedir, getattr(self, jobdef[program]["dirname"])) elif program != "unit": dirpath = utils.path_tool.composePattern("{"+jobdef[program]["basedir"]+"}", None) setattr(self, jobdef[program]["dirname"], dirpath) else: self.basedir = "debugs" def checkParameter(self): job = Job.getInstance() print (f"Parameter initialisiert {self.program}") pardef = jobdef[job.program]["pardef"] for p in pardef.split(","): print(p) if len(p) > 1 and not hasattr(self, p): job.m.setFatal("Parameter " + p + " is not set!") def setParameter(self): """ 1. Programm -- implementiert in Main-Klasse 2. anwndung -- steuert zu pruefende System [ in basis_Config ] 3. amgebung -- steuert zu pruefende Maschine [ in dir/applicationen ] 4. release -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ] 5. ~Verz -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch 6. zyklus -- optional unterscheidet echte und entwicklungsLaeufe 7. Programmspezifische Parameter 8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2) 10. modus -- steuert die Verarbeitung; default echt - echt-auto Lauf aus Automatisierung (1-7) - test Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen - echt-spez Wiederholung einer spezifischen Funktion (1-13) - unit Ausfuehrung der Unittests 11. componente -- schraenkt Verarbeitung auf parametriserte componenten ein 12. funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein 13. tool -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein """ # args = str(sys.argv) # print ("Job-Programm %s : " % args) parser = argparse.ArgumentParser() parser.add_argument('-a', '--'+B.SUBJECT_APPS, required=True, action='store') parser.add_argument('-e', '--'+B.PAR_ENV, required=True, action='store') parser.add_argument('-r', '--release', action='store') parser.add_argument('-ts', '--'+B.PAR_TSDIR, action='store') parser.add_argument('-tc', '--'+B.PAR_TCDIR, action='store') parser.add_argument('-rs', '--rsdir', action='store') parser.add_argument('-dt', '--tdtyp', action='store') # PAR_TDTYP parser.add_argument('-ds', '--tdsrc', action='store') # PAR_TDSRC parser.add_argument('-dn', '--tdname', action='store') # PAR_TDNAME parser.add_argument('-l', '--loglevel', action='store') # PAR_LOG parser.add_argument('-m', '--modus', action='store') # PAR_MODUS parser.add_argument('-c', '--component', action='store') # PAR_COMP parser.add_argument('-f', '--function', action='store') # PAR_FCT parser.add_argument('-t', '--tool', action='store') # PAR_TOOL parser.add_argument('-s', '--'+B.PAR_STEP, action='store') # PAR_STEP # parser.add_argument('-t', '--typ', default='einzel', action='store') # parser.add_argument('-d', '--dir', action='store') args = parser.parse_args() self.setParameterArgs(args) def getDirParameter(self): if hasattr(self, "tcdir"): return ("tcbase", str(self.tcdir)) if hasattr(self, "tsdir"): return ("tsbase", str(self.tsdir)) return None def setParameterArgs(self, args): job = Job.getInstance() print("setParArgs " + str(type(args))) self.parstring = "python " + self.program if "dict" in str(type(args)): for k in args: self.setJobAttr(k, args[k]) else: for k in vars(args): if getattr(args, k) is not None: self.setJobAttr(k , getattr(args, k)) dirpath = self.getDirParameter() if dirpath is not None: utils.path_tool.extractPath(dirpath[0], dirpath[1]) def setParameterLoaded(self): job = Job.getInstance() print("setParLoaded " ) readedPar = job.loadParameter() if readedPar is not None: for k in readedPar["par"].keys(): if not hasattr(self, k): self.setJobAttr(k, readedPar["par"][k]) def setJobAttr(self, key, val): setattr(self, key, val) self.parstring = self.parstring + " --" + key + " " + str(val) def getJobPar(self, key): a = key.split(":") if len(a) == 1: return self[a[0]] if len(a) == 2: return self[a[0]][a[1]] return # ------------------------------------------------------------------------------------------------------------------ class Configuration: def __init__ (self, program): self.program = program print (f"job initialisiert {self.program}") if program == "unit": if (os.path.exists(utils.path_tool.rejoinPath("..", "..", "config", B.BASIS_FILE))): self.setConfiguration(utils.path_tool.rejoinPath("..", "..", "config", B.BASIS_FILE)) return if (os.path.exists(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))): self.setConfiguration(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE)) return elif (os.path.exists(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))): self.setConfiguration(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE)) return elif (os.path.exists(utils.path_tool.rejoinPath("config", B.BASIS_FILE))): self.setConfiguration(utils.path_tool.rejoinPath("config", B.BASIS_FILE)) return raise Exception(B.EXP_NO_BASIS_FILE) def setConfiguration(self, path): self.confs = {} doc = utils.file_tool.readFileLines(path, None) with open(path, "r") as file: doc = yaml.full_load(file) if "basic" in doc: for i, v in doc["basic"].items(): self.confs[i] = v else: for i, v in doc.items(): self.confs[i] = v def setConfig(self, path, val): a = path.split(".") if len(a) == 1: self.confs[a[0]] = val elif len(a) == 2: self.confs[a[0]][a[1]] = val elif len(a) == 3: self.confs[a[0]][a[1]][a[2]] = val def getPath(self, key): return self.confs.get(B.SUBJECT_PATH).get(key) def getJobConf(self, key): a = key.split(":") if len(a) == 1: return self.confs[a[0]] if len(a) == 2: return self.confs[a[0]][a[1]]