Data-Test-Executer Framework speziell zum Test von Datenverarbeitungen mit Datengenerierung, Systemvorbereitungen, Einspielungen, ganzheitlicher diversifizierender Vergleich
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

527 lines
17 KiB

#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
# Template Batchrahmen
#
#import sys, getopt
import argparse
import copy
import yaml
import os
from datetime import datetime
import basic.constants as B
import basic.message
import basic.message
import basic.componentHandling
import utils.date_tool
import utils.path_tool
import utils.file_tool
import utils.config_tool
jobdef = {
"unit": {
"pardef": "",
"pfilesource": "",
"pfiletarget": "envparfile",
"basedir": "envbase",
"dirname": "envdir",
"logdir": "{job.par.envdir}/{log}/log_{time}.txt" },
"check_environment": {
"pardef": "",
"pfilesource": "",
"pfiletarget": "envparfile",
"basedir": "envbase",
"dirname": "envdir",
"logdir": "{job.par.envdir}/{log}/log_{time}.txt" },
"test_executer": {
"pardef": "",
"pfilesource": "tsparfile",
"pfiletarget": "tsparfile",
"basedir": "tsbase",
"dirname": "tsdir",
"logdir": "{job.par.tsdir}/{log}/log_{time}.txt" },
"init_testsuite": {
"pardef": "tsdir", # ,tdtyp,tdsrc,tdname",
"pfilesource": "envparfile",
"pfiletarget": "tsparfile",
"basedir": "tsbase",
"dirname": "tsdir",
"logdir": "{job.par.tsdir}/{log}/log_{tstime}.txt" },
"init_testcase": {
"pardef": "tcdir", # ",tdtyp,tdsrc,tdname",
"pfilesource": "envparfile",
"pfiletarget": "tcparfile",
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
"execute_testcase": {
"pardef": "tcdir", # ",tdtyp,tdsrc,tdname",
"pfilesource": "tcparfile",
"pfiletarget": "tcparfile",
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
"collect_testcase": {
"pardef": "tcdir", # ",tdtyp,tdsrc,tdname",
"pfilesource": "tcparfile",
"pfiletarget": "tcparfile",
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
"compare_testcase": {
"pardef": "tcdir", # ",tdtyp,tdsrc,tdname",
"pfilesource": "tcparfile",
"pfiletarget": "tcparfile",
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
"test_system": {
"pardef": "tcdir,tdtyp,tdsrc,tdname",
"pfilesource": "tsparfile",
"pfiletarget": "tsparfile",
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"},
"finish_testcase": {
"pardef": "tcdir",
"pfilesource": "tcparfile",
"pfiletarget": "tcparfile",
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"},
"finish_testsuite": {
"pardef": "tsdir",
"pfilesource": "tsparfile",
"pfiletarget": "tssarfile",
"basedir": "tsbase",
"dirname": "tsdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"}
}
def setGlobal():
pass
EXCP_CANT_POP = "cant pop this job from the instances"
class Job:
__instance = None
__instances = []
def __init__ (self, program):
print ("################# init Job ## " + program + " #################")
self.program = program
Job.__instance = self
if Job.__instances is None:
Job.__instances = []
Job.pushInstance(self)
par = Parameter(program)
self.par = par
print("prog-42 " + str(self.par.basedir))
conf = Configuration(program)
self.conf = conf
appl = utils.config_tool.getConfig("basic", B.SUBJECT_APPS)
print(appl)
if appl is not None:
self.conf.confs[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS]
print("prog-45 " + str(self.par.basedir))
dirpath = self.par.getDirParameter()
setGlobal()
if dirpath is not None:
utils.path_tool.extractPath(dirpath[0], dirpath[1])
if program == "unit": # no job will be started
self.start = datetime.now()
logTime = self.start.strftime("%Y%m%d_%H%M%S")
self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None)
print("prog-50 " + str(self.par.basedir))
@staticmethod
def popInstance(pjob=None):
""" pop and stop a subjob """
if pjob is not None and Job.__instance is not None and Job.__instance != pjob:
raise Exception(B.EXP_CANT_POP)
# for guarantee that both the stack and the original variable is empty if one is empty
if len(Job.__instances) < 1:
Job.__instance = None
return None
if Job.__instance is None:
while len(Job.__instances) > 0:
Job.__instances.pop()
return None
if len(Job.__instances) == 1:
job = Job.__instance
job.stopJob(1)
Job.__instances.pop()
Job.__instance = None
return None
job = Job.__instances.pop()
if pjob is not None and job is not None and job != pjob:
Job.__instances.append(job)
Job.__instance = job
raise Exception(B.EXP_CANT_POP)
if len(Job.__instances) > 0:
topjob = Job.__instances.pop()
Job.__instances.append(topjob)
Job.__instance = topjob
else:
Job.__instance = None
return job
@staticmethod
def pushInstance(pjob):
""" push a new created subjob """
if len(Job.__instances) > 0:
job = Job.__instances.pop()
if pjob is not None and job is not None and job != pjob:
Job.__instances.append(job)
Job.__instances.append(pjob)
Job.__instance = pjob
return pjob
@staticmethod
def resetInstance(program):
job = Job.getInstance()
if job is not None:
job.stopJob(1)
Job.__instance = None
Job(program)
return Job.__instance
def setProgram(self, program):
self.program = program
basedir = jobdef[program]["basedir"]
self.basedir = basedir
if (self.par is not None):
setattr(self.par, "program", program)
setattr(self.par, "basedir", basedir)
parstring = getattr(self.par, "parstring")
parstring = parstring[parstring.find("--"):]
parstring = "python "+program+" "+parstring
setattr(self.par, "parstring", parstring)
if not hasattr(self.par, jobdef[program]["dirname"]):
setattr(self.par, jobdef[program]["dirname"],
utils.path_tool.composePattern("{"+basedir+"}", None))
self.par.setParameterLoaded()
@staticmethod
def getInstance():
if (Job.__instance is not None):
return Job.__instance
else:
return None
def startJob(self):
self.start = datetime.now()
print("prog-68 " + str(self.par.basedir))
logTime = self.start.strftime("%Y%m%d_%H%M%S")
self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None)
print("prog-68 " + str(self.m.rc))
self.par.setParameterLoaded()
self.m.logInfo("# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
self.m.debug(basic.message.LIMIT_INFO, "# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
self.par.checkParameter()
def stopJob(self, reboot=0):
self.ende = datetime.now()
self.dumpParameter()
print("stopJob " + str(self.m.messages) + ", " + str(self.m.debugfile))
self.m.logInfo("# # " + self.m.topmessage + " # # # ")
self.m.logInfo("# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
self.m.debug(basic.message.LIMIT_INFO, "# # " + self.m.topmessage + " # # # ")
self.m.debug(basic.message.LIMIT_INFO, "# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # RC: " + str(self.m.getFinalRc()))
self.m.closeMessage()
rc = self.m.getFinalRc()
print ("rc " + str(rc))
if reboot == 0:
exit(rc)
def dumpParameter(self):
parpath = utils.path_tool.composePath(jobdef[self.program]["pfiletarget"], None)
output = {}
cconf = basic.componentHandling.getComponentDict()
output["par"] = self.par.__dict__
if len(cconf) < 1:
utils.file_tool.writeFileDict(self.m, parpath, output)
return
output[B.SUBJECT_COMPS] = {}
for c in cconf:
output[B.SUBJECT_COMPS][c] = {}
for x in ["function", "conn"]:
if x not in cconf[c]:
continue
output[B.SUBJECT_COMPS][c][x] = cconf[c][x]
if x == B.SUBJECT_CONN and "passwd" in cconf[c][x]:
cconf[B.SUBJECT_COMPS][c][x]["passwd"] = "xxxxx"
utils.file_tool.writeFileDict(self.m, parpath, output)
def loadParameter(self):
output = {}
if len(str(jobdef[self.program]["pfilesource"])) < 2:
return None
parpath = utils.path_tool.composePath(jobdef[self.program]["pfilesource"], None)
if not os.path.join(parpath):
return None
doc = utils.file_tool.readFileDict(parpath, self.m)
for k in doc.keys():
output[k] = copy.deepcopy(doc[k])
return output
def getParameter(self, parameter):
if hasattr(self.par, parameter):
return getattr(self.par, parameter)
elif "xxxtime" in parameter:
neu = utils.date_tool.getActdate(utils.date_tool.F_DIR)
# setattr(self.par, parameter, neu)
return neu
def hasElement(self, parameter, elem):
"""
the function searches in an optional job.parameter
(a) true, if the parameter does not exist (DEFAULT)
(b) true, if the element is member of the parameter
(c) false, if parameter exists and elem is not member of the parameter
:param parameter:
:param elem:
:return:
"""
if hasattr(self.par, parameter):
print (parameter + " in Parameter")
if getattr(self.par, parameter).find(elem) >= 0:
return True
return False
return True
def hascomponente(self, comp):
"""
it searches if comp is member of the optional job.parameter (default = each is member)
:param comp:
:return:
"""
return self.hasElement("componente", comp)
def hasFunction(self, fct):
"""
it searches if fct is member of the optional job.parameter (default = each is member)
:param fct:
:return:
"""
return self.hasElement("function", fct)
def hasTool(self, tool):
"""
it searches if tool is member of the optional job.parameter (default = each is member)
:param tool:
:return:
"""
return self.hasElement("tool", tool)
def getMessageLevel(self, errtyp, elem):
if (not hasattr(self, "m")) or (self.m is None):
return basic.message.LIMIT_DEBUG
elif elem.find("tool") > 1:
if not hasattr(self.par, "tool") or getattr(self.par, "tool").find(elem) <= 0:
return int(self.m.CONST_ERRTYP[errtyp]) -1
else:
return int(self.m.CONST_ERRTYP[errtyp])
else:
return int(self.m.CONST_ERRTYP[errtyp])
def getInfoLevel(self, elem):
return self.getMessageLevel("info", elem)
def getDebugLevel(self, elem):
return self.getMessageLevel("debug", elem)
def getTraceLevel(self, elem):
return self.getMessageLevel("trace", elem)
def debug(self, prio, text):
#print("job.debug "+str(prio)+" "+text)
if hasattr(self, "m"):
self.m.debug(prio, text)
else:
print(text)
# ------------------------------------------------------------------------------------------------------------------
class Parameter:
print ("class Parameter")
def __init__ (self, program):
print ("# init Parameter for " + program)
self.program = program
self.basedir = "debugs"
self.setBasedir(program)
print (f"# Parameter initialisiert {self.program} mit basedir {self.basedir}")
if (program != "unit"): self.setParameter()
def setBasedir(self, program):
if jobdef[program]:
self.basedir = jobdef[program]["basedir"]
if hasattr(self, jobdef[program]["dirname"]):
utils.path_tool.extractPath(self.basedir, getattr(self, jobdef[program]["dirname"]))
elif program != "unit":
dirpath = utils.path_tool.composePattern("{"+jobdef[program]["basedir"]+"}", None)
setattr(self, jobdef[program]["dirname"], dirpath)
else:
self.basedir = "debugs"
def checkParameter(self):
job = Job.getInstance()
print (f"Parameter initialisiert {self.program}")
pardef = jobdef[job.program]["pardef"]
for p in pardef.split(","):
print(p)
if len(p) > 1 and not hasattr(self, p):
job.m.setFatal("Parameter " + p + " is not set!")
def setParameter(self):
"""
1. Programm -- implementiert in Main-Klasse
2. anwndung -- steuert zu pruefende System [ in basis_Config ]
3. amgebung -- steuert zu pruefende Maschine [ in dir/applicationen ]
4. release -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ]
5. ~Verz -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch
6. zyklus -- optional unterscheidet echte und entwicklungsLaeufe
7. Programmspezifische Parameter
8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2)
10. modus -- steuert die Verarbeitung; default echt
- echt-auto Lauf aus Automatisierung (1-7)
- test Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen
- echt-spez Wiederholung einer spezifischen Funktion (1-13)
- unit Ausfuehrung der Unittests
11. componente -- schraenkt Verarbeitung auf parametriserte componenten ein
12. funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein
13. tool -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein
"""
# args = str(sys.argv)
# print ("Job-Programm %s : " % args)
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--'+B.SUBJECT_APPS, required=True, action='store')
parser.add_argument('-e', '--'+B.PAR_ENV, required=True, action='store')
parser.add_argument('-r', '--release', action='store')
parser.add_argument('-ts', '--'+B.PAR_TSDIR, action='store')
parser.add_argument('-tc', '--'+B.PAR_TCDIR, action='store')
parser.add_argument('-rs', '--rsdir', action='store')
parser.add_argument('-dt', '--tdtyp', action='store') # PAR_TDTYP
parser.add_argument('-ds', '--tdsrc', action='store') # PAR_TDSRC
parser.add_argument('-dn', '--tdname', action='store') # PAR_TDNAME
parser.add_argument('-l', '--loglevel', action='store') # PAR_LOG
parser.add_argument('-m', '--modus', action='store') # PAR_MODUS
parser.add_argument('-c', '--component', action='store') # PAR_COMP
parser.add_argument('-f', '--function', action='store') # PAR_FCT
parser.add_argument('-t', '--tool', action='store') # PAR_TOOL
parser.add_argument('-s', '--'+B.PAR_STEP, action='store') # PAR_STEP
# parser.add_argument('-t', '--typ', default='einzel', action='store')
# parser.add_argument('-d', '--dir', action='store')
args = parser.parse_args()
self.setParameterArgs(args)
def getDirParameter(self):
if hasattr(self, "tcdir"):
return ("tcbase", str(self.tcdir))
if hasattr(self, "tsdir"):
return ("tsbase", str(self.tsdir))
return None
def setParameterArgs(self, args):
job = Job.getInstance()
print("setParArgs " + str(type(args)))
self.parstring = "python " + self.program
if "dict" in str(type(args)):
for k in args:
self.setJobAttr(k, args[k])
else:
for k in vars(args):
if getattr(args, k) is not None:
self.setJobAttr(k , getattr(args, k))
dirpath = self.getDirParameter()
if dirpath is not None:
utils.path_tool.extractPath(dirpath[0], dirpath[1])
app = self.application
if self.application in job.conf.confs[B.SUBJECT_APPS]:
if B.ATTR_APPS_PROJECT in job.conf.confs[B.SUBJECT_APPS][self.application]:
setattr(self, B.ATTR_APPS_PROJECT, job.conf.confs[B.SUBJECT_APPS][self.application][B.ATTR_APPS_PROJECT])
proj = getattr(self, B.ATTR_APPS_PROJECT)
app2 = self.application
def setParameterLoaded(self):
job = Job.getInstance()
print("setParLoaded " )
readedPar = job.loadParameter()
if readedPar is not None:
for k in readedPar["par"].keys():
if not hasattr(self, k):
self.setJobAttr(k, readedPar["par"][k])
def setJobAttr(self, key, val):
setattr(self, key, val)
self.parstring = self.parstring + " --" + key + " " + str(val)
def getJobPar(self, key):
a = key.split(":")
if len(a) == 1:
return self[a[0]]
if len(a) == 2:
return self[a[0]][a[1]]
return
# ------------------------------------------------------------------------------------------------------------------
class Configuration:
def __init__ (self, program):
self.program = program
print (f"job initialisiert {self.program}")
if program == "unit":
if (os.path.exists(utils.path_tool.rejoinPath("..", "..", "config", B.BASIS_FILE))):
self.setConfiguration(utils.path_tool.rejoinPath("..", "..", "config", B.BASIS_FILE))
return
if (os.path.exists(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))):
self.setConfiguration(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))
return
elif (os.path.exists(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))):
self.setConfiguration(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))
return
elif (os.path.exists(utils.path_tool.rejoinPath("config", B.BASIS_FILE))):
self.setConfiguration(utils.path_tool.rejoinPath("config", B.BASIS_FILE))
return
raise Exception(B.EXP_NO_BASIS_FILE)
def setConfiguration(self, path):
self.confs = {}
doc = utils.file_tool.readFileLines(path, None)
with open(path, "r") as file:
doc = yaml.full_load(file)
if "basic" in doc:
for i, v in doc["basic"].items():
self.confs[i] = v
else:
for i, v in doc.items():
self.confs[i] = v
def setConfig(self, path, val):
a = path.split(".")
if len(a) == 1:
self.confs[a[0]] = val
elif len(a) == 2:
self.confs[a[0]][a[1]] = val
elif len(a) == 3:
self.confs[a[0]][a[1]][a[2]] = val
def getPath(self, key):
return self.confs.get(B.SUBJECT_PATH).get(key)
def getJobConf(self, key):
a = key.split(":")
if len(a) == 1:
return self.confs[a[0]]
if len(a) == 2:
return self.confs[a[0]][a[1]]