Data-Test-Executer Framework speziell zum Test von Datenverarbeitungen mit Datengenerierung, Systemvorbereitungen, Einspielungen, ganzheitlicher diversifizierender Vergleich
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

424 lines
14 KiB

#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
3 years ago
# Template Batchrahmen
#
#import sys, getopt
import argparse
import copy
import yaml
import os
3 years ago
from datetime import datetime
3 years ago
import basic.constants as B
3 years ago
import basic.message
import basic.message
import basic.componentHandling
3 years ago
import utils.date_tool
import utils.path_tool
import utils.file_tool
3 years ago
jobdef = {
"unit": {
"pardef": "",
"pfilesource": "",
"pfiletarget": "envparfile",
3 years ago
"basedir": "envbase",
"dirname": "envdir",
3 years ago
"logdir": "{job.par.envdir}/{log}/log_{time}.txt" },
"check_environment": {
"pardef": "",
"pfilesource": "",
"pfiletarget": "envparfile",
3 years ago
"basedir": "envbase",
"dirname": "envdir",
3 years ago
"logdir": "{job.par.envdir}/{log}/log_{time}.txt" },
"test_executer": {
"pardef": "",
"pfilesource": "tsparfile",
"pfiletarget": "tsparfile",
"basedir": "tsbase",
"dirname": "tsdir",
"logdir": "{job.par.tsdir}/{log}/log_{time}.txt" },
"init_testsuite": {
3 years ago
"pardef": "tsdir", # ,tdtyp,tdsrc,tdname",
"pfilesource": "envparfile",
"pfiletarget": "tsparfile",
3 years ago
"basedir": "tsbase",
"dirname": "tsdir",
3 years ago
"logdir": "{job.par.tsdir}/{log}/log_{tstime}.txt" },
"init_testcase": {
3 years ago
"pardef": "tcdir", # ",tdtyp,tdsrc,tdname",
"pfilesource": "envparfile",
"pfiletarget": "tcparfile",
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
"execute_testcase": {
"pardef": "tcdir", # ",tdtyp,tdsrc,tdname",
"pfilesource": "tcparfile",
"pfiletarget": "tcparfile",
3 years ago
"basedir": "tcbase",
"dirname": "tcdir",
3 years ago
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
3 years ago
"collect_testcase": {
"pardef": "tcdir", # ",tdtyp,tdsrc,tdname",
"pfilesource": "tcparfile",
"pfiletarget": "tcparfile",
3 years ago
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
"compare_testcase": {
"pardef": "tcdir", # ",tdtyp,tdsrc,tdname",
"pfilesource": "tcparfile",
"pfiletarget": "tcparfile",
3 years ago
"basedir": "tcbase",
"dirname": "tcdir",
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
3 years ago
"test_system": {
"pardef": "tcdir,tdtyp,tdsrc,tdname",
"pfilesource": "tsparfile",
"pfiletarget": "tsparfile",
3 years ago
"basedir": "tcbase",
"dirname": "tcdir",
3 years ago
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"},
3 years ago
"finish_testcase": {
"pardef": "tcdir",
"pfilesource": "tcparfile",
3 years ago
"pfiletarget": "tcparfile",
"basedir": "tcbase",
"dirname": "tcdir",
3 years ago
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"},
"finish_testsuite": {
"pardef": "tsdir",
3 years ago
"pfilesource": "tsparfile",
"pfiletarget": "tssarfile",
"basedir": "tsbase",
"dirname": "tsdir",
3 years ago
"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"}
3 years ago
}
def setGlobal():
pass
3 years ago
3 years ago
class Job:
__instance = None
def __init__ (self, program):
print ("################# init Job ## " + program + " #################")
self.program = program
Job.__instance = self
par = Parameter(program)
self.par = par
print("prog-42 " + str(self.par.basedir))
conf = Configuration(program)
self.conf = conf
print("prog-45 " + str(self.par.basedir))
dirpath = self.par.getDirParameter()
setGlobal()
if dirpath is not None:
utils.path_tool.extractPath(dirpath[0], dirpath[1])
if program == "unit": # no job will be started
self.start = datetime.now()
logTime = self.start.strftime("%Y%m%d_%H%M%S")
3 years ago
self.m = basic.message.Message(basic.message.LIMIT_DEBUG, logTime, None)
3 years ago
print("prog-50 " + str(self.par.basedir))
@staticmethod
def popInstance():
job = Job.getInstance()
Job.__instance = None
return job
@staticmethod
def pushInstance(job):
Job.__instance = job
return job
3 years ago
def resetInstance(program):
job = Job.getInstance()
if job is not None:
job.stopJob(1)
Job.__instance = None
Job(program)
return Job.__instance
def setProgram(self, program):
self.program = program
basedir = jobdef[program]["basedir"]
if (self.par is not None):
setattr(self.par, "program", program)
setattr(self.par, "basedir", basedir)
parstring = getattr(self.par, "parstring")
parstring = parstring[parstring.find("--"):]
parstring = "python "+program+" "+parstring
setattr(self.par, "parstring", parstring)
if not hasattr(self.par, jobdef[program]["dirname"]):
setattr(self.par, jobdef[program]["dirname"],
utils.path_tool.composePattern("{"+basedir+"}", None))
self.par.setParameterLoaded()
3 years ago
def getInstance():
if (Job.__instance is not None):
return Job.__instance
else:
return None
def startJob(self):
self.start = datetime.now()
print("prog-68 " + str(self.par.basedir))
logTime = self.start.strftime("%Y%m%d_%H%M%S")
3 years ago
self.m = basic.message.Message(basic.message.LIMIT_DEBUG, logTime, None)
3 years ago
print("prog-68 " + str(self.m.rc))
self.par.setParameterLoaded()
self.m.logInfo("# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
3 years ago
self.m.debug(basic.message.LIMIT_INFO, "# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
3 years ago
self.par.checkParameter()
def stopJob(self, reboot=0):
self.ende = datetime.now()
self.dumpParameter()
print("stopJob " + str(self.m.messages) + ", " + str(self.m.debugfile))
self.m.logInfo("# # " + self.m.topmessage + " # # # ")
self.m.logInfo("# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
3 years ago
self.m.debug(basic.message.LIMIT_INFO, "# # " + self.m.topmessage + " # # # ")
self.m.debug(basic.message.LIMIT_INFO, "# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # RC: " + str(self.m.getFinalRc()))
3 years ago
self.m.closeMessage()
rc = self.m.getFinalRc()
print ("rc " + str(rc))
if reboot == 0:
exit(rc)
def dumpParameter(self):
parpath = utils.path_tool.composePath(jobdef[self.program]["pfiletarget"], None)
if not os.path.exists(parpath):
return None
3 years ago
output = {}
cconf = basic.componentHandling.getComponentDict()
3 years ago
output["par"] = self.par.__dict__
output["comps"] = {}
for c in cconf:
output["comps"][c] = {}
for x in ["function", "conn"]:
output["comps"][c][x] = cconf[c][x]
if x == "conn" and "passwd" in cconf[c][x]:
output["comps"][c][x]["passwd"] = "xxxxx"
utils.file_tool.writeFileText(self.m, parpath, output)
3 years ago
def loadParameter(self):
output = {}
if len(str(jobdef[self.program]["pfilesource"])) < 2:
return None
parpath = utils.path_tool.composePath(jobdef[self.program]["pfilesource"], None)
if not os.path.join(parpath):
return None
doc = utils.file_tool.readFileDict(parpath, self.m)
for k in doc.keys():
output[k] = copy.deepcopy(doc[k])
3 years ago
return output
def getParameter(self, parameter):
if hasattr(self.par, parameter):
return getattr(self.par, parameter)
elif "xxxtime" in parameter:
neu = utils.date_tool.getActdate(utils.date_tool.F_DIR)
# setattr(self.par, parameter, neu)
return neu
def hasElement(self, parameter, elem):
if hasattr(self.par, parameter):
print (parameter + " in Parameter")
if getattr(self.par, parameter).find(elem) >= 0:
return True
return False
return True
def hascomponente(self, komp):
return self.hasElement("componente", komp)
def hasFunction(self, fct):
return self.hasElement("function", fct)
def hasTool(self, tool):
return self.hasElement("tool", tool)
def getMessageLevel(self, errtyp, elem):
if (not hasattr(self, "m")) or (self.m is None):
3 years ago
return basic.message.LIMIT_DEBUG
3 years ago
elif elem.find("tool") > 1:
if not hasattr(self.par, "tool") or getattr(self.par, "tool").find(elem) <= 0:
return int(self.m.CONST_ERRTYP[errtyp]) -1
else:
return int(self.m.CONST_ERRTYP[errtyp])
else:
return int(self.m.CONST_ERRTYP[errtyp])
def getInfoLevel(self, elem):
return self.getMessageLevel("info", elem)
def getDebugLevel(self, elem):
return self.getMessageLevel("debug", elem)
def getTraceLevel(self, elem):
return self.getMessageLevel("trace", elem)
def debug(self, prio, text):
#print("job.debug "+str(prio)+" "+text)
3 years ago
if hasattr(self, "m"):
self.m.debug(prio, text)
else:
print(text)
class Parameter:
print ("class Parameter")
def __init__ (self, program):
print ("# init Parameter for " + program)
self.program = program
self.basedir = "debugs"
self.setBasedir(program)
print (f"# Parameter initialisiert {self.program} mit basedir {self.basedir}")
if (program != "unit"): self.setParameter()
def setBasedir(self, program):
if jobdef[program]:
3 years ago
self.basedir = jobdef[program]["basedir"]
if hasattr(self, jobdef[program]["dirname"]):
utils.path_tool.extractPath(self.basedir, getattr(self, jobdef[program]["dirname"]))
elif program != "unit":
dirpath = utils.path_tool.composePattern("{"+jobdef[program]["basedir"]+"}", None)
setattr(self, jobdef[program]["dirname"], dirpath)
3 years ago
else:
self.basedir = "debugs"
def checkParameter(self):
job = Job.getInstance()
print (f"Parameter initialisiert {self.program}")
pardef = jobdef[job.program]["pardef"]
for p in pardef.split(","):
print(p)
if len(p) > 1 and not hasattr(self, p):
job.m.setFatal("Parameter " + p + " is not set!")
def setParameter(self):
"""
1. Programm -- implementiert in Main-Klasse
2. anwndung -- steuert zu pruefende System [ in basis_Config ]
3. amgebung -- steuert zu pruefende Maschine [ in dir/applicationen ]
4. release -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ]
5. ~Verz -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch
6. zyklus -- optional unterscheidet echte und entwicklungsLaeufe
7. Programmspezifische Parameter
8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2)
10. modus -- steuert die Verarbeitung; default echt
- echt-auto Lauf aus Automatisierung (1-7)
- test Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen
- echt-spez Wiederholung einer spezifischen Funktion (1-13)
- unit Ausfuehrung der Unittests
11. componente -- schraenkt Verarbeitung auf parametriserte componenten ein
12. funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein
13. tool -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein
"""
# args = str(sys.argv)
# print ("Job-Programm %s : " % args)
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--application', required=True, action='store')
parser.add_argument('-e', '--environment', required=True, action='store')
parser.add_argument('-r', '--release', action='store')
parser.add_argument('-ts', '--tsdir', action='store')
parser.add_argument('-tc', '--tcdir', action='store')
parser.add_argument('-rs', '--rsdir', action='store')
3 years ago
parser.add_argument('-dt', '--tdtyp', action='store') # PAR_TDTYP
parser.add_argument('-ds', '--tdsrc', action='store') # PAR_TDSRC
parser.add_argument('-dn', '--tdname', action='store') # PAR_TDNAME
parser.add_argument('-l', '--loglevel', action='store') # PAR_LOG
parser.add_argument('-m', '--modus', action='store') # PAR_MODUS
parser.add_argument('-c', '--component', action='store') # PAR_COMP
parser.add_argument('-f', '--function', action='store') # PAR_FCT
parser.add_argument('-t', '--tool', action='store') # PAR_TOOL
3 years ago
parser.add_argument('-s', '--'+B.PAR_STEP, action='store') # PAR_STEP
3 years ago
# parser.add_argument('-t', '--typ', default='einzel', action='store')
# parser.add_argument('-d', '--dir', action='store')
args = parser.parse_args()
self.setParameterArgs(args)
3 years ago
3 years ago
def getDirParameter(self):
if hasattr(self, "tcdir"):
return ("tcbase", str(self.tcdir))
if hasattr(self, "tsdir"):
return ("tsbase", str(self.tsdir))
return None
def setParameterArgs(self, args):
job = Job.getInstance()
print("setParArgs " + str(type(args)))
self.parstring = "python " + self.program
if "dict" in str(type(args)):
for k in args:
self.setJobAttr(k, args[k])
else:
for k in vars(args):
if getattr(args, k) is not None:
self.setJobAttr(k , getattr(args, k))
dirpath = self.getDirParameter()
if dirpath is not None:
utils.path_tool.extractPath(dirpath[0], dirpath[1])
3 years ago
def setParameterLoaded(self):
job = Job.getInstance()
print("setParLoaded " )
readedPar = job.loadParameter()
3 years ago
3 years ago
if readedPar is not None:
for k in readedPar["par"].keys():
if not hasattr(self, k):
self.setJobAttr(k, readedPar["par"][k])
def setJobAttr(self, key, val):
setattr(self, key, val)
3 years ago
self.parstring = self.parstring + " --" + key + " " + str(val)
3 years ago
3 years ago
def getJobPar(self, key):
a = key.split(":")
if len(a) == 1:
return self[a[0]]
if len(a) == 2:
return self[a[0]][a[1]]
return
3 years ago
class Configuration:
def __init__ (self, program):
self.program = program
print (f"job initialisiert {self.program}")
if program == "unit":
if (os.path.exists('../conf/basis.yml')):
self.setConfiguration('../conf/basis.yml')
return
elif (os.path.exists('conf/basis.yml')):
3 years ago
self.setConfiguration('../conf/basis.yml')
return
self.setConfiguration('conf/basis.yml')
def setConfiguration(self, path):
self.confs = {}
doc = utils.file_tool.readFileLines(path, None)
3 years ago
with open(path, "r") as file:
doc = yaml.full_load(file)
if "basic" in doc:
for i, v in doc["basic"].items():
self.confs[i] = v
else:
for i, v in doc.items():
self.confs[i] = v
def setConfig(self, path, val):
a = path.split(".")
if len(a) == 1:
self.confs[a[0]] = val
elif len(a) == 2:
self.confs[a[0]][a[1]] = val
elif len(a) == 3:
self.confs[a[0]][a[1]][a[2]] = val
3 years ago
def getJobConf(self, key):
a = key.split(":")
if len(a) == 1:
return self.confs[a[0]]
if len(a) == 2:
return self.confs[a[0]][a[1]]