Data-Test-Executer Framework speziell zum Test von Datenverarbeitungen mit Datengenerierung, Systemvorbereitungen, Einspielungen, ganzheitlicher diversifizierender Vergleich
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

490 lines
16 KiB

#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
# Template Batchrahmen
#
#import sys, getopt
import argparse
import copy
import subprocess
import time
import yaml
import os
from datetime import datetime
import basic.constants as B
import basic.message
import basic.componentHandling
import model.catalog
import tools.value_tool
import tools.date_tool
import tools.path_tool
import tools.file_tool
import tools.config_tool
import test.constants as T
import tools.path_const as P
import tools.job_tool
LIMIT_INFO = 16 #basic.message.LIMIT_INFO
LIMIT_DEBUG = 12 #basic.message.LIMIT_DEBUG
TOOL_NAME = "job_tool"
CTLG_NAME = "programs"
CTLG_PARDEF = "pardef"
CTLG_PARSOURCE = "pfilesource"
CTLG_PARTARGET = "pfiletarget"
CTLG_BASEDIR = "basedir"
CTLG_LOGPATH = "logpath"
CTLG_LOGLEVEL = "loglevel"
EXCP_CANT_POP = "cant pop this job from the instances"
class SimpleJob:
"""
the simple job is just used for test issues
"""
__logtime = "20200101_000000"
def __init__(self, program, username="", args=None):
self.program = program
self.username = username
path = tools.path_tool.getBasisConfigPath()
self.conf = getConfiguration(self, path)
self.jobid = str(100000)
try:
catalog = model.catalog.Catalog.getInstance()
self.programDef = catalog.getValue(self, CTLG_NAME, program, "")
except:
self.m = basic.message.TempMessage(self, "testtime")
pass
if args is not None:
if "par" in args:
self.par = Parameter(self, args["par"])
for k in args:
if k == "par":
continue
setattr(self, k, args[k])
else:
self.par == Parameter(self, None)
if not hasattr(self, "start"):
logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG)
while logTime <= SimpleJob.__logtime:
time.sleep(1)
logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG)
self.start = logTime
def getDebugLevel(self, arg):
return 12
def debug(self, prio, text):
pass
def getParameter(self, parameter):
if hasattr(self.par, parameter) and getattr(self.par, parameter) is not None:
return getattr(self.par, parameter)
else:
val = tools.value_tool.compose_pattern(self, parameter, None)
if val is None:
self.m.logError("Parameter "+parameter+" nicht in job.par ")
return
setattr(self.par, parameter, val)
return val
class SimpleParameter():
def __init__(self, job, args=None):
self.program = job.program
class Job:
__instance = None
__instances = []
__jobid = 100000
__logtime = "20200101_000000"
#catalog = model.catalog.Catalog.getInstance()
def __init__ (self, program, username="", args=None):
"""
initializing the job-object as a global structure for the actual doing
type of job:
a) (single) job for testing work [--args]
work-folder is the testcase-folder, logging into testcase-log
b) (single) job for reorganization of the testing area [--args]
working-folder is the workspace or the environment-folder, logging is there
c) webflask-job for the web-server --args
work-folder is the web-workspace,
if real jobs created these jobs runs as testing resp. reorganization-job
d) unit-testing job --args!
work-folder is the debug-area
if real jobs for unit-testing created these jobs runs with unit-mode like a unit-test
:param program:
:param args: optional with arguments otherwise the cli-parameter
"""
# postcondition
# - job.conf with basis config
# - job.par with parameter-args or cli-args
# - job.msg
# logtime muss unique sein denn logDateien und Verzeichnisse muessen eindeutig sein
logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG)
while logTime <= Job.__logtime:
time.sleep(1)
logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG)
self.start = logTime
self.m = basic.message.TempMessage(self, logTime)
Job.__jobid += 1
self.jobid = str(Job.__jobid)
if len(program) < 3:
print("FATAL: programname is missing")
exit(3)
self.program = program
if len(username) < 2:
self.username = os.getlogin()
path = tools.path_tool.getBasisConfigPath()
self.conf = getConfiguration(self, path)
catalog = model.catalog.Catalog.getInstance()
print("program "+program)
self.programDef = catalog.getValue(self, CTLG_NAME, program, "")
try:
path = tools.config_tool.select_config_path(self, P.KEY_BASIC, B.BASIS_FILE)
self.conf = self.getConfiguration(path)
except:
print("FATAL: config-file could not be loaded")
exit(3)
if args is not None:
self.setParameter(args)
if isinstance(self.programDef, dict):
self.m = basic.message.Message(self, self.programDef[CTLG_LOGLEVEL], logTime, None)
else:
self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None)
def getLogpath(self):
path = tools.path_tool.compose_path(self, self.programDef[CTLG_LOGPATH], None)
return path
def getConfiguration(self, path):
conf = {}
conf["configpath"] = []
if hasattr(self, "conf"):
conf = self.conf
conf["configpath"].append(path)
doc = tools.file_tool.read_file_dict(self, path, None)
if "basic" in doc:
for k, v in doc["basic"].items():
if k not in conf:
conf[k] = v
else:
for k, v in doc.items():
if k not in conf:
conf[k] = v
return conf
def setProgram(self, program):
self.program = program
basedir = self.programDef[CTLG_BASEDIR]
self.basedir = basedir
if (self.par is not None) and self.par in B.LIST_MAIN_PAR:
setattr(self.par, B.PAR_PROGRAM, program)
setattr(self.par, B.PAR_BASEDIR, basedir)
parstring = getattr(self.par, B.PAR_STRING)
parstring = parstring[parstring.find("--"):]
parstring = "python "+program+" "+parstring
setattr(self.par, "parstring", parstring)
if not hasattr(self.par, self.programDef[B.PAR_DIRNAME]):
setattr(self.par, self.programDef[B.PAR_DIRNAME],
tools.path_tool.compose_path(self, "{"+basedir+"}", None))
self.par.setParameterLoaded(self)
def startJob(self):
self.start = datetime.now()
logTime = self.start.strftime("%Y%m%d_%H%M%S")
self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None)
tools.job_tool.startJobProcesses(self)
if self.programDef[CTLG_PARSOURCE] != "":
self.par.setParameterLoaded(self)
header1 = "# # # Start " + self.program + " # "
header1 += tools.date_tool.formatParsedDate(str(self.start), tools.date_tool.F_DE_TSTAMP) + " # # # "
self.m.logInfo(header1)
print(header1)
self.par.checkParameter(self)
self.m.logInfo(self.par.parstring)
def stopJob(self, reboot=0):
tools.job_tool.stopJobProcesses(self)
self.ende = datetime.now()
if self.programDef[CTLG_PARTARGET] != "":
self.dumpParameter()
footer1 = "# # " + self.m.topmessage + " # # # "
footer2 = "# # # Stop " + self.program + " # "
footer2 += tools.date_tool.formatParsedDate(str(self.start), tools.date_tool.F_DE_TSTAMP)
footer2 += " # " + tools.date_tool.formatParsedDate(str(self.ende), tools.date_tool.F_DE_TSTAMP) + " # # # "
footer2 += " # # # RC: " + self.m.getFinalReturncode()
self.m.logDebug("Logpath: "+self.m.logpath)
self.m.logInfo(footer1)
self.m.logInfo(footer2)
self.m.closeMessage()
rc = self.m.getFinalRc()
print("Logpath: "+self.m.logpath)
print(footer1)
print(footer2)
if "editor" in self.conf[B.SUBJECT_PATH]:
subprocess.Popen([self.conf[B.SUBJECT_PATH]["editor"], str(self.m.logpath)])
if reboot == 0:
exit(rc)
def dumpParameter(self):
if len(self.programDef[CTLG_PARTARGET]) < 2:
return
parpath = tools.path_tool.compose_path(self, self.programDef[CTLG_PARTARGET], None)
output = {}
cconf = basic.componentHandling.getComponentDict()
output["par"] = self.par.__dict__
if len(cconf) < 1:
tools.file_tool.write_file_dict(self.m, self, parpath, output)
return
output[B.SUBJECT_COMPS] = {}
for c in cconf:
output[B.SUBJECT_COMPS][c] = {}
for x in ["function", "conn"]:
if x not in cconf[c]:
continue
output[B.SUBJECT_COMPS][c][x] = cconf[c][x]
if x == B.SUBJECT_CONN and "passwd" in cconf[c][x]:
cconf[B.SUBJECT_COMPS][c][x]["passwd"] = "xxxxx"
tools.file_tool.write_file_dict(self.m, self, parpath, output)
def loadParameter(self):
output = {}
if len(str(self.programDef[CTLG_PARSOURCE])) < 2:
return None
parpath = tools.path_tool.compose_path(self, self.programDef[CTLG_PARSOURCE], None)
if not os.path.join(parpath):
return None
doc = tools.file_tool.read_file_dict(self, parpath, None)
for k in doc.keys():
output[k] = copy.deepcopy(doc[k])
return output
def setParameter(self, args):
self.par = Parameter(self, args)
def getParameter(self, parameter):
if hasattr(self.par, parameter) and getattr(self.par, parameter) is not None:
return getattr(self.par, parameter)
elif "xxxtime" in parameter:
neu = tools.date_tool.getActdate(tools.date_tool.F_DIR)
# setattr(self.par, parameter, neu)
return neu
else:
val = tools.value_tool.compose_pattern(self, parameter, None)
if val is None:
self.m.logError("Parameter "+parameter+" nicht in job.par ")
return
setattr(self.par, parameter, val)
return val
def hasElement(self, parameter, elem):
"""
the function searches in an optional job.parameter
(a) true, if the parameter does not exist (DEFAULT)
(b) true, if the element is member of the parameter
(c) false, if parameter exists and elem is not member of the parameter
:param parameter:
:param elem:
:return:
"""
if hasattr(self.par, parameter):
if getattr(self.par, parameter).find(elem) >= 0:
return True
return False
return True
def hascomponente(self, comp):
"""
it searches if comp is member of the optional job.parameter (default = each is member)
:param comp:
:return:
"""
return self.hasElement("componente", comp)
def hasFunction(self, fct):
"""
it searches if fct is member of the optional job.parameter (default = each is member)
:param fct:
:return:
"""
return self.hasElement("function", fct)
def hasTool(self, tool):
"""
it searches if tool is member of the optional job.parameter (default = each is member)
:param tool:
:return:
"""
return self.hasElement("tool", tool)
def getMessageLevel(self, errtyp, elem):
if (not hasattr(self, "m")) or (self.m is None):
return basic.message.LIMIT_DEBUG
elif elem.find("tool") > 1 and hasattr(self, "par"):
if not hasattr(self.par, "tool") or elem not in getattr(self.par, "tool"):
return 0
else:
return 2
else:
# TODO quickfix
return 0
def getInfoLevel(self, elem):
return self.getMessageLevel("info", elem)
def getDebugLevel(self, elem):
return self.getMessageLevel("debug", elem)
def getTraceLevel(self, elem):
return self.getMessageLevel("trace", elem)
def debug(self, prio, text):
if hasattr(self, "m"):
self.m.debug(prio, text)
def getConfiguration(job, path):
conf = {}
conf["configpath"] = []
if hasattr(job, "conf"):
conf = job.conf
conf["configpath"].append(path)
doc = tools.file_tool.read_file_dict(job, path, None)
if "basic" in doc:
for k, v in doc["basic"].items():
if k not in conf:
conf[k] = v
else:
for k, v in doc.items():
if k not in conf:
conf[k] = v
return conf
# ------------------------------------------------------------------------------------------------------------------
class Parameter:
"""
parameter with source either the list args or the args of the main-program
content of parameter:
+ single arguments
+ dirname of work-folder either from dir-parameter or composed from single parameter
comparison with a parameter-file from the work-folder
"""
def __init__ (self, job, args=None):
self.program = job.program
if args is not None and isinstance(args, dict):
self.setParameterArgs(job, args)
else:
self.setParameter(job)
self.basedir = job.programDef[B.PAR_BASEDIR]
job.basedir = job.programDef[B.PAR_BASEDIR]
job.par = self
# Parameter aus Verzeichnis extraieren bzw. Verzeichnis aus Parameter erstellen
if not hasattr(self, job.programDef[B.PAR_DIRNAME]):
setattr(self, job.programDef[B.PAR_DIRNAME],
tools.value_tool.compose_pattern(job, self.basedir, None))
else:
tools.path_tool.extractPath(job, job.programDef[B.PAR_BASEDIR],
getattr(self, job.programDef[B.PAR_DIRNAME]))
# Abgleich mit zuvor gespeicherten Parametern
if len(job.programDef["pfilesource"]) > 2:
self.setParameterLoaded(job)
def checkParameter(self, job):
print (f"Parameter initialisiert {self.program}")
pardef = job.programDef[CTLG_PARDEF]
for p in pardef:
if len(p) > 1 and not hasattr(self, p):
job.m.setFatal("Parameter " + p + " is not set!")
def setParameter(self, job):
"""
1. Programm -- implementiert in Main-Klasse
2. anwndung -- steuert zu pruefende System [ in basis_Config ]
3. amgebung -- steuert zu pruefende Maschine [ in dir/applicationen ]
4. release -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ]
5. ~Verz -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch
6. zyklus -- optional unterscheidet echte und entwicklungsLaeufe
7. Programmspezifische Parameter
8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2)
10. modus -- steuert die Verarbeitung; default echt
- echt-auto Lauf aus Automatisierung (1-7)
- test Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen
- echt-spez Wiederholung einer spezifischen Funktion (1-13)
- unit Ausfuehrung der Unittests
11. componente -- schraenkt Verarbeitung auf parametriserte componenten ein
12. funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein
13. tool -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein
"""
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--'+B.SUBJECT_APPS, required=True, action='store')
parser.add_argument('-e', '--'+B.PAR_ENV, required=True, action='store')
parser.add_argument('-v', '--'+B.PAR_VAR, required=False, action='store')
parser.add_argument('-r', '--release', action='store')
parser.add_argument('-ts', '--'+B.PAR_TSDIR, action='store')
parser.add_argument('-tc', '--'+B.PAR_TCDIR, action='store')
parser.add_argument('-ws', '--'+B.PAR_WSDIR, action='store')
parser.add_argument('-rs', '--rsdir', action='store')
parser.add_argument('-dt', '--tdtyp', action='store') # PAR_TDTYP
parser.add_argument('-ds', '--tdsrc', action='store') # PAR_TDSRC
parser.add_argument('-dn', '--tdname', action='store') # PAR_TDNAME
parser.add_argument('-l', '--loglevel', action='store') # PAR_LOG
parser.add_argument('-m', '--modus', action='store') # PAR_MODUS
parser.add_argument('-c', '--component', action='store') # PAR_COMP
parser.add_argument('-f', '--function', action='store') # PAR_FCT
parser.add_argument('-t', '--tool', action='store') # PAR_TOOL
parser.add_argument('-s', '--'+B.PAR_STEP, action='store') # PAR_STEP
# parser.add_argument('-t', '--typ', default='einzel', action='store')
# parser.add_argument('-d', '--dir', action='store')
args = parser.parse_args()
self.setParameterArgs(job, args)
def getDirParameter(self):
if hasattr(self, "tcdir"):
return ("tcbase", str(self.tcdir))
if hasattr(self, "tsdir"):
return ("tsbase", str(self.tsdir))
return None
def setParameterArgs(self, job, args):
self.parstring = "python " + self.program
if "dict" in str(type(args)):
for k in args:
self.setJobAttr(k, args[k])
else:
for k in vars(args):
if getattr(args, k) is not None:
self.setJobAttr(k , getattr(args, k))
dirpath = self.getDirParameter()
if hasattr(self, "application") and B.SUBJECT_APPS in job.conf \
and getattr(self, "application") in job.conf[B.SUBJECT_APPS]:
if B.ATTR_APPS_PROJECT in job.conf[B.SUBJECT_APPS][self.application]:
setattr(self, B.ATTR_APPS_PROJECT, job.conf[B.SUBJECT_APPS][self.application][B.ATTR_APPS_PROJECT])
proj = getattr(self, B.ATTR_APPS_PROJECT)
def setParameterLoaded(self, job):
#job = Job.getInstance()
readedPar = job.loadParameter()
if readedPar is not None:
for k in readedPar["par"].keys():
if not hasattr(self, k):
self.setJobAttr(k, readedPar["par"][k])
def setJobAttr(self, key, val):
setattr(self, key, val)
self.parstring = self.parstring + " --" + key + " " + str(val)
def getJobPar(self, key):
a = key.split(":")
if len(a) == 1:
return self[a[0]]
if len(a) == 2:
return self[a[0]][a[1]]
return