You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
494 lines
17 KiB
494 lines
17 KiB
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
# ---------------------------------------------------------------------------------------------------------
|
|
# Author : Ulrich Carmesin
|
|
# Source : gitea.ucarmesin.de
|
|
# ---------------------------------------------------------------------------------------------------------
|
|
# Template Batchrahmen
|
|
#
|
|
#import sys, getopt
|
|
import argparse
|
|
import copy
|
|
import time
|
|
|
|
import yaml
|
|
import os
|
|
from datetime import datetime
|
|
import basic.constants as B
|
|
import basic.message
|
|
import basic.componentHandling
|
|
import objects.catalog
|
|
import tools.value_tool
|
|
import tools.date_tool
|
|
import tools.path_tool
|
|
import tools.file_tool
|
|
import tools.config_tool
|
|
import test.constants as T
|
|
import tools.path_const as P
|
|
import tools.job_tool
|
|
|
|
LIMIT_INFO = 16 #basic.message.LIMIT_INFO
|
|
LIMIT_DEBUG = 12 #basic.message.LIMIT_DEBUG
|
|
|
|
TOOL_NAME = "job_tool"
|
|
|
|
CTLG_NAME = "programs"
|
|
CTLG_PARDEF = "pardef"
|
|
CTLG_PARSOURCE = "pfilesource"
|
|
CTLG_PARTARGET = "pfiletarget"
|
|
CTLG_BASEDIR = "basedir"
|
|
CTLG_LOGPATH = "logpath"
|
|
CTLG_LOGLEVEL = "loglevel"
|
|
|
|
EXCP_CANT_POP = "cant pop this job from the instances"
|
|
|
|
|
|
class SimpleJob:
|
|
"""
|
|
the simple job is just used for test issues
|
|
"""
|
|
__logtime = "20200101_000000"
|
|
def __init__(self, program, username="", args=None):
|
|
self.program = program
|
|
self.username = username
|
|
path = tools.path_tool.getBasisConfigPath()
|
|
self.conf = getConfiguration(self, path)
|
|
self.jobid = str(100000)
|
|
catalog = objects.catalog.Catalog.getInstance()
|
|
self.programDef = catalog.getValue(self, CTLG_NAME, program, "")
|
|
|
|
if args is not None:
|
|
if "par" in args:
|
|
self.par = Parameter(self, args["par"])
|
|
for k in args:
|
|
if k == "par":
|
|
continue
|
|
setattr(self, k, args[k])
|
|
else:
|
|
self.par == Parameter(self, None)
|
|
if not hasattr(self, "start"):
|
|
logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG)
|
|
while logTime <= SimpleJob.__logtime:
|
|
time.sleep(1)
|
|
logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG)
|
|
self.start = logTime
|
|
|
|
def getDebugLevel(self, arg):
|
|
return 12
|
|
|
|
def debug(self, prio, text):
|
|
pass
|
|
|
|
def getParameter(self, parameter):
|
|
if hasattr(self.par, parameter) and getattr(self.par, parameter) is not None:
|
|
return getattr(self.par, parameter)
|
|
else:
|
|
val = tools.value_tool.compose_pattern(self, parameter, None)
|
|
if val is None:
|
|
self.m.logError("Parameter "+parameter+" nicht in job.par ")
|
|
return
|
|
setattr(self.par, parameter, val)
|
|
return val
|
|
|
|
class SimpleParameter():
|
|
def __init__(self, job, args=None):
|
|
self.program = job.program
|
|
|
|
class Job:
|
|
__instance = None
|
|
__instances = []
|
|
__jobid = 100000
|
|
__logtime = "20200101_000000"
|
|
#catalog = objects.catalog.Catalog.getInstance()
|
|
|
|
def __init__ (self, program, username="", args=None):
|
|
"""
|
|
initializing the job-object as a global structure for the actual doing
|
|
type of job:
|
|
a) (single) job for testing work [--args]
|
|
work-folder is the testcase-folder, logging into testcase-log
|
|
b) (single) job for reorganization of the testing area [--args]
|
|
working-folder is the workspace or the environment-folder, logging is there
|
|
c) webflask-job for the web-server --args
|
|
work-folder is the web-workspace,
|
|
if real jobs created these jobs runs as testing resp. reorganization-job
|
|
d) unit-testing job --args!
|
|
work-folder is the debug-area
|
|
if real jobs for unit-testing created these jobs runs with unit-mode like a unit-test
|
|
:param program:
|
|
:param args: optional with arguments otherwise the cli-parameter
|
|
"""
|
|
# postcondition
|
|
# - job.conf with basis config
|
|
# - job.par with parameter-args or cli-args
|
|
# - job.msg
|
|
# logtime muss unique sein denn logDateien und Verzeichnisse muessen eindeutig sein
|
|
logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG)
|
|
while logTime <= Job.__logtime:
|
|
time.sleep(1)
|
|
logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG)
|
|
self.start = logTime
|
|
self.m = basic.message.TempMessage(self, logTime)
|
|
Job.__jobid += 1
|
|
self.jobid = str(Job.__jobid)
|
|
if len(program) < 3:
|
|
print("FATAL: programname is missing")
|
|
exit(3)
|
|
self.program = program
|
|
if len(username) < 2:
|
|
self.username = os.getlogin()
|
|
path = tools.path_tool.getBasisConfigPath()
|
|
self.conf = getConfiguration(self, path)
|
|
catalog = objects.catalog.Catalog.getInstance()
|
|
self.programDef = catalog.getValue(self, CTLG_NAME, program, "")
|
|
try:
|
|
path = tools.config_tool.getConfigPath(self, P.KEY_BASIC, B.BASIS_FILE)
|
|
self.conf = self.getConfiguration(path)
|
|
except:
|
|
print("FATAL: config-file could not be loaded")
|
|
exit(3)
|
|
if args is not None:
|
|
self.setParameter(args)
|
|
self.m = basic.message.Message(self, self.programDef[CTLG_LOGLEVEL], logTime, None)
|
|
|
|
def altinit(self, program, args):
|
|
appl = tools.config_tool.getConfig(self, P.KEY_BASIC, B.SUBJECT_APPS)
|
|
if appl is not None:
|
|
self.conf[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS]
|
|
if B.SUBJECT_PROJECTS in self.conf:
|
|
for k in self.conf[B.SUBJECT_PROJECTS]:
|
|
if k not in appl[B.SUBJECT_PROJECTS]:
|
|
raise Exception("Workspace has project "+k+" which is not configured")
|
|
self.conf[B.SUBJECT_PROJECTS][k] = appl[B.SUBJECT_PROJECTS][k]
|
|
else:
|
|
self.conf[B.SUBJECT_PROJECTS] = appl[B.SUBJECT_PROJECTS]
|
|
par = Parameter(self, program, args)
|
|
self.par = par
|
|
|
|
def getLogpath(self):
|
|
path = tools.path_tool.compose_path(self, self.programDef[CTLG_LOGPATH], None)
|
|
return path
|
|
|
|
def getConfiguration(self, path):
|
|
conf = {}
|
|
conf["configpath"] = []
|
|
if hasattr(self, "conf"):
|
|
conf = self.conf
|
|
conf["configpath"].append(path)
|
|
doc = tools.file_tool.readFileDict(self, path, None)
|
|
if "basic" in doc:
|
|
for k, v in doc["basic"].items():
|
|
if k not in conf:
|
|
conf[k] = v
|
|
else:
|
|
for k, v in doc.items():
|
|
if k not in conf:
|
|
conf[k] = v
|
|
return conf
|
|
|
|
def setProgram(self, program):
|
|
self.program = program
|
|
basedir = self.programDef[CTLG_BASEDIR]
|
|
self.basedir = basedir
|
|
if (self.par is not None) and self.par in B.LIST_MAIN_PAR:
|
|
setattr(self.par, B.PAR_PROGRAM, program)
|
|
setattr(self.par, B.PAR_BASEDIR, basedir)
|
|
parstring = getattr(self.par, B.PAR_STRING)
|
|
parstring = parstring[parstring.find("--"):]
|
|
parstring = "python "+program+" "+parstring
|
|
setattr(self.par, "parstring", parstring)
|
|
if not hasattr(self.par, self.programDef[B.PAR_DIRNAME]):
|
|
setattr(self.par, self.programDef[B.PAR_DIRNAME],
|
|
tools.path_tool.compose_path(self, "{"+basedir+"}", None))
|
|
self.par.setParameterLoaded(self)
|
|
|
|
|
|
def startJob(self):
|
|
self.start = datetime.now()
|
|
logTime = self.start.strftime("%Y%m%d_%H%M%S")
|
|
self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None)
|
|
tools.job_tool.startJobProcesses(self)
|
|
if self.programDef[CTLG_PARSOURCE] != "":
|
|
self.par.setParameterLoaded(self)
|
|
header1 = "# # # Start Job " + tools.date_tool.formatParsedDate(str(self.start), tools.date_tool.F_DE_TSTAMP) + " # # # "
|
|
self.m.logInfo(header1)
|
|
self.m.debug(basic.message.LIMIT_INFO, header1)
|
|
print(header1)
|
|
self.par.checkParameter(self)
|
|
self.m.logInfo(self.par.parstring)
|
|
|
|
|
|
def stopJob(self, reboot=0):
|
|
tools.job_tool.stopJobProcesses(self)
|
|
self.ende = datetime.now()
|
|
if self.programDef[CTLG_PARTARGET] != "":
|
|
self.dumpParameter()
|
|
footer1 = "# # " + self.m.topmessage + " # # # "
|
|
footer2 = "# # # Stop Job " + tools.date_tool.formatParsedDate(str(self.start), tools.date_tool.F_DE_TSTAMP)
|
|
footer2 += " # " + tools.date_tool.formatParsedDate(str(self.ende), tools.date_tool.F_DE_TSTAMP) + " # # # "
|
|
footer2 += " # # # RC: " + str(self.m.getFinalRc())
|
|
self.m.logInfo(footer1)
|
|
self.m.logInfo(footer2)
|
|
self.m.debug(basic.message.LIMIT_INFO, footer1)
|
|
self.m.debug(basic.message.LIMIT_INFO, footer2)
|
|
self.m.closeMessage()
|
|
rc = self.m.getFinalRc()
|
|
print(footer1)
|
|
print(footer2)
|
|
if reboot == 0:
|
|
exit(rc)
|
|
|
|
|
|
def dumpParameter(self):
|
|
if len(self.programDef[CTLG_PARTARGET]) < 2:
|
|
return
|
|
parpath = tools.path_tool.compose_path(self, self.programDef[CTLG_PARTARGET], None)
|
|
output = {}
|
|
cconf = basic.componentHandling.getComponentDict()
|
|
output["par"] = self.par.__dict__
|
|
if len(cconf) < 1:
|
|
tools.file_tool.writeFileDict(self.m, self, parpath, output)
|
|
return
|
|
output[B.SUBJECT_COMPS] = {}
|
|
for c in cconf:
|
|
output[B.SUBJECT_COMPS][c] = {}
|
|
for x in ["function", "conn"]:
|
|
if x not in cconf[c]:
|
|
continue
|
|
output[B.SUBJECT_COMPS][c][x] = cconf[c][x]
|
|
if x == B.SUBJECT_CONN and "passwd" in cconf[c][x]:
|
|
cconf[B.SUBJECT_COMPS][c][x]["passwd"] = "xxxxx"
|
|
tools.file_tool.writeFileDict(self.m, self, parpath, output)
|
|
|
|
|
|
def loadParameter(self):
|
|
output = {}
|
|
if len(str(self.programDef[CTLG_PARSOURCE])) < 2:
|
|
return None
|
|
parpath = tools.path_tool.compose_path(self, self.programDef[CTLG_PARSOURCE], None)
|
|
if not os.path.join(parpath):
|
|
return None
|
|
doc = tools.file_tool.readFileDict(self, parpath, None)
|
|
for k in doc.keys():
|
|
output[k] = copy.deepcopy(doc[k])
|
|
return output
|
|
|
|
|
|
def setParameter(self, args):
|
|
self.par = Parameter(self, args)
|
|
|
|
def getParameter(self, parameter):
|
|
if hasattr(self.par, parameter) and getattr(self.par, parameter) is not None:
|
|
return getattr(self.par, parameter)
|
|
elif "xxxtime" in parameter:
|
|
neu = tools.date_tool.getActdate(tools.date_tool.F_DIR)
|
|
# setattr(self.par, parameter, neu)
|
|
return neu
|
|
else:
|
|
val = tools.value_tool.compose_pattern(self, parameter, None)
|
|
if val is None:
|
|
self.m.logError("Parameter "+parameter+" nicht in job.par ")
|
|
return
|
|
setattr(self.par, parameter, val)
|
|
return val
|
|
|
|
|
|
def hasElement(self, parameter, elem):
|
|
"""
|
|
the function searches in an optional job.parameter
|
|
(a) true, if the parameter does not exist (DEFAULT)
|
|
(b) true, if the element is member of the parameter
|
|
(c) false, if parameter exists and elem is not member of the parameter
|
|
:param parameter:
|
|
:param elem:
|
|
:return:
|
|
"""
|
|
if hasattr(self.par, parameter):
|
|
if getattr(self.par, parameter).find(elem) >= 0:
|
|
return True
|
|
return False
|
|
return True
|
|
def hascomponente(self, comp):
|
|
"""
|
|
it searches if comp is member of the optional job.parameter (default = each is member)
|
|
:param comp:
|
|
:return:
|
|
"""
|
|
return self.hasElement("componente", comp)
|
|
def hasFunction(self, fct):
|
|
"""
|
|
it searches if fct is member of the optional job.parameter (default = each is member)
|
|
:param fct:
|
|
:return:
|
|
"""
|
|
return self.hasElement("function", fct)
|
|
def hasTool(self, tool):
|
|
"""
|
|
it searches if tool is member of the optional job.parameter (default = each is member)
|
|
:param tool:
|
|
:return:
|
|
"""
|
|
return self.hasElement("tool", tool)
|
|
def getMessageLevel(self, errtyp, elem):
|
|
if (not hasattr(self, "m")) or (self.m is None):
|
|
return basic.message.LIMIT_DEBUG
|
|
elif elem.find("tool") > 1 and hasattr(self, "par"):
|
|
if not hasattr(self.par, "tool") or getattr(self.par, "tool").find(elem) <= 0:
|
|
return 4
|
|
else:
|
|
return 4
|
|
else:
|
|
# TODO quickfix
|
|
return 4
|
|
def getInfoLevel(self, elem):
|
|
return self.getMessageLevel("info", elem)
|
|
def getDebugLevel(self, elem):
|
|
return self.getMessageLevel("debug", elem)
|
|
def getTraceLevel(self, elem):
|
|
return self.getMessageLevel("trace", elem)
|
|
def debug(self, prio, text):
|
|
if hasattr(self, "m"):
|
|
self.m.debug(prio, text)
|
|
|
|
def getConfiguration(job, path):
|
|
conf = {}
|
|
conf["configpath"] = []
|
|
if hasattr(job, "conf"):
|
|
conf = job.conf
|
|
conf["configpath"].append(path)
|
|
doc = tools.file_tool.readFileDict(job, path, None)
|
|
if "basic" in doc:
|
|
for k, v in doc["basic"].items():
|
|
if k not in conf:
|
|
conf[k] = v
|
|
else:
|
|
for k, v in doc.items():
|
|
if k not in conf:
|
|
conf[k] = v
|
|
return conf
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------------------------------
|
|
class Parameter:
|
|
"""
|
|
parameter with source either the list args or the args of the main-program
|
|
content of parameter:
|
|
+ single arguments
|
|
+ dirname of work-folder either from dir-parameter or composed from single parameter
|
|
comparison with a parameter-file from the work-folder
|
|
"""
|
|
def __init__ (self, job, args=None):
|
|
self.program = job.program
|
|
if args is not None and isinstance(args, dict):
|
|
self.setParameterArgs(job, args)
|
|
else:
|
|
self.setParameter(job)
|
|
self.basedir = job.programDef[B.PAR_BASEDIR]
|
|
job.basedir = job.programDef[B.PAR_BASEDIR]
|
|
job.par = self
|
|
# Parameter aus Verzeichnis extraieren bzw. Verzeichnis aus Parameter erstellen
|
|
if not hasattr(self, job.programDef[B.PAR_DIRNAME]):
|
|
setattr(self, job.programDef[B.PAR_DIRNAME],
|
|
tools.value_tool.compose_pattern(job, self.basedir, None))
|
|
else:
|
|
tools.path_tool.extractPath(job, job.programDef[B.PAR_BASEDIR],
|
|
getattr(self, job.programDef[B.PAR_DIRNAME]))
|
|
# Abgleich mit zuvor gespeicherten Parametern
|
|
if len(job.programDef["pfilesource"]) > 2:
|
|
self.setParameterLoaded(job)
|
|
|
|
|
|
def checkParameter(self, job):
|
|
print (f"Parameter initialisiert {self.program}")
|
|
pardef = job.programDef[CTLG_PARDEF]
|
|
for p in pardef["par"]:
|
|
if len(p) > 1 and not hasattr(self, p):
|
|
job.m.setFatal("Parameter " + p + " is not set!")
|
|
|
|
def setParameter(self, job):
|
|
"""
|
|
1. Programm -- implementiert in Main-Klasse
|
|
2. anwndung -- steuert zu pruefende System [ in basis_Config ]
|
|
3. amgebung -- steuert zu pruefende Maschine [ in dir/applicationen ]
|
|
4. release -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ]
|
|
5. ~Verz -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch
|
|
6. zyklus -- optional unterscheidet echte und entwicklungsLaeufe
|
|
7. Programmspezifische Parameter
|
|
8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2)
|
|
10. modus -- steuert die Verarbeitung; default echt
|
|
- echt-auto Lauf aus Automatisierung (1-7)
|
|
- test Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen
|
|
- echt-spez Wiederholung einer spezifischen Funktion (1-13)
|
|
- unit Ausfuehrung der Unittests
|
|
11. componente -- schraenkt Verarbeitung auf parametriserte componenten ein
|
|
12. funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein
|
|
13. tool -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein
|
|
"""
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('-a', '--'+B.SUBJECT_APPS, required=True, action='store')
|
|
parser.add_argument('-e', '--'+B.PAR_ENV, required=True, action='store')
|
|
parser.add_argument('-v', '--'+B.PAR_VAR, required=False, action='store')
|
|
parser.add_argument('-r', '--release', action='store')
|
|
parser.add_argument('-ts', '--'+B.PAR_TSDIR, action='store')
|
|
parser.add_argument('-tc', '--'+B.PAR_TCDIR, action='store')
|
|
parser.add_argument('-ws', '--'+B.PAR_WSDIR, action='store')
|
|
parser.add_argument('-rs', '--rsdir', action='store')
|
|
parser.add_argument('-dt', '--tdtyp', action='store') # PAR_TDTYP
|
|
parser.add_argument('-ds', '--tdsrc', action='store') # PAR_TDSRC
|
|
parser.add_argument('-dn', '--tdname', action='store') # PAR_TDNAME
|
|
parser.add_argument('-l', '--loglevel', action='store') # PAR_LOG
|
|
parser.add_argument('-m', '--modus', action='store') # PAR_MODUS
|
|
parser.add_argument('-c', '--component', action='store') # PAR_COMP
|
|
parser.add_argument('-f', '--function', action='store') # PAR_FCT
|
|
parser.add_argument('-t', '--tool', action='store') # PAR_TOOL
|
|
parser.add_argument('-s', '--'+B.PAR_STEP, action='store') # PAR_STEP
|
|
# parser.add_argument('-t', '--typ', default='einzel', action='store')
|
|
# parser.add_argument('-d', '--dir', action='store')
|
|
args = parser.parse_args()
|
|
self.setParameterArgs(job, args)
|
|
|
|
|
|
def getDirParameter(self):
|
|
if hasattr(self, "tcdir"):
|
|
return ("tcbase", str(self.tcdir))
|
|
if hasattr(self, "tsdir"):
|
|
return ("tsbase", str(self.tsdir))
|
|
return None
|
|
|
|
|
|
def setParameterArgs(self, job, args):
|
|
self.parstring = "python " + self.program
|
|
if "dict" in str(type(args)):
|
|
for k in args:
|
|
self.setJobAttr(k, args[k])
|
|
else:
|
|
for k in vars(args):
|
|
if getattr(args, k) is not None:
|
|
self.setJobAttr(k , getattr(args, k))
|
|
dirpath = self.getDirParameter()
|
|
if hasattr(self, "application") and B.SUBJECT_APPS in job.conf \
|
|
and getattr(self, "application") in job.conf[B.SUBJECT_APPS]:
|
|
if B.ATTR_APPS_PROJECT in job.conf[B.SUBJECT_APPS][self.application]:
|
|
setattr(self, B.ATTR_APPS_PROJECT, job.conf[B.SUBJECT_APPS][self.application][B.ATTR_APPS_PROJECT])
|
|
proj = getattr(self, B.ATTR_APPS_PROJECT)
|
|
|
|
|
|
def setParameterLoaded(self, job):
|
|
#job = Job.getInstance()
|
|
readedPar = job.loadParameter()
|
|
if readedPar is not None:
|
|
for k in readedPar["par"].keys():
|
|
if not hasattr(self, k):
|
|
self.setJobAttr(k, readedPar["par"][k])
|
|
|
|
def setJobAttr(self, key, val):
|
|
setattr(self, key, val)
|
|
self.parstring = self.parstring + " --" + key + " " + str(val)
|
|
|
|
def getJobPar(self, key):
|
|
a = key.split(":")
|
|
if len(a) == 1:
|
|
return self[a[0]]
|
|
if len(a) == 2:
|
|
return self[a[0]][a[1]]
|
|
return
|
|
|