datestex/basic/program.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
# Template Batchrahmen
#
#import sys, getopt
import argparse
import copy
import yaml
import os
from datetime import datetime
import basic.constants as B
import basic.message
import basic.message
import basic.componentHandling
import utils.date_tool
import utils.path_tool
import utils.file_tool
import utils.config_tool
import test.constants as T


jobdef = {
	"webflask": {
		"pardef": "",
		"pfilesource": "",
		"pfiletarget": "",
		"basedir": "workbase",
		"dirname": "workdir",
		"logdir": "{job.par.debugs}/webflask/log_{time}.txt" },
	"unit": {
		"pardef": "",
		"pfilesource": "",
		"pfiletarget": "envparfile",
		"basedir": "envbase",
		"dirname": "envdir",
		"logdir": "{job.par.envdir}/{log}/log_{time}.txt" },
	"check_environment": {
		"pardef": "",
		"pfilesource": "",
		"pfiletarget": "envparfile",
		"basedir": "envbase",
		"dirname": "envdir",
		"logdir": "{job.par.envdir}/{log}/log_{time}.txt" },
	"test_executer": {
		"pardef": "",
		"pfilesource": "tsparfile",
		"pfiletarget": "tsparfile",
		"basedir": "tsbase",
		"dirname": "tsdir",
		"logdir": "{job.par.tsdir}/{log}/log_{time}.txt" },
	"init_testsuite": {
		"pardef": "tsdir",   # ,tdtyp,tdsrc,tdname",
		"pfilesource": "envparfile",
		"pfiletarget": "tsparfile",
		"basedir": "tsbase",
		"dirname": "tsdir",
		"logdir": "{job.par.tsdir}/{log}/log_{tstime}.txt" },
	"init_testcase": {
		"pardef": "tcdir",   # ",tdtyp,tdsrc,tdname",
		"pfilesource": "envparfile",
		"pfiletarget": "tcparfile",
		"basedir": "tcbase",
		"dirname": "tcdir",
		"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
	"execute_testcase": {
		"pardef": "tcdir",   # ",tdtyp,tdsrc,tdname",
		"pfilesource": "tcparfile",
		"pfiletarget": "tcparfile",
		"basedir": "tcbase",
		"dirname": "tcdir",
		"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
	"collect_testcase": {
		"pardef": "tcdir",   # ",tdtyp,tdsrc,tdname",
		"pfilesource": "tcparfile",
		"pfiletarget": "tcparfile",
		"basedir": "tcbase",
		"dirname": "tcdir",
		"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
	"compare_testcase": {
		"pardef": "tcdir",   # ",tdtyp,tdsrc,tdname",
		"pfilesource": "tcparfile",
		"pfiletarget": "tcparfile",
		"basedir": "tcbase",
		"dirname": "tcdir",
		"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" },
	"test_system": {
		"pardef": "tcdir,tdtyp,tdsrc,tdname",
		"pfilesource": "tsparfile",
		"pfiletarget": "tsparfile",
		"basedir": "tcbase",
		"dirname": "tcdir",
		"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"},
	"finish_testcase": {
		"pardef": "tcdir",
		"pfilesource": "tcparfile",
		"pfiletarget": "tcparfile",
		"basedir": "tcbase",
		"dirname": "tcdir",
		"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"},
	"finish_testsuite": {
		"pardef": "tsdir",
		"pfilesource": "tsparfile",
		"pfiletarget": "tssarfile",
		"basedir": "tsbase",
		"dirname": "tsdir",
		"logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"}
}


def setGlobal():
	pass

EXCP_CANT_POP = "cant pop this job from the instances"

DEFAULT_ARCHIV_DIR = T.DATA_PATH + "/lauf"
DEFAULT_GRAN = "ws"
DEFAULT_PRG = "webflask"
DEFAULT_GRAN = "ws"
DEFAULT_APP = "WEBFLASK"
DEFAULT_ENV = "Workspace"
DEFAULT_MODE = "test"
DEFAULT_TIME = "2022-08-29_17-29-59"

def createJob(pprg="", pgran="", papp="", penv="", ptstamp="", pmode=""):
	"""
	this creates a Job-Object with the main arguments.
	:param pprg:  program-name
	:param pgran: tc|ts|
	:param papp:  application
	:param penv:  environment
	:param ptstamp: timestamp - part of specific testfolder
	:param pmode:   if it is a productive or development execution
	:return:
	"""
	if len(pprg) < 1:
		prgname = DEFAULT_PRG
	else:
		prgname = pprg
	if len(pgran) < 1:
		gran = DEFAULT_GRAN
	else:
		gran = pgran
	if len(pgran) < 1:
		gran = DEFAULT_GRAN
	else:
		gran = pgran
	if len(papp) < 1:
		app = DEFAULT_APP
	else:
		app = papp
	if len(penv) < 1:
		env = DEFAULT_ENV
	else:
		env = penv
	if len(ptstamp) < 1:
		tstamp = DEFAULT_TIME
	else:
		tstamp = ptstamp
	if len(pmode) < 1:
		mode = DEFAULT_MODE
	else:
		mode = pmode
	if gran == "tc":
		path = DEFAULT_ARCHIV_DIR + "/TC0001/" + tstamp
	elif gran == "ts":
		path = DEFAULT_ARCHIV_DIR + "/testlauf/TST001_" + tstamp
	else:
		path = T.DATA_PATH + "/workspace/"

	job = basic.program.Job(prgname)
	# job.conf.confs[B.SUBJECT_PATH]["components"] = T.COMP_PATH
	args = {"application": app, "environment": env, "modus": mode, gran + "time": tstamp,
			gran + "dir": path,
			"step": 1}
	print(str(args))
	#         "usecase": "TST001", "tstime": "2022-03-17_17-28"}
	job.par.setParameterArgs(job, args)
	return job


class Job:
	__instance = None
	__instances = []
	__jobid = 100000

	def __init__ (self, program):
		print ("################# init Job ## " + program + " #################")
		Job.__jobid += 1
		self.jobid = str(Job.__jobid)
		self.program = program
		Job.__instance = self
		if Job.__instances is None:
			Job.__instances = []
		#Job.pushInstance(self)
		par = Parameter(self, program)
		self.par = par
		print("prog-42 " + str(self.par.basedir))
		conf = Configuration(self, program)
		self.conf = conf
		appl = utils.config_tool.getConfig(self, "basic", B.SUBJECT_APPS)
		print(appl)
		if appl is not None:
			self.conf.confs[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS]
		print("prog-45 " + str(self.par.basedir))
		dirpath = self.par.getDirParameter()
		setGlobal()
		if dirpath is not None:
			utils.path_tool.extractPath(dirpath[0], dirpath[1])
		if program == "unit": # no job will be started
			self.start = datetime.now()
			logTime = self.start.strftime("%Y%m%d_%H%M%S")
			self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None)
		print("prog-50 " + str(self.par.basedir))


	def setProgram(self, program):
		self.program = program
		basedir = jobdef[program]["basedir"]
		self.basedir = basedir
		if (self.par is not None):
			setattr(self.par, "program", program)
			setattr(self.par, "basedir", basedir)
			parstring = getattr(self.par, "parstring")
			parstring = parstring[parstring.find("--"):]
			parstring = "python "+program+" "+parstring
			setattr(self.par, "parstring", parstring)
		if not hasattr(self.par, jobdef[program]["dirname"]):
			setattr(self.par, jobdef[program]["dirname"],
					utils.path_tool.composePattern(self, "{"+basedir+"}", None))
		self.par.setParameterLoaded(self)


	def startJob(self):
		self.start = datetime.now()
		print("prog-68 " + str(self.par.basedir))
		logTime = self.start.strftime("%Y%m%d_%H%M%S")
		self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None)
		print("prog-68 " + str(self.m.rc))
		self.par.setParameterLoaded(self)
		self.m.logInfo("# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
		self.m.debug(basic.message.LIMIT_INFO, "# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
		self.par.checkParameter(self)


	def stopJob(self, reboot=0):
		self.ende = datetime.now()
		self.dumpParameter()
		print("stopJob " + str(self.m.messages) + ", " + str(self.m.debugfile))
		self.m.logInfo("# # " + self.m.topmessage + " # # # ")
		self.m.logInfo("# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # ")
		self.m.debug(basic.message.LIMIT_INFO, "# # " + self.m.topmessage + " # # # ")
		self.m.debug(basic.message.LIMIT_INFO, "# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # RC: " + str(self.m.getFinalRc()))
		self.m.closeMessage()
		rc = self.m.getFinalRc()
		print ("rc " + str(rc))
		if reboot == 0:
			exit(rc)


	def dumpParameter(self):
		parpath = utils.path_tool.composePath(self, jobdef[self.program]["pfiletarget"], None)
		output = {}
		cconf = basic.componentHandling.getComponentDict()
		output["par"] = self.par.__dict__
		if len(cconf) < 1:
			utils.file_tool.writeFileDict(self.m, self, parpath, output)
			return
		output[B.SUBJECT_COMPS] = {}
		for c in cconf:
			output[B.SUBJECT_COMPS][c] = {}
			for x in ["function", "conn"]:
				if x not in cconf[c]:
					continue
				output[B.SUBJECT_COMPS][c][x] = cconf[c][x]
				if x == B.SUBJECT_CONN and "passwd" in cconf[c][x]:
					cconf[B.SUBJECT_COMPS][c][x]["passwd"] = "xxxxx"
		utils.file_tool.writeFileDict(self.m, self, parpath, output)


	def loadParameter(self):
		output = {}
		if len(str(jobdef[self.program]["pfilesource"])) < 2:
			return None
		parpath = utils.path_tool.composePath(self, jobdef[self.program]["pfilesource"], None)
		if not os.path.join(parpath):
			return None
		doc = utils.file_tool.readFileDict(self, parpath, self.m)
		for k in doc.keys():
			output[k] = copy.deepcopy(doc[k])
		return output


	def getParameter(self, parameter):
		if hasattr(self.par, parameter):
			return getattr(self.par, parameter)
		elif "xxxtime" in parameter:
			neu = utils.date_tool.getActdate(utils.date_tool.F_DIR)
			# setattr(self.par, parameter, neu)
			return neu


	def hasElement(self, parameter, elem):
		"""
		the function searches in an optional job.parameter
		(a) true, if the parameter does not exist (DEFAULT)
		(b) true, if the element is member of the parameter
		(c) false, if parameter exists and elem is not member of the parameter
		:param parameter:
		:param elem:
		:return:
		"""
		if hasattr(self.par, parameter):
			print (parameter + " in Parameter")
			if getattr(self.par, parameter).find(elem) >= 0:
				return True
			return False
		return True
	def hascomponente(self, comp):
		"""
		it searches if comp is member of the optional job.parameter (default = each is member)
		:param comp:
		:return:
		"""
		return self.hasElement("componente", comp)
	def hasFunction(self, fct):
		"""
		it searches if fct is member of the optional job.parameter (default = each is member)
		:param fct:
		:return:
		"""
		return self.hasElement("function", fct)
	def hasTool(self, tool):
		"""
		it searches if tool is member of the optional job.parameter (default = each is member)
		:param tool:
		:return:
		"""
		return self.hasElement("tool", tool)
	def getMessageLevel(self, errtyp, elem):
		if (not hasattr(self, "m")) or (self.m is None):
			return basic.message.LIMIT_DEBUG
		elif elem.find("tool") > 1:
			if not hasattr(self.par, "tool") or getattr(self.par, "tool").find(elem) <= 0:
				return int(self.m.CONST_ERRTYP[errtyp]) -1
			else:
				return int(self.m.CONST_ERRTYP[errtyp])
		else:
			return int(self.m.CONST_ERRTYP[errtyp])
	def getInfoLevel(self, elem):
		return self.getMessageLevel("info", elem)
	def getDebugLevel(self, elem):
		return self.getMessageLevel("debug", elem)
	def getTraceLevel(self, elem):
		return self.getMessageLevel("trace", elem)
	def debug(self, prio, text):
		#print("job.debug "+str(prio)+" "+text)
		if hasattr(self, "m"):
			self.m.debug(prio, text)
		else:
			print(text)

# ------------------------------------------------------------------------------------------------------------------
class Parameter:
	print ("class Parameter")
	def __init__ (self, job, program):
		print ("# init Parameter for " + program)
		self.program = program
		self.basedir = "debugs"
		self.setBasedir(program)
		print (f"# Parameter initialisiert {self.program} mit basedir {self.basedir}")
		if (program not in ["unit", "webflask"] ): self.setParameter(job)


	def setBasedir(self, program):
		if jobdef[program]:
			self.basedir = jobdef[program]["basedir"]
			if hasattr(self, jobdef[program]["dirname"]):
				utils.path_tool.extractPath(self.basedir, getattr(self, jobdef[program]["dirname"]))
			elif self.basedir == "workbase":
				home = utils.path_tool.getHome()
				dirpath = os.path.join(home, "data", "workspace")
				setattr(self, jobdef[program]["dirname"], dirpath)
			elif program != "unit":
				dirpath = utils.path_tool.composePattern(None, "{"+jobdef[program]["basedir"]+"}", None)
				setattr(self, jobdef[program]["dirname"], dirpath)
		else:
			self.basedir = "debugs"


	def checkParameter(self, job):
		print (f"Parameter initialisiert {self.program}")
		pardef = jobdef[job.program]["pardef"]
		for p in pardef.split(","):
			print(p)
			if len(p) > 1 and not hasattr(self, p):
				job.m.setFatal("Parameter " + p + " is not set!")

	def setParameter(self, job):
		"""
		1. Programm -- implementiert in Main-Klasse
		2. anwndung -- steuert zu pruefende System [ in basis_Config ]
		3. amgebung -- steuert zu pruefende Maschine [ in dir/applicationen ]
		4. release  -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ]
		5. ~Verz    -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch
		6. zyklus   -- optional unterscheidet echte und entwicklungsLaeufe
		7. Programmspezifische Parameter
		8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2)
		10. modus -- steuert die Verarbeitung; default echt
		- echt-auto  Lauf aus Automatisierung (1-7)
		- test       Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen
		- echt-spez  Wiederholung einer spezifischen Funktion (1-13)
		- unit       Ausfuehrung der Unittests
		11. componente -- schraenkt Verarbeitung auf parametriserte componenten ein
		12. funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein
		13. tool     -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein
		"""
		# args = str(sys.argv)
		# print ("Job-Programm %s :	" % args)
		parser = argparse.ArgumentParser()
		parser.add_argument('-a', '--'+B.SUBJECT_APPS, required=True, action='store')
		parser.add_argument('-e', '--'+B.PAR_ENV, required=True, action='store')
		parser.add_argument('-r', '--release', action='store')
		parser.add_argument('-ts', '--'+B.PAR_TSDIR, action='store')
		parser.add_argument('-tc', '--'+B.PAR_TCDIR, action='store')
		parser.add_argument('-ws', '--'+B.PAR_WSDIR, action='store')
		parser.add_argument('-rs', '--rsdir', action='store')
		parser.add_argument('-dt', '--tdtyp', action='store')    # PAR_TDTYP
		parser.add_argument('-ds', '--tdsrc', action='store')    # PAR_TDSRC
		parser.add_argument('-dn', '--tdname', action='store')   # PAR_TDNAME
		parser.add_argument('-l', '--loglevel', action='store')  # PAR_LOG
		parser.add_argument('-m', '--modus', action='store')     # PAR_MODUS
		parser.add_argument('-c', '--component', action='store') # PAR_COMP
		parser.add_argument('-f', '--function', action='store')  # PAR_FCT
		parser.add_argument('-t', '--tool', action='store')      # PAR_TOOL
		parser.add_argument('-s', '--'+B.PAR_STEP, action='store')      # PAR_STEP
		# parser.add_argument('-t', '--typ', default='einzel', action='store')
		# parser.add_argument('-d', '--dir', action='store')
		args = parser.parse_args()
		self.setParameterArgs(job, args)


	def getDirParameter(self):
		if hasattr(self, "tcdir"):
			return ("tcbase", str(self.tcdir))
		if hasattr(self, "tsdir"):
			return ("tsbase", str(self.tsdir))
		return None


	def setParameterArgs(self, job, args):
		print("setParArgs " + str(type(args)))
		self.parstring = "python " + self.program
		if "dict" in str(type(args)):
			for k in args:
				self.setJobAttr(k, args[k])
		else:
			for k in vars(args):
				if getattr(args, k) is not None:
					self.setJobAttr(k , getattr(args, k))
		dirpath = self.getDirParameter()
		if dirpath is not None:
			utils.path_tool.extractPath(job, dirpath[0], dirpath[1])
		app = self.application
		if self.application in job.conf.confs[B.SUBJECT_APPS]:
			if B.ATTR_APPS_PROJECT in job.conf.confs[B.SUBJECT_APPS][self.application]:
				setattr(self, B.ATTR_APPS_PROJECT, job.conf.confs[B.SUBJECT_APPS][self.application][B.ATTR_APPS_PROJECT])
				proj = getattr(self, B.ATTR_APPS_PROJECT)
		app2 = self.application


	def setParameterLoaded(self, job):
		#job = Job.getInstance()
		print("setParLoaded " )
		readedPar = job.loadParameter()
		if readedPar is not None:
			for k in readedPar["par"].keys():
				if not hasattr(self, k):
					self.setJobAttr(k, readedPar["par"][k])

	def setJobAttr(self, key, val):
		setattr(self, key, val)
		self.parstring = self.parstring + " --" + key + " " + str(val)

	def getJobPar(self, key):
		a = key.split(":")
		if len(a) == 1:
			return self[a[0]]
		if len(a) == 2:
			return self[a[0]][a[1]]
		return


# ------------------------------------------------------------------------------------------------------------------
class Configuration:
	def __init__ (self, job, program):
		self.program = program
		print (f"job initialisiert {self.program}")
		path = utils.path_tool.getBasisConfigPath()
		self.setConfiguration(job, path)
		return

	def setConfiguration(self, job, path):
		self.confs = {}
		doc = utils.file_tool.readFileDict(job, path, None)
		self.confs["configpath"] = path
		if "basic" in doc:
			for i, v in doc["basic"].items():
				self.confs[i] = v
		else:
			for i, v in doc.items():
				self.confs[i] = v

	def setConfig(self, path, val):
		a = path.split(".")
		if len(a) == 1:
			self.confs[a[0]] = val
		elif len(a) == 2:
			self.confs[a[0]][a[1]] = val
		elif len(a) == 3:
			self.confs[a[0]][a[1]][a[2]] = val


	def getPath(self, key):
		return self.confs.get(B.SUBJECT_PATH).get(key)

	def getJobConf(self, key):
		a = key.split(":")
		if len(a) == 1:
			return self.confs[a[0]]
		if len(a) == 2:
			return self.confs[a[0]][a[1]]