From 067e0c7aca3c190a054218052ffec18424dfaab5 Mon Sep 17 00:00:00 2001 From: Ulrich Date: Mon, 26 Dec 2022 16:08:35 +0100 Subject: [PATCH] program-config via catalog --- basic/component.py | 60 ++++++- basic/componentHandling.py | 14 +- basic/constants.py | 10 +- basic/message.py | 23 +-- basic/program.py | 70 ++++---- basic/toolHandling.py | 62 ++++--- catalog/programs.csv | 21 +++ objects/__init__.py | 0 objects/catalog.py | 106 +++++++++++ test/constants.py | 6 +- test/test_01date.py | 33 ++-- test/test_07catalog.py | 29 +-- test/test_10job.py | 46 +++++ test/testtools.py | 8 +- tools/__init__.py | 0 tools/config_tool.py | 349 +++++++++++++++++++++++++++++++++++++ tools/conn_tool.py | 81 +++++++++ tools/data_const.py | 116 ++++++++++++ tools/date_tool.py | 175 +++++++++++++++++++ tools/file_abstract.py | 113 ++++++++++++ tools/file_tool.py | 295 +++++++++++++++++++++++++++++++ tools/filecsv_fcts.py | 246 ++++++++++++++++++++++++++ tools/job_tool.py | 102 +++++++++++ tools/path_const.py | 115 ++++++++++++ tools/path_tool.py | 320 ++++++++++++++++++++++++++++++++++ tools/value_tool.py | 76 ++++++++ 26 files changed, 2348 insertions(+), 128 deletions(-) create mode 100644 catalog/programs.csv create mode 100644 objects/__init__.py create mode 100644 objects/catalog.py create mode 100644 test/test_10job.py create mode 100644 tools/__init__.py create mode 100644 tools/config_tool.py create mode 100644 tools/conn_tool.py create mode 100644 tools/data_const.py create mode 100644 tools/date_tool.py create mode 100644 tools/file_abstract.py create mode 100644 tools/file_tool.py create mode 100644 tools/filecsv_fcts.py create mode 100644 tools/job_tool.py create mode 100644 tools/path_const.py create mode 100644 tools/path_tool.py create mode 100644 tools/value_tool.py diff --git a/basic/component.py b/basic/component.py index 9df6a0a..ea1f559 100644 --- a/basic/component.py +++ b/basic/component.py @@ -8,12 +8,19 @@ # --------------------------------------------------------------------- from datetime import datetime -import basic.compexec +#from basic.compexec import Testexecuter import basic.message import basic.program +#import basic.entity import basic.constants as B +import tools.path_const as P import inspect +import tools.config_tool +#import tools.git_tool +import basic.toolHandling +TABLE_NAMES = ["component", "co_table", "co_variant"] +DEFAULT_SYNC = "" #basic.entity.SYNC_HEAD_GIT2DB class CompData: def __init__(self): @@ -21,13 +28,56 @@ class CompData: self.m = None self.conf = None +def syncEnitity(job, elem): + """ + synchronize the configuration with the database + :param job: + :return: + """ + syncMethod = DEFAULT_SYNC + if B.SUBJECT_ENTITY in job.conf.confs: + syncMethod = job.conf.confs["entity"][TABLE_NAMES[0]]["storage"] + if syncMethod.count("-") < 2: + return + fileTime = basic.entity.VAL_ZERO_TIME + dbTime = basic.entity.VAL_ZERO_TIME + # get git-commit + if "git" in syncMethod: + comppath = tools.config_tool.getConfigPath(job, P.KEY_COMP, elem) + repopath = comppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:] + gitresult = tools.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1) + fileTime = gitresult[0]["date"] + print(str(gitresult)) + if "db" in syncMethod: + if B.TOPIC_NODE_DB in job.conf.confs: + dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) + else: + return "No DB in job-config" + data = dbi.selectRows(TABLE_NAMES[0], job) + print(str(data[B.DATA_NODE_DATA])) + if len(data[B.DATA_NODE_DATA]) > 0: + dbTime = data[B.DATA_NODE_DATA][0]["updtime"] + + if fileTime == dbTime: + print("gleich") + elif fileTime < dbTime: + print("db vorne") + #(appObjects, appDict) = selectEntities(job, dbi) + #print(str(appDict)) + #applPath = tools.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS) + #tools.file_tool.writeFileDict(job.m, job, applPath, appDict) + # + elif fileTime > dbTime: + print("git vorne") + compData = tools.config_tool.getConfig(job, P.KEY_COMP, elem) + #insertEntities(job, compData, dbTime, dbi) #class Component(components.sysmonitor.SystemMonitor, components.testexec.Testexecuter, components.report.Report, # components.maintain.Maintainer, components.catalog, threading.Thread): -class Component(basic.compexec.Testexecuter): +class Component(): """ - A component represents an application of the system-under-test or a data-artifact which is created from the system-under-test. + A component represents an application of the system-under-test or a data-artifact which is created from the system-under-test.g As the representation it has to knowlegde of the url, which other components depends on this component. During a test-run the component must be checked, prepared, appfiles has to be collected, etc. For this doing there are some standard-methods implemented. """ @@ -171,8 +221,8 @@ class Component(basic.compexec.Testexecuter): testreport = "" # if job.par.context == "tset": # for tc in testcases: - # header = utils.report_tool.getTcHeader() - # body = utils.report_tool.getTcExtraction() + # header = tools.report_tool.getTcHeader() + # body = tools.report_tool.getTcExtraction() # if job.par.context == "tcontext": # for ts in testsets: reportheader = reportheader +'<\head>' diff --git a/basic/componentHandling.py b/basic/componentHandling.py index c5caaa1..ef7bdd7 100644 --- a/basic/componentHandling.py +++ b/basic/componentHandling.py @@ -15,15 +15,15 @@ Each componente could be created mostly once, but not everytime: Each kind of instance has its component-class and for each use should be an object be created. Each crated component-onject are documented in the parameter-file. """ -import utils.config_tool -import utils.conn_tool +import tools.config_tool +import tools.conn_tool import basic.program import basic.message import basic.component import importlib import copy import basic.constants as B -import utils.data_const as D +import tools.data_const as D comps = {} PARAM_NOSUBNODE = [B.SUBJECT_ARTS, "components", "instance"] @@ -141,8 +141,8 @@ class ComponentManager: verify = job.getDebugLevel("job_tool") componentName = componentName.lower() job.debug(verify, "createComponent " + componentName) - confs = utils.config_tool.getConfig(job, "comp", componentName) - conns = utils.conn_tool.getConnections(job, componentName) + confs = tools.config_tool.getConfig(job, "comp", componentName) + conns = tools.conn_tool.getConnections(job, componentName) instAttr = getInstanceAttributes(confs) job.debug(verify, "createComponent -91- " + componentName + " : " + str(confs)) if nr > 0 and int(instAttr[B.ATTR_INST_CNT]) > 1: @@ -190,7 +190,7 @@ class ComponentManager: c.classname = compName c.m = basic.message.Message(job, basic.message.LIMIT_DEBUG, "logTime", name) - c.conf = utils.config_tool.mergeConn(c.m, confs["conf"], conns[i]) + c.conf = tools.config_tool.mergeConn(c.m, confs["conf"], conns[i]) c.conf[B.SUBJECT_CONN] = conns[i] c.init(job) if parContent is not None: @@ -204,7 +204,7 @@ class ComponentManager: for table in c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]: if table in B.LIST_DB_ATTR: continue - conf = utils.config_tool.getConfig(job, D.DDL_FILENAME, compName, table) + conf = tools.config_tool.getConfig(job, D.DDL_FILENAME, compName, table) if B.DATA_NODE_TABLES in conf and table in conf[B.DATA_NODE_TABLES]: c.conf[B.DATA_NODE_DDL][table] = conf[B.DATA_NODE_TABLES][table] elif table in conf: diff --git a/basic/constants.py b/basic/constants.py index 9931c88..ab9b56e 100644 --- a/basic/constants.py +++ b/basic/constants.py @@ -7,9 +7,9 @@ """ The constants desribes the keywords of the main datastructures, these are * the configurations of - * program-configuration for general technical variables in tools - stored in internal conf-folder + * program-configuration for general technical variables in xtools - stored in internal conf-folder it represents the general knowledge of this test-automatism - * basic-configuration for global variables in job - stored in external conf-folder + * xbasic-configuration for global variables in job - stored in external conf-folder it represents the knowledge of your test-topic * comp-configuration for component-attributes - stored in internal component-folder in + ddl-configuration of an entity of the component as attributes - stored in internal component-folder @@ -114,7 +114,7 @@ ATTR_ARTS_TYPE = "type" ATTR_ARTS_NAME = "name" """ optional attribute just for information """ ATTR_ARTS_PATH = "path" -""" optional attribute for the basic folder if the artifact is stored in the filesystem """ +""" optional attribute for the xbasic folder if the artifact is stored in the filesystem """ ATTR_ARTS_RESET = "reset" """ optional attribute if the artefact should be reset in the initializing-phase """ ATTR_ARTS_PRESTEP = "prestep" @@ -172,12 +172,12 @@ ATTR_CONN_PASSWD = "password" LIST_CONN_ATTR = [ATTR_CONN_HOST, ATTR_CONN_IP, ATTR_CONN_PORT, ATTR_CONN_DOMPATH, ATTR_CONN_USER, ATTR_CONN_PASSWD] # the configuration of a component or tool -# entity { : variable name of the group, basic, component-name or tool-name +# entity { : variable name of the group, xbasic, component-name or tool-name # + subject { : variable subject-name - it correspondends to a tool # + + sub-subject { : variable subject-name - it correspondends to a tool # + attributes : constant of the tool in which the attribute ist implemented -# the main subjects # prog basic envir tool comp testcase main implentation module +# the main subjects # prog xbasic envir tool comp testcase main implentation module SUBJECT_PATH = "paths" # | x | | x | | path_tool, config_tool """ This constant defines the subject in order to define paths of filesystem of any testuse """ ATTR_PATH_MODE = "mode" diff --git a/basic/message.py b/basic/message.py index 514aa9f..b8f631b 100644 --- a/basic/message.py +++ b/basic/message.py @@ -20,9 +20,9 @@ import basic.program import os import math from datetime import datetime -import utils.path_tool -import utils.i18n_tool -import basic.text_const as T +import tools.path_tool +#import tools.i18n_tool +#import basic.text_const as T import basic.constants as B LIMIT_FATAL = 0 @@ -72,7 +72,7 @@ class Message: self.debugfile = job.m.debugfile self.debug(verify, "> > > debugfile uebernommen zu " + str(componente)) else: - debugpath = job.conf.confs["paths"]["debugs"] + "/debug_" + logTime[0:-4] + "00.txt" + debugpath = job.conf["paths"]["debugs"] + "/debug_" + logTime[0:-4] + "00.txt" print ("debugpathx "+debugpath) if os.path.exists(debugpath): self.debugfile = open(debugpath, "a") @@ -80,7 +80,7 @@ class Message: self.debugfile = open(debugpath, "w") self.debug(verify, "> > > debugfile geoeffnet zu " + job.program + " mit " + debugpath) # init logfile - except for components or unittest - self.logDebug("logfile " + str(componente) + ", " + str(job.par.basedir)) + #self.logDebug("logfile " + str(componente) + ", " + str(job.par.basedir)) if componente is not None: # self.logfile = self.debugfile elif job.program == "unit": @@ -98,8 +98,8 @@ class Message: basedir = job.par.basedir basedir = basedir.replace("base", "log") os.makedirs(basedir, exist_ok=True) - # basedir = utils.path_tool.composePath(basedir, None) - # basedir = utils.path_tool.composePath(job, basedir, None) + # basedir = tools.path_tool.composePath(basedir, None) + # basedir = tools.path_tool.composePath(job, basedir, None) logpath = job.getLogpath() basedir = os.path.dirname(logpath) os.makedirs(basedir, exist_ok=True) @@ -205,14 +205,7 @@ class Message: self.logInfo(text) def getMessageText(self, job, text, args): - out = "" - constName = "" - for i in range(0, len(T.LIST_EXP_TEXT)): - if text == T.LIST_EXP_TEXT[i]: - constName = T.LIST_EXP_CONST[i] - txt = utils.i18n_tool.I18n.getInstance(job).getMessage(self.job, constName, args) - out = txt.format(args) - return out + return text def logFatal(self, text): self.log(LIMIT_FATAL, "FATAL: " + text) diff --git a/basic/program.py b/basic/program.py index 221aba3..b85347a 100644 --- a/basic/program.py +++ b/basic/program.py @@ -15,13 +15,13 @@ from datetime import datetime import basic.constants as B import basic.message import basic.componentHandling -import utils.date_tool -import utils.path_tool -import utils.file_tool -import utils.config_tool +import tools.date_tool +import tools.path_tool +import tools.file_tool +import tools.config_tool import test.constants as T -import utils.path_const as P -import utils.job_tool +import tools.path_const as P +import tools.job_tool LIMIT_INFO = 16 #basic.message.LIMIT_INFO LIMIT_DEBUG = 12 #basic.message.LIMIT_DEBUG @@ -254,14 +254,19 @@ class Job: self.jobid = str(Job.__jobid) self.program = program conf = Configuration(self, program) - self.conf = conf + self.conf = conf.confs try: - path = utils.config_tool.getConfigPath(self, P.KEY_BASIC, B.BASIS_FILE) + path = tools.config_tool.getConfigPath(self, P.KEY_BASIC, B.BASIS_FILE) print("comps.basispath "+path) self.conf.setConfiguration(self, path) except: pass # the special path is not necessary - appl = utils.config_tool.getConfig(self, P.KEY_BASIC, B.SUBJECT_APPS) + logTime = tools.date_tool.getActdate(tools.date_tool.F_LOG) + self.start = logTime + self.m = basic.message.Message(self, jobdef[program]["loglevel"], logTime, None) + + def altinit(self, program, args): + appl = tools.config_tool.getConfig(self, P.KEY_BASIC, B.SUBJECT_APPS) if appl is not None: self.conf.confs[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS] if B.SUBJECT_PROJECTS in self.conf.confs: @@ -273,12 +278,9 @@ class Job: self.conf.confs[B.SUBJECT_PROJECTS] = appl[B.SUBJECT_PROJECTS] par = Parameter(self, program, args) self.par = par - logTime = utils.date_tool.getActdate(utils.date_tool.F_LOG) - self.start = logTime - self.m = basic.message.Message(self, jobdef[program]["loglevel"], logTime, None) def getLogpath(self): - path = utils.path_tool.composePattern(self, jobdef[self.program]["logpath"], None) + path = tools.path_tool.composePattern(self, jobdef[self.program]["logpath"], None) return path def murks(self, par, program): @@ -286,7 +288,7 @@ class Job: dirpath = self.par.getDirParameter() setGlobal() if dirpath is not None: - utils.path_tool.extractPath(dirpath[0], dirpath[1]) + tools.path_tool.extractPath(dirpath[0], dirpath[1]) if program == "unit": # no job will be started self.start = datetime.now() logTime = self.start.strftime("%Y%m%d_%H%M%S") @@ -307,7 +309,7 @@ class Job: setattr(self.par, "parstring", parstring) if not hasattr(self.par, jobdef[program]["dirname"]): setattr(self.par, jobdef[program]["dirname"], - utils.path_tool.composePattern(self, "{"+basedir+"}", None)) + tools.path_tool.composePattern(self, "{"+basedir+"}", None)) print(parstring) self.par.setParameterLoaded(self) @@ -318,7 +320,7 @@ class Job: logTime = self.start.strftime("%Y%m%d_%H%M%S") self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None) print("prog-68 " + str(self.m.rc)) - utils.job_tool.startJobProcesses(self) + tools.job_tool.startJobProcesses(self) self.par.setParameterLoaded(self) self.m.logInfo("# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") self.m.debug(basic.message.LIMIT_INFO, "# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") @@ -327,12 +329,12 @@ class Job: def stopJob(self, reboot=0): - utils.job_tool.stopJobProcesses(self) + tools.job_tool.stopJobProcesses(self) self.ende = datetime.now() self.dumpParameter() footer1 = "# # " + self.m.topmessage + " # # # " - footer2 = "# # # Stop Job " + utils.date_tool.formatParsedDate(str(self.start), utils.date_tool.F_DE_TSTAMP) - footer2 += " # " + utils.date_tool.formatParsedDate(str(self.ende), utils.date_tool.F_DE_TSTAMP) + " # # # " + footer2 = "# # # Stop Job " + tools.date_tool.formatParsedDate(str(self.start), tools.date_tool.F_DE_TSTAMP) + footer2 += " # " + tools.date_tool.formatParsedDate(str(self.ende), tools.date_tool.F_DE_TSTAMP) + " # # # " footer2 += " # # # RC: " + str(self.m.getFinalRc()) self.m.logInfo(footer1) self.m.logInfo(footer2) @@ -349,12 +351,12 @@ class Job: def dumpParameter(self): if len(jobdef[self.program]["pfiletarget"]) < 2: return - parpath = utils.path_tool.composePath(self, jobdef[self.program]["pfiletarget"], None) + parpath = tools.path_tool.composePath(self, jobdef[self.program]["pfiletarget"], None) output = {} cconf = basic.componentHandling.getComponentDict() output["par"] = self.par.__dict__ if len(cconf) < 1: - utils.file_tool.writeFileDict(self.m, self, parpath, output) + tools.file_tool.writeFileDict(self.m, self, parpath, output) return output[B.SUBJECT_COMPS] = {} for c in cconf: @@ -365,19 +367,19 @@ class Job: output[B.SUBJECT_COMPS][c][x] = cconf[c][x] if x == B.SUBJECT_CONN and "passwd" in cconf[c][x]: cconf[B.SUBJECT_COMPS][c][x]["passwd"] = "xxxxx" - utils.file_tool.writeFileDict(self.m, self, parpath, output) + tools.file_tool.writeFileDict(self.m, self, parpath, output) def loadParameter(self): output = {} if len(str(jobdef[self.program]["pfilesource"])) < 2: return None - parpath = utils.path_tool.composePath(self, jobdef[self.program]["pfilesource"], None) + parpath = tools.path_tool.composePath(self, jobdef[self.program]["pfilesource"], None) print("parpath "+parpath) if not os.path.join(parpath): return None print("parpath "+parpath) - doc = utils.file_tool.readFileDict(self, parpath, None) + doc = tools.file_tool.readFileDict(self, parpath, None) for k in doc.keys(): output[k] = copy.deepcopy(doc[k]) return output @@ -387,7 +389,7 @@ class Job: if hasattr(self.par, parameter): return getattr(self.par, parameter) elif "xxxtime" in parameter: - neu = utils.date_tool.getActdate(utils.date_tool.F_DIR) + neu = tools.date_tool.getActdate(tools.date_tool.F_DIR) # setattr(self.par, parameter, neu) return neu else: @@ -436,7 +438,7 @@ class Job: def getMessageLevel(self, errtyp, elem): if (not hasattr(self, "m")) or (self.m is None): return basic.message.LIMIT_DEBUG - elif elem.find("tool") > 1: + elif elem.find("tool") > 1 and hasattr(self, "par"): if not hasattr(self.par, "tool") or getattr(self.par, "tool").find(elem) <= 0: return int(self.m.CONST_ERRTYP[errtyp]) -1 else: @@ -479,9 +481,9 @@ class Parameter: job.par = self if not hasattr(self, jobdef[program]["dirname"]): setattr(self, jobdef[program]["dirname"], - utils.path_tool.composePattern(job, "{"+self.basedir+"}", None)) + tools.path_tool.composePattern(job, "{"+self.basedir+"}", None)) else: - utils.path_tool.extractPath(job, jobdef[program]["basedir"], getattr(self, jobdef[program]["dirname"])) + tools.path_tool.extractPath(job, jobdef[program]["basedir"], getattr(self, jobdef[program]["dirname"])) self.setParameterLoaded(job) @@ -489,14 +491,14 @@ class Parameter: if jobdef[program]: self.basedir = jobdef[program]["basedir"] if hasattr(self, jobdef[program]["dirname"]): - utils.path_tool.extractPath(self.basedir, getattr(self, jobdef[program]["dirname"])) + tools.path_tool.extractPath(self.basedir, getattr(self, jobdef[program]["dirname"])) elif self.basedir == "workbase": - home = utils.path_tool.getHome() + home = tools.path_tool.getHome() dirpath = os.path.join(home, "data", "workspace") setattr(self, jobdef[program]["dirname"], dirpath) elif program != "unit": # compose after setargs - dirpath = utils.path_tool.composePattern(job, "{"+jobdef[program]["basedir"]+"}", None) + dirpath = tools.path_tool.composePattern(job, "{"+jobdef[program]["basedir"]+"}", None) setattr(self, jobdef[program]["dirname"], dirpath) else: self.basedir = "debugs" @@ -575,7 +577,7 @@ class Parameter: self.setJobAttr(k , getattr(args, k)) dirpath = self.getDirParameter() #if dirpath is not None: - # utils.path_tool.extractPath(job, dirpath[0], dirpath[1]) + # tools.path_tool.extractPath(job, dirpath[0], dirpath[1]) if hasattr(self, "application") and self.application in job.conf.confs[B.SUBJECT_APPS]: if B.ATTR_APPS_PROJECT in job.conf.confs[B.SUBJECT_APPS][self.application]: setattr(self, B.ATTR_APPS_PROJECT, job.conf.confs[B.SUBJECT_APPS][self.application][B.ATTR_APPS_PROJECT]) @@ -609,7 +611,7 @@ class Parameter: class Configuration: def __init__ (self, job, program): self.program = program - path = utils.path_tool.getBasisConfigPath() + path = tools.path_tool.getBasisConfigPath() print ("conf initialisieren "+self.program+" > "+path) self.setConfiguration(job, path) return @@ -622,7 +624,7 @@ class Configuration: if not hasattr(self, "confs"): self.confs = {} self.confs["configpath"] = [] - doc = utils.file_tool.readFileDict(job, path, None) + doc = tools.file_tool.readFileDict(job, path, None) self.confs["configpath"].append(path) if "basic" in doc: for k, v in doc["basic"].items(): diff --git a/basic/toolHandling.py b/basic/toolHandling.py index e830e80..86268fb 100644 --- a/basic/toolHandling.py +++ b/basic/toolHandling.py @@ -9,7 +9,7 @@ import os import basic.program import basic.constants as B # ------------------------------------------------- -import utils.config_tool +import tools.config_tool def hasAttr(o, name): @@ -17,36 +17,40 @@ def hasAttr(o, name): if (name in o.keys()): return True elif (isinstance(o, list)): - print("hasAttr list "+str(type(o))) + print("hasAttr list " + str(type(o))) elif hasattr(o, name): return True return False + def getAttr(o, name): if (isinstance(o, dict)): if (name in o.keys()): return o[name] elif (isinstance(o, list)): - print("getAttr c list "+str(type(o))) + print("getAttr c list " + str(type(o))) elif hasattr(o, name): return o.get(name) + """ Toolmanager """ + def getCompAttr(comp, topic, attr, table=""): out = "" - print(topic+" "+attr+" "+str(comp)) + print(topic + " " + attr + " " + str(comp)) if hasAttr(comp.conf[B.SUBJECT_CONN], topic) and hasAttr(comp.conf[B.SUBJECT_CONN][topic], attr): return getAttr(comp.conf[B.SUBJECT_CONN][topic], attr) if len(table) > 1 and hasAttr(comp.conf[B.SUBJECT_ARTS][topic], table) \ and hasAttr(comp.conf[B.SUBJECT_ARTS][topic][table], attr): return getAttr(comp.conf[B.SUBJECT_ARTS][topic][table], attr) if hasAttr(comp.conf[B.SUBJECT_ARTS], topic) and hasAttr(comp.conf[B.SUBJECT_ARTS][topic], attr): - print("attr "+attr+" vorhanden") + print("attr " + attr + " vorhanden") return getAttr(comp.conf[B.SUBJECT_ARTS][topic], attr) - raise LookupError(topic+"."+attr+" is not set in comp " + comp.name) + raise LookupError(topic + "." + attr + " is not set in comp " + comp.name) + def getTool(technicType, comp, job): if technicType == B.TOPIC_NODE_DB: @@ -60,64 +64,68 @@ def getTool(technicType, comp, job): # denn zu einer Komponente koennen unterschiedliche Dateien vorkommen return getFileTool(job, comp, "") + # class ToolManager: def getDbTool(job, comp, dbtype=""): verify = int(job.getDebugLevel("db_tool")) if len(dbtype) < 3: dbtype = getCompAttr(comp, B.TOPIC_NODE_DB, B.ATTR_TYPE, "") - toolname = "db"+dbtype+"_tool" - filepath = os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_PROGRAM], "utils", toolname+".py") - #comp.m.debug(verify, "toolname "+filepath) + toolname = "db" + dbtype + "_tool" + filepath = os.path.join(job.conf[B.SUBJECT_PATH][B.ATTR_PATH_PROGRAM], "tools", toolname + ".py") + # comp.m.debug(verify, "toolname "+filepath) if not os.path.exists(filepath): - raise FileNotFoundError("file for tool "+toolname+" does not exist "+filepath) - cmodul = importlib.import_module("utils."+toolname) + raise FileNotFoundError("file for tool " + toolname + " does not exist " + filepath) + cmodul = importlib.import_module("tools." + toolname) class_ = getattr(cmodul, "DbFcts") c = class_() c.setComp(job, comp) return c + def getCliTool(job, comp): verify = int(job.getDebugLevel("db_tool")) clitype = getCompAttr(comp, B.TOPIC_NODE_CLI, B.ATTR_TYPE, "") - toolname = "cli"+clitype+"_tool" - filepath = os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_PROGRAM], "utils", toolname+".py") - #comp.m.debug(verify, "toolname "+filepath) + toolname = "cli" + clitype + "_tool" + filepath = os.path.join(job.conf[B.SUBJECT_PATH][B.ATTR_PATH_PROGRAM], "tools", toolname + ".py") + # comp.m.debug(verify, "toolname "+filepath) if not os.path.exists(filepath): - raise FileNotFoundError("file for tool "+toolname+" does not exist "+filepath) - cmodul = importlib.import_module("utils."+toolname) + raise FileNotFoundError("file for tool " + toolname + " does not exist " + filepath) + cmodul = importlib.import_module("tools." + toolname) class_ = getattr(cmodul, "CliFcts") c = class_() c.setComp(job, comp) return c + def getApiTool(job, comp): verify = int(job.getDebugLevel("db_tool")) apitype = getCompAttr(comp, B.TOPIC_NODE_API, B.ATTR_TYPE, "") - toolname = "api"+apitype+"_tool" - filepath = os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_PROGRAM], "utils", toolname+".py") - #comp.m.debug(verify, "toolname "+filepath) + toolname = "api" + apitype + "_tool" + filepath = os.path.join(job.conf[B.SUBJECT_PATH][B.ATTR_PATH_PROGRAM], "tools", toolname + ".py") + # comp.m.debug(verify, "toolname "+filepath) if not os.path.exists(filepath): - raise FileNotFoundError("file for tool "+toolname+" does not exist "+filepath) - cmodul = importlib.import_module("utils."+toolname) + raise FileNotFoundError("file for tool " + toolname + " does not exist " + filepath) + cmodul = importlib.import_module("tools." + toolname) class_ = getattr(cmodul, "ApiFcts") c = class_() c.setComp(job, comp) return c + def getFileTool(job, comp, filenode=""): verify = int(job.getDebugLevel("file_tool")) if len(filenode) > 3 and "." in filenode and filenode[-1:] != ".": - filetype = utils.config_tool.getAttribute(comp, filenode, B.ATTR_ARTS_TYPE, job) + filetype = tools.config_tool.getAttribute(comp, filenode, B.ATTR_ARTS_TYPE, job) elif len(filenode) > 2 and len(filenode) < 5: filetype = filenode else: filetype = getCompAttr(comp, B.TOPIC_NODE_FILE, B.ATTR_TYPE, "") - toolname = "file"+filetype+"_tool" - filepath = os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_PROGRAM], "utils", toolname+".py") - #comp.m.debug(verify, "toolname "+filepath) + toolname = "file" + filetype + "_fcts" + filepath = os.path.join(job.conf[B.SUBJECT_PATH][B.ATTR_PATH_PROGRAM], "tools", toolname + ".py") + # comp.m.debug(verify, "toolname "+filepath) if not os.path.exists(filepath): - raise FileNotFoundError("file for tool "+toolname+" does not exist "+filepath) - cmodul = importlib.import_module("utils."+toolname) + raise FileNotFoundError("file for tool " + toolname + " does not exist " + filepath) + cmodul = importlib.import_module("tools." + toolname) class_ = getattr(cmodul, "FileFcts") c = class_() c.setComp(job, comp) diff --git a/catalog/programs.csv b/catalog/programs.csv new file mode 100644 index 0000000..30eafe5 --- /dev/null +++ b/catalog/programs.csv @@ -0,0 +1,21 @@ +_key;name;;;;;;;;;;;; +table:programs;name;objtype;objname;time;env;app;variant;pardef;pfilesource;pfiletarget;basedir;loglevel;logpath +;test_executer;tp,ts,tc;m;m;m;m;o;;;;{objtype}base;info;{job.par.wsdir}/{log}/log_{job.start}.txt +;init_testsuite;ts;m;o;m;m;o;;envparfile;tsparfile;tsbase;info;{job.par.tsdir}/{log}/{job.program}_{job.start}.txt +;execute_testsuite;ts;m;m;m;m;o;;tsparfile;tsparfile;tsbase;info;{job.par.tsdir}/{log}/{job.program}_{job.start}.txt +;collect_testsuite;ts;m;m;m;m;o;;tsparfile;tsparfile;tsbase;info;{job.par.tsdir}/{log}/{job.program}_{job.start}.txt +;finish_testsuite;ts;m;m;m;m;o;;tsparfile;tsparfile;tsbase;info;{job.par.tsdir}/{log}/{job.program}_{job.start}.txt +;unzip_testsuite;ts;m;m;m;m;o;;;;;info;{job.par.tsdir}/{log}/{job.program}_{job.start}.txt +;init_testcase;tc;m;o;m;m;o;;tsparfile;tcparfile;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt +;exec_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt +;collect_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt +;compare_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt +;check_environment;env;;;m;;o;;;envparfile;envbase;info;{job.par.envdir}/{log}/log_{job.start}.txt +;check_specification;tp,ts,tc;o;;;;;;;;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt +;check_configuration;ws;o;;;o;;;;;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt +;make_workspace;ws;;;_;;;;;;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt +;unit_tester;ws;o;;d;;;;;;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt +;declare_expection;ts,tc;m;m;m;m;o;;;;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt +;clean_workspace;ws;o ;;_;;;;;;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt +;start_dialog;ws;input;;input;input;input;;;;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt +;web_start;ws;;;;;;;;;wsbase;warn;{job.par.wsdir}/{log}/log_{job.start}.txt diff --git a/objects/__init__.py b/objects/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/objects/catalog.py b/objects/catalog.py new file mode 100644 index 0000000..ca69318 --- /dev/null +++ b/objects/catalog.py @@ -0,0 +1,106 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +import os +import basic.program +import basic.constants as B +import tools.path_const as P +import tools.data_const as D +import tools.config_tool +import tools.path_tool +import tools.file_tool +# import tools.tdata_tool + +EXP_KEY_MISSING = "key is missing {}" +EXP_KEY_DOESNT_EXIST = "key doesnt exist in domain {}" + +class Catalog: + __instance = None + """ + in this class there should be managed each defined key-value-pairs + the pairs ara loaded from the path testdata/catalog: + * initially the csv-file catalog.csv + * on demand other csv-files in the path + """ + + def __init__(self): + self.catalog = {} + Catalog.__instance = self + pass + + + @staticmethod + def getInstance(): + if Catalog.__instance == None: + return Catalog() + return Catalog.__instance + + + def getValue(self, domain, key, job): + """ + this function gets the value of the domain an key + :param domain: + :param key: + :return: + """ + if not (isinstance(domain, str) or len(domain) < 1): + raise Exception(EXP_KEY_MISSING, (domain, key)) + if not (isinstance(key, str) or len(key) < 1): + job.m.setError(EXP_KEY_MISSING+" ("+domain+", "+key+")") + return "" + + if domain not in self.catalog: + self.readDomain(domain, job) + if key not in self.catalog[domain]: + job.m.setError(EXP_KEY_DOESNT_EXIST+" ("+domain+", "+key+")") + return "" + return self.catalog[domain][key] + + + def getKeys(self, domain, job): + """ + this function gets the value of the domain an key + :param domain: + :param key: + :return: + """ + if not (isinstance(domain, str) or len(domain) < 1): + raise Exception(EXP_KEY_MISSING, (domain)) + + if domain not in self.catalog: + self.readDomain(domain, job) + if domain not in self.catalog: + return [] + out = [] + for x in self.catalog[domain].keys(): + out.append(x) + return out + + + def readDomain(self, domain, job): + """ + this function reads the domain-entries + :param domain: + :return: + """ + if not (isinstance(domain, str) or len(domain) < 1): + raise Exception(EXP_KEY_MISSING, (domain)) + if domain in self.catalog: + return + pathname = tools.config_tool.getConfigPath(job, P.KEY_CATALOG, domain) + if pathname is None: + raise Exception(EXP_KEY_MISSING, (domain)) + data = tools.file_tool.readFileDict(job, pathname, job.m) + self.catalog[domain] = data[B.DATA_NODE_TABLES][domain][B.DATA_NODE_KEYS] + return data + + + def exportXSD(self, domain): + """ + this function exports the domain into xsd-declaration of simple types + :return: + """ + pass diff --git a/test/constants.py b/test/constants.py index ff8c576..fcbccf9 100644 --- a/test/constants.py +++ b/test/constants.py @@ -5,16 +5,16 @@ # Source : gitea.ucarmesin.de # --------------------------------------------------------------------------------------------------------- """ -constants +constants """ import os home = os.getcwd() prgdir = "" -if home[-4:] == "test" and home[-6:] != "datest": +if home[-5:] == "xtests" and home[-9:] != "program": home = home[0:-5] if home[-10:] == "components": home = home[0:-11] -if home[-6:] == "datest": +if home[-9:] == "program": prgdir = home[-6:] home = home[0:-7] elif home[-7:] == "program": diff --git a/test/test_01date.py b/test/test_01date.py index de47f5f..d589904 100644 --- a/test/test_01date.py +++ b/test/test_01date.py @@ -2,16 +2,15 @@ import json import inspect import unittest import datetime -import utils.date_tool +import tools.date_tool -TEST_FUNCTIONS = ["test_dateformat", "test_parseFormula", "test_parseDate"] -TEST_FUNCTIONS = ["test_parseDate"] +TEST_FUNCTIONS = ["test_01dateformat", "test_10parseFormula", "test_11parseDate"] +#TEST_FUNCTIONS = ["test_11parseDate"] verbose = True # class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase): - TEST_FUNCTIONS = ["test_dateformat", "test_parseFormula", "test_parseDate"] mymsg = "--------------------------------------------------------------" """ @@ -26,7 +25,7 @@ class MyTestCase(unittest.TestCase): if verbose: print("run own test " + str(suite)) """ - def test_dateformat(self, result=None): + def test_01dateformat(self, result=None): if verbose: print(str(result)) actfunction = str(inspect.currentframe().f_code.co_name) cnttest = 0 @@ -42,31 +41,31 @@ class MyTestCase(unittest.TestCase): MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) - def test_parseFormula(self, result=None): + def test_10parseFormula(self, result=None): actfunction = str(inspect.currentframe().f_code.co_name) cnttest = 0 if actfunction not in TEST_FUNCTIONS: return - res = utils.date_tool.parseFormula("{(21.12.2012 +1Y)}") + res = tools.date_tool.parseFormula("{(21.12.2012 +1Y)}") if verbose: print(str(res)) self.assertEqual(res[0], 2013) self.assertEqual(res[3], 0) - res = utils.date_tool.parseFormula("{(21.12.2012 +1Y -1M)}") + res = tools.date_tool.parseFormula("{(21.12.2012 +1Y -1M)}") if verbose: print(str(res)) self.assertEqual(res[0], 2013) self.assertEqual(res[1], 11) self.assertEqual(res[3], 0) - res = utils.date_tool.parseFormula("{(21.12.2012 +1 Jahre +20 Tage)}") + res = tools.date_tool.parseFormula("{(21.12.2012 +1 Jahre +20 Tage)}") if verbose: print(str(res)) self.assertEqual(res[0], 2014) self.assertEqual(res[1], 1) self.assertEqual(res[2], 10) self.assertEqual(res[3], 0) - res = utils.date_tool.parseFormula("{(21.12.2012_11:12:43 +1Y)}") + res = tools.date_tool.parseFormula("{(21.12.2012_11:12:43 +1Y)}") if verbose: print(str(res)) self.assertEqual(res[0], 2013) self.assertEqual(res[5], 43) - res = utils.date_tool.parseFormula("{(21.12.2012 -60M)}") + res = tools.date_tool.parseFormula("{(21.12.2012 -60M)}") if verbose: print(str(res)) self.assertEqual(res[0], 2007) self.assertEqual(res[1], 12) @@ -74,28 +73,28 @@ class MyTestCase(unittest.TestCase): MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) - def test_parseDate(self, result=None): + def test_11parseDate(self, result=None): actfunction = str(inspect.currentframe().f_code.co_name) cnttest = 0 if actfunction not in TEST_FUNCTIONS: return - res = utils.date_tool.parseDate("21.12.2012") + res = tools.date_tool.parseDate("21.12.2012") if verbose: print(str(res)) self.assertEqual(res[0], 2012) self.assertEqual(res[3], 0) - res = utils.date_tool.parseDate("{(21.12.2012 +1Y)}") + res = tools.date_tool.parseDate("{(21.12.2012 +1Y)}") if verbose: print(str(res)) self.assertEqual(res[0], 2013) self.assertEqual(res[3], 0) - res = utils.date_tool.parseDate("Wed May 18 23:32:55 2022 +0200") + res = tools.date_tool.parseDate("Wed May 18 23:32:55 2022 +0200") if verbose: print(str(res)) self.assertEqual(res[0], 2022) self.assertEqual(res[1], 5) - res = utils.date_tool.parseDate("Mit Dez 18 23:32:55 2021 +0200") + res = tools.date_tool.parseDate("Mit Dez 18 23:32:55 2021 +0200") if verbose: print(str(res)) self.assertEqual(res[0], 2021) self.assertEqual(res[1], 12) - res = utils.date_tool.parseDate("Sun Sep 4 15:12:21 2022 +0200") + res = tools.date_tool.parseDate("Sun Sep 4 15:12:21 2022 +0200") if verbose: print(str(res)) self.assertEqual(res[0], 2022) self.assertEqual(res[1], 9) diff --git a/test/test_07catalog.py b/test/test_07catalog.py index 21371d3..f082c77 100644 --- a/test/test_07catalog.py +++ b/test/test_07catalog.py @@ -1,21 +1,23 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- import unittest -import os import inspect -import utils.path_tool import basic.message import basic.program -import basic.constants as B import test.constants import test.testtools -import utils.path_const as P -import basic.catalog +import objects.catalog HOME_PATH = test.constants.HOME_PATH OS_SYSTEM = test.constants.OS_SYSTEM # here you can select single testfunction for developping the tests TEST_FUNCTIONS = ["test_01class", "test_02read", "test_03key"] -#TEST_FUNCTIONS = [ "test_03key"] +# TEST_FUNCTIONS = [ "test_02read"] verbose = False class MyTestCase(unittest.TestCase): @@ -28,7 +30,7 @@ class MyTestCase(unittest.TestCase): if actfunction not in TEST_FUNCTIONS: return job = test.testtools.getJob() - catalog = basic.catalog.Catalog.getInstance() + catalog = objects.catalog.Catalog.getInstance() self.assertIsNotNone(catalog) cnttest += 1 MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) @@ -41,11 +43,16 @@ class MyTestCase(unittest.TestCase): if actfunction not in TEST_FUNCTIONS: return job = test.testtools.getJob() - catalog = basic.catalog.Catalog.getInstance() + catalog = objects.catalog.Catalog.getInstance() self.assertRaises(Exception, catalog.readDomain, ("xxx", job)) cnttest += 1 - res = catalog.readDomain("countries", job) - self.assertEqual(isinstance(res, dict), True) + try: + res = catalog.readDomain("countries", job) + self.assertEqual(isinstance(res, dict), True) + res = catalog.readDomain("programs", job) + self.assertEqual(isinstance(res, dict), True) + except: + print ("except") cnttest += 1 countries = catalog.getKeys("countries", job) self.assertEqual(len(countries), 21) @@ -60,7 +67,7 @@ class MyTestCase(unittest.TestCase): if actfunction not in TEST_FUNCTIONS: return job = test.testtools.getJob() - catalog = basic.catalog.Catalog.getInstance() + catalog = objects.catalog.Catalog.getInstance() res = catalog.getValue("countries", "key", job) self.assertEqual(res, "") self.assertEqual(job.m.rc, basic.message.RC_ERROR) diff --git a/test/test_10job.py b/test/test_10job.py new file mode 100644 index 0000000..20524c8 --- /dev/null +++ b/test/test_10job.py @@ -0,0 +1,46 @@ +import unittest +import os +import inspect +import shutil + +import tools.path_tool +import basic.program +from basic.componentHandling import ComponentManager +import test.constants +import basic.constants as B +import test.constants as T +import basic.componentHandling +import tools.file_tool + +HOME_PATH = test.constants.HOME_PATH +PYTHON_CMD = "python" +TEST_FUNCTIONS = ["test_00init"] + +PROGRAM_NAME = "clean_workspace" + +class MyTestCase(unittest.TestCase): + mymsg = "--------------------------------------------------------------" + + + def test_00init(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return + # simple job instantiate - without parameter and only simple messaging + job = basic.program.Job(PROGRAM_NAME) + print(str(job.__dict__)) + self.checkSimpleJob(job) + + def checkSimpleJob(self, job): + self.assertIn("conf", job.__dict__) + self.assertIn("jobid", job.__dict__) + self.assertIn("program", job.__dict__) + self.assertIn("start", job.__dict__) + self.assertIn("configpath", job.conf) + self.assertIn("paths", job.conf) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/testtools.py b/test/testtools.py index 97a47a7..3dd6d1e 100644 --- a/test/testtools.py +++ b/test/testtools.py @@ -1,9 +1,9 @@ import basic.program import basic.constants as B import basic.component -import utils.data_const as D +import tools.data_const as D import test.constants as T -import utils.config_tool +import tools.config_tool DEFAULT_GRAN = "tc" DEFAULT_APP = "TESTAPP" @@ -89,8 +89,8 @@ def getComp(job, componentName=""): componentName = DEFAULT_COMP comp.conf = {} comp.name = componentName - confs = utils.config_tool.getConfig(job, "comp", componentName) - conns = utils.conn_tool.getConnections(job, componentName) + confs = tools.config_tool.getConfig(job, "comp", componentName) + conns = tools.conn_tool.getConnections(job, componentName) comp.conf = confs["conf"] comp.conf[B.SUBJECT_CONN] = conns[0] return comp \ No newline at end of file diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/config_tool.py b/tools/config_tool.py new file mode 100644 index 0000000..e549b3d --- /dev/null +++ b/tools/config_tool.py @@ -0,0 +1,349 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# https://ucarmesin.de/index.php/it/testautomatisierung-fuer-daten-test/225-konfiguration-der-testanwendung +# --------------------------------------------------------------------------------------------------------- +import sys +import basic.constants as B + +try: + import basic.program +except ImportError: + print("ImportError: " + str(ImportError.with_traceback())) + pass +import basic.componentHandling +import tools.path_tool +import tools.file_tool +import os.path +import basic.constants as B +import tools.data_const as D +import tools.path_const as P + +COMP_FILES = [D.DDL_FILENAME] +CONFIG_FORMAT = [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV] + +def getExistingPath(job, pathnames): + if isinstance(pathnames, str): + pathnames = [pathnames] + for p in pathnames: + if p[-1:] == ".": + p = p[0:-1] + for format in CONFIG_FORMAT: + pathname = p+"."+format + if os.path.exists(pathname): + return pathname + return None + +def getConfigPath(job, modul, name, subname=""): + """ + gets the most specified configuration of different sources + Parameter: + * typ -- (basic, comp, tool) + * name -- the specific class + sources: + * programm << + * install << + * environ << basis-conf + * release << basis-conf + * testset << parameter/environ + * testcase << parameter + the parameter-files could be one of these file-types: + * yaml, json, csv + """ + if job is None: + verify = False # job = basic.program.Job.getInstance() + else: + verify = job.getDebugLevel("config_tool")-4 + if verify: job.debug(verify, "getConfig " + modul + ", " + name) + if modul == P.KEY_TOOL: + path = getExistingPath(job, os.path.join(job.conf["paths"][P.ATTR_PATH_COMPONENTS], + P.VAL_CONFIG, P.KEY_TOOL+"_"+name)) + if path is not None: + return path + path = getExistingPath(job, os.path.join(job.conf["paths"][P.ATTR_PATH_HOME], + P.VAL_CONFIG, P.KEY_TOOL+"_"+name)) + if path is not None: + return path + path = getExistingPath(job, os.path.join(job.conf["paths"][P.ATTR_PATH_PROGRAM], + P.VAL_UTIL, P.VAL_CONFIG, name)) + if path is not None: + return path + path = getExistingPath(job, os.path.join(job.conf["paths"][P.ATTR_PATH_ENV], + job.par.environment, P.VAL_CONFIG, P.KEY_TOOL+"_"+ name)) + if path is not None: + return path + raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name) + elif modul == P.KEY_COMP: + for format in CONFIG_FORMAT: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_HOME], + P.VAL_CONFIG, P.KEY_COMP+"_" + name + "."+format) + if verify: job.debug(verify, "4 " + pathname) + if os.path.exists(pathname): + return pathname + for format in CONFIG_FORMAT: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_COMPONENTS], + basic.componentHandling.getComponentFolder(name), "CONFIG." + format) + if verify: job.debug(verify, "5 " + pathname) + if os.path.exists(pathname): + return pathname + if verify: job.debug(verify, "6 " + pathname) + raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name) + elif modul in COMP_FILES: + # for example DATASTRUCURE or the table + pathnames = [] + pathnames.append(os.path.join(job.conf["paths"][P.ATTR_PATH_COMPONENTS], + basic.componentHandling.getComponentFolder(name), modul)) + pathnames.append(os.path.join(job.conf["paths"][P.ATTR_PATH_COMPONENTS], + basic.componentHandling.getComponentFolder(subname), modul)) + pathnames.append(os.path.join(job.conf["paths"][P.ATTR_PATH_PROGRAM], P.VAL_BASIC, modul)) + pathnames.append(os.path.join(job.conf["paths"][P.ATTR_PATH_PROGRAM], P.VAL_BASIC, subname)) + configpath = getExistingPath(job, pathnames) + if configpath is not None: + return configpath + for format in CONFIG_FORMAT: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_COMPONENTS], + basic.componentHandling.getComponentFolder(name), modul+"."+format) + if os.path.exists(pathname): + return pathname + for format in CONFIG_FORMAT: + if len(subname) > 1: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_COMPONENTS], + basic.componentHandling.getComponentFolder(name), subname+"."+format) + if os.path.exists(pathname): + return pathname + raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name) + elif modul == P.KEY_BASIC: + for format in CONFIG_FORMAT: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_COMPONENTS], + P.VAL_CONFIG , name + "."+format) + if verify: job.debug(verify, "4 " + pathname) + if os.path.exists(pathname): + return pathname + elif modul == P.KEY_TESTCASE: + for format in CONFIG_FORMAT: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_TDATA], + name, D.DFILE_TESTCASE_NAME + "."+format) + if verify: job.debug(verify, "4 " + pathname) + if os.path.exists(pathname): + return pathname + elif modul == P.KEY_TESTSUITE: + for format in CONFIG_FORMAT: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_TDATA], + name, D.DFILE_TESTSUITE_NAME + "." + format) + if verify: job.debug(verify, "4 " + pathname) + if os.path.exists(pathname): + return pathname + elif modul == P.KEY_CATALOG: + path = getExistingPath(job, os.path.join(job.conf["paths"][P.ATTR_PATH_TDATA], + P.KEY_CATALOG, name)) + if path is not None: + return path + path = getExistingPath(job, os.path.join(job.conf["paths"][P.ATTR_PATH_COMPONENTS], + P.KEY_CATALOG, name)) + if path is not None: + return path + path = getExistingPath(job, os.path.join(job.conf["paths"][P.ATTR_PATH_PROGRAM], + P.KEY_CATALOG, name)) + if path is not None: + return path + raise Exception(P.EXP_CONFIG_MISSING, name) + else: + pathname = tools.path_tool.composePath(job, P.P_TCPARFILE) + if verify: job.debug(verify, "7 " + pathname) + if os.path.exists(pathname): + return pathname + pathname = tools.path_tool.composePath(job, P.P_TSPARFILE) + if verify: job.debug(verify, "8 " + pathname) + if os.path.exists(pathname): + return pathname + for format in CONFIG_FORMAT: + if len(subname) > 1: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_RELEASE], + P.VAL_CONFIG, "basis."+format) + if verify: job.debug(verify, "9 " + pathname) + if os.path.exists(pathname): + return pathname + for format in CONFIG_FORMAT: + if len(subname) > 1: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_ENV], + P.VAL_CONFIG, "basis."+format) + if verify: job.debug(verify, "9 " + pathname) + if os.path.exists(pathname): + return pathname + for format in CONFIG_FORMAT: + if len(subname) > 1: + pathname = os.path.join(job.conf["paths"][P.ATTR_PATH_HOME], + P.VAL_CONFIG, "basis."+format) + if verify: job.debug(verify, "9 " + pathname) + if os.path.exists(pathname): + return pathname + raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name) + + +def getConfValue(attribute, comp): + if attribute == B.ATTR_CONN_DBTYPE: + if not hasAttr(comp.conf[B.SUBJECT_CONN], "dbtype"): + if hasAttr(comp.conf[B.SUBJECT_CONN], "types") and hasAttr(comp.conf[B.SUBJECT_CONN]["types"], "dbtype"): + dbtype = comp.conf[B.SUBJECT_CONN]["types"]["dbtype"] + else: + raise LookupError("dbtype is not set in comp " + comp.name) + else: + dbtype = comp.conf["conn"]["dbtype"] + return "" + return "" + + +def getAttr(o, name): + if (isinstance(o, dict)): + if (name in o.keys()): + return o[name] + elif (isinstance(o, list)): + pass + elif hasattr(o, name): + return getattr(o, name) + return False + + +def hasAttr(o, name): + if (isinstance(o, dict)): + if (name in o.keys()): + return True + elif (isinstance(o, list)): + pass + + + elif hasattr(o, name): + return True + return False + + +def getConfig(job, modul, name, subname=""): + if job is None: + verify = 24 + else: + verify = job.getDebugLevel("config_tool")-4 + msg = None + if hasattr(job, "m"): msg = job.m + pathname = getConfigPath(job, modul, name, subname) + confs = {} + job.debug(verify, "getConfig " + pathname) + if len(pathname) < 1: + return confs + doc = tools.file_tool.readFileDict(job, pathname, msg) + if modul == D.DDL_FILENAME: + # in csv the root is the subname + # from the Dict-structure of DDL_FILENAME pick the substructure of the subname + keys = list(doc.keys()) + if subname not in keys and len(keys) == 1: + doc0 = doc[keys[0]] + doc = doc0 + keys = list(doc.keys()) + if subname in keys: + doc0 = doc[subname] + doc = doc0 + for i, v in doc.items(): + confs[i] = v + return confs + + +def getAttribute(comp, path, attr, job): + attrList = getAttributeList(comp, path, job) + if attr in attrList: + return attrList[attr] + else: + return "" + + +def getAttributeList(comp, path, job): + """ + gets a concrete attribute-list for an arteifact-element from the config-attributes from the connection-attributes + https://ucarmesin.de/index.php/it/testautomatisierung-fuer-daten-test/225-konfiguration-der-testanwendung#konfigurationshierarchie + :param comp: + :param path: artifact-type.artifact-name for example: DB.person + :return: list of all attributes for the artifact-element + """ + attrList = {} + a = path.split(".") + artType = a[0] + artName = a[1] + if B.SUBJECT_CONN not in comp.conf: + raise Exception ("Environment is not configured") + if artType in comp.conf[B.SUBJECT_CONN]: + if artName in comp.conf[B.SUBJECT_CONN][artType]: + for attr, val in comp.conf[B.SUBJECT_CONN][artType][artName].items(): + if attr not in B.LIST_ATTR[artType]: + continue + attrList[attr] = val + for attr, val in comp.conf[B.SUBJECT_CONN][artType].items(): + if attr not in B.LIST_ATTR[artType]: + continue + if attr in attrList: + continue + attrList[attr] = val + if artType in comp.conf[B.SUBJECT_ARTS]: + if artName in comp.conf[B.SUBJECT_ARTS][artType]: + for attr, val in comp.conf[B.SUBJECT_ARTS][artType][artName].items(): + if attr not in B.LIST_ATTR[artType]: + continue + if attr in attrList: + continue + attrList[attr] = val + for attr, val in comp.conf[B.SUBJECT_ARTS][artType].items(): + if attr not in B.LIST_ATTR[artType]: + continue + if attr in attrList: + continue + attrList[attr] = val + return attrList + +def mergeConn(msg, conf, conn): + """ + merges the config-attributes from the connection-attributes + because the connection-attributes has to overwrite the config-attributes if the subject is configured + https://ucarmesin.de/index.php/it/testautomatisierung-fuer-daten-test/225-konfiguration-der-testanwendung#konfigurationshierarchie + :param conf: + :param conn: + :return: + """ + if B.SUBJECT_INST not in conf: + conf[B.SUBJECT_INST] = {} + for a in conn[B.SUBJECT_INST]: + conf[B.SUBJECT_INST][a] = conn[B.SUBJECT_INST][a] + for topic in [B.TOPIC_NODE_DB, B.TOPIC_NODE_CLI, B.TOPIC_NODE_API, B.TOPIC_NODE_FILE]: + if topic not in conf[B.SUBJECT_ARTS]: + continue + if topic == B.TOPIC_NODE_DB: + list = B.LIST_DB_ATTR + if topic == B.TOPIC_NODE_CLI: + list = B.LIST_CLI_ATTR + if topic == B.TOPIC_NODE_API: + list = B.LIST_API_ATTR + if topic == B.TOPIC_NODE_FILE: + list = B.LIST_FILE_ATTR + print(" --- merge-conn " + topic + " " + str(list)) + for a in conf[B.SUBJECT_ARTS][topic]: + if topic not in conn: + continue + if a in list: + if a in conn[topic]: + conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] + else: + for b in conf[B.SUBJECT_ARTS][topic][a]: + print(" --- merge-conn b " + topic + " " + a+" "+b) + if b not in list: + msg.logError("not-topic-attribute in topic-connection: "+topic+", "+b) + continue + if a not in conn[topic]: + continue + if b in conn[topic][a]: + conf[B.SUBJECT_ARTS][topic][a][b] = conn[topic][a][b] + for a in list: + if topic not in conn: + break + if topic not in conn: + continue + if a in conn[topic]: + conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] + return conf \ No newline at end of file diff --git a/tools/conn_tool.py b/tools/conn_tool.py new file mode 100644 index 0000000..6453de7 --- /dev/null +++ b/tools/conn_tool.py @@ -0,0 +1,81 @@ +# functions about connections to other instances +# ------------------------------------------------------------------- +""" + +""" +import basic.program +import tools.config_tool +import basic.constants as B +import tools.data_const as D + + +def getConnection(job, comp, nr): + #job = basic.program.Job.getInstance() + verify = job.getDebugLevel("conn_tool") + conn = {} + if job.conf.confs.get(B.SUBJECT_TOOL).get("connsrc") == D.DFILE_TYPE_YML: + conn = tools.config_tool.getConfig(job, "tool", B.SUBJECT_CONN) + xtypes = None + if ("types" in conn["env"][comp]): + xtypes = conn["env"][comp]["types"] + instnr = "inst" + str(nr) + if conn["env"][comp][instnr]: + if (xtypes is not None): + conn["env"][comp][instnr]["types"] = xtypes + return conn["env"][comp][instnr] + else: + job.m.setFatal("Conn-Tool: Comp not configured " + comp + " " + str(nr)) + return None + + +def getConnections(job, comp): + """ + it reads the connection-attributes for each instances of this component + general attributes are added to the connection-attributes + :param comp: + :return: + """ + #job = basic.program.Job.getInstance() + verify = job.getDebugLevel("conn_tool") + msg = None + if hasattr(comp, "m") and comp.m is not None: + msg = comp.m + elif hasattr(job, "m") and job.m is not None: + msg = job.m + else: + raise Exception("message-object is missing") + msg.debug(verify, "getConnections " + comp) + conn = {} + conns = [] + # if a datest-database exists read the connections + conndb = {} + if job.conf.confs.get("db"): + # select + pass + + conn = tools.config_tool.getConfig(job, "tool", B.SUBJECT_CONN) + if not comp in conn[B.SUBJECT_ENV]: + job.m.setFatal("Conn-Tool: Comp not configured " + comp) + + attr = {} + if B.CONF_NODE_GENERAL in conn[B.SUBJECT_ENV]: + for a in conn[B.SUBJECT_ENV][B.CONF_NODE_GENERAL]: + attr[a] = conn[B.SUBJECT_ENV][B.CONF_NODE_GENERAL] + for a in conn[B.SUBJECT_ENV][comp]: + if "inst" in a and a != B.SUBJECT_INST: + continue + attr[a] = conn["env"][comp][a] + #if ("types" in conn["env"][comp]): + # xtypes = conn["env"][comp]["types"] + for i in range(conn[B.SUBJECT_ENV][comp][B.SUBJECT_INST][B.ATTR_INST_CNT]): + #print("range " + str(i + 1)) + instnr = "inst" + str(i + 1) + #if (xtypes is not None): + # conn["env"][comp][instnr]["types"] = xtypes + for a in attr: + if a in conn["env"][comp][instnr]: + continue # dont overwrite an instance-specific value + conn["env"][comp][instnr][a] = attr[a] + conns.append(conn["env"][comp][instnr]) + + return conns diff --git a/tools/data_const.py b/tools/data_const.py new file mode 100644 index 0000000..815a9ee --- /dev/null +++ b/tools/data_const.py @@ -0,0 +1,116 @@ +#!/usr/bin/python +""" +constants for used for api-functions +""" +import basic.constants as B + +DDL_FILENAME = "DATASTRUCTURE" + +DATA_NODE_TYPE = "type" +TYPE_STRING = "string" +TYPE_STR = "str" +TYPE_TEXT = "text" +TYPE_INT = "int" +TYPE_FLOAT = "float" +TYPE_DOUBLE = "double" +TYPE_DATE = "date" +TYPE_TIME = "time" +TYPE_PK = "pk" + +# fields in DDL +DDL_FNULLABLE = "nullable" +DDL_FNAME = "field" +DDL_ACCEPTANCE = "acceptance" +DDL_KEY = "key" +DDL_TYPE = "type" +DDL_INDEX = "index" + +DFILE_TYPE_YML = "yml" +DFILE_TYPE_JSON = "json" +DFILE_TYPE_CSV = "csv" +DFILE_TYPE_XML = "xml" +DFILE_TESTCASE_NAME = "testspec" +DFILE_TESTSUITE_NAME = "testsuite" +DFILE_TABLE_PREFIX = "table_" +LIST_DFNAME_ATTR = [DFILE_TESTCASE_NAME, DFILE_TESTSUITE_NAME, DFILE_TABLE_PREFIX] +LIST_DFNAME_CONST = ["DFILE_TESTCASE_NAME", "DFILE_TESTSUITE_NAME", "DFILE_TABLE_PREFIX"] + +DATA_SRC_DIR = "dir" +DATA_SRC_CSV = "csv" + +DATA_ATTR_COUNT = "_count" +""" statistical information of data-count """ +DATA_ATTR_DATE = "_date" +""" reference-date for computing the actual date in relation to specification or expectation """ +DATA_ATTR_COMP = "_comp" +""" reference to using componente with their object """ +DATA_ATTR_CHAR = "_char" +""" character of the data in order to delete it ión initialization """ +DATA_ATTR_KEY = "_key" +""" key for a data-specification of a catalog-list - default: the first field is the key """ +DATA_ATTR_ALIAS = "_alias" +DATA_ATTR_IDS = "_ids" +DATA_ATTR_REF = "_ref" +LIST_DATA_ATTR = [DATA_ATTR_COUNT, DATA_ATTR_DATE, DATA_ATTR_CHAR, DATA_ATTR_COMP, + DATA_ATTR_REF, DATA_ATTR_IDS, DATA_ATTR_ALIAS, DATA_ATTR_KEY] +LIST_ATTR_CONST = ["DATA_ATTR_COUNT", "DATA_ATTR_DATE", "DATA_ATTR_CHAR", "DATA_ATTR_COMP", "DATA_ATTR_ALIAS", "DATA_ATTR_KEY"] + +HEAD_ATTR_DESCR = "decription" +HEAD_ATTR_TARGET = "target" +HEAD_ATTR_USECASE = "usecase" +HEAD_ATTR_UCID = "usecase-id" +HEAD_ATTR_STORY = "story" +HEAD_ATTR_STORYID = "storyid-id" +HEAD_ATTR_APPS = B.SUBJECT_APPS +HEAD_ATTR_DEPR = "deprecated" +LIST_HEAD_ATTR = [HEAD_ATTR_DESCR, HEAD_ATTR_TARGET, HEAD_ATTR_USECASE, HEAD_ATTR_UCID, + HEAD_ATTR_STORY, HEAD_ATTR_STORYID, HEAD_ATTR_APPS, HEAD_ATTR_DEPR] +LIST_HEAD_CONST = ["HEAD_ATTR_DESCR", "HEAD_ATTR_TARGET", "HEAD_ATTR_USECASE", "HEAD_ATTR_UCID", + "HEAD_ATTR_STORY", "HEAD_ATTR_STORYID", "HEAD_ATTR_APPS", "HEAD_ATTR_DEPR"] + + +CSV_HEADER_START = ["node", "table", "tabelle"] +CSV_DELIMITER = ";" +INTERNAL_DELIMITER = "||" + +""" +internal structure of testdata +""" +CSV_SPECTYPE_DATA = "data" +CSV_SPECTYPE_TREE = "tree" +CSV_SPECTYPE_KEYS = "keys" +CSV_SPECTYPE_CONF = "conf" +CSV_NODETYPE_KEYS = "_keys" + +CSV_BLOCK_HEAD = "_head" +CSV_BLOCK_OPTION = B.DATA_NODE_OPTION +CSV_BLOCK_STEP = B.DATA_NODE_STEPS +CSV_BLOCK_TABLES = B.DATA_NODE_TABLES +CSV_BLOCK_IMPORT = "_import" +LIST_CSV_BLOCKS = [CSV_BLOCK_HEAD, CSV_BLOCK_OPTION, CSV_BLOCK_STEP, CSV_BLOCK_TABLES, CSV_BLOCK_IMPORT] +LIST_BLOCK_CONST = ["CSV_BLOCK_HEAD", "CSV_BLOCK_OPTION", "CSV_BLOCK_STEP", "CSV_BLOCK_TABLES", "CSV_BLOCK_IMPORT"] + +STEP_COMP_I = 1 +STEP_EXECNR_I = 2 +STEP_REFNR_I = 3 +STEP_VARIANT_I = 4 +STEP_ARGS_I = 5 +STEP_LIST_I = 5 +STEP_ATTR_COMP = "component" +STEP_ATTR_EXECNR = "exec-step" +STEP_ATTR_REFNR = "reference-nr" +STEP_ATTR_ARGS = "arguments" +LIST_STEP_ATTR = [STEP_ATTR_COMP, STEP_ATTR_EXECNR, STEP_ATTR_REFNR, STEP_ATTR_ARGS] +LIST_STEP_CONST = ["STEP_ATTR_COMP", "STEP_ATTR_EXECNR", "STEP_ATTR_REFNR", "STEP_ATTR_ARGS"] + +EXCP_MALFORMAT = "malformated line: " + +ATTR_SRC_TYPE = "tdtyp" +ATTR_SRC_DATA = "tdsrc" +ATTR_SRC_NAME = "tdname" + +DEFAULT_DB_PARTITION = "n" +""" attribute if table is partitioned - partitions are parametrized """ +DEFAULT_DB_CONN_JAR = "n" +""" attribute for connection-jar-file instead of connection by ip, port """ + diff --git a/tools/date_tool.py b/tools/date_tool.py new file mode 100644 index 0000000..dc7103e --- /dev/null +++ b/tools/date_tool.py @@ -0,0 +1,175 @@ +# functions related to Date-fields +# ----------------------------------------------------- +""" +additionally functions for calculating date with formulas like [DATE+2M] and for comparison of date related on two reference-dates +""" +import datetime +import re +import tools.data_const as D + + +F_DIR = "%Y-%m-%d_%H-%M-%S" +F_DB_DATE = "%Y-%m-%d" +F_DB_TIME = "%Y-%m-%d %H:%M:%S" +F_DE = "%d.%m.%Y" +F_N8 = "%Y%m%d" +F_LOG = "%Y%m%d_%H%M%S" +F_DE_TSTAMP = "%d.%m.%Y %H:%M:%S" +MONTH_EN = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"] +MONTH_DE = ["jan", "feb", "mar", "apr", "mai", "jun", "jul", "aug", "sep", "okt", "nov", "dez"] +F_TIME_DEFAULT = F_DIR + +def getActdate(format): + return getFormatdate(datetime.datetime.now(), format) + + +def getFormatdate(date, format): + """ it return the date as string in the format """ + return date.strftime(format) + + +def getFormatDatetupel(dtupel, format): + """ it return the date as string in the format """ + if format == F_N8: + return f'{dtupel[0]:04}'+f'{dtupel[1]:02}'+f'{dtupel[2]:02}' + return getFormatdate(datetime.datetime(dtupel[0], dtupel[1], dtupel[2], + dtupel[3], dtupel[4], dtupel[5]) ,format) + +def formatParsedDate(instring, format): + dtupel = parseDate(instring) + #print ("---------------"+str(dtupel)) + return getFormatDatetupel(dtupel, format) + +def parseFormula(instring): + """ + the function parses the string as a formula. In the formula the input-date - actdate or explicite date - + will be added resp. subtracted with years, months or dates which are specified in the formula. + The structure of the formula: DATE +/- mY +/-nM +/-qD + :param instring: + :return: + """ + instring = instring.upper() + if instring[2:6] == "DATE": + refdate = datetime.datetime.today() + formula = instring[7:-2].upper() + else: + dstring = instring[2:instring.find(" ")] + res = parseDate(dstring) + refdate = datetime.datetime(res[0], res[1], res[2], res[3], res[4], res[5]) + formula = instring[2+len(dstring):-2] + formula = re.sub(r' ', '', formula) + year = refdate.year + mon = refdate.month + day = refdate.day + hour = refdate.hour + min = refdate.minute + sec = refdate.second + if re.match(r"[-+]\d+[JYMDT]", formula): + ress = re.compile(r"([-+])(\d+)([JYMDT])") + for res in ress.finditer(formula): + summand = int(res.group(2)) + if res.group(1) == "-": + summand = summand * (-1) + if res.group(3) in "JY": + year = year + summand + if res.group(3) in "M": + mon = mon + summand + while mon <= 0: + mon = mon + 12 + year = year - 1 + while mon > 12: + mon = mon - 12 + year = year + 1 + if res.group(3) in "DT": + refdate = datetime.datetime(year, mon, day, hour, min, sec) + refdate = refdate + datetime.timedelta(days=summand) + year = refdate.year + mon = refdate.month + day = refdate.day + hour = refdate.hour + min = refdate.minute + sec = refdate.second + return (year, mon, day, hour, min, sec) + else: + print("re matcht nicht") + return (year, mon, day, hour, min, sec) + +def getMonthInt(instring): + i = 0 + j = 0 + for l in [MONTH_EN, MONTH_DE]: + i = 0 + for m in l: + i += 1 + if instring.lower() == m: + j = i + break + if j > 0: + break + return j + +def parseDate(instring): + """ + the function parses the string as a date or timestamp which is formed in one of the typical formates + :param the string to be parse: + :return timestamp as tupel (y, m, d, H, M ,S): + """ + year = 0 + mon = 0 + day = 0 + hour = 0 + min = 0 + sec = 0 + #print(instring) + if instring[0:2] == "{(" and instring[-2:] == ")}": + return parseFormula(instring) + if re.match(r"\d{8}_\d{6}", instring): + year = int(instring[0:4]) + mon = int(instring[4:6]) + day = int(instring[6:8]) + hour = int(instring[9:11]) + min = int(instring[11:13]) + sec = int(instring[13:]) + return (year, mon, day, hour, min, sec) + if len(instring) > 8: + for d in ["_", " "]: + if d in instring and instring.find(d) > 8: + dstring = instring[0:instring.find(d)] + tstring = instring[instring.find(d)+1:] + dres = parseDate(dstring) + tres = parseDate(tstring) + return (dres[0], dres[1], dres[2], tres[3], tres[4], tres[5]) + if re.match(r"\d{4}[-./]\d{2}[-./]\d{2}", instring): + res = re.match(r"(\d{4})[-./](\d{2})[-./](\d{2})", instring) + year = int(res.group(1)) + mon = int(res.group(2)) + day = int(res.group(3)) + return (year, mon, day, hour, min, sec) + if re.match(r"\d{1,2}[-./]\d{1,2}[-./]\d{4}", instring): + res = re.match(r"(\d{1,2})[-./](\d{1,2})[-./](\d{4})", instring) + year = int(res.group(3)) + mon = int(res.group(2)) + day = int(res.group(1)) + return (year, mon, day, hour, min, sec) + if re.match(r"\w{3} \w{3}\s+\d{1,2} \d{1,2}[:]\d{1,2}[:]\d{2} \d{4}", instring.strip()): + res = re.search(r"\w{3} (\w{3})\s+(\d{1,2}) (\d{1,2})[:](\d{1,2})[:](\d{2}) (\d{4})", instring.strip()) + month = res.group(1) + mon = getMonthInt(month) + day = int(res.group(2)) + hour = int(res.group(3)) + min = int(res.group(4)) + sec = int(res.group(5)) + year = int(res.group(6)) + return (year, mon, day, hour, min, sec) + if re.match(r"\d{8}", instring): + year = instring[0:4] + mon = instring[4:6] + day = instring[6:8] + return (year, mon, day, hour, min, sec) + if re.match(r"\d{2}[-:]\d{2}[-:/]\d{2}", instring): + res = re.match(r"(\d{2})[-:/](\d{2})[-:/](\d{2})", instring) + hour = int(res.group(1)) + min = int(res.group(2)) + sec = int(res.group(3)) + return (year, mon, day, hour, min, sec) + return (year, mon, day, hour, min, sec) \ No newline at end of file diff --git a/tools/file_abstract.py b/tools/file_abstract.py new file mode 100644 index 0000000..d76864c --- /dev/null +++ b/tools/file_abstract.py @@ -0,0 +1,113 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +import os +import basic.program +import tools.config_tool +import basic.constants as B +import basic.toolHandling +import tools.file_tool +import tools.path_tool + + +class FileFcts(): + """ + this is an abstract class + """ + def __init__(self): + pass + + def setComp(self, job, comp=None): + self.job = job + self.comp = comp + + def getMsg(self): + if self.comp is not None: + return self.comp.m + if self.job is not None: + return self.job.m + return None + + def loadFile(self, path): + """ + this function parses the text and translates it to dict + :param text: + :return: + """ + raise Exception(B.EXCEPT_NOT_IMPLEMENT) + + + def file2dict(self): + pass + + def reset_TData(self, job): + pass + + # Funktionen + #=== init_testcase === + def removeFiles(self): + tools.file_tool.removeFiles(self.comp.m, "envpath", "pattern", self.comp.conf["conn"]) + + def copyFiles(self): + fileList = [] + srcpath = "" + envpath = "" + pattern = "" + tools.file_tool.copyFiles(self.job, fileList, srcpath, envpath, pattern) + + def readEnvFiles(self, job): + envpath = "" + pattern = "" + fileList = tools.file_tool.getFiles(self.comp.m, job, envpath, pattern, self.comp.conf["conn"]) + + + # === execute_testcase === + def create_request(self, job, tdata, step): + mapping = "" + schema = "" + archivpath = "" + filename = step.args["filename"] + txt = "" + for o in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_FILE]: + if o["name"] != filename: + continue + mapping = o["mapping"] + schema = o["schema"] + archivpath = os.path.join(tools.path_tool.composePattern(job, "{tcresult}/request", self.comp), filename) # ergebnisse/comp/request ) + #txt = self.createDict() + tools.file_tool.writeFileText(self.comp.m, job, archivpath, txt) + + def send_request(self, job, step): + archivpath = "" + filename = step.args["filename"] + technique = step.args["technique"] + archivpath = os.path.join(tools.path_tool.composePattern(job, "{tcresult}/request", self.comp), filename) + if technique == "cli": + for o in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_FILE]: + if o["name"] != filename: + continue + envpath = o["envpath"] + envpath = tools.path_tool.composePattern(job, envpath, self.comp) + fct = basic.toolHandling.getCliTool(job, self.comp) + fct.copy(self.job, archivpath, envpath) + elif technique == "api": + txt = tools.file_tool.readFileText(job, archivpath, self.comp.m) + fct = basic.toolHandling.getApiTool(job, self.comp) + response = fct.send(self.job, self.comp, txt) + archivpath = os.path.join(tools.path_tool.composePattern(job, "{tcresult}/response", self.comp), filename) + +""" +get_response: +- channel (sync/async) + ... implement +- archivpath ( ergebnisse/comp/response ) +- envpath ( ./log) / envconn ( = in Request empfangen ) + +=== collect_testcase === +- envpath +- pattern +> readfiles +""" \ No newline at end of file diff --git a/tools/file_tool.py b/tools/file_tool.py new file mode 100644 index 0000000..09c5c70 --- /dev/null +++ b/tools/file_tool.py @@ -0,0 +1,295 @@ +# Funktionen zum Dateizugriff mit Suchen, Lesen, Schreiben +# ------------------------------------------------------------ +""" + +""" +import codecs +import json +import os +import os.path +import re +import time + +import xmltodict +import yaml +import platform +import basic.toolHandling + +import basic.program +import tools.data_const as D +#import tools.tdata_tool +import tools.date_tool + + +def getDump(obj): + result = "" + print(str(type(obj))) + result = vars(obj) + return str(result) + + +# if type(obj) == "__dict__" + + +def getFiles(msg, job, path, pattern, conn): + """ + search filenames in the directory - if conn is set search remote + :param msg: -- msg-Objekt + :param path: -- path - String + :param pattern: -- filename or pattern + :param conn: + :return: Array with filenames + """ + if conn is not None: + return getRemoteFiles(msg, path, pattern, conn) + # job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + out = [] + msg.debug(verify, "getFiles " + path + " , " + pattern) + if not os.path.exists(path): + return out + for f in os.listdir(path): + msg.debug(verify, "getFiles " + f) + if re.search(pattern, f): + msg.debug(verify, "match " + f) + out.append(f) + return out + + +def removeFiles(msg, path, pattern, conn): + """ + search filenames in the directory and removes it + - if conn is set search remote + :param msg: -- msg-Objekt + :param path: -- path - String + :param pattern: -- filename as Pattern + :param conn: + :return: Array filenames + """ + pass + + +def copyFiles(job, fileList, source, target, comp): + """ + copies files from source to target + :param job: + :param fileList: + :param source: + :param target: + :param comp: + :return: + """ + pass + + +def getRemoteFiles(msg, path, pattern, conn): + """ + search filenames in the directory - if conn is set search remote + :param msg: -- msg-Objekt + :param path: -- path - String + :param pattern: -- filename as Pattern + :param conn: + :return: Array filenames + """ + pass + + +def getFilesRec(msg, job, path, pattern): + """ + Sucht Dateien im Verzeichnis rekursiv + :param msg: -- msg-Objekt + :param path: -- Pfad - String + :param pattern: -- Dateiname als Pattern + :return: Array mit gefundenen Dateien, absoluter Pfad + """ + # job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + out = [] + msg.debug(verify, "getFilesRec " + path + " , " + pattern) + for (r, dirs, files) in os.walk(path): + for f in files: + msg.debug(verify, "getFilesRec " + f) + if re.search(pattern, f): + msg.debug(verify, "match " + f) + out.append(os.path.join(r, f)) + return out + + +def getTree(msg, job, pfad): + # job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + msg.debug(verify, "getTree " + pfad) + tree = {} + files = [] + for f in os.listdir(pfad): + if os.path.isDir(os.path.join(pfad, f)): + tree[f] = getTree(msg, job, os.path.join(pfad, f)) + elif os.path.isFile(os.path.join(pfad, f)): + files.append(f) + tree["_files_"] = files + return tree + + +def mkPaths(job, path, msg): + # job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + # modus = job.conf.paths["mode"] + dirname = os.path.dirname(path) + if os.path.exists(dirname): + return + os.makedirs(dirname, exist_ok=True) + + +def getFileEncoding(msg, job, path): + print("--- getFileEncoding " + path) + encodings = ['utf-8', 'iso-8859-1'] # add more + for e in encodings: + print(e) + try: + fh = codecs.open(path, 'r', encoding=e) + fh.readlines() + fh.seek(0) + except UnicodeDecodeError: + print('got unicode error with %s , trying different encoding' % e) + except: + print("except") + else: + print('opening the file with encoding: %s ' % e) + return e + return detectFileEncode(job, path, msg) + + +def detectFileEncode(job, path, msg): # return "" + # job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + print(path) + cntIso = 0 + cntUtf = 0 + j = 0 + CHAR_ISO = [196, 228, 214, 246, 220, 252, 191] + with open(path, 'rb') as file: + byte = file.read(1) + while (byte): + i = int.from_bytes(byte, "little") + # byte = file.read(1) + if (i in CHAR_ISO): + cntIso += 1 + if (i == 160): + pass + elif (i > 127): + cntUtf += 1 + j += 1 + l = i + byte = file.read(1) + file.close() + if (cntIso > cntUtf): + return 'iso-8859-1' + return 'utf-8' + + +def readFileLines(job, path, msg): + lines = readFileText(job, path, msg) + if isinstance(lines, (str)): + return lines.splitlines() + return [] + + +def readFileText(job, path, msg): + # job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + if not os.path.exists(path): + return "" + enc = detectFileEncode(job, path, msg) + with open(path, 'r', encoding=enc) as file: + text = file.read() + file.close() + return text + + +def getModTime(job, filepath): + out = "" + mtime = os.path.getmtime(filepath) + out = tools.date_tool.formatParsedDate(time.ctime(mtime), tools.date_tool.F_TIME_DEFAULT) + return out + + +def readFileDict(job, path, msg): + """ + reads and gets general a dict from any kind of filetyp + :param path: with extension of filetype + :param msg: optionally + :return: + """ + # 20220329 generalize + # job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + doc = {} + if not os.path.exists(path): + return doc + enc = detectFileEncode(job, path, msg) + if D.DFILE_TYPE_YML in path[-4:]: + with open(path, 'r', encoding=enc) as file: + doc = yaml.full_load(file) + file.close() + elif D.DFILE_TYPE_JSON in path[-5:]: + with open(path, 'r', encoding=enc) as file: + doc = json.load(file) + file.close() + elif D.DFILE_TYPE_XML in path[-4:]: + with open(path, 'r', encoding=enc) as file: + res = xmltodict.parse(file.read()) + # doc = dict(res) + doc = castOrderedDict(res) + file.close() + elif D.DFILE_TYPE_CSV in path[-5:]: + ffcts = basic.toolHandling.getFileTool(job, None, D.DFILE_TYPE_CSV) + #doc = tools.tdata_tool.getCsvSpec(msg, job, path, D.CSV_SPECTYPE_CONF) + doc = ffcts.loadFile(path) + # tools.tdata_tool.getCsvSpec(msg, job, path, D.CSV_SPECTYPE_CONF) + return doc + + +def castOrderedDict(res, job=None, key=""): + if isinstance(res, dict): + doc = dict(res) + for x in doc: + doc[x] = castOrderedDict(doc[x], job, x) + elif isinstance(res, list): + sublist = [] + for i in range(0, len(res)): + sublist.append(castOrderedDict(res[i], job, "")) + doc = sublist + else: + doc = res + return doc + + +def writeFileText(msg, job, path, text, enc="utf-8"): + # job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + mkPaths(job, path, msg) + with open(path, 'w', encoding=enc) as file: + file.write(text) + file.close() + + +def writeFileDict(msg, job, path, dict, enc="utf-8"): + # job = basic.program.Job.getInstance() + mkPaths(job, path, msg) + if D.DFILE_TYPE_YML in path[-5:]: + with open(path, 'w', encoding=enc) as file: + yaml.dump(dict, file) + file.close() + elif D.DFILE_TYPE_JSON in path[-5:]: + with open(path, 'w', encoding=enc) as file: + doc = json.dumps(dict, indent=4) + file.write(doc) + file.close() + elif D.DFILE_TYPE_XML in path[-4:]: + with open(path, 'w', encoding=enc) as file: + text = xmltodict.unparse(dict, pretty=True) + if "\n" + text + file.write(text) + file.close() + diff --git a/tools/filecsv_fcts.py b/tools/filecsv_fcts.py new file mode 100644 index 0000000..58ac16f --- /dev/null +++ b/tools/filecsv_fcts.py @@ -0,0 +1,246 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +import re +import basic.program +import tools.file_abstract +import basic.constants as B +import tools.data_const as D +import tools.file_tool + + +class FileFcts(tools.file_abstract.FileFcts): + + def __init__(self): + pass + + def loadFile(self, path): + """ + this function parses the text and translates it to dict + :param text: + :return: + """ + lines = tools.file_tool.readFileLines(self.job, path, self.getMsg()) + return self.parseCsv(self.getMsg(), self.job, lines) + + def parseCsv(self, msg, job, lines, ttype=""): + """ + + :param msg: + :param lines: + :param type: + :param job: + :return: + """ + tdata = {} + status = "start" + verbose = False + tableAttr = {} # table + tableDict = {} # table + for l in lines: + if verbose: print("lines "+l) + fields = splitFields(l, D.CSV_DELIMITER, job) + # check empty line, comment + if (len(fields) < 1) or (len(l.strip().replace(D.CSV_DELIMITER,"")) < 1): + status = "start" + continue + if (fields[0][0:1] == "#"): + continue + a = fields[0].lower().split(":") + # keywords option, step, table + if verbose: print(str(a)+" -- "+str(fields)) + tableAttr = setTableAttribute(tableAttr, a[0], fields[1], job) + if a[0].lower() in D.LIST_DATA_ATTR: + status = "TABLE_ALIAS" + if a[0].lower() == D.DATA_ATTR_KEY: + ttype = D.CSV_SPECTYPE_KEYS + continue + if (a[0].lower() in [D.CSV_BLOCK_HEAD]): + if verbose: print("head "+l) + setTdataLine(tdata, fields, D.CSV_BLOCK_HEAD, job) + status = "start" + continue + elif (a[0].lower() == D.CSV_BLOCK_OPTION): + if verbose: print("option " + l) + setTdataLine(tdata, fields, D.CSV_BLOCK_OPTION, job) + status = "start" + continue + elif (a[0].lower() == D.CSV_BLOCK_STEP): + if verbose: print("step "+l) + step = basic.step.parseStep(job, fields) + if D.CSV_BLOCK_STEP not in tdata: + tdata[D.CSV_BLOCK_STEP] = [] + tdata[D.CSV_BLOCK_STEP].append(step) + status = "start" + continue + elif (a[0].lower() == D.CSV_BLOCK_IMPORT): + if verbose: print("includes " + l) + if D.CSV_BLOCK_IMPORT not in tdata: + tdata[D.CSV_BLOCK_IMPORT] = [] + tdata[D.CSV_BLOCK_IMPORT].append(fields[1]) + status = "start" + continue + elif (a[0].lower() == D.CSV_BLOCK_TABLES) or (a[0].lower() in D.CSV_HEADER_START): + if verbose: print("tables "+l) + h = a + h[0] = B.DATA_NODE_TABLES + if ttype == D.CSV_SPECTYPE_CONF: + del h[0] + tableDict = getTdataContent(msg, tdata, h) + setTableHeader(tableDict, tableAttr, fields, ttype, job) + status = D.CSV_SPECTYPE_DATA + elif (status == D.CSV_SPECTYPE_DATA): + tableDict = getTdataContent(msg, tdata, h) + if verbose: print("setTableData "+str(h)+" "+str(tableDict)) + setTableData(tableDict, fields, ttype, job) + elif (status == "TABLE_ALIAS") and D.DATA_ATTR_ALIAS in tdata: + alias = tdata[D.DATA_ATTR_ALIAS] + b = alias.split(":") + h = [B.DATA_NODE_TABLES] + b + tableDict = getTdataContent(msg, tdata, h) + tableDict[D.DATA_ATTR_ALIAS] = alias + fields = [alias] + fields + setTableHeader(tableDict, tableAttr, fields, ttype, job) + status = D.CSV_SPECTYPE_DATA + if ttype == D.CSV_SPECTYPE_CONF: + header = [] + for k in tdata: + if k in D.LIST_DATA_ATTR: + continue + if B.DATA_NODE_DATA in tdata[k]: + tdata[k].pop(B.DATA_NODE_DATA) + for f in tdata[k]: + if f in [B.DATA_NODE_HEADER, "_hit"] + D.LIST_DATA_ATTR: + continue + header.append(f) + tdata[k][B.DATA_NODE_HEADER] = header + header = [] + if B.DATA_NODE_TABLES in tdata and B.DATA_NODE_TABLES in tdata[B.DATA_NODE_TABLES]: + for k in tdata[B.DATA_NODE_TABLES][B.DATA_NODE_TABLES]: + if k in tdata[B.DATA_NODE_TABLES]: + if verbose: print("Error") + else: + tdata[B.DATA_NODE_TABLES][k] = tdata[B.DATA_NODE_TABLES][B.DATA_NODE_TABLES][k] + tdata[B.DATA_NODE_TABLES].pop(B.DATA_NODE_TABLES) + return tdata + +def splitFields(line, delimiter, job): + out = [] + fields = line.split(delimiter) + for i in range(0, len(fields)): + if fields[i][0:1] == "#": + break + if re.match(r"^\"(.*)\"$", fields[i]): + fields[i] = fields[i][1:-1] + out.append(fields[i]) + return out + +def setTableAttribute(tableAttr, key, val, job): + for attr in D.LIST_DATA_ATTR: + if (key.lower() == attr): + tableAttr[attr] = val.strip() + tableAttr["_hit"] = True + return tableAttr + tableAttr["_hit"] = False + return tableAttr + +def setTdataLine(tdata, fields, block, job): + """ + sets field(s) into tdata as a key-value-pair + additional fields will be concatenate to a intern separated list + :param tdata: + :param fields: + :param block: + :param job: + :return: + """ + a = fields[0].lower().split(":") + a[0] = block # normalized key + val = "" + for i in range(1, len(fields)-1): + val += D.INTERNAL_DELIMITER+fields[i] + if len(val) > len(D.INTERNAL_DELIMITER): + val = val[len(D.INTERNAL_DELIMITER):] + setTdataContent(job.m, tdata, val, a) + return tdata + +def setTdataContent(msg, data, tabledata, path): + setTdataStructure(msg, data, path) + if len(path) == 2: + data[path[0]][path[1]] = tabledata + elif len(path) == 3: + data[path[0]][path[1]][path[2]] = tabledata + elif len(path) == 4: + data[path[0]][path[1]][path[2]][path[3]] = tabledata + +def setTdataStructure(msg, data, path): + if len(path) >= 1 and path[0] not in data: + data[path[0]] = {} + if len(path) >= 2 and path[1] not in data[path[0]]: + data[path[0]][path[1]] = {} + if len(path) >= 3 and path[2] not in data[path[0]][path[1]]: + data[path[0]][path[1]][path[2]] = {} + if len(path) >= 4 and path[3] not in data[path[0]][path[1]][path[2]]: + data[path[0]][path[1]][path[2]][path[3]] = {} + return data + +def getTdataContent(msg, data, path): + setTdataStructure(msg, data, path) + if len(path) == 2: + return data[path[0]][path[1]] + elif len(path) == 3: + return data[path[0]][path[1]][path[2]] + elif len(path) == 4: + return data[path[0]][path[1]][path[2]][path[3]] + elif len(path) == 1: + return data[path[0]] + else: + return None + +def setTableHeader(tableDict, tableAttr, fields, ttype, job): + header = [] + for i in range(1, len(fields)): + header.append(fields[i].strip()) + tableDict[B.DATA_NODE_HEADER] = header + for attr in tableAttr: + tableDict[attr] = tableAttr[attr] + # preparate the sub-structure for row-data + if ttype == D.CSV_SPECTYPE_TREE: + tableDict[B.DATA_NODE_DATA] = {} + elif ttype == D.CSV_SPECTYPE_KEYS: + tableDict[D.CSV_NODETYPE_KEYS] = {} + tableDict[D.DATA_ATTR_KEY] = 1 + if D.DATA_ATTR_KEY in tableAttr: + tableDict[D.DATA_ATTR_KEY] = header.index(tableAttr[D.DATA_ATTR_KEY]) + 1 + else: + tableDict[B.DATA_NODE_DATA] = [] + return tableDict + +def setTableData(tableDict, fields, ttype, job): + row = {} + if ttype == D.CSV_SPECTYPE_DATA and ":" not in fields[0] and D.DATA_ATTR_ALIAS in tableDict: + fields = [tableDict[D.DATA_ATTR_ALIAS]] + fields + i = 1 + for f in tableDict[B.DATA_NODE_HEADER]: + row[f] = fields[i].strip() + i += 1 + if ttype == D.CSV_SPECTYPE_DATA: + if B.ATTR_DATA_COMP in tableDict: + tcomps = tableDict[B.ATTR_DATA_COMP] + else: + tcomps = {} + row[B.ATTR_DATA_COMP] = {} + for c in fields[0].split(","): + a = c.split(":") + tcomps[a[0]] = a[1] + row[B.ATTR_DATA_COMP][a[0]] = a[1].strip() + tableDict[B.DATA_NODE_DATA].append(row) + tableDict[B.ATTR_DATA_COMP] = tcomps + elif ttype == D.CSV_SPECTYPE_KEYS: + tableDict[D.CSV_NODETYPE_KEYS][fields[tableDict[D.DATA_ATTR_KEY]].strip()] = row + elif ttype == D.CSV_SPECTYPE_CONF: + tableDict[fields[1]] = row + return tableDict diff --git a/tools/job_tool.py b/tools/job_tool.py new file mode 100644 index 0000000..7727b17 --- /dev/null +++ b/tools/job_tool.py @@ -0,0 +1,102 @@ +# GrundFunktionen zur Ablaufsteuerung +# +# -------------------------------------------------------- +""" +1. Programm -- implementiert in Main-Klasse +2. Anwndung -- steuert zu pruefende System [ in basis_Config ] +3. application -- steuert zu pruefende Maschine [ in dir/applicationen ] +4. release -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ] +5. ~Verz -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch +6. zyklus -- optional unterscheidet echte und entwicklungsLaeufe +7. Programmspezifische Parameter +8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2) +10. Laufart -- steuert die Verarbeitung; default echt +- echt-auto Lauf aus Automatisierung (1-7) +- test Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen +- echt-spez Wiederholung einer spezifischen Funktion (1-13) +- unit Ausfuehrung der Unittests +11. Modul -- schraenkt Verarbeitung auf parametriserte componenten ein +12. Funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein +13. Tool -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein +""" +import basic.program +import basic.constants as B +try: + import collect_testcase + import compare_testcase + import execute_testcase + import finish_testsuite + import init_testcase + import init_testsuite + import test_executer +except Exception as e: + pass +import tools.path_tool +import tools.file_tool +import components.tools.job_tool + + +def hasModul(komp): + #job = Job.getInstance() + return False +def hasFunction(fct): + #job = Job.getInstance() + return False +def hasTool(tool): + #job = Job.getInstance() + return False + +def createJob(parentJob, jobargs): + job = basic.program.Job("temp") # meaning temp + job.par.setParameterArgs(job, jobargs) + job.startJob() + return + +def startJobProcesses(job): + """ function to open processes like db-connection """ + components.tools.job_tool.startJobProcesses(job) + pass + +def stopJobProcesses(job): + """ function to close processes like db-connection """ + components.tools.job_tool.stopJobProcesses(job) + pass + +def startProcess(job, process): + print(str(process)) + path = tools.path_tool.getActualJsonPath(job) + print("------- "+path) + tools.file_tool.writeFileDict(job.m, job, path, process) + jobargs = {} + jobargs[B.PAR_APP] = process["app"] + jobargs[B.PAR_ENV] = process["env"] + if B.PAR_STEP in process: + jobargs[B.PAR_STEP] = process[B.PAR_STEP] + if B.PAR_TCDIR in process: + jobargs[B.PAR_TCDIR] = process[B.PAR_TCDIR] + jobargs[B.PAR_TESTCASE] = process["entity"] + elif B.PAR_TSDIR in process: + jobargs[B.PAR_TSDIR] = process[B.PAR_TSDIR] + jobargs[B.PAR_TESTSUITE] = process["entity"] + print("process-programm "+process["program"]) + myjob = basic.program.Job(process["program"], jobargs) + myjob.startJob() + if process["program"] == "init_testcase": + init_testcase.startPyJob(myjob) + elif process["program"] == "execute_testcase": + execute_testcase.startPyJob(myjob) + elif process["program"] == "collect_testcase": + collect_testcase.startPyJob(myjob) + elif process["program"] == "compare_testcase": + compare_testcase.startPyJob(myjob) + elif process["program"] == "init_testsuite": + init_testsuite.startPyJob(myjob) + elif process["program"] == "execute_testsuite": + print("execute_testsuite.startPyJob(myjob) not implemented") + elif process["program"] == "collect_testsuite": + print("collect_testsuite.startPyJob(myjob) not implemented") + elif process["program"] == "finish_testsuite": + finish_testsuite.startPyJob(myjob) + elif process["program"] == "test_executer": + test_executer.startPyJob(myjob) + diff --git a/tools/path_const.py b/tools/path_const.py new file mode 100644 index 0000000..5c6a0be --- /dev/null +++ b/tools/path_const.py @@ -0,0 +1,115 @@ +import basic.constants as B +# ------------------------------------------------------------- +# values and keywords +KEY_PRECOND = "precond" +KEY_POSTCOND = "postcond" +KEY_RESULT = "result" +KEY_ORIGIN = "origin" +KEY_PARTS = "parts" +KEY_SUMFILE = "sumfile" +KEY_BACKUP = "backup" +KEY_REFFILE = "reffile" +KEY_TESTCASE = "tc" +KEY_TESTSUITE = "ts" +KEY_CATALOG = "catalog" +KEY_DEBUGNAME = "debugname" +KEY_LOGNAME = "logname" +KEY_BASIC = "basic" +""" keyword for basic config in components """ +KEY_COMP = "comp" +""" keyword for individual component """ +KEY_TOOL = "tool" +""" keyword for technical tools """ +VAL_UTIL = "utils" +""" subdir for any technical tools """ +VAL_CONFIG = "config" +""" subdir for any place of config-files """ +VAL_COMPS = "components" +""" subdir for the plugin components """ +VAL_BASIC = "basic" +""" subdir for the basic job-framework """ +VAL_BASE_DATA = "data" +""" subdir for the basis data-folder """ +VAL_TDATA = "testdata" +""" subdir for the basis data-folder """ +# ------------------------------------------------------------- +# parameter with arguments +PAR_APP = "job.par." + B.PAR_APP +PAR_ENV = "job.par." + B.PAR_ENV +PAR_REL = "job.par." + B.PAR_REL +PAR_TSDIR = "job.par." + B.PAR_TSDIR +PAR_TCDIR = "job.par." + B.PAR_TCDIR +PAR_XPDIR = "job.par." + B.PAR_XPDIR +PAR_TDTYP = "job.par." + B.PAR_TDTYP +PAR_TDSRC = "job.par." + B.PAR_TDSRC +PAR_TDNAME = "job.par." + B.PAR_TDNAME +PAR_LOG = "job.par." + B.PAR_LOG +PAR_MODUS = "job.par." + B.PAR_MODUS +PAR_COMP = "job.par." + B.PAR_COMP +PAR_FCT = "job.par." + B.PAR_FCT +PAR_TOOL = "job.par." + B.PAR_TOOL +PAR_STEP = "job.par." + B.PAR_STEP +PAR_DESCRIPT = "job.par." + B.PAR_DESCRIPT +PAR_TESTCASE = "job.par." + B.PAR_TESTCASE +PAR_TESTCASES = "job.par." + B.PAR_TESTCASES +PAR_TESTSUITE = "job.par." + B.PAR_TESTSUITE +PAR_TCTIME = "job.par." + B.PAR_TCTIME +PAR_TSTIME = "job.par." + B.PAR_TSTIME +PAR_TESTINSTANCES = "job.par." + B.PAR_TESTINSTANCES + +# ------------------------------------------------------------- +# attributes +ATTR_PATH_MODE = "mode" +""" This constant defines the home-folder in filesystem of test """ +ATTR_PATH_HOME = "home" +""" This constant defines the home-folder in testing-filesystem """ +ATTR_PATH_DEBUG = "debugs" +""" This constant defines the debug-folder in testing-filesystem """ +ATTR_PATH_ARCHIV = "archiv" +""" This constant defines the folder in testing-filesystem for results and log of execution """ +ATTR_PATH_PROGRAM = "program" +""" This constant defines the program-folder in the workspace """ +ATTR_PATH_COMPONENTS = "components" +""" This constant defines the program-folder in the workspace """ +ATTR_PATH_ENV = "environment" +""" This constant defines the folder in testing-filesystem, used for configs related to environments """ +ATTR_PATH_RELEASE = "release" +""" This constant defines the folder in testing-filesystem, used for configs related to release """ +ATTR_PATH_TDATA = "testdata" +""" This constant defines the folder in testing-filesystem with the testcase-specifications """ +ATTR_PATH_TEMP = "temp" +""" This constant defines the debug-folder in testing-filesystem """ +ATTR_PATH_PATTN = "pattern" +""" This constant defines the debug-folder in testing-filesystem """ + + + +# ------------------------------------------------------------- +# structure - nodes +P_DEBUGS = "debugs" +P_ENVBASE = "envbase" +P_ENVLOG = "envlog" +P_ENVPARFILE = "envparfile" +P_TCBASE = "tcbase" +P_TCLOG = "tclog" +P_TCRESULT = "tcresult" +P_TCPARFILE = "tcparfile" +P_TCDIFF = "tcdiff" +P_TCPREDIFF = "tcprediff" +P_TCRUNDIFF = "tcrundiff" +P_TCPRECOND = "tcprecond" +P_TCPOSTCOND = "tcpostcond" +P_TSBASE = "tsbase" +P_TSLOG = "tslog" +P_TSPARFILE = "tsparfile" +P_TSSUM = "tssum" +P_XPBASE = "xpbase" +P_XPRESULT = "xpresult" +P_XPBACKUP = "xpbackup" + +# ------------------------------------------------------------- +# exception texts +EXP_COMP_MISSING = "Component is missing for {}" +""" excetion for the case that a specific component doesnt exist, 1 parameter (context) """ +EXP_CONFIG_MISSING = "Configuration is missing for {}" +""" excetion for the case that a specific configuration is missing, 1 parameter (context) """ \ No newline at end of file diff --git a/tools/path_tool.py b/tools/path_tool.py new file mode 100644 index 0000000..5cf0d74 --- /dev/null +++ b/tools/path_tool.py @@ -0,0 +1,320 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +""" In diesem Modul werden alle Funktionen zusammengefasst zur Generierung und Ermittlung von pathsn """ +import os.path +import sys +import basic.program +import tools.config_tool +import re +import basic.constants as B +import tools.path_const as P +import tools.date_tool +import getpass +TOOL_NAME = "path_tool" + +def getHome(): + home = os.getcwd() + if home[-4:] == "test" and home[-6:] != "datest": + home = home[0:-5] + if home[-10:] == "components": + home = home[0:-11] + if home[-6:] == "datest": + prgdir = home[-6:] + home = home[0:-7] + elif home[-7:] == "program": + prgdir = home[-7:] + home = home[0:-8] + return home + + +def getBasisConfigPath(): + home = os.getcwd() + a = home.split(os.path.sep) + for i in range(1, len(a)): + path = os.path.sep.join(a[0:-i]) + path = os.path.join(path, P.VAL_CONFIG, B.BASIS_FILE) + for format in tools.config_tool.CONFIG_FORMAT: + filepath = path+"."+format + if os.path.isfile(filepath): + return filepath + if os.path.exists(filepath): + return filepath + raise Exception("no basis-configuration found") + + +def getActualJsonPath(job): + username = getpass.getuser() + path = os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_DEBUG], username+"Job.json") + print("------ path "+path) + #if os.path.exists(path): + # return path + return path + +def getKeyValue(job, key, comp=None): + """ + this function gets the value for the key which relates to an attribute in the job or in the component + :param key: + :param comp: + :return: + """ + #job = basic.program.Job.getInstance() + try: + verify = job.getDebugLevel(TOOL_NAME)-4 + except: + verify = False + pt = PathConf.getInstance(job) + if verify: job.debug(verify, "getKeyValue " + key) + if 'job.par' in key: + val = job.getParameter(key[8:]) + return val + elif 'job.conf' in key: + val = job.conf.confs[B.SUBJECT_PATH][key[9:]] + if verify: job.debug(verify, val) + return val + elif 'job.' in key: + a = key[4:].split(":") + val = getattr(job, a[0]) + # only date with hours + if a[0] in ["start"]: + print("++++++++++++++"+str(val)) + val = tools.date_tool.formatParsedDate(str(val), tools.date_tool.F_LOG) + print("++++++++++++++"+val) + if len(a) > 1 and a[1] == "H": + val = val[0:-4]+"00" + if verify: job.debug(verify, val) + return val + # return job.conf.paths[key[9:]] + elif 'comp.' in key: + if comp is None: + raise Exception(P.EXP_COMP_MISSING.format(key)) + if tools.config_tool.hasAttr(comp.conf, key[5:]): + return tools.config_tool.getAttr(comp.conf, key[5:]) + if tools.config_tool.hasAttr(comp, key[5:]): + return tools.config_tool.getAttr(comp, key[5:]) + return "" + elif 'env.' in key: + if key[4:] in comp.conf["conn"]: + return comp.conf["conn"][key[4:]] + pass + elif key in pt.pattern: + return pt.pattern[key] + elif "time" in key and hasattr(job, "start"): + return getattr(job, "start") + else: + return "xx-"+key+"-xx" + + +def composePath(job, pathname, comp): + """ + this function composes a concrete path by the structured pathname + - the key of pathname is declared in path_const and the structure is configurated in config/path.yml. + :param pathname - plain keyword + :param comp: + :return: + """ + #job = basic.program.Job.getInstance() + verify = job.getDebugLevel(TOOL_NAME) + pt = PathConf.getInstance(job) + job.debug(verify, "composePath " + pathname + " zu " + str(pt) + "mit ") + job.debug(verify, str(pt.pattern)) + if pt.pattern[pathname]: + return composePattern(job, pt.pattern[pathname], comp) + else: + job.debug(verify, "in Pattern nicht vorhanden: " + pathname) + + +def composePattern(job, pattern, comp): + """ + the function composes the pattern to the standardarized path with the attributes + which are stored in the job and the component + - the key of pathname is declared in path_const and the structure is configurated in config/path.yml. + :param pattern: - keyword surroundet with {} + :param comp: + :return: path + """ + #job = basic.program.Job.getInstance() + try: + verify = job.getDebugLevel(TOOL_NAME) + except: + verify = False + verbose = not False + #job.debug(verify, "composePattern " + pattern) + max=5 + l = re.findall('\{.*?\}', pattern) + #job.debug(verify, l) + print(l) + for pat in l: + if verbose: print(str(max) + ": " + pattern + ": " + pat) + pit = getKeyValue(job, pat[1:-1], comp) + if verbose: print(str(pit) + ": " + pattern + ": " + pat) + #job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit) + pattern = pattern.replace(pat, pit) + #job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit) + while ("{" in pattern): + max = max-1 + #job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit) + pattern = composePattern(job, pattern, comp) + #job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit) + if (max < 3) : + break + return pattern + + +def rejoinPath(a, b="", c="", d="", e="", f=""): + """ + this function concatenates the arguments to a path in the correct format for the operating-system + :param a: + :param b: optional + :param c: optional + :param d: optional + :param e: optional + :param f: optional + :return: path + """ + work = a+"/"+b+"/"+c+"/"+d+"/"+e+"/"+f + if a.find("://") > 1: + protocol = True + else: + protocol = False + work = re.sub(r'\\', '/', work) + work = re.sub(r'\/', '/', work) + work = re.sub(r'//', '/', work) + while work[-1:] == "/": + work = work[0:-1] + l = work.split("/") + out = "" + for x in l: + if len(x) < 1: + continue + if protocol: + if len(out) < 1: + out = x + else: + out = out+"/"+x + else: + out = os.path.join(out, x) + if out[1:2] == ":" and out[2:3] != "\\": + out = out[0:2]+"\\"+out[2:] + elif protocol: + if "://" not in out or out.index("://") > 8 or out.index("://") < 1: + i = out.index(":/") + out = out[0:i+1] + "/" + out[i+1:] + pass + if not protocol and out.count("\\") < 1 and out[0:1] != "/" and out[0:2] != "..": + out = "/"+out + return out + + +def extractPattern(job, pathtyp, comp=None): + """ + this function extracts recoursively all parts of the pathstrucure as key and gets the values from the + job-parameter and job-configuration + :param pathtyp: the name of the path-structure + :param comp: + :return: dictionary of all part (key) with their valuess + """ + #job = basic.program.Job.getInstance() + verify = job.getDebugLevel(TOOL_NAME) + out = [] + pt = PathConf.getInstance(job) + pattern = pt.pattern[pathtyp] + work = pattern + while "{" in work: + i = work.index("{") + j = work.index("}") + pre = work[0:i] + pat = work[i+1:j] + job.debug(verify, work + " von " + str(i) + "-" + str(j) + " pre " + pre + "pat " + pat) + pit = getKeyValue(job, pat, comp) + tup = (pre, pat, pit) + out.append(tup) + work = work[j+1:] + return out + +def extractPath(job, pathtyp, path): + """ + this function extracts parts of a concrete structered path and stores the parts + as attributes into the actual job. So these attributes can read from the concrete + path instead of the related parameter-arguments. + It stores the values into the job-parameter + :param pathtyp: the structure of the concrete path + :param path: the concrete path - it should be the directory in the parameter of the job + :return: + """ + #job = basic.program.Job.getInstance() + patterlist = extractPattern(job, pathtyp) + verbose = False + work = path + i = 0 + if verbose: print("-- extractPatternList -- " + pathtyp + ":" + str(patterlist)) + for p in patterlist: + if len(p) < 1 : continue + delim = p[0] + key = p[1] + val = p[2] + nextdelim = "" + if i >= len(patterlist) - 1: + nextdelim = "" + else: + nextdelim = patterlist[i+1][0] + if verbose: print("xPath delim " + delim + " " + str(len(delim)) + ", " + nextdelim + " work " + work) + work = work[len(delim):] + if verbose: print("xPath key " + key + " i " + str(i) + " work " + work) + if val is not None: + if verbose: print("val not none " + val) + if val in work: + if verbose: print("val ok") + work = work.replace(val, "") + elif "time" in key and "job.par" in key: + prop = "" + if i < len(patterlist) - 1: + prop = work[0:work.index(nextdelim)] + else: + prop = work + key = key[8:] + if verbose: print("setprop " + key + " = " + prop) + if hasattr(job.par, key): delattr(job.par, key) + setattr(job.par, key, val) + else: + if verbose: print("val not not ok " + val + " zu " + key) + elif "job.par" in key: + prop = "" + if i < len(patterlist) - 1: + if verbose: print("job.par nextdelim " + nextdelim) + prop = work[0:work.index(nextdelim)] + else: + prop = work + key = key[8:] + if verbose: print("setprop " + key + " = " + prop) + if hasattr(job.par, key): delattr(job.par, key) + setattr(job.par, key, prop) + work = work.replace(prop, "") + else: + if verbose: print("val is none " + key) + i = i +1 + + +class PathConf: + """ + this class contains the structure-informations of the testrelevant directories + """ + __instance = None + def __init__(self, job=None): + #print('init pathConf') + confs = tools.config_tool.getConfig(job, "tool", "path") + self.pattern = confs["pattern"] + #print(self.pattern) + PathConf.__instance = self + + @staticmethod + def getInstance(job = None): + #print("PathConf getInstance " + str(PathConf.__instance)) + if (PathConf.__instance is None): + PathConf(job) + #print("PathConf getInstance " + str(PathConf.__instance)) + return PathConf.__instance + diff --git a/tools/value_tool.py b/tools/value_tool.py new file mode 100644 index 0000000..6c1773d --- /dev/null +++ b/tools/value_tool.py @@ -0,0 +1,76 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +""" In diesem Modul werden alle Funktionen zusammengefasst zur Generierung und Ermittlung von pathsn """ +import os.path +import sys +import basic.program +import tools.config_tool +import re +import basic.constants as B +import tools.path_const as P +import tools.date_tool +import getpass +TOOL_NAME = "value_tool" + +DOM_JOB = "job" +DOM_PAR = "par" +DOM_COMP = "comp" +DOM_CONF = "conf" +DOM_ENV = "env" + +def getKeyValue(job, key, comp=None): + """ + this function gets the value for the key which relates to an attribute in the job or in the component + :param key: + :param comp: + :return: + """ + #job = basic.program.Job.getInstance() + try: + verify = job.getDebugLevel(TOOL_NAME)-4 + except: + verify = False + #pt = PathConf.getInstance(job) + if verify: job.debug(verify, "getKeyValue " + key) + if DOM_JOB == key[0:3]: + if DOM_PAR in key[4:7]: + val = job.getParameter(key[8:]) + return val + elif DOM_CONF in key[4:8]: + val = job.conf.confs[B.SUBJECT_PATH][key[9:]] + if verify: job.debug(verify, val) + return val + elif 'job.' in key: + a = key[4:].split(":") + val = getattr(job, a[0]) + # only date with hours + if a[0] in ["start"]: + print("++++++++++++++"+str(val)) + val = tools.date_tool.formatParsedDate(str(val), tools.date_tool.F_LOG) + print("++++++++++++++"+val) + if len(a) > 1 and a[1] == "H": + val = val[0:-4]+"00" + if verify: job.debug(verify, val) + return val + # return job.conf.paths[key[9:]] + elif DOM_COMP in key: + if comp is None: + raise Exception(P.EXP_COMP_MISSING.format(key)) + if tools.config_tool.hasAttr(comp.conf, key[5:]): + return tools.config_tool.getAttr(comp.conf, key[5:]) + if tools.config_tool.hasAttr(comp, key[5:]): + return tools.config_tool.getAttr(comp, key[5:]) + return "" + elif DOM_ENV in key: + if key[4:] in comp.conf["conn"]: + return comp.conf["conn"][key[4:]] + pass + elif "time" in key and hasattr(job, "start"): + return getattr(job, "start") + else: + return "xx-"+key+"-xx" +