From 5d60dcc348d0e79e33f51bd2e9765240c225b31b Mon Sep 17 00:00:00 2001 From: Ulrich Carmesin Date: Sat, 9 Jul 2022 01:44:33 +0200 Subject: [PATCH] refactorings and bugfixes --- basic/componentHandling.py | 14 ++++++------ basic/constants.py | 12 +++++++++-- basic/program.py | 13 ++++++----- basic/testexec.py | 41 ++++++++++++++++++++--------------- test/test_toolhandling.py | 2 +- test/testtools.py | 7 +++--- utils/config/path.yml | 44 ++++++++++++++++++++++++++++++++++++++ utils/config_tool.py | 29 +++++++++++++++++++++---- utils/conn_tool.py | 4 ++-- utils/data_const.py | 3 +-- utils/date_tool.py | 18 +++++++++------- utils/file_abstract.py | 31 +++++++++++++++++++++++++++ utils/file_tool.py | 1 + utils/path_tool.py | 3 ++- 14 files changed, 171 insertions(+), 51 deletions(-) create mode 100644 utils/config/path.yml diff --git a/basic/componentHandling.py b/basic/componentHandling.py index 3baf196..697636e 100644 --- a/basic/componentHandling.py +++ b/basic/componentHandling.py @@ -23,6 +23,7 @@ import basic.component import importlib import copy import basic.constants as B +import utils.data_const as D comps = {} PARAM_NOSUBNODE = [B.SUBJECT_ARTS, "components", "instance"] @@ -86,6 +87,7 @@ class ComponentManager: return for k in job.conf.confs[B.SUBJECT_APPS].get(anw): job.m.logDebug("applicationscomponente -- " + k + ":") + print("applicationscomponente -- " + k + ":") self.createComponent(k, 0, "") @@ -192,17 +194,17 @@ class ComponentManager: c.conf[B.SUBJECT_CONN] = conns[i] c.init() if parContent is not None: - print("createComponent 5 a " + name + " : " + str(parContent)) - if name in parContent["comps"]: - for k in parContent["comps"][name].keys(): - c.conf[k] = parContent["comps"][name][k] + print("createComponent 5 a " + compName + " : " + str(parContent)) + if B.SUBJECT_COMPS in parContent and compName in parContent[B.SUBJECT_COMPS]: + for k in parContent[B.SUBJECT_COMPS][compName].keys(): + c.conf[k] = parContent[B.SUBJECT_COMPS][compName][k] if B.SUBJECT_ARTS in c.conf and B.TOPIC_NODE_DB in c.conf[B.SUBJECT_ARTS]: if not B.DATA_NODE_DDL in c.conf: c.conf[B.DATA_NODE_DDL] = {} for table in c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]: - if table in ["type"]: + if table in B.LIST_DB_ATTR: continue - conf = utils.config_tool.getConfig("DATASTRUCTURE", c.name, table) + conf = utils.config_tool.getConfig(D.DDL_FILENAME, compName, table) if B.DATA_NODE_TABLES in conf and table in conf[B.DATA_NODE_TABLES]: c.conf[B.DATA_NODE_DDL][table] = conf[B.DATA_NODE_TABLES][table] elif table in conf: diff --git a/basic/constants.py b/basic/constants.py index eed3d06..66f5f67 100644 --- a/basic/constants.py +++ b/basic/constants.py @@ -102,6 +102,9 @@ DATA_NODE_PAR = "par" TOPIC_NODE_DB = "db" # testexec, db_abstr ATTR_DB_TYPE = "type" +""" must attribute for the type of the database """ +ATTR_DB_RESET = "reset" +""" optional attribute in order to use a different technical name for the db-table """ ATTR_DB_PARTITION = "partitioned" """ optional attribute if table is partitioned - this keyword delimited by "+" will be replaced by partition-names which are parametrized """ @@ -111,7 +114,10 @@ ATTR_DB_SCHEMA = "schema" """ optional attribute for technical name of the schema """ ATTR_DB_TABNAME = "tabname" """ optional attribute in order to use a different technical name for the db-table """ -LIST_DB_ATTR = [ATTR_DB_TYPE, ATTR_DB_PARTITION, ATTR_DB_DATABASE, ATTR_DB_SCHEMA, ATTR_DB_TABNAME] +ATTR_DB_PRESTEP = "prestep" +""" optional attribute to define a source-table for this table """ +LIST_DB_ATTR = [ATTR_DB_TYPE, ATTR_DB_RESET, ATTR_DB_PARTITION, + ATTR_DB_DATABASE, ATTR_DB_SCHEMA, ATTR_DB_TABNAME, ATTR_DB_PRESTEP] ATTR_DB_CONN_JAR = "conn_jar_name" """ optional attribute for connection-jar-file instead of connection by ip, port """ ATTR_CONN_HOST = "hostname" @@ -171,7 +177,8 @@ SUBJECT_INST = "instance" # | | | | x | CompHanldin ATTR_INST_CNT = "count" # | | | | x | CompHanlding ATTR_INST_SGL = "single" LIST_INST_ATTR = [ATTR_INST_CNT, ATTR_INST_SGL] -ATTR_INST_SUBCOMP = "components" +SUBJECT_COMPS = "components" +ATTR_INST_SUBCOMP = SUBJECT_COMPS #SUBJECT_FCT = "function" # | | | | x | main-programs SUBJECT_ARTS = "artifact" # | | | | x | Component @@ -195,6 +202,7 @@ RULE_ACCEPTANCE = "acceptance" # | | | | x | tools_match ATTR_STEP_ARGS = "args" ATTR_DATA_REF = "_nr" +ATTR_DATA_COMP = "_comp" # ------------------------------------------------------------- # exception texts diff --git a/basic/program.py b/basic/program.py index 65706f0..9744228 100644 --- a/basic/program.py +++ b/basic/program.py @@ -245,13 +245,13 @@ class Job: if len(cconf) < 1: utils.file_tool.writeFileDict(self.m, parpath, output) return - output["comps"] = {} + output[B.SUBJECT_COMPS] = {} for c in cconf: - output["comps"][c] = {} + output[B.SUBJECT_COMPS][c] = {} for x in ["function", "conn"]: - output["comps"][c][x] = cconf[c][x] - if x == "conn" and "passwd" in cconf[c][x]: - cconf["comps"][c][x]["passwd"] = "xxxxx" + output[B.SUBJECT_COMPS][c][x] = cconf[c][x] + if x == B.SUBJECT_CONN and "passwd" in cconf[c][x]: + cconf[B.SUBJECT_COMPS][c][x]["passwd"] = "xxxxx" utils.file_tool.writeFileDict(self.m, parpath, output) @@ -441,6 +441,9 @@ class Configuration: self.program = program print (f"job initialisiert {self.program}") if program == "unit": + if (os.path.exists(utils.path_tool.rejoinPath("..", "..", "config", B.BASIS_FILE))): + self.setConfiguration(utils.path_tool.rejoinPath("..", "..", "config", B.BASIS_FILE)) + return if (os.path.exists(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))): self.setConfiguration(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE)) return diff --git a/basic/testexec.py b/basic/testexec.py index f514fa8..89c61d2 100644 --- a/basic/testexec.py +++ b/basic/testexec.py @@ -74,7 +74,7 @@ class Testexecuter(): + datetime.now().strftime("%Y%m%d_%H%M%S")+" for " + str(self.name).upper()) if B.ATTR_ARTS_LOG in self.conf[B.SUBJECT_ARTS]: self.m.logInfo("log rotate in "+ self.name) - if "db" in self.conf[B.SUBJECT_ARTS]: + if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS]: self.m.logInfo("delete content "+ self.name) dbi = basic.toolHandling.getDbTool(self) dbi.deleteTables() @@ -85,7 +85,7 @@ class Testexecuter(): self.m.setMsg("resetInstance for " + self.name + " is OK") self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) - def load_TData(self, granularity, testdata): + def load_TData(self, granularity, tdata): """ the testdata will be loaded into the componend especially into databses or with import-functions of the component @@ -96,13 +96,14 @@ class Testexecuter(): job = basic.program.Job.getInstance() verify = -1+job.getDebugLevel(self.name) self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) - plainname = basic.componentHandling.getPlainCompname(self.name) - if plainname in testdata: - print("plainname in testdata "+plainname) - if "db" in self.conf[B.SUBJECT_ARTS]: - self.m.logInfo("delete content "+ self.name) - dbi = basic.toolHandling.getDbTool(self) - dbi.insertTables(testdata) + if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS] and B.DATA_NODE_TABLES in tdata: + for t in tdata[B.DATA_NODE_TABLES]: + print (t) + if utils.db_abstract.isCompTable(self, tdata, t): + self.m.logInfo("insert content "+ self.name) + dbi = basic.toolHandling.getDbTool(self) + dbi.insertTables(tdata) + break self.m.setMsg("data loaded for " + self.name + " is OK") self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) @@ -125,11 +126,17 @@ class Testexecuter(): job = basic.program.Job.getInstance() verify = -1+job.getDebugLevel(self.name) self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) - if "db" in self.conf[B.SUBJECT_ARTS]: + if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS]: self.m.logInfo("select db-content "+ self.name) dbi = basic.toolHandling.getDbTool(self) - tdata = dbi.selectTables() - utils.tdata_tool.writeCsvData("", tdata, self) + data = dbi.selectTables(subdir) + print("ppp") + #data = {} + for t in data[subdir]: + data[B.DATA_NODE_TABLES] = {} + data[B.DATA_NODE_TABLES][t] = data[subdir][t] + utils.tdata_tool.writeCsvData(utils.path_tool.rejoinPath( + utils.path_tool.composePattern("{tcresult}", self), subdir, t+".csv"), data, self) if B.ATTR_ARTS_LOB in self.conf[B.SUBJECT_ARTS]: self.m.logInfo("check lob if is deleted with flaskdb "+ self.name) self.m.setMsg("readInstance for " + self.name + " is OK") @@ -305,15 +312,15 @@ class Testexecuter(): cm = basic.componentHandling.ComponentManager.getInstance() data = {} matching = utils.match_tool.Matching() - if "db" in self.conf[B.SUBJECT_ARTS]: - for t in self.conf[B.SUBJECT_ARTS]["db"]: - if t in ["type"]: + if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS]: + for t in self.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]: + if t in B.LIST_DB_ATTR: continue # fill each data into matching-object for side in M.MATCH_SIDES: if side == M.MATCH_SIDE_PRESTEP: - if "prestep" in self.conf[B.SUBJECT_ARTS]["db"][t]: - a = self.conf[B.SUBJECT_ARTS]["db"][t]["prestep"].split(":") + if B.ATTR_DB_PRESTEP in self.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][t]: + a = self.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][t][B.ATTR_DB_PRESTEP].split(":") if a[0] != self.name: comp = cm.getComponent(a[0]) else: diff --git a/test/test_toolhandling.py b/test/test_toolhandling.py index ace4419..de455a0 100644 --- a/test/test_toolhandling.py +++ b/test/test_toolhandling.py @@ -15,7 +15,7 @@ class MyTestCase(unittest.TestCase): def test_toolhandling(self): job = basic.program.Job("unit") - args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", + args = {B.PAR_APP: "TEST", B.PAR_ENV: "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", "modus": "unit"} job.par.setParameterArgs(args) #t = basic.toolHandling.ToolManager() diff --git a/test/testtools.py b/test/testtools.py index db5b407..ef93e4b 100644 --- a/test/testtools.py +++ b/test/testtools.py @@ -7,7 +7,8 @@ import test.constants as T DEFAULT_GRAN = "tc" DEFAULT_APP = "TESTAPP" DEFAULT_ENV = "ENV01" -DEFAULT_DIR = T.DATA_PATH + "/tdata" +DEFAULT_DATA_DIR = T.DATA_PATH + "/tdata" +DEFAULT_ARCHIV_DIR = T.DATA_PATH + "/lauf" DEFAULT_TIME = "2022-03-19_12-09-09" DEFAULT_MODE = "unit" gran = "" @@ -68,9 +69,9 @@ def getJob(pgran="", papp="", penv="", ptstamp="", pmode=""): else: mode = pmode if gran == "tc": - path = DEFAULT_DIR+"/TC0001/"+tstamp + path = DEFAULT_ARCHIV_DIR + "/TC0001/" + tstamp elif gran == "ts": - path = DEFAULT_DIR + "/testlauf/TST001_" + tstamp + path = DEFAULT_ARCHIV_DIR + "/testlauf/TST001_" + tstamp job = basic.program.Job("unit") job.conf.confs[B.SUBJECT_PATH]["components"] = T.COMP_PATH args = {"application": app, "environment": env, "modus": mode, gran+"time": tstamp, diff --git a/utils/config/path.yml b/utils/config/path.yml new file mode 100644 index 0000000..4c6a9c7 --- /dev/null +++ b/utils/config/path.yml @@ -0,0 +1,44 @@ +# +pattern: + # Keywords + log: log + parfile: PARAMETER_{job.par.application}_{job.par.environment}.yml + precond: vorher + postcond: nachher + diff: diff_fach + prediff: diff_init + rundiff: diff_ablauf + result: Ergebnisse/{comp.name} + origin: original + parts: teilergebnisse + sumfile: xxx + backup: backup + reffile: Herkunft.txt + tc: testfall + ts: testlauf + debugname: debug + logname: log + debugs: "{job.conf.home}/test/log" + # environment + envbase: "{job.conf.environment}/{job.par.environment}" + envlog: "{envbase}/{log}" + envparfile: "{envbase}/{parfile}" + # testcase + tcbase: "{job.conf.archiv}/{job.par.testcase}/{job.par.tctime}" + tclog: "{tcbase}/{log}" + tcresult: "{tcbase}/{result}" + tcparfile: "{tcbase}/{parfile}" + tcdiff: "{tcresult}/{diff}" + tcprediff: "{tcresult}/{prediff}" + tcrundiff: "{tcresult}/{rundiff}" + tcprecond: "{tcresult}/{precond}" + tcpostcond: "{tcresult}/{postcond}" + # testset + tsbase: "{job.conf.archiv}/{ts}/{job.par.usecase}_{job.par.tstime}" + tslog: "{tsbase}/{log}" + tsparfile: "{tsbase}/{parfile}" + tssum: "{tsbase}/Ergebnis" + # expectation-result rs + xpbase: "{job.conf.expect}/{job.par.branch}" + xpresult: "{xpbase}/{result}" + xpbackup: "{xpbase}/{result}" diff --git a/utils/config_tool.py b/utils/config_tool.py index 29640a2..816a64c 100644 --- a/utils/config_tool.py +++ b/utils/config_tool.py @@ -72,8 +72,8 @@ def getConfigPath(modul, name, subname=""): if os.path.exists(pathname): return pathname for format in CONFIG_FORMAT: - pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), - P.VAL_COMPS, basic.componentHandling.getComponentFolder(name), "CONFIG." + format) + pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), + basic.componentHandling.getComponentFolder(name), "CONFIG." + format) job.debug(verify, "5 " + pathname) if os.path.exists(pathname): return pathname @@ -82,13 +82,13 @@ def getConfigPath(modul, name, subname=""): elif modul in COMP_FILES: # for example DATASTRUCURE or the table for format in CONFIG_FORMAT: - pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_COMPS, + pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), basic.componentHandling.getComponentFolder(name), modul+"."+format) if os.path.exists(pathname): return pathname for format in CONFIG_FORMAT: if len(subname) > 1: - pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_COMPS, + pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), basic.componentHandling.getComponentFolder(name), subname+"."+format) if os.path.exists(pathname): return pathname @@ -184,6 +184,17 @@ def getConfig(modul, name, subname=""): if len(pathname) < 1: return confs doc = utils.file_tool.readFileDict(pathname, msg) + if modul == D.DDL_FILENAME: + # in csv the root is the subname + # from the Dict-structure of DDL_FILENAME pick the substructure of the subname + keys = list(doc.keys()) + if subname not in keys and len(keys) == 1: + doc0 = doc[keys[0]] + doc = doc0 + keys = list(doc.keys()) + if subname in keys: + doc0 = doc[subname] + doc = doc0 for i, v in doc.items(): confs[i] = v return confs @@ -197,6 +208,8 @@ def mergeConn(msg, conf, conn): :param conn: :return: """ + if B.SUBJECT_INST not in conf: + conf[B.SUBJECT_INST] = {} for a in conn[B.SUBJECT_INST]: conf[B.SUBJECT_INST][a] = conn[B.SUBJECT_INST][a] for topic in [B.TOPIC_NODE_DB, B.TOPIC_NODE_CLI, B.TOPIC_NODE_API, B.TOPIC_NODE_FILE]: @@ -210,20 +223,28 @@ def mergeConn(msg, conf, conn): list = B.LIST_API_ATTR if topic == B.TOPIC_NODE_FILE: list = B.LIST_FILE_ATTR + print(" --- merge-conn " + topic + " " + str(list)) for a in conf[B.SUBJECT_ARTS][topic]: + if topic not in conn: + continue if a in list: if a in conn[topic]: conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] else: for b in conf[B.SUBJECT_ARTS][topic][a]: + print(" --- merge-conn b " + topic + " " + a+" "+b) if b not in list: msg.logError("not-topic-attribute in topic-connection: "+topic+", "+b) continue + if a not in conn[topic]: + continue if b in conn[topic][a]: conf[B.SUBJECT_ARTS][topic][a][b] = conn[topic][a][b] for a in list: if topic not in conn: break + if topic not in conn: + continue if a in conn[topic]: conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] return conf \ No newline at end of file diff --git a/utils/conn_tool.py b/utils/conn_tool.py index 6380634..001acaf 100644 --- a/utils/conn_tool.py +++ b/utils/conn_tool.py @@ -14,7 +14,7 @@ def getConnection(comp, nr): verify = job.getDebugLevel("conn_tool") conn = {} if job.conf.confs.get("tools").get("connsrc") == D.DFILE_TYPE_YML: - conn = utils.config_tool.getConfig("tool", "conn") + conn = utils.config_tool.getConfig("tool", B.SUBJECT_CONN) xtypes = None if ("types" in conn["env"][comp]): xtypes = conn["env"][comp]["types"] @@ -52,7 +52,7 @@ def getConnections(comp): conn = {} conns = [] if job.conf.confs.get("tools").get("connsrc") in [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV]: - conn = utils.config_tool.getConfig("tool", "conn") + conn = utils.config_tool.getConfig("tool", B.SUBJECT_CONN) if not comp in conn["env"]: job.m.setFatal("Conn-Tool: Comp not configured " + comp) elif job.conf.confs.get("tools").get("connsrc") == "flaskdb": diff --git a/utils/data_const.py b/utils/data_const.py index 1080ca7..4f194a1 100644 --- a/utils/data_const.py +++ b/utils/data_const.py @@ -12,6 +12,7 @@ TYPE_FLOAT = "float" TYPE_DOUBLE = "double" TYPE_DATE = "date" TYPE_TIME = "time" + # fields in DDL DDL_FNULLABLE = "nullable" DDL_FNAME = "field" @@ -54,5 +55,3 @@ DEFAULT_DB_PARTITION = "n" DEFAULT_DB_CONN_JAR = "n" """ attribute for connection-jar-file instead of connection by ip, port """ -ATTR_TABLE_DATE = "date" -ATTR_TABLE_CNT = "count" diff --git a/utils/date_tool.py b/utils/date_tool.py index fd0918a..226e90c 100644 --- a/utils/date_tool.py +++ b/utils/date_tool.py @@ -99,13 +99,14 @@ def parseDate(instring): sec = 0 if instring[0:2] == "{(" and instring[-2:] == ")}": return parseFormula(instring) - for d in ["_", " "]: - if d in instring and instring.find(d) > 8: - dstring = instring[0:instring.find(d)] - tstring = instring[instring.find(d)+1:] - dres = parseDate(dstring) - tres = parseDate(tstring) - return (dres[0], dres[1], dres[2], tres[3], tres[4], tres[5]) + if len(instring) > 8: + for d in ["_", " "]: + if d in instring and instring.find(d) > 8: + dstring = instring[0:instring.find(d)] + tstring = instring[instring.find(d)+1:] + dres = parseDate(dstring) + tres = parseDate(tstring) + return (dres[0], dres[1], dres[2], tres[3], tres[4], tres[5]) if re.match(r"\d{4}[-./]\d{2}[-./]\d{2}", instring): res = re.match(r"(\d{4})[-./](\d{2})[-./](\d{2})", instring) year = int(res.group(1)) @@ -128,4 +129,5 @@ def parseDate(instring): hour = int(res.group(1)) min = int(res.group(2)) sec = int(res.group(3)) - return (year, mon, day, hour, min, sec) \ No newline at end of file + return (year, mon, day, hour, min, sec) + return (year, mon, day, hour, min, sec) \ No newline at end of file diff --git a/utils/file_abstract.py b/utils/file_abstract.py index e69de29..6a57471 100644 --- a/utils/file_abstract.py +++ b/utils/file_abstract.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +import basic.program +import basic.catalog +import utils.config_tool +import basic.constants as B +import utils.data_const as D +import utils.date_tool + +class FileFcts(): + """ + this is an abstract class + """ + def __init__(self): + pass + + + def parseText(self, text): + """ + this function parses the text and translates it to dict + :param text: + :return: + """ + + + def file2dict(self): + pass \ No newline at end of file diff --git a/utils/file_tool.py b/utils/file_tool.py index 2514de7..44cfb97 100644 --- a/utils/file_tool.py +++ b/utils/file_tool.py @@ -222,3 +222,4 @@ def writeFileDict(msg, path, dict, enc="utf-8"): doc = json.dumps(file, indent=4) file.write(doc) file.close() + diff --git a/utils/path_tool.py b/utils/path_tool.py index 1f2387c..10b93a3 100644 --- a/utils/path_tool.py +++ b/utils/path_tool.py @@ -43,7 +43,8 @@ def getKeyValue(key, comp=None): return utils.config_tool.getAttr(comp, key[5:]) return "" elif 'env.' in key: - #if key[4:] + if key[4:] in comp.conf["conn"]: + return comp.conf["conn"][key[4:]] pass elif (pt.pattern): return pt.pattern[key]