From 44dfb287fdae72d7fc48628ce834cb10308a3424 Mon Sep 17 00:00:00 2001 From: Ulrich Carmesin Date: Wed, 6 Apr 2022 17:35:08 +0200 Subject: [PATCH] further correction after tests --- basic/program.py | 99 +++++++++++++++-------- finish_testcase.py => collect_testcase.py | 22 +++-- components/testexec.py | 12 ++- finish_testsuite.py | 34 +++++++- init_testcase.py | 2 +- init_testset.py => init_testsuite.py | 2 +- test/test_date.py | 17 ++++ test_executer.py | 43 ++++++++-- utils/config_tool.py | 22 ++--- utils/db_abstract.py | 5 +- 10 files changed, 189 insertions(+), 69 deletions(-) rename finish_testcase.py => collect_testcase.py (65%) rename init_testset.py => init_testsuite.py (95%) diff --git a/basic/program.py b/basic/program.py index 38c9d5f..86e79fa 100644 --- a/basic/program.py +++ b/basic/program.py @@ -9,8 +9,8 @@ #import sys, getopt import argparse import copy - -import yaml, os +import yaml +import os from datetime import datetime import basic.constants as B import basic.message @@ -18,76 +18,84 @@ import basic.message import basic.componentHandling import utils.date_tool import utils.path_tool +import utils.file_tool jobdef = { "unit": { "pardef": "", - "pfilesource" : "", - "pfiletarget" : "envparfile", + "pfilesource": "", + "pfiletarget": "envparfile", "basedir": "envbase", "dirname": "envdir", "logdir": "{job.par.envdir}/{log}/log_{time}.txt" }, "check_environment": { "pardef": "", - "pfilesource" : "", - "pfiletarget" : "envparfile", + "pfilesource": "", + "pfiletarget": "envparfile", "basedir": "envbase", "dirname": "envdir", "logdir": "{job.par.envdir}/{log}/log_{time}.txt" }, "test_executer": { "pardef": "", - "pfilesource" : "tsparfile", - "pfiletarget" : "tsparfile", + "pfilesource": "tsparfile", + "pfiletarget": "tsparfile", "basedir": "tsbase", "dirname": "tsdir", "logdir": "{job.par.tsdir}/{log}/log_{time}.txt" }, "init_testsuite": { "pardef": "tsdir", # ,tdtyp,tdsrc,tdname", - "pfilesource" : "envparfile", - "pfiletarget" : "tsparfile", + "pfilesource": "envparfile", + "pfiletarget": "tsparfile", "basedir": "tsbase", "dirname": "tsdir", "logdir": "{job.par.tsdir}/{log}/log_{tstime}.txt" }, "init_testcase": { "pardef": "tcdir", # ",tdtyp,tdsrc,tdname", - "pfilesource" : "envparfile", - "pfiletarget" : "tcparfile", + "pfilesource": "envparfile", + "pfiletarget": "tcparfile", + "basedir": "tcbase", + "dirname": "tcdir", + "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" }, + "execute_testcase": { + "pardef": "tcdir", # ",tdtyp,tdsrc,tdname", + "pfilesource": "tcparfile", + "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" }, "collect_testcase": { "pardef": "tcdir", # ",tdtyp,tdsrc,tdname", - "pfilesource" : "envparfile", - "pfiletarget" : "tcparfile", + "pfilesource": "tcparfile", + "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" }, "compare_testcase": { "pardef": "tcdir", # ",tdtyp,tdsrc,tdname", - "pfilesource" : "envparfile", - "pfiletarget" : "tcparfile", + "pfilesource": "tcparfile", + "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" }, "test_system": { "pardef": "tcdir,tdtyp,tdsrc,tdname", "pfilesource": "tsparfile", - "pfiletarget": "tcparfile", + "pfiletarget": "tsparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"}, "finish_testcase": { - "pardef": "tcdir,tdtyp,tdsrc,tdname", - "pfilesource": "tsparfile", + "pardef": "tcdir", + "pfilesource": "tcparfile", "pfiletarget": "tcparfile", "basedir": "tcbase", "dirname": "tcdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"}, "finish_testsuite": { - "pardef": "tcdir,tdtyp,tdsrc,tdname", + "pardef": "tsdir", "pfilesource": "tsparfile", - "pfiletarget": "tcparfile", - "basedir": "tcbase", + "pfiletarget": "tssarfile", + "basedir": "tsbase", "dirname": "tsdir", "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt"} } @@ -141,9 +149,18 @@ class Job: def setProgram(self, program): self.program = program - self.par.program = program - self.par.setBasedir(program) - self.m.setLogdir("logTime") + basedir = jobdef[program]["basedir"] + if (self.par is not None): + setattr(self.par, "program", program) + setattr(self.par, "basedir", basedir) + parstring = getattr(self.par, "parstring") + parstring = parstring[parstring.find("--"):] + parstring = "python "+program+" "+parstring + setattr(self.par, "parstring", parstring) + if not hasattr(self.par, jobdef[program]["dirname"]): + setattr(self.par, jobdef[program]["dirname"], + utils.path_tool.composePattern("{"+basedir+"}", None)) + self.par.setParameterLoaded() def getInstance(): if (Job.__instance is not None): @@ -178,23 +195,30 @@ class Job: def dumpParameter(self): parpath = utils.path_tool.composePath(jobdef[self.program]["pfiletarget"], None) + if not os.path.exists(parpath): + return None output = {} - output["comps"] = basic.componentHandling.getComponentDict() + cconf = basic.componentHandling.getComponentDict() output["par"] = self.par.__dict__ - # print(str(output)) - with open(parpath, "w") as file: - doc = yaml.dump(output, file) - file.close() + output["comps"] = {} + for c in cconf: + output["comps"][c] = {} + for x in ["function", "conn"]: + output["comps"][c][x] = cconf[c][x] + if x == "conn" and "passwd" in cconf[c][x]: + output["comps"][c][x]["passwd"] = "xxxxx" + utils.file_tool.writeFileText(self.m, parpath, output) def loadParameter(self): output = {} if len(str(jobdef[self.program]["pfilesource"])) < 2: return None parpath = utils.path_tool.composePath(jobdef[self.program]["pfilesource"], None) - with open(parpath, "r") as file: - doc = yaml.full_load(file) - for k in doc.keys(): - output[k] = copy.deepcopy(doc[k]) + if not os.path.join(parpath): + return None + doc = utils.file_tool.readFileDict(parpath, self.m) + for k in doc.keys(): + output[k] = copy.deepcopy(doc[k]) return output def getParameter(self, parameter): @@ -256,7 +280,7 @@ class Parameter: if hasattr(self, jobdef[program]["dirname"]): utils.path_tool.extractPath(self.basedir, getattr(self, jobdef[program]["dirname"])) elif program != "unit": - dirpath = utils.path_tool.composePatttern("{"+jobdef[program]["basedir"]+"}", None) + dirpath = utils.path_tool.composePattern("{"+jobdef[program]["basedir"]+"}", None) setattr(self, jobdef[program]["dirname"], dirpath) else: self.basedir = "debugs" @@ -331,6 +355,9 @@ class Parameter: for k in vars(args): if getattr(args, k) is not None: self.setJobAttr(k , getattr(args, k)) + dirpath = self.getDirParameter() + if dirpath is not None: + utils.path_tool.extractPath(dirpath[0], dirpath[1]) def setParameterLoaded(self): job = Job.getInstance() @@ -369,6 +396,7 @@ class Configuration: def setConfiguration(self, path): self.confs = {} + doc = utils.file_tool.readFileLines(path, None) with open(path, "r") as file: doc = yaml.full_load(file) if "basic" in doc: @@ -377,6 +405,7 @@ class Configuration: else: for i, v in doc.items(): self.confs[i] = v + def setConfig(self, path, val): a = path.split(".") if len(a) == 1: diff --git a/finish_testcase.py b/collect_testcase.py similarity index 65% rename from finish_testcase.py rename to collect_testcase.py index 70c6f37..70f646d 100644 --- a/finish_testcase.py +++ b/collect_testcase.py @@ -3,11 +3,23 @@ import sys# # import jsonpickle # pip install jsonpickle import yaml # pip install pyyaml import basic.program -from basic.componentHandling import ComponentManager +import basic.componentHandling import basic.message import utils.tdata_tool -PROGRAM_NAME = "finish_testcase" +PROGRAM_NAME = "collect_testcase" + +def startPyJob(job): + cm = basic.componentHandling.ComponentManager.getInstance("init") + cm.initComponents() + comps = cm.getComponents(PROGRAM_NAME) + print(" relevant components for this job: " + str(comps)) + tdata = utils.tdata_tool.getTestdata() + job.m.setMsg("# components initialized with "+str(comps)) + for c in comps: + comp = cm.getComponent(c) + comp.collect_Artifacts("testcase") + job.m.merge(comp.m) if __name__ == '__main__': x = basic.program.Job(PROGRAM_NAME) @@ -20,10 +32,6 @@ if __name__ == '__main__': x.stopJob() exit(x.m.rc * (-1) + 3) # now in theory the program is runnable - cm = ComponentManager() - cm.initComponents() - comps = getComponents(PROGRAM_NAME) - print(" relevant components for this job: " + str(comps)) - tdata = utils.tdata_tool.getTestdata() + startPyJob(x) x.stopJob() # See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/components/testexec.py b/components/testexec.py index 9623897..00d4c8a 100644 --- a/components/testexec.py +++ b/components/testexec.py @@ -106,7 +106,17 @@ class Testexecuter(): self.m.setMsg("data loaded for " + self.name + " is OK") self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) - def read_TData(self, granularity): + def collect_TcArtifacts(self): + """ + collects the artifacts from the test-system. + the result is written as original in subfolder {tsorigin} + :return: + """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.read_TData("nachher", "testcase") + + def read_TData(self, subdir, granularity): """ :param granularity: diff --git a/finish_testsuite.py b/finish_testsuite.py index 4d3d006..5a4d216 100644 --- a/finish_testsuite.py +++ b/finish_testsuite.py @@ -1,15 +1,41 @@ # This is a sample Python script. +import os.path import sys# # import jsonpickle # pip install jsonpickle -import yaml # pip install pyyaml import basic.program as program -from basic.componentHandling import ComponentManager -import basic.message as message +import basic.componentHandling +import utils.file_tool +import utils.path_tool +import utils.report_tool +import utils.zip_tool # Press Umschalt+F10 to execute it or replace it with your code. # Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. +PROGRAM_NAME = "finish_testsuite" +def startPyJob(job): + cm = basic.componentHandling.ComponentManager.getInstance("init") + report = utils.report_tool.Report() + testinstances = {} + if hasattr(job.par, "testinstances"): + testinstances = getattr(job.par, "testinstances") + for tc in testinstances: + path = os.path.join(testinstances[tc], "Result.html") + text = utils.file_tool.readFileText(path, job.m) + report.extractTestcase(tc, text) + text = report.reportTestsuite() + path = os.path.join(job.par.tsdir, "Result.html") + utils.file_tool.writeFileText(job.m, path, text) + archivFolder = job.conf.confs["paths"]["archiv"] + tsFolder = os.path.join(archivFolder, "Testlauf") + tarfile = utils.zip_tool.openNewTarFile(tsFolder, job.par.usecase+"_"+job.par.tstime+".tar.gz") + utils.zip_tool.appendFolderIntoTarFile(tsFolder, job.par.usecase+"_"+job.par.tstime, tarfile) + for tc in testinstances: + tcSubFolder = testinstances[tc][len(archivFolder)+1:] + utils.zip_tool.appendFolderIntoTarFile(archivFolder, tcSubFolder, tarfile) + tarfile.close() + pass # Press the green button in the gutter to run the script. if __name__ == '__main__': @@ -20,9 +46,9 @@ if __name__ == '__main__': x.m.logInfo("hier eine LogInfo") x.m.logDebug("hier eine DbugMeldung") x.m.logDebug(str(vars(x.par)) + "\n" + str(vars(x.conf))) - cm = ComponentManager() if x.m.isRc("fatal"): x.stopJob() exit(x.m.rc * (-1) + 3) + startPyJob(x) x.stopJob() # See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/init_testcase.py b/init_testcase.py index 04ff7fe..685a70b 100644 --- a/init_testcase.py +++ b/init_testcase.py @@ -24,7 +24,7 @@ def startPyJob(job): comp.m.logInfo("------- "+comp.name+" ----------------------------------------") comp.reset_TData("testcase") comp.load_TData("testcase", testdata) - comp.read_TData("testcase") + comp.read_TData("vorher", "testcase") comp.m.logInfo("------- "+comp.name+" ----------------------------------------") job.m.merge(comp.m) print(str(comp)) diff --git a/init_testset.py b/init_testsuite.py similarity index 95% rename from init_testset.py rename to init_testsuite.py index bcab071..d31fd16 100644 --- a/init_testset.py +++ b/init_testsuite.py @@ -22,7 +22,7 @@ if __name__ == '__main__': # now in theory the program is runnable cm = ComponentManager() cm.initComponents() - comps = getComponents(PROGRAM_NAME) + comps = cm.getComponents(PROGRAM_NAME) print(" relevant components for this job: " + str(comps)) tdata = utils.tdata_tool.getTestdata() x.stopJob() diff --git a/test/test_date.py b/test/test_date.py index e69de29..66ba9c4 100644 --- a/test/test_date.py +++ b/test/test_date.py @@ -0,0 +1,17 @@ +import json +import unittest +import datetime +import utils.date_tool + +class MyTestCase(unittest.TestCase): + + def test_dateformat(self): + stime = datetime.datetime.now() + print(stime) + tdate = (2022, 2, 10) + sdate = datetime.datetime(tdate[0], tdate[1],tdate[2], 3, 32, 23) + sdate = datetime.datetime(stime) + print(sdate) + +if __name__ == '__main__': + unittest.main() diff --git a/test_executer.py b/test_executer.py index 2627a19..4e41631 100644 --- a/test_executer.py +++ b/test_executer.py @@ -2,47 +2,74 @@ from datetime import datetime import basic.program import basic.program as program import utils.tdata_tool - +import init_testcase +import init_testsuite +import execute_testcase +import finish_testsuite PROGRAM_NAME = "test_executer" +myjob = None +testcases = [] +testinstances = {} def getTime(): time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") return time -def start(job): - job.m.setMsg("# # # # # start executer # # # # # ") +def start(myjob): + myjob.m.setMsg("# # # # # start executer # # # # # ") tdata = utils.tdata_tool.getTestdata() - myjob = job + job = basic.program.Job("unit") + testcases = getattr(myjob.par, "testcases") + if not hasattr(myjob.par, "step"): + raise Exception("Parameter step is missing") for step in tdata["_steps"]: + if int(step["_nr"]) != int(getattr(myjob.par, "step")): + continue for arg in step["args"]: if arg == "start": if "testsuite" in step["args"][arg]: - jobargs = {"application": job.par.application, "environment": job.par.environment, - "usecase": job.par.usecase, "tstime": getTime()} + jobargs = {"application": myjob.par.application, "environment": myjob.par.environment, + "usecase": myjob.par.usecase, "tstime": utils.date_tool.getActdate(utils.date_tool.F_DIR)} job.popInstance() job = basic.program.Job("unit") job.par.setParameterArgs(jobargs) job.setProgram(step["args"][arg]) print("Job initialisiert "+job.program+" in Verzeichnis "+getattr(job.par, "tsdir")) + myjob.m.logInfo("Job initialisiert "+job.program+" in Verzeichnis "+getattr(job.par, "tsdir")) + dirname = getattr(job.par, "tsdir") job.stopJob(1) job.popInstance() basic.program.Job.pushInstance(myjob) print("ende") job = myjob if "testcase" in step["args"][arg]: - jobargs = {"application": job.par.application, "environment": job.par.environment, - "testcase": step["comp"], "tctime": getTime()} + if step["comp"] in testcases: + jobargs = {"application": myjob.par.application, "environment": myjob.par.environment, + "tcdir": testcases[step["comp"]] } + else: + jobargs = {"application": myjob.par.application, "environment": myjob.par.environment, + "testcase": step["comp"], "tctime": utils.date_tool.getActdate(utils.date_tool.F_DIR)} job.popInstance() job = basic.program.Job("unit") job.par.setParameterArgs(jobargs) job.setProgram(step["args"][arg]) print("Job initialisiert "+job.program+" in Verzeichnis "+getattr(job.par, "tcdir")) + myjob.m.logInfo("Job initialisiert "+job.program+" in Verzeichnis "+getattr(job.par, "tcdir")) + dirname = getattr(job.par, "tcdir") + testcases[step["comp"]] = dirname job.stopJob(1) job.popInstance() basic.program.Job.pushInstance(myjob) print("ende") job = myjob + if arg == "report": # testsuite + basic.program.Job.pushInstance(myjob) + pass + + basic.program.Job.pushInstance(myjob) + setattr(myjob.par, "testcases", testcases) + # myjob.stopJob(1) if __name__ == '__main__': diff --git a/utils/config_tool.py b/utils/config_tool.py index fcbcd56..caffcaf 100644 --- a/utils/config_tool.py +++ b/utils/config_tool.py @@ -12,9 +12,9 @@ try: except ImportError: print("ImportError: " + str(ImportError.with_traceback())) pass -import yaml import basic.componentHandling import utils.path_tool +import utils.file_tool import os.path import basic.constants as B @@ -69,7 +69,8 @@ def getConfigPath(modul, name, subname=""): job.debug(verify, "6 " + pathname) elif modul in COMP_FILES: for format in CONFIG_FORMAT: - pathname = os.path.join(job.conf.confs.get("paths").get("program"), "components", basic.componentHandling.getComponentFolder(name), modul+"."+format) + pathname = os.path.join(job.conf.confs.get("paths").get("program"), "components", + basic.componentHandling.getComponentFolder(name), modul+"."+format) if os.path.exists(pathname): return pathname if len(subname) > 1: @@ -145,16 +146,17 @@ def hasAttr(o, name): return True return False -def getConfig(modul, name): +def getConfig(modul, name, subname=""): job = basic.program.Job.getInstance() verify = job.getDebugLevel("config_tool")-4 - pathname = getConfigPath(modul, name) + msg = None + if hasattr(job, "m"): msg = job.m + pathname = getConfigPath(modul, name, subname) confs = {} job.debug(verify, "getConfig " + pathname) - with open(pathname, "r") as file: - job.debug(verify, "openfile") - doc = yaml.full_load(file) - for i, v in doc.items(): - job.debug(verify, "item " + str(i) + ": " + str(v)) - confs[i] = v + if len(pathname) < 1: + return confs + doc = utils.file_tool.readFileDict(pathname, msg) + for i, v in doc.items(): + confs[i] = v return confs \ No newline at end of file diff --git a/utils/db_abstract.py b/utils/db_abstract.py index 394da81..75c345d 100644 --- a/utils/db_abstract.py +++ b/utils/db_abstract.py @@ -142,13 +142,14 @@ class DbFcts(): return out - def selectTables(self): + def selectTables(self, subdir): """ method to delete rows from a database statement written in sql """ self.loadDdl() tdata = {} + tdata[subdir] = {} for t in self.comp.conf[B.DATA_NODE_DDL]: - tdata[t] = self.selectRows(t) + tdata[subdir][t] = self.selectRows(t) return tdata def selectRows(self, statement):