From 21d0813e9bcd9253a6c6d00e876a11a0ac0e0c0c Mon Sep 17 00:00:00 2001 From: Ulrich Carmesin Date: Wed, 6 Apr 2022 17:08:18 +0200 Subject: [PATCH] test and corrections --- basic/componentHandling.py | 19 ++++++---- basic/message.py | 8 ++++- compare_testcase.py | 24 +++++++------ init_testcase.py | 16 ++++----- utils/file_tool.py | 73 +++++++++++++++++++++++++++++++++----- utils/path_tool.py | 14 ++++---- utils/report_tool.py | 25 ++++++++----- utils/zip_tool.py | 57 +++++++++++++++++------------ 8 files changed, 164 insertions(+), 72 deletions(-) diff --git a/basic/componentHandling.py b/basic/componentHandling.py index 0439398..222b7dd 100644 --- a/basic/componentHandling.py +++ b/basic/componentHandling.py @@ -70,6 +70,7 @@ class ComponentManager: job = basic.program.Job.getInstance() job.m.logDebug("applicationscomponente -- " + str(type(job.par))) self.components = {} + ComponentManager.__instance = self print ("init ComponentHandling "+str(self)) def initComponents(self): @@ -109,9 +110,11 @@ class ComponentManager: return out @staticmethod - def getInstance(): + def getInstance(init="N"): if (ComponentManager.__instance is not None): return ComponentManager.__instance + elif (init != "N"): + return ComponentManager() else: raise Exception("Klasse noch nicht initialisiert") @@ -186,9 +189,14 @@ class ComponentManager: if B.SUBJECT_ARTS in c.conf and B.TOPIC_NODE_DB in c.conf[B.SUBJECT_ARTS]: if not B.DATA_NODE_DDL in c.conf: c.conf[B.DATA_NODE_DDL] = {} - for table in c.conf[B.SUBJECT_ARTS][B.SUBJECT_ARTS]: + for table in c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]: + if table in ["type"]: + continue conf = utils.config_tool.getConfig("DATASTRUCTURE", c.name, table) - c.conf[B.DATA_NODE_DDL][table] = conf + if "tabelle" in conf and table in conf["tabelle"]: + c.conf[B.DATA_NODE_DDL][table] = conf["tabelle"][table] + else: + c.conf[B.DATA_NODE_DDL][table] = conf comps[name] = c return c @@ -225,11 +233,8 @@ def getComponentDict(): out = {} for c in comps: out[comps[c].name] = {} - print("getCompDict " + comps[c].name) for k in comps[c].conf.keys(): - print("getCompDict " + k) if isParameterSubnode(k): # "artifact" in k or "components" in k or "instance" in k: - print("getCompDict -b- " + k) out[comps[c].name][k] = copy.deepcopy(comps[c].conf[k]) return out @@ -244,4 +249,4 @@ def isParameterSubnode(key): def getPlainCompname(name): if "_0" in name: return name[0:-3] - return name \ No newline at end of file + return name diff --git a/basic/message.py b/basic/message.py index c9b4397..b378d88 100644 --- a/basic/message.py +++ b/basic/message.py @@ -75,7 +75,10 @@ class Message: else: debugpath = job.conf.confs["paths"]["debugs"] + "/debug_" + logTime[0:-4] + "00.txt" print ("debugpathx "+debugpath) - self.debugfile = open(debugpath, "a") + if os.path.exists(debugpath): + self.debugfile = open(debugpath, "a") + else: + self.debugfile = open(debugpath, "w") self.debug(verify, "> > > debugfile geoeffnet zu " + job.program + " mit " + debugpath) # init logfile - except for components or unittest self.logDebug("logfile " + str(componente) + ", " + str(job.par.basedir)) @@ -231,6 +234,9 @@ class Message: if (int(prio) < int(self.level)+1) : #and self.debugfile.closed == False: self.debugfile.write(text + "\n") + def resetLog(self): + self.messages = [] + def merge(self, submsg): self.setRc(submsg.getFinalRc(), submsg.topmessage) text = "\n".join(submsg.messages) diff --git a/compare_testcase.py b/compare_testcase.py index 7c4e5d1..b78da2d 100644 --- a/compare_testcase.py +++ b/compare_testcase.py @@ -1,34 +1,38 @@ # This is a sample Python script. import sys# -# import jsonpickle # pip install jsonpickle -import yaml # pip install pyyaml +import os import basic.program as program import utils.tdata_tool -from basic.componentHandling import ComponentManager +import utils.report_tool +import utils.path_tool +import utils.file_tool +import basic.componentHandling import basic.message as message # Press Umschalt+F10 to execute it or replace it with your code. # Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. -PROGRAM_NAME = "init_testcase" +PROGRAM_NAME = "compare_testcase" -def start(job): - cm = ComponentManager() +def startPyJob(job): + cm = basic.componentHandling.ComponentManager("init") print("cm "+str(cm)) cm.initComponents() comps = cm.getComponents(PROGRAM_NAME) job.m.setMsg("# Components initialized with these relevant components " + str(comps)) + report = utils.report_tool.Report() testdata = utils.tdata_tool.getTestdata() for c in comps: comp = cm.getComponent(c) comp.m.logInfo("------- "+comp.name+" ----------------------------------------") - comp.reset_TData("testcase") - comp.load_TData("testcase", testdata) - comp.read_TData("testcase") + comp.compare_TcResults(report) comp.m.logInfo("------- "+comp.name+" ----------------------------------------") job.m.merge(comp.m) print(str(comp)) comp.conf["function"][PROGRAM_NAME] = comp.m.topmessage + text = report.reportTestcase(job.par.testcase) + path = os.path.join(job.par.tcdir, "Result.html") + utils.file_tool.writeFileText(job.m, path, text) # Press the green button in the gutter to run the script. if __name__ == '__main__': @@ -41,6 +45,6 @@ if __name__ == '__main__': if x.m.isRc("fatal"): x.stopJob() exit(x.m.rc * (-1) + 3) - start(x) + startPyJob(x) x.stopJob() # See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/init_testcase.py b/init_testcase.py index d2c8c0d..04ff7fe 100644 --- a/init_testcase.py +++ b/init_testcase.py @@ -1,10 +1,10 @@ # This is a sample Python script. -import sys# +import os # import jsonpickle # pip install jsonpickle import yaml # pip install pyyaml import basic.program as program import utils.tdata_tool -from basic.componentHandling import ComponentManager +import basic.componentHandling import basic.message as message # Press Umschalt+F10 to execute it or replace it with your code. # Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. @@ -12,12 +12,12 @@ import basic.message as message PROGRAM_NAME = "init_testcase" -def start(x): - cm = ComponentManager() +def startPyJob(job): + cm = basic.componentHandling.ComponentManager.getInstance("init") print("cm "+str(cm)) cm.initComponents() comps = cm.getComponents(PROGRAM_NAME) - x.m.setMsg("# Components initialized with these relevant components " + str(comps)) + job.m.setMsg("# Components initialized with these relevant components " + str(comps)) testdata = utils.tdata_tool.getTestdata() for c in comps: comp = cm.getComponent(c) @@ -26,7 +26,7 @@ def start(x): comp.load_TData("testcase", testdata) comp.read_TData("testcase") comp.m.logInfo("------- "+comp.name+" ----------------------------------------") - x.m.merge(comp.m) + job.m.merge(comp.m) print(str(comp)) comp.conf["function"][PROGRAM_NAME] = comp.m.topmessage @@ -34,8 +34,6 @@ def start(x): if __name__ == '__main__': print(PROGRAM_NAME) x = program.Job(PROGRAM_NAME) - #m = message.Message(3) - #m.closeMessage() x.startJob() x.m.logInfo("hier eine LogInfo") x.m.logDebug("hier eine DbugMeldung") @@ -43,6 +41,6 @@ if __name__ == '__main__': if x.m.isRc("fatal"): x.stopJob() exit(x.m.rc * (-1) + 3) - start(x) + startPyJob(x) x.stopJob() # See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/utils/file_tool.py b/utils/file_tool.py index 7e80d71..eab27e5 100644 --- a/utils/file_tool.py +++ b/utils/file_tool.py @@ -4,12 +4,18 @@ """ import codecs +import json import os import os.path import re + +import yaml + import basic.message import basic.program from pprint import pp +import utils.tdata_tool + def getDump(obj): result="" print (str(type(obj))) @@ -84,11 +90,14 @@ def getTree(msg, pfad): tree["_files_"] = files return tree -def mkPaths(msg, pfad): +def mkPaths(path, msg): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("file_tool")) modus = job.conf.confs["paths"]["mode"] - os.makedirs(pfad, exist_ok=True) + dirname = os.path.dirname(path) + if os.path.exists(dirname): + return + os.makedirs(dirname, exist_ok=True) def getFileEncoding(msg, path): print("--- getFileEncoding "+path) @@ -106,9 +115,9 @@ def getFileEncoding(msg, path): else: print('opening the file with encoding: %s ' % e) return e - return detectFileEncode(msg, path) + return detectFileEncode(path, msg) -def detectFileEncode(msg, path): # return "" +def detectFileEncode(path, msg): # return "" job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("file_tool")) print(path) @@ -128,18 +137,66 @@ def detectFileEncode(msg, path): # return "" return 'iso-8859-1' return 'utf-8' -def readFileLines(msg, path): +def readFileLines(path, msg): + lines = readFileText(path, msg) + if isinstance(lines, (str)): + return lines.splitlines() + return [] + +def readFileText(path, msg): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("file_tool")) - enc = detectFileEncode(msg, path) + if not os.path.exists(path): + return "" + enc = detectFileEncode(path, msg) with open(path, 'r', encoding=enc) as file: - lines = file.read().splitlines() + text = file.read() file.close() - return lines + return text + +def readFileDict(path, msg): + """ + reads and gets general a dict from any kind of filetyp + :param path: with extension of filetype + :param msg: optionally + :return: + """ + # 20220329 generalize + job = basic.program.Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + doc = {} + if not os.path.exists(path): + return doc + enc = detectFileEncode(path, msg) + if "yml" in path[-5:]: + with open(path, 'r', encoding=enc) as file: + doc = yaml.full_load(file) + file.close() + elif "json" in path[-5:]: + with open(path, 'r', encoding=enc) as file: + doc = json.load(file) + file.close() + elif "csv" in path[-5:]: + doc = utils.tdata_tool.getCsvSpec(msg, path, "conf") + return doc def writeFileText(msg, path, text, enc="utf-8"): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("file_tool")) + mkPaths(path, msg) with open(path, 'w', encoding=enc) as file: file.write(text) file.close() + +def writeFileDict(msg, path, dict, enc="utf-8"): + job = basic.program.Job.getInstance() + mkPaths(path, msg) + if "yml" in path[-5:]: + with open(path, 'r', encoding=enc) as file: + doc = yaml.dump(dict, file) + file.close() + elif "json" in path[-5:]: + with open(path, 'w', encoding=enc) as file: + doc = json.dumps(file, indent=4) + file.write(doc) + file.close() diff --git a/utils/path_tool.py b/utils/path_tool.py index 1bdc00b..78fda18 100644 --- a/utils/path_tool.py +++ b/utils/path_tool.py @@ -28,9 +28,11 @@ def getKeyValue(key, comp=None): elif 'comp.' in key: if comp is None: raise Exception("Component is missing for "+key) - if not utils.config_tool.hasAttr(comp, key[5:]): - pass - return utils.config_tool.getAttr(comp, key[5:]) + if utils.config_tool.hasAttr(comp.conf, key[5:]): + return utils.config_tool.gasAttr(comp.conf, key[5:]) + if utils.config_tool.hasAttr(comp, key[5:]): + return utils.config_tool.gasAttr(comp, key[5:]) + return "" elif 'env.' in key: #if key[4:] pass @@ -48,12 +50,12 @@ def composePath(pathname, comp): job.debug(verify, "composePath " + pathname + " zu " + str(pt) + "mit ") job.debug(verify, str(pt.pattern)) if pt.pattern[pathname]: - return composePatttern(pt.pattern[pathname], comp) + return composePattern(pt.pattern[pathname], comp) else: job.debug(verify, "in Pattern nicht vorhanden: " + pathname) -def composePatttern(pattern, comp): +def composePattern(pattern, comp): job = basic.program.Job.getInstance() verify = job.getDebugLevel("path_tool") job.debug(verify, "composePattern " + pattern) @@ -69,7 +71,7 @@ def composePatttern(pattern, comp): while ("{" in pattern): max = max-1 job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit) - pattern = composePatttern(pattern, comp) + pattern = composePattern(pattern, comp) job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit) if (max < 3) : break diff --git a/utils/report_tool.py b/utils/report_tool.py index ca43bff..fdad0ea 100644 --- a/utils/report_tool.py +++ b/utils/report_tool.py @@ -102,7 +102,8 @@ class Report(): elif len(testcase) > 1: html = ""+REP_TITLE+" "+REP_TC+" "+testcase+"

"+REP_TITLE+" "+REP_TC+" "+testcase+"

" else: - html = ""+REP_TITLE+" "+REP_TS+" "+getattr(job.par, "testsuite")+"

"+REP_TITLE+" "+REP_TS+" "+getattr(job.par, "testsuite")+"

" + html = ""+REP_TITLE+" "+REP_TS+" "+getattr(job.par, "usecase")+"" + html += "

"+REP_TITLE+" "+REP_TS+" "+getattr(job.par, "usecase")+"

" if hasattr(job.par, B.PAR_DESCRIPT): html += "

"+getattr(job.par, B.PAR_DESCRIPT)+"

" return html @@ -117,12 +118,18 @@ class Report(): :return: """ cssClass = "result0" - if len(match) > 0: + if len(match) > 0 and len(artefact) > 0: if match in self.report[testcase][component][artefact]: if "css" in self.report[testcase][component][artefact][match]: if cssClass < self.report[testcase][component][artefact][match]["css"]: cssClass = self.report[testcase][component][artefact][match]["css"] return cssClass + elif len(match) > 0: + for a in self.report[testcase][component]: + val = self.getCssClass(testcase, component, a, match) + if cssClass < val: + cssClass = val + return cssClass elif len(artefact) > 0: for match in self.report[testcase][component][artefact]: if "css" in self.report[testcase][component][artefact][match]: @@ -197,16 +204,16 @@ class Report(): for side in ["A", "B"]: path = self.report[testcase][component][artefact][matchtype]["path"+side] href = path.replace(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ARCHIV], os.path.join("..", "..")) - html += M.MATCH[M.MATCH[M.MATCH_POSTCOND][side]]["long"]+" " + html += M.MATCH[M.MATCH[matchtype][side]]["long"]+" " html += ""+path+"\n" if side == "A": html += "
" html += "

" return html def getFilepath(self, testcase, component, artefact, matchtype): - cm = basic.componentHandling.ComponentManager() + cm = basic.componentHandling.ComponentManager.getInstance("init") comp = cm.getComponent(component) - path = os.path.join(utils.path_tool.composePatttern("{tcresult}", comp), artefact+"_"+M.MATCH[matchtype]["filename"]+".html") + path = os.path.join(utils.path_tool.composePattern("{tcresult}", comp), artefact+"_"+M.MATCH[matchtype]["filename"]+".html") return path def getComparisonBlock(self, testcase, component, artefact, matchtype): @@ -233,6 +240,8 @@ class Report(): def extractTestcase(self, testcase, html): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("report_tool")) - 1 + if not testcase in self.report: + self.report = {} overview = re.findall("", html) if len(overview) == 1: self.report[testcase]["overview"] = overview[0] @@ -245,16 +254,16 @@ class Report(): def reportTestsuite(self): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("report_tool")) - 1 - testcases = getattr(job.par, "testcases") + testinstances = getattr(job.par, "testinstances") html = self.getHeader() html += self.getTitle() i = 0 - for testcase in testcases: + for testcase in testinstances: if i == 0: html += self.getOverviewHead(testcase) html += self.report[testcase]["overview"] i += 1 html += "" - for testcase in testcases: + for testcase in testinstances: html += self.report[testcase]["block"] html += "" return html diff --git a/utils/zip_tool.py b/utils/zip_tool.py index 022b117..ff2be8a 100644 --- a/utils/zip_tool.py +++ b/utils/zip_tool.py @@ -1,23 +1,39 @@ import zipfile import tarfile import os +import basic.program -ZIEL = '/home/ulrich/tmp' -QUELLE = '/home/ulrich/1_privat' -FOLDER = '64-UMKER' - -def untarFolder(): - tar_file = tarfile.open(os.path.join(ZIEL, 'tartemp.tar.gz'), 'r:gz') - tar_file.extractall(path=os.path.join(ZIEL, 'tarliste')) +def untarFolder(target, targetFile): + tar_file = tarfile.open(os.path.join(target, targetFile), 'r:gz') + tar_file.extractall(path=os.path.join(target, 'tarliste')) tar_file.close() pass -def tarFolder(): - with tarfile.open(os.path.join(ZIEL, 'tartemp.tar.gz'), 'w:gz') as tar_file: - for folderName, subfolders, filenames in os.walk(os.path.join(QUELLE, FOLDER)): +def openNewTarFile(target, targetFile): + job = basic.program.Job.getInstance() + tarfilename = os.path.join(target, targetFile) + if os.path.exists(tarfilename): + os.remove(tarfilename) + job.m.logInfo("Archiv angelegt "+tarfilename) + return tarfile.open(tarfilename, 'w:gz') + +def appendFolderIntoTarFile(source, sourceFolder, tarFile): + workFolder = os.path.join(source, sourceFolder) + for folderName, subfolders, filenames in os.walk(workFolder): + for filename in filenames: + folderShort = folderName[len(source)+1:] + # create complete filepath of file in directory + filePath = os.path.join(folderName, filename) + # Add file to zip + tarFile.add(filePath, os.path.join(folderShort, filename)) + + +def tarFolder(source, sourceFolder, target, targetFile): + with tarfile.open(os.path.join(target, targetFile), 'w:gz') as tar_file: + for folderName, subfolders, filenames in os.walk(os.path.join(source, sourceFolder)): for filename in filenames: - folderShort = folderName.replace(QUELLE + '/', '') + folderShort = folderName.replace(source + '/', '') # create complete filepath of file in directory filePath = os.path.join(folderName, filename) # Add file to zip @@ -25,18 +41,18 @@ def tarFolder(): tar_file.close() -def unzipFolder(): - zip_file = zipfile.ZipFile(os.path.join(ZIEL, 'temp.zip'), 'r') - zip_file.extractall(path=os.path.join(ZIEL, 'liste')) +def unzipFolder(target, targetFile): + zip_file = zipfile.ZipFile(os.path.join(target, targetFile), 'r') + zip_file.extractall(path=os.path.join(target, 'liste')) zip_file.close() pass -def zipFolder(): - with zipfile.ZipFile(os.path.join(ZIEL, 'temp.zip'), 'w') as zip_file: +def zipFolder(source, sourceFolder, target, targetFile): + with zipfile.ZipFile(os.path.join(target, targetFile), 'w') as zip_file: # Iterate over all the files in directory - for folderName, subfolders, filenames in os.walk(os.path.join(QUELLE, FOLDER)): + for folderName, subfolders, filenames in os.walk(os.path.join(source, sourceFolder)): for filename in filenames: - folderShort = folderName.replace(QUELLE+'/', '') + folderShort = folderName.replace(source+'/', '') # create complete filepath of file in directory filePath = os.path.join(folderName, filename) # Add file to zip @@ -44,8 +60,3 @@ def zipFolder(): zip_file.close() return "" -if __name__ == '__main__': - zipFolder() - unzipFolder() - tarFolder() - untarFolder()