test and corrections

basic/componentHandling.py

@@ -70,6 +70,7 @@ class ComponentManager:
         job = basic.program.Job.getInstance()
         job.m.logDebug("applicationscomponente -- " + str(type(job.par)))
         self.components = {}
+        ComponentManager.__instance = self
         print ("init ComponentHandling "+str(self))
 
     def initComponents(self):
@@ -109,9 +110,11 @@ class ComponentManager:
         return out
 
     @staticmethod
-    def getInstance():
+    def getInstance(init="N"):
         if (ComponentManager.__instance is not None):
             return ComponentManager.__instance
+        elif (init != "N"):
+            return ComponentManager()
         else:
             raise Exception("Klasse noch nicht initialisiert")
 
@@ -186,9 +189,14 @@ class ComponentManager:
         if B.SUBJECT_ARTS in c.conf and B.TOPIC_NODE_DB in c.conf[B.SUBJECT_ARTS]:
             if not B.DATA_NODE_DDL in c.conf:
                 c.conf[B.DATA_NODE_DDL] = {}
-                for table in c.conf[B.SUBJECT_ARTS][B.SUBJECT_ARTS]:
+                for table in c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]:
+                    if table in ["type"]:
+                        continue
                     conf = utils.config_tool.getConfig("DATASTRUCTURE", c.name, table)
-                    c.conf[B.DATA_NODE_DDL][table] = conf
+                    if "tabelle" in conf and table in conf["tabelle"]:
+                        c.conf[B.DATA_NODE_DDL][table] = conf["tabelle"][table]
+                    else:
+                        c.conf[B.DATA_NODE_DDL][table] = conf
         comps[name] = c
         return c
 
@@ -225,11 +233,8 @@ def getComponentDict():
     out = {}
     for c in comps:
         out[comps[c].name] = {}
-        print("getCompDict " + comps[c].name)
         for k in comps[c].conf.keys():
-            print("getCompDict " + k)
             if isParameterSubnode(k):  # "artifact" in k or "components" in k or "instance" in k:
-                print("getCompDict -b- " + k)
                 out[comps[c].name][k] = copy.deepcopy(comps[c].conf[k])
     return out
 
@@ -244,4 +249,4 @@ def isParameterSubnode(key):
 def getPlainCompname(name):
     if "_0" in name:
         return name[0:-3]
-    return name
+    return name

basic/message.py

@@ -75,7 +75,10 @@ class Message:
         else:
             debugpath = job.conf.confs["paths"]["debugs"] + "/debug_" + logTime[0:-4] + "00.txt"
             print ("debugpathx "+debugpath)
-            self.debugfile = open(debugpath, "a")
+            if os.path.exists(debugpath):
+                self.debugfile = open(debugpath, "a")
+            else:
+                self.debugfile = open(debugpath, "w")
             self.debug(verify, "> > > debugfile geoeffnet zu " + job.program + " mit " + debugpath)
         # init logfile - except for components or unittest
         self.logDebug("logfile " + str(componente) + ", " + str(job.par.basedir))
@@ -231,6 +234,9 @@ class Message:
         if (int(prio) < int(self.level)+1) : #and self.debugfile.closed == False:
             self.debugfile.write(text + "\n")
 
+    def resetLog(self):
+        self.messages = []
+
     def merge(self, submsg):
         self.setRc(submsg.getFinalRc(), submsg.topmessage)
         text = "\n".join(submsg.messages)

compare_testcase.py

@@ -1,34 +1,38 @@
 # This is a sample Python script.
 import sys#
-# import jsonpickle # pip install jsonpickle
-import yaml # pip install pyyaml
+import os
 import basic.program as program
 import utils.tdata_tool
-from basic.componentHandling import ComponentManager
+import utils.report_tool
+import utils.path_tool
+import utils.file_tool
+import basic.componentHandling
 import basic.message as message
 # Press Umschalt+F10 to execute it or replace it with your code.
 # Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
 
 
-PROGRAM_NAME = "init_testcase"
+PROGRAM_NAME = "compare_testcase"
 
-def start(job):
-    cm = ComponentManager()
+def startPyJob(job):
+    cm = basic.componentHandling.ComponentManager("init")
     print("cm "+str(cm))
     cm.initComponents()
     comps = cm.getComponents(PROGRAM_NAME)
     job.m.setMsg("# Components initialized with these relevant components " + str(comps))
+    report = utils.report_tool.Report()
     testdata = utils.tdata_tool.getTestdata()
     for c in comps:
         comp = cm.getComponent(c)
         comp.m.logInfo("------- "+comp.name+" ----------------------------------------")
-        comp.reset_TData("testcase")
-        comp.load_TData("testcase", testdata)
-        comp.read_TData("testcase")
+        comp.compare_TcResults(report)
         comp.m.logInfo("------- "+comp.name+" ----------------------------------------")
         job.m.merge(comp.m)
         print(str(comp))
         comp.conf["function"][PROGRAM_NAME] = comp.m.topmessage
+    text = report.reportTestcase(job.par.testcase)
+    path = os.path.join(job.par.tcdir, "Result.html")
+    utils.file_tool.writeFileText(job.m, path, text)
 
 # Press the green button in the gutter to run the script.
 if __name__ == '__main__':
@@ -41,6 +45,6 @@ if __name__ == '__main__':
     if x.m.isRc("fatal"):
         x.stopJob()
         exit(x.m.rc * (-1) + 3)
-    start(x)
+    startPyJob(x)
     x.stopJob()
 # See PyCharm help at https://www.jetbrains.com/help/pycharm/

init_testcase.py

@@ -1,10 +1,10 @@
 # This is a sample Python script.
-import sys#
+import os
 # import jsonpickle # pip install jsonpickle
 import yaml # pip install pyyaml
 import basic.program as program
 import utils.tdata_tool
-from basic.componentHandling import ComponentManager
+import basic.componentHandling
 import basic.message as message
 # Press Umschalt+F10 to execute it or replace it with your code.
 # Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
@@ -12,12 +12,12 @@ import basic.message as message
 
 PROGRAM_NAME = "init_testcase"
 
-def start(x):
-    cm = ComponentManager()
+def startPyJob(job):
+    cm = basic.componentHandling.ComponentManager.getInstance("init")
     print("cm "+str(cm))
     cm.initComponents()
     comps = cm.getComponents(PROGRAM_NAME)
-    x.m.setMsg("# Components initialized with these relevant components " + str(comps))
+    job.m.setMsg("# Components initialized with these relevant components " + str(comps))
     testdata = utils.tdata_tool.getTestdata()
     for c in comps:
         comp = cm.getComponent(c)
@@ -26,7 +26,7 @@ def start(x):
         comp.load_TData("testcase", testdata)
         comp.read_TData("testcase")
         comp.m.logInfo("------- "+comp.name+" ----------------------------------------")
-        x.m.merge(comp.m)
+        job.m.merge(comp.m)
         print(str(comp))
         comp.conf["function"][PROGRAM_NAME] = comp.m.topmessage
 
@@ -34,8 +34,6 @@ def start(x):
 if __name__ == '__main__':
     print(PROGRAM_NAME)
     x = program.Job(PROGRAM_NAME)
-    #m = message.Message(3)
-    #m.closeMessage()
     x.startJob()
     x.m.logInfo("hier eine LogInfo")
     x.m.logDebug("hier eine DbugMeldung")
@@ -43,6 +41,6 @@ if __name__ == '__main__':
     if x.m.isRc("fatal"):
         x.stopJob()
         exit(x.m.rc * (-1) + 3)
-    start(x)
+    startPyJob(x)
     x.stopJob()
 # See PyCharm help at https://www.jetbrains.com/help/pycharm/

utils/file_tool.py

@@ -4,12 +4,18 @@
 
 """
 import codecs
+import json
 import os
 import os.path
 import re
+
+import yaml
+
 import basic.message
 import basic.program
 from pprint import pp
+import utils.tdata_tool
+
 def getDump(obj):
     result=""
     print (str(type(obj)))
@@ -84,11 +90,14 @@ def getTree(msg, pfad):
     tree["_files_"] = files
     return tree
 
-def mkPaths(msg, pfad):
+def mkPaths(path, msg):
     job = basic.program.Job.getInstance()
     verify = int(job.getDebugLevel("file_tool"))
     modus = job.conf.confs["paths"]["mode"]
-    os.makedirs(pfad, exist_ok=True)
+    dirname = os.path.dirname(path)
+    if os.path.exists(dirname):
+        return
+    os.makedirs(dirname, exist_ok=True)
 
 def getFileEncoding(msg, path):
     print("--- getFileEncoding "+path)
@@ -106,9 +115,9 @@ def getFileEncoding(msg, path):
         else:
             print('opening the file with encoding:  %s ' % e)
             return e
-    return detectFileEncode(msg, path)
+    return detectFileEncode(path, msg)
 
-def detectFileEncode(msg, path):  # return ""
+def detectFileEncode(path, msg):  # return ""
     job = basic.program.Job.getInstance()
     verify = int(job.getDebugLevel("file_tool"))
     print(path)
@@ -128,18 +137,66 @@ def detectFileEncode(msg, path):  # return ""
         return 'iso-8859-1'
     return 'utf-8'
 
-def readFileLines(msg, path):
+def readFileLines(path, msg):
+    lines = readFileText(path, msg)
+    if isinstance(lines, (str)):
+        return lines.splitlines()
+    return []
+
+def readFileText(path, msg):
     job = basic.program.Job.getInstance()
     verify = int(job.getDebugLevel("file_tool"))
-    enc = detectFileEncode(msg, path)
+    if not os.path.exists(path):
+        return ""
+    enc = detectFileEncode(path, msg)
     with open(path, 'r', encoding=enc) as file:
-        lines = file.read().splitlines()
+        text = file.read()
     file.close()
-    return lines
+    return text
+
+def readFileDict(path, msg):
+    """
+    reads and gets general a dict from any kind of filetyp
+    :param path: with extension of filetype
+    :param msg: optionally
+    :return:
+    """
+    # 20220329 generalize
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("file_tool"))
+    doc = {}
+    if not os.path.exists(path):
+        return doc
+    enc = detectFileEncode(path, msg)
+    if "yml" in path[-5:]:
+        with open(path, 'r', encoding=enc) as file:
+            doc = yaml.full_load(file)
+        file.close()
+    elif "json" in path[-5:]:
+        with open(path, 'r', encoding=enc) as file:
+            doc = json.load(file)
+        file.close()
+    elif "csv" in path[-5:]:
+        doc = utils.tdata_tool.getCsvSpec(msg, path, "conf")
+    return doc
 
 def writeFileText(msg, path, text, enc="utf-8"):
     job = basic.program.Job.getInstance()
     verify = int(job.getDebugLevel("file_tool"))
+    mkPaths(path, msg)
     with open(path, 'w', encoding=enc) as file:
         file.write(text)
     file.close()
+
+def writeFileDict(msg, path, dict, enc="utf-8"):
+    job = basic.program.Job.getInstance()
+    mkPaths(path, msg)
+    if "yml" in path[-5:]:
+        with open(path, 'r', encoding=enc) as file:
+            doc = yaml.dump(dict, file)
+        file.close()
+    elif "json" in path[-5:]:
+        with open(path, 'w', encoding=enc) as file:
+            doc = json.dumps(file, indent=4)
+            file.write(doc)
+        file.close()

utils/path_tool.py

@@ -28,9 +28,11 @@ def getKeyValue(key, comp=None):
     elif 'comp.' in key:
         if comp is None:
             raise Exception("Component is missing for "+key)
-        if not utils.config_tool.hasAttr(comp, key[5:]):
-            pass
-        return utils.config_tool.getAttr(comp, key[5:])
+        if utils.config_tool.hasAttr(comp.conf, key[5:]):
+            return utils.config_tool.gasAttr(comp.conf, key[5:])
+        if utils.config_tool.hasAttr(comp, key[5:]):
+            return utils.config_tool.gasAttr(comp, key[5:])
+        return ""
     elif 'env.' in key:
         #if key[4:]
         pass
@@ -48,12 +50,12 @@ def composePath(pathname, comp):
     job.debug(verify, "composePath " + pathname + " zu " + str(pt) + "mit ")
     job.debug(verify, str(pt.pattern))
     if pt.pattern[pathname]:
-        return composePatttern(pt.pattern[pathname], comp)
+        return composePattern(pt.pattern[pathname], comp)
     else:
         job.debug(verify, "in Pattern nicht vorhanden: " + pathname)
 
 
-def composePatttern(pattern, comp):
+def composePattern(pattern, comp):
     job = basic.program.Job.getInstance()
     verify = job.getDebugLevel("path_tool")
     job.debug(verify, "composePattern " + pattern)
@@ -69,7 +71,7 @@ def composePatttern(pattern, comp):
     while ("{" in pattern):
         max = max-1
         job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
-        pattern = composePatttern(pattern, comp)
+        pattern = composePattern(pattern, comp)
         job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
         if (max < 3) :
             break

utils/report_tool.py

@@ -102,7 +102,8 @@ class Report():
         elif len(testcase) > 1:
             html = "<title>"+REP_TITLE+" "+REP_TC+" "+testcase+"</title></head><body><h1>"+REP_TITLE+" "+REP_TC+" "+testcase+"</h1>"
         else:
-            html = "<title>"+REP_TITLE+" "+REP_TS+" "+getattr(job.par, "testsuite")+"</title></head><body><h1>"+REP_TITLE+" "+REP_TS+" "+getattr(job.par, "testsuite")+"</h1>"
+            html = "<title>"+REP_TITLE+" "+REP_TS+" "+getattr(job.par, "usecase")+"</title>"
+            html += "</head><body><h1>"+REP_TITLE+" "+REP_TS+" "+getattr(job.par, "usecase")+"</h1>"
         if hasattr(job.par, B.PAR_DESCRIPT):
             html += "<p>"+getattr(job.par, B.PAR_DESCRIPT)+"</p>"
         return html
@@ -117,12 +118,18 @@ class Report():
         :return:
         """
         cssClass = "result0"
-        if len(match) > 0:
+        if len(match) > 0 and len(artefact) > 0:
             if match in self.report[testcase][component][artefact]:
                 if "css" in self.report[testcase][component][artefact][match]:
                     if cssClass <  self.report[testcase][component][artefact][match]["css"]:
                         cssClass = self.report[testcase][component][artefact][match]["css"]
             return cssClass
+        elif len(match) > 0:
+            for a in self.report[testcase][component]:
+                val = self.getCssClass(testcase, component, a, match)
+                if cssClass < val:
+                    cssClass = val
+            return cssClass
         elif len(artefact) > 0:
             for match in self.report[testcase][component][artefact]:
                 if "css" in self.report[testcase][component][artefact][match]:
@@ -197,16 +204,16 @@ class Report():
         for side in ["A", "B"]:
             path = self.report[testcase][component][artefact][matchtype]["path"+side]
             href = path.replace(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ARCHIV], os.path.join("..", ".."))
-            html += M.MATCH[M.MATCH[M.MATCH_POSTCOND][side]]["long"]+" "
+            html += M.MATCH[M.MATCH[matchtype][side]]["long"]+" "
             html += "<a href=\""+href+"\">"+path+"<a>\n"
             if side == "A": html += "<br>"
         html += "</p>"
         return html
 
     def getFilepath(self, testcase, component, artefact, matchtype):
-        cm = basic.componentHandling.ComponentManager()
+        cm = basic.componentHandling.ComponentManager.getInstance("init")
         comp = cm.getComponent(component)
-        path = os.path.join(utils.path_tool.composePatttern("{tcresult}", comp), artefact+"_"+M.MATCH[matchtype]["filename"]+".html")
+        path = os.path.join(utils.path_tool.composePattern("{tcresult}", comp), artefact+"_"+M.MATCH[matchtype]["filename"]+".html")
         return path
 
     def getComparisonBlock(self, testcase, component, artefact, matchtype):
@@ -233,6 +240,8 @@ class Report():
     def extractTestcase(self, testcase, html):
         job = basic.program.Job.getInstance()
         verify = int(job.getDebugLevel("report_tool")) - 1
+        if not testcase in self.report:
+            self.report = {}
         overview = re.findall("<tr class.+tc-overview.*</tr>", html)
         if len(overview) == 1:
             self.report[testcase]["overview"] = overview[0]
@@ -245,16 +254,16 @@ class Report():
     def reportTestsuite(self):
         job = basic.program.Job.getInstance()
         verify = int(job.getDebugLevel("report_tool")) - 1
-        testcases = getattr(job.par, "testcases")
+        testinstances = getattr(job.par, "testinstances")
         html = self.getHeader()
         html += self.getTitle()
         i = 0
-        for testcase in testcases:
+        for testcase in testinstances:
             if i == 0: html += self.getOverviewHead(testcase)
             html += self.report[testcase]["overview"]
             i += 1
         html += "</table>"
-        for testcase in testcases:
+        for testcase in testinstances:
             html += self.report[testcase]["block"]
         html += "</body></html>"
         return html

utils/zip_tool.py

@@ -1,23 +1,39 @@
 import zipfile
 import tarfile
 import os
+import basic.program
 
-ZIEL = '/home/ulrich/tmp'
-QUELLE = '/home/ulrich/1_privat'
-FOLDER = '64-UMKER'
 
-
-def untarFolder():
-    tar_file = tarfile.open(os.path.join(ZIEL, 'tartemp.tar.gz'), 'r:gz')
-    tar_file.extractall(path=os.path.join(ZIEL, 'tarliste'))
+def untarFolder(target, targetFile):
+    tar_file = tarfile.open(os.path.join(target, targetFile), 'r:gz')
+    tar_file.extractall(path=os.path.join(target, 'tarliste'))
     tar_file.close()
     pass
 
-def tarFolder():
-    with tarfile.open(os.path.join(ZIEL, 'tartemp.tar.gz'), 'w:gz') as tar_file:
-        for folderName, subfolders, filenames in os.walk(os.path.join(QUELLE, FOLDER)):
+def openNewTarFile(target, targetFile):
+    job = basic.program.Job.getInstance()
+    tarfilename = os.path.join(target, targetFile)
+    if os.path.exists(tarfilename):
+        os.remove(tarfilename)
+    job.m.logInfo("Archiv angelegt "+tarfilename)
+    return tarfile.open(tarfilename, 'w:gz')
+
+def appendFolderIntoTarFile(source, sourceFolder, tarFile):
+    workFolder = os.path.join(source, sourceFolder)
+    for folderName, subfolders, filenames in os.walk(workFolder):
+        for filename in filenames:
+            folderShort = folderName[len(source)+1:]
+            # create complete filepath of file in directory
+            filePath = os.path.join(folderName, filename)
+            # Add file to zip
+            tarFile.add(filePath, os.path.join(folderShort, filename))
+
+
+def tarFolder(source, sourceFolder, target, targetFile):
+    with tarfile.open(os.path.join(target, targetFile), 'w:gz') as tar_file:
+        for folderName, subfolders, filenames in os.walk(os.path.join(source, sourceFolder)):
             for filename in filenames:
-                folderShort = folderName.replace(QUELLE + '/', '')
+                folderShort = folderName.replace(source + '/', '')
                 # create complete filepath of file in directory
                 filePath = os.path.join(folderName, filename)
                 # Add file to zip
@@ -25,18 +41,18 @@ def tarFolder():
     tar_file.close()
 
 
-def unzipFolder():
-    zip_file = zipfile.ZipFile(os.path.join(ZIEL, 'temp.zip'), 'r')
-    zip_file.extractall(path=os.path.join(ZIEL, 'liste'))
+def unzipFolder(target, targetFile):
+    zip_file = zipfile.ZipFile(os.path.join(target, targetFile), 'r')
+    zip_file.extractall(path=os.path.join(target, 'liste'))
     zip_file.close()
     pass
 
-def zipFolder():
-    with zipfile.ZipFile(os.path.join(ZIEL, 'temp.zip'), 'w') as zip_file:
+def zipFolder(source, sourceFolder, target, targetFile):
+    with zipfile.ZipFile(os.path.join(target, targetFile), 'w') as zip_file:
         # Iterate over all the files in directory
-        for folderName, subfolders, filenames in os.walk(os.path.join(QUELLE, FOLDER)):
+        for folderName, subfolders, filenames in os.walk(os.path.join(source, sourceFolder)):
             for filename in filenames:
-                folderShort = folderName.replace(QUELLE+'/', '')
+                folderShort = folderName.replace(source+'/', '')
                 # create complete filepath of file in directory
                 filePath = os.path.join(folderName, filename)
                 # Add file to zip
@@ -44,8 +60,3 @@ def zipFolder():
     zip_file.close()
     return ""
 
-if __name__ == '__main__':
-    zipFolder()
-    unzipFolder()
-    tarFolder()
-    untarFolder()