matching and comparing of tables

basic/program.py

@@ -310,5 +310,11 @@ class Configuration:
 			doc = yaml.full_load(file)
 			for i, v in doc.items():
 				self.confs[i] = v
-		print ("set conf")
+	def setConfig(self, path, val):
-		print ("set configuration")
+		a = path.split(".")
+		if len(a) == 1:
+			self.confs[a[0]] = val
+		elif len(a) == 2:
+			self.confs[a[0]][a[1]] = val
+		elif len(a) == 3:
+			self.confs[a[0]][a[1]][a[2]] = val

test/test_css.py

@@ -0,0 +1,53 @@
+import unittest
+import utils.css_tool
+import basic.program
+import json
+
+class MyTestCase(unittest.TestCase):
+    def test_css(self):
+        job = basic.program.Job("unit")
+        args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug",
+                "tool": "job_tool", "tdtyp": "csv", "tdsrc": "implement", "tdname": "firstunit",
+                "modus": "unit"}
+        job.par.setParameterArgs(args)
+        print("eeeeeeeee")
+        print(json.dumps(job.conf.confs))
+        # ------- inline ---------------
+        job.conf.setConfig("tools.csstyp", "inline")
+        job.conf.confs.get("tools")["csstyp"] == "inline"
+        text = utils.css_tool.getInlineStyle("diffFiles", "diffA")
+        self.assertEqual(len(text), 37)
+        self.assertEqual(("style" in text), True)
+        text = utils.css_tool.getInternalStyle("diffFiles")
+        self.assertEqual(len(text), 0)
+        text = utils.css_tool.getExternalStyle("diffFiles")
+        self.assertEqual(len(text), 0)
+        # ------- internal ---------------
+        job.conf.setConfig("tools.csstyp", "internal")
+        text = utils.css_tool.getInlineStyle("diffFiles", "diffA")
+        self.assertEqual(len(text), 13)
+        self.assertEqual(("class" in text), True)
+        text = utils.css_tool.getInternalStyle("diffFiles")
+        print(text)
+        self.assertEqual(len(text), 173)
+        self.assertEqual(("<style>" in text), True)
+        text = utils.css_tool.getExternalStyle("diffFiles")
+        self.assertEqual(len(text), 0)
+        print(text)
+        print(str(len(text)))
+        # ------- external ---------------
+        job.conf.setConfig("tools.csstyp", "external")
+        text = utils.css_tool.getInlineStyle("diffFiles", "diffA")
+        self.assertEqual(len(text), 13)
+        self.assertEqual(("class" in text), True)
+        text = utils.css_tool.getInternalStyle("diffFiles")
+        print(text)
+        self.assertEqual(len(text), 55)
+        self.assertEqual(("<link " in text and "stylesheet" in text), True)
+        text = utils.css_tool.getExternalStyle("diffFiles")
+        self.assertEqual(len(text), 189)
+        print(text)
+        print(str(len(text)))
+
+if __name__ == '__main__':
+    unittest.main()

test/test_main.py

@@ -5,7 +5,7 @@ import basic.program as program
 class MyTestCase(unittest.TestCase):
     def test_pathTool(self):
         x = program.Job("test:application=TEST:application=ENV01")
-        self.assertEqual(path_tool.generatePath("program", "komp", "testA", "CONFIG.yml"), "/home/basic/6_Projekte/PythonProject/komponents/testA/COFIG.yml")
+        self.assertEqual(path_tool.generatePath("program", "komp", "testA", "CONFIG.yml"), "/home/ulrich/6_Projekte/holtz/components/testA/CONFIG.yml")
 
 
 if __name__ == '__main__':

utils/css_tool.py

@@ -0,0 +1,59 @@
+import basic.program
+CSS_CLASS = {
+    "general":  {
+        "table, td, th": "border: 1px solid grey;font-family:sans-serif"
+    },
+    "diffFiles": {
+        "diffA": "color:green;font-weight:bold;",
+        "diffB": "color:crimson;font-weight:bold;",
+        "acceptA": "color:darkblue;",
+        "acceptB": "color:darkmagenta;"
+    }
+}
+
+def getInlineStyle(filetype, cssclass):
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("css_tool")) - 4
+    job.debug(verify, "getDiffHeader ")
+    print(job.conf.confs.get("tools").get("csstyp"))
+    if job.conf.confs.get("tools").get("csstyp") == "inline":
+        out = "style=\""+CSS_CLASS[filetype][cssclass]+"\""
+    else:
+        out = "class=\"" + cssclass + "\""
+    return out
+
+def getInternalStyle(filetype):
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool")) - 4
+    job.debug(verify, "getDiffHeader ")
+    out = ""
+    if job.conf.confs.get("tools").get("csstyp") == "internal":
+        out = "<style>"
+        for c in CSS_CLASS["general"]:
+            line = "\n"+c+" { "+CSS_CLASS["general"][c]+"} "
+            line.replace(":", ": ").replace(";", "; ").replace("_Q_", ", ")
+            out += line
+        for c in CSS_CLASS[filetype]:
+            out += "\n."+c+" { "+CSS_CLASS[filetype][c].replace(":", ": ").replace(";", "; ")+"} "
+        out += "\n</style>"
+    elif job.conf.confs.get("tools").get("csstyp") == "external":
+        out = " <link rel=\"stylesheet\" href=\""+job.conf.confs.get("tools").get("cssfile")+"\"> "
+    else:
+        out = "<style>"
+        for c in CSS_CLASS["general"]:
+            line = "\n"+c+" { "+CSS_CLASS["general"][c]+"} "
+            line.replace(":", ": ").replace(";", "; ").replace("_Q_", ", ")
+            out += line
+        out += "\n</style>"
+    return out
+
+def getExternalStyle(filetype):
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool")) - 4
+    job.debug(verify, "getDiffHeader ")
+    out = ""
+    if job.conf.confs.get("tools").get("csstyp") == "external":
+        for c in CSS_CLASS[filetype]:
+            out += c+" {\n    "+CSS_CLASS[filetype][c].replace(":", ": ").replace(";", ";\n    ")+"}\n"
+        out.replace("\n    \n}", "\n}")
+    return out

utils/match_tool.py

@@ -1,16 +1,145 @@
 #
+import json
+import utils.css_tool
+import basic.program
 # ------------------------------------------------------------
 """
 
 """
+MATCH = {
+    "preconditions": {
+        "A": "preReq",
+        "B": "preAct",
+        "shortA": "SV",
+        "shortB": "IV",
+        "longA": "Soll-Vorher",
+        "longB": "Ist-Vorher",
+        "mode": "info",
+        "title": "Pruefung Vorbedingung (Soll-Vorher - Ist-Vorher)"
+    },
+    "postconditions": {
+        "A": "postReq",
+        "B": "postAct",
+        "shortA": "SN",
+        "shortB": "IN",
+        "longA": "Soll-Nachher",
+        "longB": "Ist-Nachher",
+        "mode": "hard",
+        "title": "Fachliche Auswertung (Soll-Nachher - Ist-Nachher)"
+    },
+    "prepost": {
+        "A": "preAct",
+        "B": "postAct",
+        "shortA": "IV",
+        "shortB": "IN",
+        "longA": "Ist-Vorher",
+        "longB": "Ist-Nachher",
+        "mode": "action",
+        "title": "Ablauf-Differenz (Ist-Vorher - Ist-Nachher)"
+    },
+    "prestep": {
+        "A": "preStep",
+        "B": "postAct",
+        "shortA": "VS",
+        "shortB": "IN",
+        "longA": "Vor-Schritt",
+        "longB": "Ist-Nachher",
+        "mode": "action",
+        "title": "Schritt-Differenz (Vorschritt-Nachher - Ist-Nachher)"
+    }
+}
+MATCH_PREPOST = "prepost"
+""" matches the action between pre- and postcondition of the actual testexecution """
+MATCH_PRECOND = "preconditions"
+""" matches the preconditions betwenn the required result the the actual testexecution 
+ - just for info if the both executions have the same precondition """
+MATCH_POSTCOND = "postconditions"
+""" matches the postconditions betwenn the required result the the actual testexecution 
+ - it is the main comparison """
+
+
 class Matching():
     def __init__(self, comp):
         self.comp = comp
         self.elements = {}
-        self.resultFiles = comp.resultFiles
+        self.matchfiles = comp.files
-        self.targetFiles = comp.targetFiles
         self.assignedFiles = {}
-        pass
+        self.linksA = {}
+        self.linksB = {}
+        self.nomatch = {}
+        self.matchkeys = {}
+        self.htmltext = ""
+        self.sideA = []
+        self.sideB = []
+        self.mode = ""
+        self.matchtype = ""
+
+    def setData(self, tdata, match):
+        """
+        selects testdata for actual matching
+        :param tdata:
+        :param match: kind of actual match
+        :return:
+        """
+        self.sideA = tdata[MATCH[match]["A"]]
+        self.sideB = tdata[MATCH[match]["B"]]
+        self.matchtype = match
+        self.mode = MATCH[match]["mode"]
+        self.setDiffHeader()
+
+    def resetHits(self):
+        self.linksA = {}
+        self.linksB = {}
+        self.nomatch = {}
+        self.matchkeys = {}
+
+    def isHitA(self, key):
+        return ((key in self.linksA) and (self.linksA[key] != "null"))
+    def isHitB(self, key):
+        return ((key in self.linksB) and (self.linksB[key] != "null"))
+    def setHit(self, keyA, keyB):
+        if (not self.isHitA(keyA)) and (not self.isHitB(keyB)):
+            if (keyA != "null"): self.linksA[keyA] = keyB
+            if (keyB != "null"): self.linksB[keyB] = keyA
+            return "OK"
+        raise Exception("one of the links are set")
+    def setNohit(self, similarity, keyA, keyB):
+        """ The similarity must be unique. Only a mismatch will be set. """
+        if similarity in self.nomatch:
+            raise Exception("similarity "+similarity+" exists")
+        if (self.isHitA(keyA) or self.isHitB(keyB)):
+            return
+        self.nomatch[similarity] = [keyA, keyB]
+    def getTableDdl(self, path):
+        a = path.split(":")
+        ddl = self.comp.conf["ddl"]
+        for x in a:
+            if (len(x) < 2): continue
+            if (x == "_data"): break
+            if x in ddl: ddl = ddl[x]
+        return ddl
+    def setDiffHeader(matching):
+        job = basic.program.Job.getInstance()
+        verify = int(job.getDebugLevel("match_tool"))-4
+        job.debug(verify, "getDiffHeader ")
+        htmltxt = "<!DOCTYPE html>"
+        htmltxt += "<html><head>"
+        htmltxt += "<title>"+MATCH[matching.matchtype]["title"]+"</title>"
+        htmltxt += utils.css_tool.getInternalStyle("diffFiles")
+        htmltxt += "</head>"
+        htmltxt += "<body>"
+        htmltxt += "<h1>"+MATCH[matching.matchtype]["title"]+"</h1>"
+        htmltxt += "<h4>" + MATCH[matching.matchtype]["longA"] +": " + matching.matchfiles["A"] + "</h4>"
+        htmltxt += "<h4>" + MATCH[matching.matchtype]["longB"] +": " + matching.matchfiles["B"] + "</h4><br>"
+        matching.htmltext = htmltxt
+
+    def setDiffFooter(self):
+        job = basic.program.Job.getInstance()
+        verify = int(job.getDebugLevel("match_tool")) - 4
+        job.debug(verify, "getDiffFooter ")
+        htmltext = self.htmltext
+        htmltext += "</body></html>"
+        self.htmltext = htmltext
 
 def matchFiles(matching):
     """
@@ -19,9 +148,8 @@ def matchFiles(matching):
     :param matching:
     :return:
     """
-    pass
 
-def matchBestfit(matching):
+def matchBestfit(matching, path):
     """
     in this strategy the difference-score of all elements of both sides will be calculated.
     the best fit will assigned together until nothing is
@@ -30,35 +158,98 @@ def matchBestfit(matching):
     :param matching:
     :return:
     """
-    hits = {}
+    i=0
-    output = []
+    for r in matching.sideA:
-    results = {}
+        k = composeKey("a", i)
-    targets = {}
+        matching.setHit(k, "null")
-    for tg in matching.elements["tgelement"]:
+        i += 1
-        targets[tg] = True
+    i = 0
-    for rs in matching.elements["rselement"]:
+    for r in matching.sideB:
-        results[rs] = True
+        k = composeKey("b", i)
-        for tg in matching.elements["tgelement"]:
+        matching.setHit("null", k)
-            acthit = matching.comp.getHitscore(matching.elementtyp, matching.elements["rselement"][rs], matching.elements["tgelement"][tg])
+        i += 1
-            while acthit in hits.keys():
+    ia = 0
-                acthit = getNextHitscore(acthit)
+    ix = 1
-            hits[acthit] = {}
+    for rA in matching.sideA:
-            hits[acthit]["result"] = rs
+        ib = 0
-            hits[acthit]["target"] = tg
+        for rB in matching.sideB:
-    for h in sorted(hits.keys()):
+            if (matching.isHitB(composeKey("b", ib))):
-        if results[h]["result"] and targets[h]["target"]:
+                ib += 1
-            results[h]["result"] = False
+                continue
-            targets[h]["target"] = False
+            similarity=getSimilarity(matching, path, rA, rB, ix)
-            output.append((hits[acthit]["result"], hits[acthit]["target"]))
+            if (similarity == "MATCH"):
-    for rs in results.keys():
+                matching.setHit(composeKey("a", ia), composeKey("b", ib))
-        if results[rs]:
+                continue
-            output.append((matching.elements["rselement"][rs], None))
+            else:
-    for tg in targets.keys():
+                matching.setNohit(similarity, composeKey("a", ia), composeKey("b", ib))
-        if results[rs]:
+            ib += 1
-            output.append((matching.elements["tgelement"][tg], None))
+            ix += 1
-    return output
+        ia += 1
 
-# --------------------------------------------------------------------------
+def matchRestfit(matching):
+    """ """
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool"))-1
+    job.debug(verify, "matchRestfit ")
+    for x in sorted(matching.nomatch, reverse=True):
+        job.debug(verify, "matchRestfit " +x)
+        pair = matching.nomatch[x]
+        if (matching.isHitA(pair[0])):
+            print("A "+pair[0]+" bereits zugeordnet mit "+matching.linksA[pair[0]])
+        if (matching.isHitB(pair[1])):
+            print("B " + pair[1] + " bereits zugeordnet mit " + matching.linksB[pair[1]])
+            continue
+        if (matching.isHitA(pair[0])):
+            continue
+        print("neues Matching "+pair[0]+" "+pair[1])
+        matching.setHit(pair[0], pair[1])
+
+def composeKey(side, i):
+    return side.lower()+str(i+1).zfill(4)
+def extractKeyI(key):
+    return int(key[1:])-1
+
+def getSimilarity(matching, path, rA, rB, i):
+    """ it calculates the similarity between both rows by:
+    concat each criteria with single-similarity 00..99 and i with 999..000 """
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool"))-1
+    job.debug(verify, "getSimilarity "+path+" "+str(i))
+    if len(matching.matchkeys) > 0:
+        keys = matching.matchkeys
+    elif ("ddl" in matching.comp.conf):
+        job.debug(verify, "ddl " + path + " " + str(i))
+        a = path.split(":")
+        ddl = matching.comp.conf["ddl"]
+        for x in a:
+            if (len(x) < 2): continue
+            if (x == "_data"): break
+            if x in ddl: ddl = ddl[x]
+        job.debug(verify, "ddl " + json.dumps(ddl) + " " + str(i))
+        keys = {}
+        for f in ddl:
+            job.debug(verify, "ddl " + f + " " + str(i))
+            if ("key" in ddl[f]) and (len(ddl[f]["key"])>0):
+                b = ddl[f]["key"].split(":")
+                if (len(b)!=2): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"])
+                if (not b[1].isnumeric()): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"])
+                k = "k"+b[1].zfill(2)
+                job.debug(verify, "ddl " + f + " " + str(i))
+                keys[k]={ "ktyp": b[0], "field": ddl[f]["feld"], "type": ddl[f]["type"], "rule": ddl[f]["acceptance"]}
+        matching.matchkeys=keys
+        job.debug(verify, "ddl " + json.dumps(keys) + " " + str(i))
+    msim=""
+    topsim=""
+    for k in sorted(matching.matchkeys):
+        msim += getStringSimilarity(str(rA[matching.matchkeys[k]["field"]]), str(rB[matching.matchkeys[k]["field"]]))
+        topsim += "99"
+    if msim == topsim:
+        job.debug(verify, "Treffer ")
+        return "MATCH"
+    else:
+        job.debug(verify, "nomatch S"+msim+str(i).zfill(3))
+        return "S"+msim+str(i).zfill(3)
+    pass
 
 def matchTree(matching):
     """
@@ -66,20 +257,156 @@ def matchTree(matching):
     :param matching:
     :return:
     """
-    pass
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool"))-4
+    job.debug(verify, "matching "+matching.mode)
+    matchElement(matching, matching.sideA, matching.sideB, "")
+    matching.setDiffFooter()
+    return matching.htmltext
+
+def matchElement(matching, A, B, path):
+    """ travers through the datatree """
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool"))-4
+    job.debug(verify, "matchElem "+path+" A "+str(type(A))+" B "+str(type(B)))
+    if ((A is not None) and (isinstance(A, list))) \
+        or ((B is not None) and (isinstance(B, list))):
+        return matchArray(matching, A, B, path)
+    elif ((A is not None) and (isinstance(A, dict))) \
+        or ((B is not None) and (isinstance(B, dict))):
+        return matchDict(matching, A, B, path)
+    else:
+        return matching
+
+def getStringSimilarity(strA, strB):
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool"))-1
+    job.debug(verify, "getStringSimilarity "+strA+" ?= "+strB)
+    if (strA == strB): return "99"
+    if (strA.strip() == strB.strip()): return "77"
+    if (strA.lower() == strB.lower()): return "66"
+    if (strA.strip().lower() == strB.strip().lower()): return "55"
+    return "00"
+
+def getEvaluation(matching, type, acceptance, sideA, sideB):
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool"))-1
+    job.debug(verify, "getEvaluation "+str(sideA)+" ?= "+str(sideB))
+    match = getStringSimilarity(str(sideA), str(sideB))
+    colorA = "black"
+    colorB = "black"
+    classA = "black"
+    classB = "black"
+    result = "test"
+    if match == "99": return ["MATCH", "black", "black", "black", "black"]
+    if acceptance == "ignore": result = "ignore"
+    if (matching.matchtype == MATCH_POSTCOND) and (result == "test"):
+        result = "hard"
+        colorA = "green"
+        colorB = "crimson"
+        classA = "diffA"
+        classB = "diffB"
+    else:
+        colorA = "darkblue"
+        colorB = "darkmagenta"
+        classA = "acceptA"
+        classB = "acceptB"
+    return [result, colorA, colorB, classA, classB]
 
-    def matchElement(matching, rs, tg):
+def matchDict(matching, A, B, path):
-        if "array" in type(rs) and "array" in type(tg):
+    """ travers through the datatree """
-            return matchArray(matching, rs, tg)
+    job = basic.program.Job.getInstance()
-        elif "dict" in type(rs) and "dict" in type(tg):
+    verify = int(job.getDebugLevel("match_tool"))-4
-            return matchDict(matching, rs, tg)
+    job.debug(verify, "matchDict "+path)
+    for k in A:
+        print(k)
+        if k == "_match":
+            continue
+        if (B is not None) and (B[k]):
+            if (isinstance(A[k], dict)): A[k]["_match"] = "Y"
+            if (isinstance(B[k], dict)): B[k]["_match"] = "Y"
+            matchElement(matching, A[k], B[k], path+":"+k)
         else:
-            pass
+            if (isinstance(A[k], dict)): A[k]["_march"] = "N"
+            matchElement(matching, A[k], None, path+":"+k)
+    for k in B:
+        if k == "_match":
+            continue
+        if (A is not None) and (A[k]):
+            continue
+        else:
+            if (isinstance(A[k], dict)): B[k]["_match"] = "N"
+            matchElement(matching, None, B[k], path+":"+k)
+    return matching
+
+def matchArray(matching, A, B, path):
+    """ matches the datarows of the datatree """
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool"))-1
+    job.debug(verify, "matchArray "+path)
+    matching.sideA = A
+    matching.sideB = B
+    matchBestfit(matching, path)
+    matchRestfit(matching)
+    htmltext = compareRows(matching, path)
+    matching.htmltext += htmltext
 
-    def matchDict(matching, rs, tg):
+def compareRows(matching, path):
-        pass
+    """ traverse through matched rows """
-    def matchArray(matching, rs, tg):
+    job = basic.program.Job.getInstance()
-        pass # matchBest
+    verify = int(job.getDebugLevel("match_tool"))-1
+    ddl = matching.getTableDdl(path)
+    table = ""
+    a = path.split(":")
+    for x in a:
+        if (x == "_data"): break
+        table = x
+    header = []
+    htmltext = "<p>Tabelle : "+table+"</p>"
+    htmltext += "<table>"
+    htmltext += "<tr><th></th>"
+    for f in ddl["_header"]:
+        job.debug(verify, "ddl " + f + " ")
+        header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
+        htmltext += "<th>"+f+"</th>"
+    htmltext += "</tr>"
+    for k in sorted(matching.linksA):
+        htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))],
+                   matching.sideB[int(extractKeyI(matching.linksA[k]))])
+    htmltext += "</table>"
+    return htmltext
+
+def compareRow(matching, header, rA, rB):
+    """ traverse through matched rows """
+    job = basic.program.Job.getInstance()
+    verify = int(job.getDebugLevel("match_tool"))-4
+    allident = True
+    textA = ""
+    textB = ""
+    for f in header:
+        print(f)
+        res = getEvaluation(matching, f["type"], f["acceptance"], rA[f["field"]], rB[f["field"]])
+        match = res[0]
+        colorA = res[1]
+        colorB = res[2]
+        classA = res[3]
+        classB = res[4]
+        if (match == "MATCH"):
+            textA += "<td>"+str(rA[f["field"]])+"</td>"
+            textB += "<td>"+str(rB[f["field"]])+"</td>"
+        elif (match == "hard"):
+            allident = False
+            textA += "<td "+utils.css_tool.getInlineStyle("diffFiles", classA)+">" + str(rA[f["field"]]) + "</td>"
+            textB += "<td "+utils.css_tool.getInlineStyle("diffFiles", classB)+">" + str(rB[f["field"]]) + "</td>"
+            #textB += "<td style=\"color:" + colorB + ";\">" + str(rB[f["field"]]) + "</td>"
+        else:
+            allident = False
+            textA += "<td "+utils.css_tool.getInlineStyle("diffFiles", classA)+">"+str(rA[f["field"]])+" ("+match+")</td>"
+            textB += "<td "+utils.css_tool.getInlineStyle("diffFiles", classB)+">"+str(rB[f["field"]])+" ("+match+")</td>"
+            #textB += "<td style=\"color:"+colorB+";\">"+str(rA[f["field"]])+" ("+match+")</td>"
+    if allident:
+        return "<tr><td/>"+textA+"</tr>"
+    return "<tr><td>"+MATCH[matching.matchtype]["shortA"]+"</td>"+textA+"</tr><tr><td>"+MATCH[matching.matchtype]["shortB"]+"</td>"+textB+"</tr>"
 
 # --------------------------------------------------------------------------
 def matchLines(matching):