Browse Source

matching and comparing of tables

master
Ulrich Carmesin 3 years ago
parent
commit
8de7af5321
  1. 10
      basic/program.py
  2. 0
      test/test_compare.py
  3. 53
      test/test_css.py
  4. 2
      test/test_main.py
  5. 59
      utils/css_tool.py
  6. 415
      utils/match_tool.py

10
basic/program.py

@ -310,5 +310,11 @@ class Configuration:
doc = yaml.full_load(file) doc = yaml.full_load(file)
for i, v in doc.items(): for i, v in doc.items():
self.confs[i] = v self.confs[i] = v
print ("set conf") def setConfig(self, path, val):
print ("set configuration") a = path.split(".")
if len(a) == 1:
self.confs[a[0]] = val
elif len(a) == 2:
self.confs[a[0]][a[1]] = val
elif len(a) == 3:
self.confs[a[0]][a[1]][a[2]] = val

0
test/test_compare.py

53
test/test_css.py

@ -0,0 +1,53 @@
import unittest
import utils.css_tool
import basic.program
import json
class MyTestCase(unittest.TestCase):
def test_css(self):
job = basic.program.Job("unit")
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug",
"tool": "job_tool", "tdtyp": "csv", "tdsrc": "implement", "tdname": "firstunit",
"modus": "unit"}
job.par.setParameterArgs(args)
print("eeeeeeeee")
print(json.dumps(job.conf.confs))
# ------- inline ---------------
job.conf.setConfig("tools.csstyp", "inline")
job.conf.confs.get("tools")["csstyp"] == "inline"
text = utils.css_tool.getInlineStyle("diffFiles", "diffA")
self.assertEqual(len(text), 37)
self.assertEqual(("style" in text), True)
text = utils.css_tool.getInternalStyle("diffFiles")
self.assertEqual(len(text), 0)
text = utils.css_tool.getExternalStyle("diffFiles")
self.assertEqual(len(text), 0)
# ------- internal ---------------
job.conf.setConfig("tools.csstyp", "internal")
text = utils.css_tool.getInlineStyle("diffFiles", "diffA")
self.assertEqual(len(text), 13)
self.assertEqual(("class" in text), True)
text = utils.css_tool.getInternalStyle("diffFiles")
print(text)
self.assertEqual(len(text), 173)
self.assertEqual(("<style>" in text), True)
text = utils.css_tool.getExternalStyle("diffFiles")
self.assertEqual(len(text), 0)
print(text)
print(str(len(text)))
# ------- external ---------------
job.conf.setConfig("tools.csstyp", "external")
text = utils.css_tool.getInlineStyle("diffFiles", "diffA")
self.assertEqual(len(text), 13)
self.assertEqual(("class" in text), True)
text = utils.css_tool.getInternalStyle("diffFiles")
print(text)
self.assertEqual(len(text), 55)
self.assertEqual(("<link " in text and "stylesheet" in text), True)
text = utils.css_tool.getExternalStyle("diffFiles")
self.assertEqual(len(text), 189)
print(text)
print(str(len(text)))
if __name__ == '__main__':
unittest.main()

2
test/test_main.py

@ -5,7 +5,7 @@ import basic.program as program
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
def test_pathTool(self): def test_pathTool(self):
x = program.Job("test:application=TEST:application=ENV01") x = program.Job("test:application=TEST:application=ENV01")
self.assertEqual(path_tool.generatePath("program", "komp", "testA", "CONFIG.yml"), "/home/basic/6_Projekte/PythonProject/komponents/testA/COFIG.yml") self.assertEqual(path_tool.generatePath("program", "komp", "testA", "CONFIG.yml"), "/home/ulrich/6_Projekte/holtz/components/testA/CONFIG.yml")
if __name__ == '__main__': if __name__ == '__main__':

59
utils/css_tool.py

@ -0,0 +1,59 @@
import basic.program
CSS_CLASS = {
"general": {
"table, td, th": "border: 1px solid grey;font-family:sans-serif"
},
"diffFiles": {
"diffA": "color:green;font-weight:bold;",
"diffB": "color:crimson;font-weight:bold;",
"acceptA": "color:darkblue;",
"acceptB": "color:darkmagenta;"
}
}
def getInlineStyle(filetype, cssclass):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("css_tool")) - 4
job.debug(verify, "getDiffHeader ")
print(job.conf.confs.get("tools").get("csstyp"))
if job.conf.confs.get("tools").get("csstyp") == "inline":
out = "style=\""+CSS_CLASS[filetype][cssclass]+"\""
else:
out = "class=\"" + cssclass + "\""
return out
def getInternalStyle(filetype):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "getDiffHeader ")
out = ""
if job.conf.confs.get("tools").get("csstyp") == "internal":
out = "<style>"
for c in CSS_CLASS["general"]:
line = "\n"+c+" { "+CSS_CLASS["general"][c]+"} "
line.replace(":", ": ").replace(";", "; ").replace("_Q_", ", ")
out += line
for c in CSS_CLASS[filetype]:
out += "\n."+c+" { "+CSS_CLASS[filetype][c].replace(":", ": ").replace(";", "; ")+"} "
out += "\n</style>"
elif job.conf.confs.get("tools").get("csstyp") == "external":
out = " <link rel=\"stylesheet\" href=\""+job.conf.confs.get("tools").get("cssfile")+"\"> "
else:
out = "<style>"
for c in CSS_CLASS["general"]:
line = "\n"+c+" { "+CSS_CLASS["general"][c]+"} "
line.replace(":", ": ").replace(";", "; ").replace("_Q_", ", ")
out += line
out += "\n</style>"
return out
def getExternalStyle(filetype):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "getDiffHeader ")
out = ""
if job.conf.confs.get("tools").get("csstyp") == "external":
for c in CSS_CLASS[filetype]:
out += c+" {\n "+CSS_CLASS[filetype][c].replace(":", ": ").replace(";", ";\n ")+"}\n"
out.replace("\n \n}", "\n}")
return out

415
utils/match_tool.py

@ -1,16 +1,145 @@
# #
import json
import utils.css_tool
import basic.program
# ------------------------------------------------------------ # ------------------------------------------------------------
""" """
""" """
MATCH = {
"preconditions": {
"A": "preReq",
"B": "preAct",
"shortA": "SV",
"shortB": "IV",
"longA": "Soll-Vorher",
"longB": "Ist-Vorher",
"mode": "info",
"title": "Pruefung Vorbedingung (Soll-Vorher - Ist-Vorher)"
},
"postconditions": {
"A": "postReq",
"B": "postAct",
"shortA": "SN",
"shortB": "IN",
"longA": "Soll-Nachher",
"longB": "Ist-Nachher",
"mode": "hard",
"title": "Fachliche Auswertung (Soll-Nachher - Ist-Nachher)"
},
"prepost": {
"A": "preAct",
"B": "postAct",
"shortA": "IV",
"shortB": "IN",
"longA": "Ist-Vorher",
"longB": "Ist-Nachher",
"mode": "action",
"title": "Ablauf-Differenz (Ist-Vorher - Ist-Nachher)"
},
"prestep": {
"A": "preStep",
"B": "postAct",
"shortA": "VS",
"shortB": "IN",
"longA": "Vor-Schritt",
"longB": "Ist-Nachher",
"mode": "action",
"title": "Schritt-Differenz (Vorschritt-Nachher - Ist-Nachher)"
}
}
MATCH_PREPOST = "prepost"
""" matches the action between pre- and postcondition of the actual testexecution """
MATCH_PRECOND = "preconditions"
""" matches the preconditions betwenn the required result the the actual testexecution
- just for info if the both executions have the same precondition """
MATCH_POSTCOND = "postconditions"
""" matches the postconditions betwenn the required result the the actual testexecution
- it is the main comparison """
class Matching(): class Matching():
def __init__(self, comp): def __init__(self, comp):
self.comp = comp self.comp = comp
self.elements = {} self.elements = {}
self.resultFiles = comp.resultFiles self.matchfiles = comp.files
self.targetFiles = comp.targetFiles
self.assignedFiles = {} self.assignedFiles = {}
pass self.linksA = {}
self.linksB = {}
self.nomatch = {}
self.matchkeys = {}
self.htmltext = ""
self.sideA = []
self.sideB = []
self.mode = ""
self.matchtype = ""
def setData(self, tdata, match):
"""
selects testdata for actual matching
:param tdata:
:param match: kind of actual match
:return:
"""
self.sideA = tdata[MATCH[match]["A"]]
self.sideB = tdata[MATCH[match]["B"]]
self.matchtype = match
self.mode = MATCH[match]["mode"]
self.setDiffHeader()
def resetHits(self):
self.linksA = {}
self.linksB = {}
self.nomatch = {}
self.matchkeys = {}
def isHitA(self, key):
return ((key in self.linksA) and (self.linksA[key] != "null"))
def isHitB(self, key):
return ((key in self.linksB) and (self.linksB[key] != "null"))
def setHit(self, keyA, keyB):
if (not self.isHitA(keyA)) and (not self.isHitB(keyB)):
if (keyA != "null"): self.linksA[keyA] = keyB
if (keyB != "null"): self.linksB[keyB] = keyA
return "OK"
raise Exception("one of the links are set")
def setNohit(self, similarity, keyA, keyB):
""" The similarity must be unique. Only a mismatch will be set. """
if similarity in self.nomatch:
raise Exception("similarity "+similarity+" exists")
if (self.isHitA(keyA) or self.isHitB(keyB)):
return
self.nomatch[similarity] = [keyA, keyB]
def getTableDdl(self, path):
a = path.split(":")
ddl = self.comp.conf["ddl"]
for x in a:
if (len(x) < 2): continue
if (x == "_data"): break
if x in ddl: ddl = ddl[x]
return ddl
def setDiffHeader(matching):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-4
job.debug(verify, "getDiffHeader ")
htmltxt = "<!DOCTYPE html>"
htmltxt += "<html><head>"
htmltxt += "<title>"+MATCH[matching.matchtype]["title"]+"</title>"
htmltxt += utils.css_tool.getInternalStyle("diffFiles")
htmltxt += "</head>"
htmltxt += "<body>"
htmltxt += "<h1>"+MATCH[matching.matchtype]["title"]+"</h1>"
htmltxt += "<h4>" + MATCH[matching.matchtype]["longA"] +": " + matching.matchfiles["A"] + "</h4>"
htmltxt += "<h4>" + MATCH[matching.matchtype]["longB"] +": " + matching.matchfiles["B"] + "</h4><br>"
matching.htmltext = htmltxt
def setDiffFooter(self):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "getDiffFooter ")
htmltext = self.htmltext
htmltext += "</body></html>"
self.htmltext = htmltext
def matchFiles(matching): def matchFiles(matching):
""" """
@ -19,9 +148,8 @@ def matchFiles(matching):
:param matching: :param matching:
:return: :return:
""" """
pass
def matchBestfit(matching): def matchBestfit(matching, path):
""" """
in this strategy the difference-score of all elements of both sides will be calculated. in this strategy the difference-score of all elements of both sides will be calculated.
the best fit will assigned together until nothing is the best fit will assigned together until nothing is
@ -30,35 +158,98 @@ def matchBestfit(matching):
:param matching: :param matching:
:return: :return:
""" """
hits = {} i=0
output = [] for r in matching.sideA:
results = {} k = composeKey("a", i)
targets = {} matching.setHit(k, "null")
for tg in matching.elements["tgelement"]: i += 1
targets[tg] = True i = 0
for rs in matching.elements["rselement"]: for r in matching.sideB:
results[rs] = True k = composeKey("b", i)
for tg in matching.elements["tgelement"]: matching.setHit("null", k)
acthit = matching.comp.getHitscore(matching.elementtyp, matching.elements["rselement"][rs], matching.elements["tgelement"][tg]) i += 1
while acthit in hits.keys(): ia = 0
acthit = getNextHitscore(acthit) ix = 1
hits[acthit] = {} for rA in matching.sideA:
hits[acthit]["result"] = rs ib = 0
hits[acthit]["target"] = tg for rB in matching.sideB:
for h in sorted(hits.keys()): if (matching.isHitB(composeKey("b", ib))):
if results[h]["result"] and targets[h]["target"]: ib += 1
results[h]["result"] = False continue
targets[h]["target"] = False similarity=getSimilarity(matching, path, rA, rB, ix)
output.append((hits[acthit]["result"], hits[acthit]["target"])) if (similarity == "MATCH"):
for rs in results.keys(): matching.setHit(composeKey("a", ia), composeKey("b", ib))
if results[rs]: continue
output.append((matching.elements["rselement"][rs], None)) else:
for tg in targets.keys(): matching.setNohit(similarity, composeKey("a", ia), composeKey("b", ib))
if results[rs]: ib += 1
output.append((matching.elements["tgelement"][tg], None)) ix += 1
return output ia += 1
# -------------------------------------------------------------------------- def matchRestfit(matching):
""" """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-1
job.debug(verify, "matchRestfit ")
for x in sorted(matching.nomatch, reverse=True):
job.debug(verify, "matchRestfit " +x)
pair = matching.nomatch[x]
if (matching.isHitA(pair[0])):
print("A "+pair[0]+" bereits zugeordnet mit "+matching.linksA[pair[0]])
if (matching.isHitB(pair[1])):
print("B " + pair[1] + " bereits zugeordnet mit " + matching.linksB[pair[1]])
continue
if (matching.isHitA(pair[0])):
continue
print("neues Matching "+pair[0]+" "+pair[1])
matching.setHit(pair[0], pair[1])
def composeKey(side, i):
return side.lower()+str(i+1).zfill(4)
def extractKeyI(key):
return int(key[1:])-1
def getSimilarity(matching, path, rA, rB, i):
""" it calculates the similarity between both rows by:
concat each criteria with single-similarity 00..99 and i with 999..000 """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-1
job.debug(verify, "getSimilarity "+path+" "+str(i))
if len(matching.matchkeys) > 0:
keys = matching.matchkeys
elif ("ddl" in matching.comp.conf):
job.debug(verify, "ddl " + path + " " + str(i))
a = path.split(":")
ddl = matching.comp.conf["ddl"]
for x in a:
if (len(x) < 2): continue
if (x == "_data"): break
if x in ddl: ddl = ddl[x]
job.debug(verify, "ddl " + json.dumps(ddl) + " " + str(i))
keys = {}
for f in ddl:
job.debug(verify, "ddl " + f + " " + str(i))
if ("key" in ddl[f]) and (len(ddl[f]["key"])>0):
b = ddl[f]["key"].split(":")
if (len(b)!=2): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"])
if (not b[1].isnumeric()): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"])
k = "k"+b[1].zfill(2)
job.debug(verify, "ddl " + f + " " + str(i))
keys[k]={ "ktyp": b[0], "field": ddl[f]["feld"], "type": ddl[f]["type"], "rule": ddl[f]["acceptance"]}
matching.matchkeys=keys
job.debug(verify, "ddl " + json.dumps(keys) + " " + str(i))
msim=""
topsim=""
for k in sorted(matching.matchkeys):
msim += getStringSimilarity(str(rA[matching.matchkeys[k]["field"]]), str(rB[matching.matchkeys[k]["field"]]))
topsim += "99"
if msim == topsim:
job.debug(verify, "Treffer ")
return "MATCH"
else:
job.debug(verify, "nomatch S"+msim+str(i).zfill(3))
return "S"+msim+str(i).zfill(3)
pass
def matchTree(matching): def matchTree(matching):
""" """
@ -66,20 +257,156 @@ def matchTree(matching):
:param matching: :param matching:
:return: :return:
""" """
pass job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-4
job.debug(verify, "matching "+matching.mode)
matchElement(matching, matching.sideA, matching.sideB, "")
matching.setDiffFooter()
return matching.htmltext
def matchElement(matching, A, B, path):
""" travers through the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-4
job.debug(verify, "matchElem "+path+" A "+str(type(A))+" B "+str(type(B)))
if ((A is not None) and (isinstance(A, list))) \
or ((B is not None) and (isinstance(B, list))):
return matchArray(matching, A, B, path)
elif ((A is not None) and (isinstance(A, dict))) \
or ((B is not None) and (isinstance(B, dict))):
return matchDict(matching, A, B, path)
else:
return matching
def getStringSimilarity(strA, strB):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-1
job.debug(verify, "getStringSimilarity "+strA+" ?= "+strB)
if (strA == strB): return "99"
if (strA.strip() == strB.strip()): return "77"
if (strA.lower() == strB.lower()): return "66"
if (strA.strip().lower() == strB.strip().lower()): return "55"
return "00"
def getEvaluation(matching, type, acceptance, sideA, sideB):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-1
job.debug(verify, "getEvaluation "+str(sideA)+" ?= "+str(sideB))
match = getStringSimilarity(str(sideA), str(sideB))
colorA = "black"
colorB = "black"
classA = "black"
classB = "black"
result = "test"
if match == "99": return ["MATCH", "black", "black", "black", "black"]
if acceptance == "ignore": result = "ignore"
if (matching.matchtype == MATCH_POSTCOND) and (result == "test"):
result = "hard"
colorA = "green"
colorB = "crimson"
classA = "diffA"
classB = "diffB"
else:
colorA = "darkblue"
colorB = "darkmagenta"
classA = "acceptA"
classB = "acceptB"
return [result, colorA, colorB, classA, classB]
def matchElement(matching, rs, tg): def matchDict(matching, A, B, path):
if "array" in type(rs) and "array" in type(tg): """ travers through the datatree """
return matchArray(matching, rs, tg) job = basic.program.Job.getInstance()
elif "dict" in type(rs) and "dict" in type(tg): verify = int(job.getDebugLevel("match_tool"))-4
return matchDict(matching, rs, tg) job.debug(verify, "matchDict "+path)
for k in A:
print(k)
if k == "_match":
continue
if (B is not None) and (B[k]):
if (isinstance(A[k], dict)): A[k]["_match"] = "Y"
if (isinstance(B[k], dict)): B[k]["_match"] = "Y"
matchElement(matching, A[k], B[k], path+":"+k)
else: else:
pass if (isinstance(A[k], dict)): A[k]["_march"] = "N"
matchElement(matching, A[k], None, path+":"+k)
for k in B:
if k == "_match":
continue
if (A is not None) and (A[k]):
continue
else:
if (isinstance(A[k], dict)): B[k]["_match"] = "N"
matchElement(matching, None, B[k], path+":"+k)
return matching
def matchArray(matching, A, B, path):
""" matches the datarows of the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-1
job.debug(verify, "matchArray "+path)
matching.sideA = A
matching.sideB = B
matchBestfit(matching, path)
matchRestfit(matching)
htmltext = compareRows(matching, path)
matching.htmltext += htmltext
def matchDict(matching, rs, tg): def compareRows(matching, path):
pass """ traverse through matched rows """
def matchArray(matching, rs, tg): job = basic.program.Job.getInstance()
pass # matchBest verify = int(job.getDebugLevel("match_tool"))-1
ddl = matching.getTableDdl(path)
table = ""
a = path.split(":")
for x in a:
if (x == "_data"): break
table = x
header = []
htmltext = "<p>Tabelle : "+table+"</p>"
htmltext += "<table>"
htmltext += "<tr><th></th>"
for f in ddl["_header"]:
job.debug(verify, "ddl " + f + " ")
header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
htmltext += "<th>"+f+"</th>"
htmltext += "</tr>"
for k in sorted(matching.linksA):
htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))],
matching.sideB[int(extractKeyI(matching.linksA[k]))])
htmltext += "</table>"
return htmltext
def compareRow(matching, header, rA, rB):
""" traverse through matched rows """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-4
allident = True
textA = ""
textB = ""
for f in header:
print(f)
res = getEvaluation(matching, f["type"], f["acceptance"], rA[f["field"]], rB[f["field"]])
match = res[0]
colorA = res[1]
colorB = res[2]
classA = res[3]
classB = res[4]
if (match == "MATCH"):
textA += "<td>"+str(rA[f["field"]])+"</td>"
textB += "<td>"+str(rB[f["field"]])+"</td>"
elif (match == "hard"):
allident = False
textA += "<td "+utils.css_tool.getInlineStyle("diffFiles", classA)+">" + str(rA[f["field"]]) + "</td>"
textB += "<td "+utils.css_tool.getInlineStyle("diffFiles", classB)+">" + str(rB[f["field"]]) + "</td>"
#textB += "<td style=\"color:" + colorB + ";\">" + str(rB[f["field"]]) + "</td>"
else:
allident = False
textA += "<td "+utils.css_tool.getInlineStyle("diffFiles", classA)+">"+str(rA[f["field"]])+" ("+match+")</td>"
textB += "<td "+utils.css_tool.getInlineStyle("diffFiles", classB)+">"+str(rB[f["field"]])+" ("+match+")</td>"
#textB += "<td style=\"color:"+colorB+";\">"+str(rA[f["field"]])+" ("+match+")</td>"
if allident:
return "<tr><td/>"+textA+"</tr>"
return "<tr><td>"+MATCH[matching.matchtype]["shortA"]+"</td>"+textA+"</tr><tr><td>"+MATCH[matching.matchtype]["shortB"]+"</td>"+textB+"</tr>"
# -------------------------------------------------------------------------- # --------------------------------------------------------------------------
def matchLines(matching): def matchLines(matching):

Loading…
Cancel
Save