You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
510 lines
18 KiB
510 lines
18 KiB
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
# ---------------------------------------------------------------------------------------------------------
|
|
# Author : Ulrich Carmesin
|
|
# Source : gitea.ucarmesin.de
|
|
# ---------------------------------------------------------------------------------------------------------
|
|
import json
|
|
import utils.css_tool
|
|
import basic.program
|
|
import basic.constants as B
|
|
# ------------------------------------------------------------
|
|
"""
|
|
|
|
"""
|
|
MATCH_SIDE_PREEXPECT = "preexpect"
|
|
""" it implies the precondition of the expectation """
|
|
MATCH_DICT_PREEXPECT = {
|
|
"short": "SV",
|
|
"long": "Soll-Vorher",
|
|
"filepattern": "rsprecond"
|
|
}
|
|
MATCH_SIDE_POSTEXPECT = "postexpect"
|
|
""" it implies the postcondition of the expectation - it is the expectation"""
|
|
MATCH_DICT_POSTEXPECT = {
|
|
"short": "SN",
|
|
"long": "Soll-Nachher",
|
|
"filepattern": "rsprecond"
|
|
}
|
|
MATCH_SIDE_PREACTUAL = "preactual"
|
|
""" it implies the precondition of the actual execution """
|
|
MATCH_DICT_PREACTUAL = {
|
|
"short": "IV",
|
|
"long": "Ist-Vorher",
|
|
"filepattern": "rsprecond"
|
|
}
|
|
MATCH_SIDE_POSTACTUAL = "postactual"
|
|
""" it implies the postondition of the actual execution - it is the result """
|
|
MATCH_DICT_POSTACTUAL = {
|
|
"short": "IN",
|
|
"long": "Ist-Nachher",
|
|
"filepattern": "rsprecond"
|
|
}
|
|
MATCH_SIDE_PRESTEP = "prestep"
|
|
""" it implies the postcondition of a preceding step of the actual execution - the preceding step must be configured in the component"""
|
|
MATCH_DICT_PRESTEP = {
|
|
"short": "VS",
|
|
"long": "Vorhergehender Schritt (Nachher)",
|
|
"filepattern": "rsprecond"
|
|
}
|
|
MATCH_SIDE_TESTCASE = "testexample"
|
|
""" it implies the postcondition of an exemplary testcase - the exemplary testcase must be parametrized """
|
|
MATCH_DICT_TESTCASE = {
|
|
"short": "VT",
|
|
"long": "Vergleichstestfall (Nachher)",
|
|
"filepattern": "rsprecond"
|
|
}
|
|
MATCH_SIDES = [MATCH_SIDE_PREEXPECT, MATCH_SIDE_POSTEXPECT, MATCH_SIDE_PREACTUAL, MATCH_SIDE_POSTACTUAL, MATCH_SIDE_PRESTEP, MATCH_SIDE_TESTCASE]
|
|
|
|
MATCH_SUCCESS = "success"
|
|
""" matches the action between pre- and postcondition of the actual testexecution """
|
|
MATCH_PRECOND = "preconditions"
|
|
""" matches the preconditions betwenn the required result the the actual testexecution
|
|
- just for info if the both executions have the same precondition """
|
|
MATCH_POSTCOND = "postconditions"
|
|
""" matches the postconditions betwenn the required result the the actual testexecution
|
|
- it is the main comparison """
|
|
MATCH_PRESTEP = "prestep"
|
|
MATCH_TESTEXAMPLE = "testeample"
|
|
MATCH_TYPES = [MATCH_PRECOND, MATCH_PRESTEP, MATCH_TESTEXAMPLE, MATCH_SUCCESS, MATCH_POSTCOND]
|
|
MATCH = {
|
|
MATCH_SIDE_PREEXPECT : MATCH_DICT_PREEXPECT,
|
|
MATCH_SIDE_POSTEXPECT : MATCH_DICT_POSTEXPECT,
|
|
MATCH_SIDE_PREACTUAL : MATCH_DICT_PREACTUAL,
|
|
MATCH_SIDE_POSTACTUAL : MATCH_DICT_POSTACTUAL,
|
|
MATCH_SIDE_PRESTEP : MATCH_DICT_PRESTEP,
|
|
MATCH_SIDE_TESTCASE : MATCH_DICT_TESTCASE,
|
|
MATCH_PRECOND: {
|
|
"A": MATCH_SIDE_PREEXPECT,
|
|
"B": MATCH_SIDE_PREACTUAL,
|
|
"shortA": "SV",
|
|
"shortB": "IV",
|
|
"longA": "Soll-Vorher",
|
|
"longB": "Ist-Vorher",
|
|
"mode": "info",
|
|
"title": "Pruefung Vorbedingung (Soll-Vorher - Ist-Vorher)"
|
|
},
|
|
MATCH_POSTCOND: {
|
|
"A": MATCH_SIDE_POSTEXPECT,
|
|
"B": MATCH_SIDE_POSTACTUAL,
|
|
"shortA": "SN",
|
|
"shortB": "IN",
|
|
"longA": "Soll-Nachher",
|
|
"longB": "Ist-Nachher",
|
|
"mode": "hard",
|
|
"title": "Fachliche Auswertung (Soll-Nachher - Ist-Nachher)"
|
|
},
|
|
MATCH_SUCCESS: {
|
|
"A": MATCH_SIDE_PREACTUAL,
|
|
"B": MATCH_SIDE_POSTACTUAL,
|
|
"shortA": "IV",
|
|
"shortB": "IN",
|
|
"longA": "Ist-Vorher",
|
|
"longB": "Ist-Nachher",
|
|
"mode": "action",
|
|
"title": "Ablauf-Differenz (Ist-Vorher - Ist-Nachher)"
|
|
},
|
|
MATCH_PRESTEP: {
|
|
"A": MATCH_SIDE_PRESTEP,
|
|
"B": MATCH_SIDE_POSTACTUAL,
|
|
"shortA": "VS",
|
|
"shortB": "IN",
|
|
"longA": "Vor-Schritt",
|
|
"longB": "Ist-Nachher",
|
|
"mode": "action",
|
|
"title": "Schritt-Differenz (Vorschritt-Nachher - Ist-Nachher)"
|
|
},
|
|
MATCH_TESTEXAMPLE: {
|
|
"A": MATCH_SIDE_TESTCASE,
|
|
"B": MATCH_SIDE_POSTACTUAL,
|
|
"shortA": "VS",
|
|
"shortB": "IN",
|
|
"longA": "Vor-Schritt",
|
|
"longB": "Ist-Nachher",
|
|
"mode": "action",
|
|
"title": "Schritt-Differenz (Vorschritt-Nachher - Ist-Nachher)"
|
|
},
|
|
}
|
|
|
|
|
|
class Matching():
|
|
def __init__(self, comp):
|
|
self.comp = comp
|
|
self.elements = {}
|
|
self.matchfiles = comp.files
|
|
self.assignedFiles = {}
|
|
self.linksA = {}
|
|
self.linksB = {}
|
|
self.nomatch = {}
|
|
self.matchkeys = {}
|
|
self.htmltext = ""
|
|
self.sideA = []
|
|
self.sideB = []
|
|
self.mode = ""
|
|
self.matchtype = ""
|
|
|
|
def setData(self, tdata, match):
|
|
"""
|
|
selects testdata for actual matching
|
|
:param tdata:
|
|
:param match: kind of actual match
|
|
:return:
|
|
"""
|
|
self.sideA = tdata[MATCH[match]["A"]]
|
|
self.sideB = tdata[MATCH[match]["B"]]
|
|
self.matchtype = match
|
|
self.mode = MATCH[match]["mode"]
|
|
self.setDiffHeader()
|
|
|
|
def resetHits(self):
|
|
self.linksA = {}
|
|
self.linksB = {}
|
|
self.nomatch = {}
|
|
self.matchkeys = {}
|
|
|
|
def isHitA(self, key):
|
|
print("isHitA "+str(key))
|
|
for k in self.linksA: print(k)
|
|
return ((key in self.linksA) and (self.linksA[key] != "null"))
|
|
def isHitB(self, key):
|
|
return ((key in self.linksB) and (self.linksB[key] != "null"))
|
|
def setHit(self, keyA, keyB):
|
|
if (not self.isHitA(keyA)) and (not self.isHitB(keyB)):
|
|
if (keyA != "null"): self.linksA[keyA] = keyB
|
|
if (keyB != "null"): self.linksB[keyB] = keyA
|
|
return "OK"
|
|
raise Exception("one of the links are set")
|
|
def setNohit(self, similarity, keyA, keyB):
|
|
""" The similarity must be unique. Only a mismatch will be set. """
|
|
if similarity in self.nomatch:
|
|
raise Exception("similarity "+similarity+" exists")
|
|
if (self.isHitA(keyA) or self.isHitB(keyB)):
|
|
return
|
|
self.nomatch[similarity] = [keyA, keyB]
|
|
def getTableDdl(self, path):
|
|
a = path.split(":")
|
|
ddl = self.comp.conf[B.DATA_NODE_DDL]
|
|
for x in a:
|
|
if (len(x) < 2): continue
|
|
if (x == B.DATA_NODE_DATA): break
|
|
if x in ddl: ddl = ddl[x]
|
|
return ddl
|
|
def setDiffHeader(matching):
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-1
|
|
job.debug(verify, "getDiffHeader ")
|
|
htmltxt = "<!DOCTYPE html>"
|
|
htmltxt += "<html><head>"
|
|
htmltxt += "<title>"+MATCH[matching.matchtype]["title"]+"</title>"
|
|
htmltxt += utils.css_tool.getInternalStyle("diffFiles")
|
|
htmltxt += "</head>"
|
|
htmltxt += "<body>"
|
|
htmltxt += "<h1>"+MATCH[matching.matchtype]["title"]+"</h1>"
|
|
htmltxt += "<h4>" + MATCH[matching.matchtype]["longA"] +": " + matching.matchfiles["A"] + "</h4>"
|
|
htmltxt += "<h4>" + MATCH[matching.matchtype]["longB"] +": " + matching.matchfiles["B"] + "</h4><br>"
|
|
matching.htmltext = htmltxt
|
|
|
|
def setDiffFooter(self):
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool")) - 4
|
|
job.debug(verify, "getDiffFooter ")
|
|
htmltext = self.htmltext
|
|
htmltext += "</body></html>"
|
|
self.htmltext = htmltext
|
|
|
|
def matchFiles(matching):
|
|
"""
|
|
|
|
post:
|
|
:param matching:
|
|
:return:
|
|
"""
|
|
|
|
def matchBestfit(matching, path):
|
|
"""
|
|
in this strategy the difference-score of all elements of both sides will be calculated.
|
|
the best fit will assigned together until nothing is
|
|
* the elements can be each kind of object
|
|
* the count of elements should not be high
|
|
:param matching:
|
|
:return:
|
|
"""
|
|
i=0
|
|
if (matching.sideA is not None):
|
|
for r in matching.sideA:
|
|
k = composeKey("a", i)
|
|
matching.setHit(k, "null")
|
|
i += 1
|
|
i = 0
|
|
if (matching.sideB is not None):
|
|
for r in matching.sideB:
|
|
k = composeKey("b", i)
|
|
matching.setHit("null", k)
|
|
i += 1
|
|
ia = 0
|
|
ix = 1
|
|
if (matching.sideA is None) or (matching.sideB is None):
|
|
return
|
|
for rA in matching.sideA:
|
|
ib = 0
|
|
for rB in matching.sideB:
|
|
if (matching.isHitB(composeKey("b", ib))):
|
|
ib += 1
|
|
continue
|
|
similarity=getSimilarity(matching, path, rA, rB, ix)
|
|
if (similarity == "MATCH"):
|
|
matching.setHit(composeKey("a", ia), composeKey("b", ib))
|
|
continue
|
|
else:
|
|
matching.setNohit(similarity, composeKey("a", ia), composeKey("b", ib))
|
|
ib += 1
|
|
ix += 1
|
|
ia += 1
|
|
|
|
def matchRestfit(matching):
|
|
""" """
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-1
|
|
job.debug(verify, "matchRestfit ")
|
|
for x in sorted(matching.nomatch, reverse=True):
|
|
job.debug(verify, "matchRestfit " +x)
|
|
pair = matching.nomatch[x]
|
|
if (matching.isHitA(pair[0])):
|
|
print("A "+pair[0]+" bereits zugeordnet mit "+matching.linksA[pair[0]])
|
|
if (matching.isHitB(pair[1])):
|
|
print("B " + pair[1] + " bereits zugeordnet mit " + matching.linksB[pair[1]])
|
|
continue
|
|
if (matching.isHitA(pair[0])):
|
|
continue
|
|
print("neues Matching "+pair[0]+" "+pair[1])
|
|
matching.setHit(pair[0], pair[1])
|
|
|
|
def composeKey(side, i):
|
|
return side.lower()+str(i+1).zfill(4)
|
|
def extractKeyI(key):
|
|
return int(key[1:])-1
|
|
|
|
def getSimilarity(matching, path, rA, rB, i):
|
|
""" it calculates the similarity between both rows by:
|
|
concat each criteria with single-similarity 00..99 and i with 999..000 """
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-1
|
|
job.debug(verify, "getSimilarity "+path+" "+str(i))
|
|
if len(matching.matchkeys) > 0:
|
|
keys = matching.matchkeys
|
|
elif (B.DATA_NODE_DDL in matching.comp.conf):
|
|
job.debug(verify, "ddl " + path + " " + str(i))
|
|
a = path.split(":")
|
|
ddl = matching.comp.conf[B.DATA_NODE_DDL]
|
|
for x in a:
|
|
if (len(x) < 2): continue
|
|
if (x == B.DATA_NODE_DATA): break
|
|
if x in ddl: ddl = ddl[x]
|
|
job.debug(verify, "ddl " + json.dumps(ddl) + " " + str(i))
|
|
keys = {}
|
|
for f in ddl:
|
|
job.debug(verify, "ddl " + f + " " + str(i))
|
|
if ("key" in ddl[f]) and (len(ddl[f]["key"])>0):
|
|
b = ddl[f]["key"].split(":")
|
|
if (len(b)!=2): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"])
|
|
if (not b[1].isnumeric()): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"])
|
|
k = "k"+b[1].zfill(2)
|
|
job.debug(verify, "ddl " + f + " " + str(i))
|
|
keys[k]={ "ktyp": b[0], "field": ddl[f]["feld"], "type": ddl[f]["type"], "rule": ddl[f]["acceptance"]}
|
|
matching.matchkeys=keys
|
|
job.debug(verify, "ddl " + json.dumps(keys) + " " + str(i))
|
|
msim=""
|
|
topsim=""
|
|
for k in sorted(matching.matchkeys):
|
|
msim += getStringSimilarity(str(rA[matching.matchkeys[k]["field"]]), str(rB[matching.matchkeys[k]["field"]]))
|
|
topsim += "99"
|
|
if msim == topsim:
|
|
job.debug(verify, "Treffer ")
|
|
return "MATCH"
|
|
else:
|
|
job.debug(verify, "nomatch S"+msim+str(i).zfill(3))
|
|
return "S"+msim+str(i).zfill(3)
|
|
pass
|
|
|
|
def matchTree(matching):
|
|
"""
|
|
:param matching:
|
|
:return:
|
|
"""
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-4
|
|
job.debug(verify, "matching "+matching.mode)
|
|
matchElement(matching, matching.sideA, matching.sideB, "")
|
|
matching.setDiffFooter()
|
|
return matching.htmltext
|
|
|
|
def matchElement(matching, A, B, path):
|
|
""" travers through the datatree """
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-4
|
|
job.debug(verify, "matchElem "+path+" A "+str(type(A))+" B "+str(type(B)))
|
|
if ((A is not None) and (isinstance(A, list))) \
|
|
or ((B is not None) and (isinstance(B, list))):
|
|
return matchArray(matching, A, B, path)
|
|
elif ((A is not None) and (isinstance(A, dict))) \
|
|
or ((B is not None) and (isinstance(B, dict))):
|
|
return matchDict(matching, A, B, path)
|
|
else:
|
|
return matching
|
|
|
|
def getStringSimilarity(strA, strB):
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-1
|
|
job.debug(verify, "getStringSimilarity "+strA+" ?= "+strB)
|
|
if (strA == strB): return "99"
|
|
if (strA.strip() == strB.strip()): return "77"
|
|
if (strA.lower() == strB.lower()): return "66"
|
|
if (strA.strip().lower() == strB.strip().lower()): return "55"
|
|
return "00"
|
|
|
|
def getEvaluation(matching, type, acceptance, sideA, sideB):
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-1
|
|
job.debug(verify, "getEvaluation "+str(sideA)+" ?= "+str(sideB))
|
|
match = getStringSimilarity(str(sideA), str(sideB))
|
|
classA = "black"
|
|
classB = "black"
|
|
result = "test"
|
|
if match == "99": return ["MATCH", "black", "black", "black", "black"]
|
|
if acceptance == "ignore": result = "ignore"
|
|
if (matching.matchtype == MATCH_POSTCOND) and (result == "test"):
|
|
result = "hard"
|
|
classA = "diffA"
|
|
classB = "diffB"
|
|
else:
|
|
classA = "acceptA"
|
|
classB = "acceptB"
|
|
return [result, classA, classB]
|
|
|
|
def matchDict(matching, A, B, path):
|
|
""" travers through the datatree """
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-4
|
|
job.debug(verify, "matchDict "+path)
|
|
if (A is not None):
|
|
for k in A:
|
|
print(k)
|
|
if k == "_match":
|
|
continue
|
|
if (B is not None) and (B[k]):
|
|
if (isinstance(A[k], dict)): A[k]["_match"] = "Y"
|
|
if (isinstance(B[k], dict)): B[k]["_match"] = "Y"
|
|
matchElement(matching, A[k], B[k], path+":"+k)
|
|
else:
|
|
if (isinstance(A[k], dict)): A[k]["_match"] = "N"
|
|
matchElement(matching, A[k], None, path+":"+k)
|
|
if (B is not None):
|
|
for k in B:
|
|
if k == "_match":
|
|
continue
|
|
if (A is not None) and (A[k]):
|
|
continue
|
|
else:
|
|
if (A is not None) and (isinstance(A[k], dict)): B[k]["_match"] = "N"
|
|
matchElement(matching, None, B[k], path+":"+k)
|
|
return matching
|
|
|
|
def matchArray(matching, A, B, path):
|
|
""" matches the datarows of the datatree """
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-1
|
|
job.debug(verify, "matchArray "+path)
|
|
matching.sideA = A
|
|
matching.sideB = B
|
|
matchBestfit(matching, path)
|
|
matchRestfit(matching)
|
|
htmltext = compareRows(matching, path)
|
|
matching.htmltext += htmltext
|
|
|
|
def compareRows(matching, path):
|
|
""" traverse through matched rows """
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-1
|
|
ddl = matching.getTableDdl(path)
|
|
table = ""
|
|
a = path.split(":")
|
|
for x in a:
|
|
if (x == "_data"): break
|
|
table = x
|
|
header = []
|
|
htmltext = "<p>Tabelle : "+table+"</p>"
|
|
htmltext += "<table>"
|
|
htmltext += "<tr><th></th>"
|
|
for f in ddl[B.DATA_NODE_HEADER]:
|
|
job.debug(verify, "ddl " + f + " ")
|
|
header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
|
|
htmltext += "<th>"+f+"</th>"
|
|
htmltext += "</tr>"
|
|
matching.difftext = htmltext
|
|
for k in sorted(matching.linksA):
|
|
print(k)
|
|
if (matching.isHitA(k)):
|
|
htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))],
|
|
matching.sideB[int(extractKeyI(matching.linksA[k]))])
|
|
else:
|
|
htmltext += markRow(matching, header, matching.sideA[int(extractKeyI(k))], "A")
|
|
for k in sorted(matching.linksB):
|
|
if (not matching.isHitB(k)):
|
|
htmltext += markRow(matching, header, matching.sideB[int(extractKeyI(k))], "B")
|
|
htmltext += "</table>"
|
|
matching.difftext += "</table>"
|
|
return htmltext
|
|
|
|
def markRow(matching, header, row, side):
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-4
|
|
text = ""
|
|
cssClass = ""
|
|
for f in header:
|
|
if side == "A":
|
|
res = getEvaluation(matching, f["type"], f["acceptance"], row[f["field"]], "")
|
|
cssClass = res[1]
|
|
else:
|
|
res = getEvaluation(matching, f["type"], f["acceptance"], "", row[f["field"]])
|
|
cssClass = res[2]
|
|
text += "<td "+utils.css_tool.getInlineStyle("diffFiles", cssClass)+">" + str(row[f["field"]]) + "</td>"
|
|
text = "<tr><td "+utils.css_tool.getInlineStyle("diffFiles", cssClass)+">" \
|
|
+ MATCH[matching.matchtype]["short"+side] + "</td>" + text + "</tr>"
|
|
matching.difftext += text
|
|
return text
|
|
|
|
def compareRow(matching, header, rA, rB):
|
|
""" traverse through matched rows """
|
|
job = basic.program.Job.getInstance()
|
|
verify = int(job.getDebugLevel("match_tool"))-4
|
|
allident = True
|
|
textA = ""
|
|
textB = ""
|
|
text = ""
|
|
for f in header:
|
|
print(f)
|
|
res = getEvaluation(matching, f["type"], f["acceptance"], rA[f["field"]], rB[f["field"]])
|
|
match = res[0]
|
|
classA = res[1]
|
|
classB = res[2]
|
|
if (match == "MATCH"):
|
|
textA += "<td>"+str(rA[f["field"]])+"</td>"
|
|
textB += "<td>"+str(rB[f["field"]])+"</td>"
|
|
elif (match == "hard"):
|
|
allident = False
|
|
textA += "<td "+utils.css_tool.getInlineStyle("diffFiles", classA)+">" + str(rA[f["field"]]) + "</td>"
|
|
textB += "<td "+utils.css_tool.getInlineStyle("diffFiles", classB)+">" + str(rB[f["field"]]) + "</td>"
|
|
else:
|
|
allident = False
|
|
textA += "<td "+utils.css_tool.getInlineStyle("diffFiles", classA)+">"+str(rA[f["field"]])+" ("+match+")</td>"
|
|
textB += "<td "+utils.css_tool.getInlineStyle("diffFiles", classB)+">"+str(rB[f["field"]])+" ("+match+")</td>"
|
|
if allident:
|
|
return "<tr><td/>"+textA+"</tr>"
|
|
text = "<tr><td>"+MATCH[matching.matchtype]["shortA"]+"</td>"+textA+"</tr><tr><td>"+MATCH[matching.matchtype]["shortB"]+"</td>"+textB+"</tr>"
|
|
matching.difftext += text
|
|
return text
|
|
|
|
# --------------------------------------------------------------------------
|
|
def matchLines(matching):
|
|
pass
|
|
|
|
|