#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import json
import utils.css_tool
import utils.report_tool
import basic.program
import basic.constants as B
import utils.match_const as M
# ------------------------------------------------------------
"""
"""
class Matching():
def __init__(self, comp):
self.comp = comp
self.elements = {}
self.matchfiles = {}
self.assignedFiles = {}
self.linksA = {}
self.linksB = {}
self.nomatch = {}
self.matchkeys = {}
self.htmltext = ""
self.sideA = []
self.sideB = []
self.mode = ""
self.matchtype = ""
self.cssClass = "result0"
def setData(self, tdata, match):
"""
selects testdata for actual matching
:param tdata:
:param match: kind of actual match
:return:
"""
sideA = M.MATCH[match]["A"]
sideB = M.MATCH[match]["B"]
self.sideA = tdata[sideA]["data"]
self.sideB = tdata[sideB]["data"]
self.matchfiles["A"] = tdata[sideA]["path"]
self.matchfiles["B"] = tdata[sideB]["path"]
self.matchtype = match
self.mode = M.MATCH[match]["mode"]
self.setDiffHeader()
self.report = utils.report_tool.Report.getInstance()
self.resetHits()
def resetHits(self):
self.linksA = {}
self.linksB = {}
self.nomatch = {}
self.matchkeys = {}
self.cssClass = "result0"
def setCssClass(self, cssClass):
if cssClass > self.cssClass:
self.cssClass = cssClass
def isHitA(self, key):
return ((key in self.linksA) and (self.linksA[key] != B.SVAL_NULL))
def isHitB(self, key):
return ((key in self.linksB) and (self.linksB[key] != B.SVAL_NULL))
def setHit(self, keyA, keyB):
if (not self.isHitA(keyA)) and (not self.isHitB(keyB)):
if (keyA != B.SVAL_NULL): self.linksA[keyA] = keyB
if (keyB != B.SVAL_NULL): self.linksB[keyB] = keyA
return "OK"
raise Exception("one of the links are set")
def setNohit(self, similarity, keyA, keyB):
""" The similarity must be unique. Only a mismatch will be set. """
if similarity in self.nomatch:
raise Exception("similarity " + similarity + " exists")
if (self.isHitA(keyA) or self.isHitB(keyB)):
return
self.nomatch[similarity] = [keyA, keyB]
def getTableDdl(self, path):
a = path.split(":")
ddl = self.comp.conf[B.DATA_NODE_DDL]
for x in a:
if (len(x) < 2): continue
if (x == B.DATA_NODE_DATA): break
if x in ddl: ddl = ddl[x]
return ddl
def setDiffHeader(matching):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getDiffHeader ")
htmltxt = ""
htmltxt += ""
htmltxt += "" + M.MATCH[matching.matchtype]["title"] + ""
htmltxt += utils.css_tool.getInternalStyle("diffFiles")
htmltxt += ""
htmltxt += ""
htmltxt += "" + M.MATCH[matching.matchtype]["title"] + "
"
htmltxt += "" + M.MATCH[M.MATCH[matching.matchtype]["A"]]["long"] + ": " + matching.matchfiles[
"A"] + "
"
htmltxt += "" + M.MATCH[M.MATCH[matching.matchtype]["B"]]["long"] + ": " + matching.matchfiles[
"B"] + "
"
matching.htmltext = htmltxt
def setDiffFooter(self):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "getDiffFooter ")
htmltext = self.htmltext
htmltext += ""
self.htmltext = htmltext
def matchFiles(matching):
"""
post:
:param matching:
:return:
"""
def matchBestfit(matching, path):
"""
in this strategy the difference-score of all elements of both sides will be calculated.
the best fit will assigned together until nothing is
* the elements can be each kind of object
* the count of elements should not be high
:param matching:
:return:
"""
# initialize all potential links with null
i = 0
if (matching.sideA is not None):
for r in matching.sideA:
k = composeKey("a", i)
matching.setHit(k, B.SVAL_NULL)
i += 1
i = 0
if (matching.sideB is not None):
for r in matching.sideB:
k = composeKey("b", i)
matching.setHit(B.SVAL_NULL, k)
i += 1
ia = 0
ix = 1
# CASE: one side is None
if (matching.sideA is None) or (matching.sideB is None):
return
# CASE: to match
for rA in matching.sideA:
ib = 0
for rB in matching.sideB:
if (matching.isHitA(composeKey("a", ia))):
continue
if (matching.isHitB(composeKey("b", ib))):
ib += 1
continue
similarity = getSimilarity(matching, rA, rB, ix, M.MATCH[matching.matchtype]["simorder"])
if (similarity == "MATCH"):
matching.setHit(composeKey("a", ia), composeKey("b", ib))
continue
else:
matching.setNohit(similarity, composeKey("a", ia), composeKey("b", ib))
ib += 1
ix += 1
ia += 1
def matchRestfit(matching):
""" """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "matchRestfit ")
for x in sorted(matching.nomatch, reverse=True):
job.debug(verify, "matchRestfit " + x)
pair = matching.nomatch[x]
if (matching.isHitA(pair[0])):
print("A " + pair[0] + " bereits zugeordnet mit " + matching.linksA[pair[0]])
if (matching.isHitB(pair[1])):
print("B " + pair[1] + " bereits zugeordnet mit " + matching.linksB[pair[1]])
continue
if (matching.isHitA(pair[0])):
continue
print("neues Matching " + pair[0] + " " + pair[1])
matching.setHit(pair[0], pair[1])
def composeKey(side, i):
return side.lower() + str(i + 1).zfill(4)
def extractKeyI(key):
return int(key[1:]) - 1
def setMatchkeys(matching, path):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getSimilarity " + path)
if len(matching.matchkeys) > 0:
return
if (B.DATA_NODE_DDL in matching.comp.conf):
job.debug(verify, "ddl " + path)
a = path.split(":")
ddl = matching.comp.conf[B.DATA_NODE_DDL]
for x in a:
if (len(x) < 2):
continue
if (x == B.DATA_NODE_DATA):
break
if x in ddl:
ddl = ddl[x]
job.debug(verify, "ddl " + json.dumps(ddl))
keys = {}
for f in ddl:
job.debug(verify, "ddl " + f)
if ("key" in ddl[f]) and (len(ddl[f]["key"]) > 0):
b = ddl[f]["key"].split(":")
if (len(b) != 2):
raise Exception("falsch formatierter Schluessel " + ddl[f]["key"])
if (not b[1].isnumeric()):
raise Exception("falsch formatierter Schluessel " + ddl[f]["key"])
k = "k"+b[0]+""+b[1].zfill(2)
job.debug(verify, "ddl " + f)
keys[k] = {"ktyp": b[0], "field": ddl[f]["feld"], "type": ddl[f]["type"], "rule": ddl[f]["acceptance"]}
matching.matchkeys = keys
job.debug(verify, "ddl " + json.dumps(keys))
def getSimilarity(matching, rA, rB, i, simorder=M.SIM_DEFAULT):
""" it calculates the similarity between both rows by:
concat each criteria with single-similarity 00..99 and i with 999..000 """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getSimilarity ")
mBsim = ""
mTsim = ""
topBsim = ""
topTsim = ""
for k in sorted(matching.matchkeys):
if M.SIM_TECHNICAL in k:
mTsim += getStringSimilarity(str(rA[matching.matchkeys[k]["field"]]), str(rB[matching.matchkeys[k]["field"]]))
topTsim += "99"
if M.SIM_BUSINESS in k:
mTsim += getStringSimilarity(str(rA[matching.matchkeys[k]["field"]]), str(rB[matching.matchkeys[k]["field"]]))
topTsim += "99"
if mBsim == topBsim and mTsim == topTsim:
job.debug(verify, "Treffer ")
return "MATCH"
elif simorder[0:1] == M.SIM_TECHNICAL and mTsim == topTsim:
job.debug(verify, "Treffer ")
return "MATCH"
elif simorder[0:1] == M.SIM_BUSINESS and mBsim == topBsim:
job.debug(verify, "Treffer ")
return "MATCH"
elif simorder[0:1] == M.SIM_TECHNICAL:
return "S"+mTsim+mBsim+str(i).zfill(3)
else: # if simorder[0:1] == M.SIM_BUSINESS:
return "S" + mBsim + mTsim + str(i).zfill(3)
def matchTree(matching):
"""
:param matching:
:return:
"""
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "..>> start matching " + matching.mode)
matchElement(matching, matching.sideA, matching.sideB, "")
matching.setDiffFooter()
job.debug(verify, "..>> ende matching " + matching.htmltext)
return matching.htmltext
def matchElement(matching, A, B, path):
""" travers through the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "matchElem " + path + " A " + str(type(A)) + " B " + str(type(B)))
if ((A is not None) and (isinstance(A, list))) \
or ((B is not None) and (isinstance(B, list))):
return matchArray(matching, A, B, path)
elif ((A is not None) and (isinstance(A, dict))) \
or ((B is not None) and (isinstance(B, dict))):
return matchDict(matching, A, B, path)
else:
return matching
def getStringSimilarity(strA, strB):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getStringSimilarity " + strA + " ?= " + strB)
if (strA == strB): return "99"
if (strA.strip() == strB.strip()): return "77"
if (strA.lower() == strB.lower()): return "66"
if (strA.strip().lower() == strB.strip().lower()): return "55"
return "00"
def getEvaluation(matching, type, acceptance, sideA, sideB):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getEvaluation " + str(sideA) + " ?= " + str(sideB))
match = getStringSimilarity(str(sideA), str(sideB))
classA = "novalue"
classB = "novalue"
result = "test"
if match == "99": return ["MATCH", "novalue", "novalue", "novalue", "novalue"]
if acceptance == "ignore": result = "ignore"
if (matching.matchtype == M.MATCH_POSTCOND) and (result == "test"):
result = "hard"
classA = "diffA"
classB = "diffB"
else:
classA = "acceptA"
classB = "acceptB"
return [result, classA, classB]
def matchDict(matching, A, B, path):
""" travers through the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "matchDict " + path)
if (A is not None):
for k in A:
job.debug(verify, "matchDict 400 " + k + ".")
if k in ["_match"]:
continue
if (B is not None) and (k in B):
if (isinstance(A[k], dict)): A[k]["_match"] = "Y"
if (isinstance(B[k], dict)): B[k]["_match"] = "Y"
job.debug(verify, "matchDict 404 " + k + "." + path)
matchElement(matching, A[k], B[k], path + ":" + k)
else:
if (isinstance(A[k], dict)): A[k]["_match"] = "N"
job.debug(verify, "matchDict 408 " + path)
matchElement(matching, A[k], None, path + ":" + k)
if (B is not None):
for k in B:
job.debug(verify, "matchDict 412 " + k + ".")
if k in ["_match"]:
continue
if (A is not None) and (k in A):
continue
elif (A is None) or (k not in A):
if (A is not None) and (isinstance(A[k], dict)): B[k]["_match"] = "N"
job.debug(verify, "matchDict 418 " + k + "." + path)
matchElement(matching, None, B[k], path + ":" + k)
job.debug(verify, "matchDict 420 ...<<---")
return matching
def matchArray(matching, A, B, path):
""" matches the datarows of the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "matchArray " + path + "\n.." + matching.htmltext)
matching.sideA = A
matching.sideB = B
matchBestfit(matching, path)
matchRestfit(matching)
a = path.split(":")
for x in a:
if (x == "_data"): break
table = x
report = utils.report_tool.Report.getInstance()
report.setPaths(getattr(job.par, "testcase"), matching.comp.name, table, matching.matchtype,
matching.matchfiles["A"], matching.matchfiles["B"])
# report.setMatchResult("TC0001", "comp01", "arte01", m, "result" + str(i), "")
htmltext = report.getTitle(getattr(job.par, "testcase"), matching.comp.name, table, matching.matchtype)
htmltext += report.getComponentHead(getattr(job.par, "testcase"), matching.comp.name, table, matching.matchtype)
htmltext += report.getArtefactBlock(getattr(job.par, "testcase"), matching.comp.name, table, matching.matchtype)
htmltext += compareRows(matching, path)
matching.htmltext += htmltext
def compareRows(matching, path):
""" traverse through matched rows """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
ddl = matching.getTableDdl(path)
report = utils.report_tool.Report.getInstance()
table = ""
header = []
# "Tabelle : "+table+"
"
htmltext = " | "
for f in ddl[B.DATA_NODE_HEADER]:
job.debug(verify, "ddl " + f + " ")
header.append({"field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
htmltext += "" + f + " | "
htmltext += "
|---|
"
matching.difftext = htmltext
for k in sorted(matching.linksA):
print(k)
if (matching.isHitA(k)):
htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))],
matching.sideB[int(extractKeyI(matching.linksA[k]))])
else:
htmltext += markRow(matching, header, matching.sideA[int(extractKeyI(k))], "A")
matching.setCssClass("result3")
for k in sorted(matching.linksB):
if (not matching.isHitB(k)):
htmltext += markRow(matching, header, matching.sideB[int(extractKeyI(k))], "B")
matching.setCssClass("result2")
htmltext += "
"
matching.difftext += ""
return htmltext
def markRow(matching, header, row, side):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
text = ""
cssClass = ""
for f in header:
if not f["field"] in row:
val = ""
cssClass = "novalue"
elif side == "A":
res = getEvaluation(matching, f["type"], f["acceptance"], row[f["field"]], "")
val = str(row[f["field"]])
cssClass = res[1]
else:
res = getEvaluation(matching, f["type"], f["acceptance"], "", row[f["field"]])
val = str(row[f["field"]])
cssClass = res[2]
text += "" + val + " | "
text = "| " \
+ M.MATCH[M.MATCH[matching.matchtype][side]]["short"] + " | " + text + "
"
matching.difftext += text
return text
def compareRow(matching, header, rA, rB):
""" traverse through matched rows """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
allident = True
textA = ""
textB = ""
text = ""
valA = ""
valB = ""
classA = "novalue"
classB = "novalue"
for f in header:
print(f)
if f["field"] in rA and f["field"] in rB:
res = getEvaluation(matching, f["type"], f["acceptance"], rA[f["field"]], rB[f["field"]])
match = res[0]
classA = res[1]
classB = res[2]
valA = str(rA[f["field"]])
valB = str(rB[f["field"]])
elif f["field"] in rA:
valA = str(rA[f["field"]])
match = "ddl"
classA = "acceptA"
classB = "acceptB"
elif f["field"] in rB:
valB = str(rB[f["field"]])
match = "ddl"
classA = "acceptA"
classB = "acceptB"
else:
match = "MATCH"
classA = "acceptA"
classB = "acceptB"
if (match == "MATCH"):
textA += "" + valA + " | "
textB += "" + valB + " | "
matching.setCssClass("result1")
elif (match == "hard"):
allident = False
textA += "" + valA + " | "
textB += "" + valB + " | "
matching.setCssClass("result3")
else:
allident = False
textA += "" + valA + " (" + match + ") | "
textB += "" + valB + " (" + match + ") | "
matching.setCssClass("result1")
if allident:
return " | " + textA + "
"
text = "| " + M.MATCH[M.MATCH[matching.matchtype]["A"]]["short"] + " | " + textA + "
"
text += "| " + M.MATCH[matching.matchtype]["shortB"] + " | " + textB + "
"
matching.difftext += text
return text
# --------------------------------------------------------------------------
def matchLines(matching):
pass