#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import json
import utils.css_tool
import utils.report_tool
import basic.program
import basic.constants as B
import utils.match_const as M
import utils.data_const as D
# ------------------------------------------------------------
"""
"""
class Matching():
def __init__(self, comp):
self.comp = comp
self.elements = {}
self.matchfiles = {}
self.assignedFiles = {}
self.linksA = {}
self.linksB = {}
self.nomatch = {}
self.matchkeys = {}
self.htmltext = ""
self.sideA = []
self.sideB = []
self.mode = ""
self.matchtype = ""
self.cssClass = "result0"
self.preIds = {}
def setData(self, tdata, match):
"""
selects testdata for actual matching
:param tdata:
:param match: kind of actual match
:return:
"""
sideA = M.MATCH[match][M.M_SIDE_A]
sideB = M.MATCH[match][M.M_SIDE_B]
self.sideA = tdata[sideA]["data"]
self.sideB = tdata[sideB]["data"]
self.matchfiles[M.M_SIDE_A] = tdata[sideA]["path"]
self.matchfiles[M.M_SIDE_B] = tdata[sideB]["path"]
self.matchtype = match
self.mode = M.MATCH[match]["mode"]
self.setDiffHeader()
self.report = utils.report_tool.Report.getInstance()
self.resetHits()
if match == M.MATCH_PRECOND:
self.preIds = {} # structure db:scheme:table:...
# get ddl
# for _data: for ddl._header: if SIM_TECHNICAL in ddl[h][key]: preIds.append(_data[h])
def resetHits(self):
self.linksA = {}
self.linksB = {}
self.nomatch = {}
self.matchkeys = {}
self.cssClass = "result0"
def setCssClass(self, cssClass):
if cssClass > self.cssClass:
self.cssClass = cssClass
def isHitA(self, key):
return ((key in self.linksA) and (self.linksA[key] != B.SVAL_NULL))
def isHitB(self, key):
return ((key in self.linksB) and (self.linksB[key] != B.SVAL_NULL))
def setHit(self, keyA, keyB):
if (not self.isHitA(keyA)) and (not self.isHitB(keyB)):
if (keyA != B.SVAL_NULL): self.linksA[keyA] = keyB
if (keyB != B.SVAL_NULL): self.linksB[keyB] = keyA
return "OK"
raise Exception("one of the links are set")
def setNohit(self, similarity, keyA, keyB):
""" The similarity must be unique. Only a mismatch will be set. """
if similarity in self.nomatch:
raise Exception("similarity " + similarity + " exists")
if (self.isHitA(keyA) or self.isHitB(keyB)):
return
self.nomatch[similarity] = [keyA, keyB]
def getTableDdl(self, path):
a = path.split(":")
ddl = self.comp.conf[B.DATA_NODE_DDL]
for x in a:
if (len(x) < 2): continue
if (x == B.DATA_NODE_DATA): break
if x in ddl: ddl = ddl[x]
return ddl
def setDiffHeader(matching):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getDiffHeader ")
htmltxt = ""
htmltxt += ""
htmltxt += "" + M.MATCH[matching.matchtype]["title"] + ""
htmltxt += utils.css_tool.getInternalStyle("diffFiles")
htmltxt += ""
htmltxt += ""
htmltxt += "" + M.MATCH[matching.matchtype]["title"] + "
"
htmltxt += "" + M.MATCH[M.MATCH[matching.matchtype][M.M_SIDE_A]][M.M_SIDE_LONG] + ": " + matching.matchfiles[
M.M_SIDE_A] + "
"
htmltxt += "" + M.MATCH[M.MATCH[matching.matchtype][M.M_SIDE_B]][M.M_SIDE_LONG] + ": " + matching.matchfiles[
M.M_SIDE_B] + "
"
matching.htmltext = htmltxt
def setDiffFooter(self):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "getDiffFooter ")
htmltext = self.htmltext
htmltext += ""
self.htmltext = htmltext
def ignorePreid(matching, key):
if matching.matchtype == M.MATCH_POSTCOND:
key_int = extractKeyI(key)
if matching.linksB[key] != B.SVAL_NULL:
return False
if matching.preIds is None or len(matching.preIds) < 1:
return False
for ids in matching.preIds:
hit = True
for k in ids:
row = matching.sideB[key_int]
if ids[k] == row[k]:
pass
else:
hit = False
if hit:
return True
return False
def matchFiles(matching):
"""
post:
:param matching:
:return:
"""
def matchBestfit(matching, path):
"""
in this strategy the difference-score of all elements of both sides will be calculated.
the best fit will assigned together until nothing is
* the elements can be each kind of object
* the count of elements should not be high
:param matching:
:return:
"""
# initialize all potential links with null
i = 0
if (matching.sideA is not None):
for r in matching.sideA:
k = composeKey("a", i)
matching.setHit(k, B.SVAL_NULL)
i += 1
i = 0
if (matching.sideB is not None):
for r in matching.sideB:
k = composeKey("b", i)
matching.setHit(B.SVAL_NULL, k)
i += 1
ia = 0
ix = 1
# CASE: one side is None
if (matching.sideA is None) or (matching.sideB is None):
return
# CASE: to match
for rA in matching.sideA:
ib = 0
for rB in matching.sideB:
if (matching.isHitA(composeKey("a", ia))):
continue
if (matching.isHitB(composeKey("b", ib))):
ib += 1
continue
similarity = getSimilarity(matching, rA, rB, ix, M.MATCH[matching.matchtype]["simorder"])
if (similarity == "MATCH"):
matching.setHit(composeKey("a", ia), composeKey("b", ib))
continue
else:
matching.setNohit(similarity, composeKey("a", ia), composeKey("b", ib))
ib += 1
ix += 1
ia += 1
def matchRestfit(matching):
""" """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "matchRestfit ")
for x in sorted(matching.nomatch, reverse=True):
job.debug(verify, "matchRestfit " + x)
pair = matching.nomatch[x]
if (matching.isHitA(pair[0])):
job.debug(verify, "A " + pair[0] + " bereits zugeordnet mit " + matching.linksA[pair[0]])
pass
if (matching.isHitB(pair[1])):
job.debug(verify, "B " + pair[1] + " bereits zugeordnet mit " + matching.linksB[pair[1]])
continue
if (matching.isHitA(pair[0])):
continue
job.debug(verify, "neues Matching " + pair[0] + " " + pair[1])
matching.setHit(pair[0], pair[1])
def composeKey(side, i):
return side.lower() + str(i + 1).zfill(4)
def extractKeyI(key):
return int(key[1:]) - 1
def setMatchkeys(matching, path):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getSimilarity " + path)
if len(matching.matchkeys) > 0:
return
if (B.DATA_NODE_DDL in matching.comp.conf):
job.debug(verify, "ddl " + path)
a = path.split(":")
ddl = matching.comp.conf[B.DATA_NODE_DDL]
for x in a:
if (len(x) < 2):
continue
if (x == B.DATA_NODE_DATA):
break
if x in ddl:
ddl = ddl[x]
job.debug(verify, "ddl " + json.dumps(ddl))
keys = {}
for f in ddl:
job.debug(verify, "ddl " + f)
if (D.DDL_KEY in ddl[f]) and (len(ddl[f][D.DDL_KEY]) > 0):
b = ddl[f][D.DDL_KEY].split(":")
if (len(b) != 2):
raise Exception("falsch formatierter Schluessel " + ddl[f][D.DDL_KEY])
if (not b[1].isnumeric()):
raise Exception("falsch formatierter Schluessel " + ddl[f][D.DDL_KEY])
if (b[0] not in M.SIM_DEFAULT):
raise Exception("falsch formatierter Schluessel " + ddl[f][D.DDL_KEY])
k = "k"+b[0]+""+b[1].zfill(2)
job.debug(verify, "ddl " + f)
keys[k] = {"ktyp": b[0], D.DDL_FNAME: ddl[f][D.DDL_FNAME], D.DDL_TYPE: ddl[f][D.DDL_TYPE], "rule": ddl[f][D.DDL_ACCEPTANCE]}
matching.matchkeys = keys
job.debug(verify, "ddl " + json.dumps(keys))
def getSimilarity(matching, rA, rB, i, simorder=M.SIM_DEFAULT):
""" it calculates the similarity between both rows by:
concat each criteria with single-similarity 00..99 and i with 999..000 """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getSimilarity ")
mBsim = ""
mTsim = ""
topBsim = ""
topTsim = ""
for k in sorted(matching.matchkeys):
if not D.DDL_FNAME in matching.matchkeys[k]:
job.m.logError("technical Attribut is missing "+k )
continue
if M.SIM_TECHNICAL in k:
if matching.matchkeys[k][D.DDL_FNAME] in rA and matching.matchkeys[k][D.DDL_FNAME] in rB:
mTsim += getStringSimilarity(str(rA[matching.matchkeys[k][D.DDL_FNAME]]),
str(rB[matching.matchkeys[k][D.DDL_FNAME]]))
else:
mTsim += "00"
topTsim += "99"
if M.SIM_BUSINESS in k:
if matching.matchkeys[k][D.DDL_FNAME] in rA and matching.matchkeys[k][D.DDL_FNAME] in rB:
mBsim += getStringSimilarity(str(rA[matching.matchkeys[k][D.DDL_FNAME]]),
str(rB[matching.matchkeys[k][D.DDL_FNAME]]))
else:
mBsim += "55"
topBsim += "99"
if mBsim == topBsim and mTsim == topTsim:
job.debug(verify, "Treffer ")
return "MATCH"
elif simorder[0:1] == M.SIM_BUSINESS and mBsim == topBsim:
job.debug(verify, "Treffer ")
return "MATCH"
elif simorder[0:1] == M.SIM_TECHNICAL and mTsim == topTsim:
job.debug(verify, "Treffer ")
return "MATCH"
elif simorder[0:1] == M.SIM_TECHNICAL:
return "S"+mTsim+mBsim+str(i).zfill(3)
else: # if simorder[0:1] == M.SIM_BUSINESS:
return "S" + mBsim + mTsim + str(i).zfill(3)
def matchTree(matching):
"""
:param matching:
:return:
"""
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "..>> start matching " + matching.mode)
matchElement(matching, matching.sideA, matching.sideB, "")
matching.setDiffFooter()
job.debug(verify, "..>> ende matching " + matching.htmltext)
return matching.htmltext
def matchElement(matching, A, B, path):
""" travers through the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "matchElem " + path + " A " + str(type(A)) + " B " + str(type(B)))
if ((A is not None) and (isinstance(A, list))) \
or ((B is not None) and (isinstance(B, list))):
return matchArray(matching, A, B, path)
elif ((A is not None) and (isinstance(A, dict))) \
or ((B is not None) and (isinstance(B, dict))):
return matchDict(matching, A, B, path)
else:
return matching
def getStringSimilarity(strA, strB):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getStringSimilarity " + strA + " ?= " + strB)
if (strA == strB): return "99"
if (strA.strip() == strB.strip()): return "77"
if (strA.lower() == strB.lower()): return "66"
if (strA.strip().lower() == strB.strip().lower()): return "55"
return "00"
def getEvaluation(matching, type, acceptance, sideA, sideB):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
job.debug(verify, "getEvaluation " + str(sideA) + " ?= " + str(sideB))
match = getStringSimilarity(str(sideA), str(sideB))
classA = "novalue"
classB = "novalue"
result = "test"
if match == "99": return ["MATCH", "novalue", "novalue", "novalue", "novalue"]
if acceptance == "ignore": result = "ignore"
if (matching.matchtype == M.MATCH_POSTCOND) and (result == "test"):
result = "hard"
classA = "diffA"
classB = "diffB"
else:
classA = "acceptA"
classB = "acceptB"
return [result, classA, classB]
def matchDict(matching, sideA, sideB, path):
""" travers through the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "matchDict " + path)
if (sideA is not None):
for k in sideA:
job.debug(verify, "matchDict 400 " + k + ".")
if k in ["_match", D.DATA_ATTR_COUNT, D.DATA_ATTR_DATE, B.DATA_NODE_HEADER]:
continue
if (sideB is not None) and (k in sideB):
if (isinstance(sideA[k], dict)): sideA[k]["_match"] = "Y"
if (isinstance(sideB[k], dict)): sideB[k]["_match"] = "Y"
job.debug(verify, "matchDict 404 " + k + "." + path)
matchElement(matching, sideA[k], sideB[k], path + ":" + k)
else:
if (isinstance(sideA[k], dict)): sideA[k]["_match"] = "N"
job.debug(verify, "matchDict 408 " + path)
matchElement(matching, sideA[k], None, path + ":" + k)
if (sideB is not None):
for k in sideB:
job.debug(verify, "matchDict 412 " + k + ".")
if k in ["_match", D.DATA_ATTR_COUNT, D.DATA_ATTR_DATE, B.DATA_NODE_HEADER]:
continue
if (sideA is not None) and (k in sideA):
continue
elif (sideA is None) or (k not in sideA):
if (sideA is not None) and (isinstance(sideA[k], dict)): sideB[k]["_match"] = "N"
job.debug(verify, "matchDict 418 " + k + "." + path)
matchElement(matching, None, sideB[k], path + ":" + k)
job.debug(verify, "matchDict 420 ...<<---")
return matching
def matchArray(matching, sideA, sideB, path):
""" matches the datarows of the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
job.debug(verify, "matchArray " + path + "\n.." + matching.htmltext)
matching.sideA = sideA
matching.sideB = sideB
setMatchkeys(matching, path)
matchBestfit(matching, path)
matchRestfit(matching)
a = path.split(":")
for x in a:
if (x == "_data"): break
table = x
report = utils.report_tool.Report.getInstance()
report.setPaths(getattr(job.par, B.PAR_TESTCASE), matching.comp.name, table, matching.matchtype,
matching.matchfiles[M.M_SIDE_A], matching.matchfiles[M.M_SIDE_B])
# report.setMatchResult("TC0001", "comp01", "arte01", m, "result" + str(i), "")
htmltext = report.getTitle(getattr(job.par, B.PAR_TESTCASE), matching.comp.name, table, matching.matchtype)
htmltext += report.getComponentHead(getattr(job.par, B.PAR_TESTCASE), matching.comp.name, table, matching.matchtype)
htmltext += report.getArtefactBlock(getattr(job.par, B.PAR_TESTCASE), matching.comp.name, table, matching.matchtype)
htmltext += compareRows(matching, path)
matching.htmltext += htmltext
def compareRows(matching, path):
""" traverse through matched rows """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 1
ddl = matching.getTableDdl(path)
report = utils.report_tool.Report.getInstance()
table = ""
header = []
# "Tabelle : "+table+"
"
htmltext = " | "
for f in ddl[B.DATA_NODE_HEADER]:
job.debug(verify, "ddl " + f + " ")
header.append({D.DDL_FNAME: f, D.DDL_TYPE: ddl[f][D.DDL_TYPE], D.DDL_ACCEPTANCE: ddl[f][D.DDL_ACCEPTANCE]})
htmltext += "" + f + " | "
htmltext += "
|---|
"
matching.difftext = htmltext
for k in sorted(matching.linksA):
print(k)
if (matching.isHitA(k)):
htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))],
matching.sideB[int(extractKeyI(matching.linksA[k]))])
else:
htmltext += markRow(matching, header, matching.sideA[int(extractKeyI(k))], M.M_SIDE_A)
matching.setCssClass("result3")
for k in sorted(matching.linksB):
if ignorePreid(matching, k):
continue
if (not matching.isHitB(k)):
htmltext += markRow(matching, header, matching.sideB[int(extractKeyI(k))], M.M_SIDE_B)
matching.setCssClass("result2")
htmltext += "
"
matching.difftext += ""
return htmltext
def markRow(matching, header, row, side):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
text = ""
cssClass = ""
for f in header:
if not f[D.DDL_FNAME] in row:
val = ""
cssClass = "novalue"
elif side == M.M_SIDE_A:
res = getEvaluation(matching, f[D.DDL_TYPE], f[D.DDL_ACCEPTANCE], row[f[D.DDL_FNAME]], "")
val = str(row[f[D.DDL_FNAME]])
cssClass = res[1]
else:
res = getEvaluation(matching, f[D.DDL_TYPE], f[D.DDL_ACCEPTANCE], "", row[f[D.DDL_FNAME]])
val = str(row[f[D.DDL_FNAME]])
cssClass = res[2]
text += "" + val + " | "
text = "| " \
+ M.MATCH[M.MATCH[matching.matchtype][side]]["short"] + " | " + text + "
"
matching.difftext += text
return text
def compareRow(matching, header, rA, rB):
""" traverse through matched rows """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool")) - 4
allident = True
textA = ""
textB = ""
text = ""
valA = ""
valB = ""
classA = "novalue"
classB = "novalue"
for f in header:
print(f)
if f[D.DDL_FNAME] in rA and f[D.DDL_FNAME] in rB:
res = getEvaluation(matching, f[D.DDL_TYPE], f[D.DDL_ACCEPTANCE], rA[f[D.DDL_FNAME]], rB[f[D.DDL_FNAME]])
match = res[0]
classA = res[1]
classB = res[2]
valA = str(rA[f[D.DDL_FNAME]])
valB = str(rB[f[D.DDL_FNAME]])
elif f[D.DDL_FNAME] in rA:
valA = str(rA[f[D.DDL_FNAME]])
match = "ddl"
classA = "acceptA"
classB = "acceptB"
elif f[D.DDL_FNAME] in rB:
valB = str(rB[f[D.DDL_FNAME]])
match = "ddl"
classA = "acceptA"
classB = "acceptB"
else:
match = "MATCH"
classA = "acceptA"
classB = "acceptB"
if (match == "MATCH"):
textA += "" + valA + " | "
textB += "" + valB + " | "
matching.setCssClass("result1")
elif (match == "hard"):
allident = False
textA += "" + valA + " | "
textB += "" + valB + " | "
matching.setCssClass("result3")
else:
allident = False
textA += "" + valA + " (" + match + ") | "
textB += "" + valB + " (" + match + ") | "
matching.setCssClass("result1")
if allident:
return " | " + textA + "
"
text = "| " + M.MATCH[M.MATCH[matching.matchtype][M.M_SIDE_A]]["short"] + " | " + textA + "
"
text += "| " + M.MATCH[M.MATCH[matching.matchtype][M.M_SIDE_B]]["short"] + " | " + textB + "
"
matching.difftext += text
return text
# --------------------------------------------------------------------------
def matchLines(matching):
pass