#!/usr/bin/python # -*- coding: utf-8 -*- # --------------------------------------------------------------------------------------------------------- # Author : Ulrich Carmesin # Source : gitea.ucarmesin.de # --------------------------------------------------------------------------------------------------------- import json import utils.css_tool import utils.report_tool import basic.program import basic.constants as B import utils.match_const as M import utils.data_const as D # ------------------------------------------------------------ """ """ class Matching(): def __init__(self, job, comp): self.job = job self.comp = comp self.elements = {} self.matchfiles = {} self.assignedFiles = {} self.linksA = {} self.linksB = {} self.nomatch = {} self.matchkeys = {} self.htmltext = "" self.sideA = [] self.sideB = [] self.mode = "" self.matchtype = "" self.cssClass = "result0" self.preIds = {} def setData(self, tdata, match): """ selects testdata for actual matching :param tdata: :param match: kind of actual match :return: """ sideA = M.MATCH[match][M.M_SIDE_A] sideB = M.MATCH[match][M.M_SIDE_B] self.sideA = tdata[sideA]["data"] self.sideB = tdata[sideB]["data"] self.matchfiles[M.M_SIDE_A] = tdata[sideA]["path"] self.matchfiles[M.M_SIDE_B] = tdata[sideB]["path"] self.matchtype = match self.mode = M.MATCH[match]["mode"] self.setDiffHeader() self.report = utils.report_tool.Report.getInstance(self.job) self.resetHits() if match == M.MATCH_PRECOND: self.preIds = {} # structure db:scheme:table:... # get ddl # for _data: for ddl._header: if SIM_TECHNICAL in ddl[h][key]: preIds.append(_data[h]) def resetHits(self): self.linksA = {} self.linksB = {} self.nomatch = {} self.matchkeys = {} self.cssClass = "result0" def setCssClass(self, cssClass): if cssClass > self.cssClass: self.cssClass = cssClass def isHitA(self, key): return ((key in self.linksA) and (self.linksA[key] != B.SVAL_NULL)) def isHitB(self, key): return ((key in self.linksB) and (self.linksB[key] != B.SVAL_NULL)) def setHit(self, keyA, keyB): if (not self.isHitA(keyA)) and (not self.isHitB(keyB)): if (keyA != B.SVAL_NULL): self.linksA[keyA] = keyB if (keyB != B.SVAL_NULL): self.linksB[keyB] = keyA return "OK" raise Exception("one of the links are set") def setNohit(self, similarity, keyA, keyB): """ The similarity must be unique. Only a mismatch will be set. """ if similarity in self.nomatch: raise Exception("similarity " + similarity + " exists") if (self.isHitA(keyA) or self.isHitB(keyB)): return self.nomatch[similarity] = [keyA, keyB] def getTableDdl(self, path): a = path.split(":") ddl = self.comp.conf[B.DATA_NODE_DDL] for x in a: if (len(x) < 2): continue if (x == B.DATA_NODE_DATA): break if x in ddl: ddl = ddl[x] return ddl def setDiffHeader(matching): job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 1 job.debug(verify, "getDiffHeader ") htmltxt = "" htmltxt += "" htmltxt += "" + M.MATCH[matching.matchtype]["title"] + "" htmltxt += utils.css_tool.getInternalStyle(job, "diffFiles") htmltxt += "" htmltxt += "" htmltxt += "

" + M.MATCH[matching.matchtype]["title"] + "

" htmltxt += "

" + M.MATCH[M.MATCH[matching.matchtype][M.M_SIDE_A]][M.M_SIDE_LONG] + ": " + matching.matchfiles[ M.M_SIDE_A] + "

" htmltxt += "

" + M.MATCH[M.MATCH[matching.matchtype][M.M_SIDE_B]][M.M_SIDE_LONG] + ": " + matching.matchfiles[ M.M_SIDE_B] + "


" matching.htmltext = htmltxt def setDiffFooter(self): job = self.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 4 job.debug(verify, "getDiffFooter ") htmltext = self.htmltext htmltext += "" self.htmltext = htmltext def ignorePreid(matching, key): if matching.matchtype == M.MATCH_POSTCOND: key_int = extractKeyI(key) if matching.linksB[key] != B.SVAL_NULL: return False if matching.preIds is None or len(matching.preIds) < 1: return False for ids in matching.preIds: hit = True for k in ids: row = matching.sideB[key_int] if ids[k] == row[k]: pass else: hit = False if hit: return True return False def matchFiles(matching): """ post: :param matching: :return: """ def matchBestfit(matching, path): """ in this strategy the difference-score of all elements of both sides will be calculated. the best fit will assigned together until nothing is * the elements can be each kind of object * the count of elements should not be high :param matching: :return: """ # initialize all potential links with null i = 0 if (matching.sideA is not None): for r in matching.sideA: k = composeKey("a", i) matching.setHit(k, B.SVAL_NULL) i += 1 i = 0 if (matching.sideB is not None): for r in matching.sideB: k = composeKey("b", i) matching.setHit(B.SVAL_NULL, k) i += 1 ia = 0 ix = 1 # CASE: one side is None if (matching.sideA is None) or (matching.sideB is None): return # CASE: to match for rA in matching.sideA: ib = 0 for rB in matching.sideB: if (matching.isHitA(composeKey("a", ia))): continue if (matching.isHitB(composeKey("b", ib))): ib += 1 continue similarity = getSimilarity(matching, rA, rB, ix, M.MATCH[matching.matchtype]["simorder"]) if (similarity == "MATCH"): matching.setHit(composeKey("a", ia), composeKey("b", ib)) continue else: matching.setNohit(similarity, composeKey("a", ia), composeKey("b", ib)) ib += 1 ix += 1 ia += 1 def matchRestfit(matching): """ """ job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 1 job.debug(verify, "matchRestfit ") for x in sorted(matching.nomatch, reverse=True): job.debug(verify, "matchRestfit " + x) pair = matching.nomatch[x] if (matching.isHitA(pair[0])): job.debug(verify, "A " + pair[0] + " bereits zugeordnet mit " + matching.linksA[pair[0]]) pass if (matching.isHitB(pair[1])): job.debug(verify, "B " + pair[1] + " bereits zugeordnet mit " + matching.linksB[pair[1]]) continue if (matching.isHitA(pair[0])): continue job.debug(verify, "neues Matching " + pair[0] + " " + pair[1]) matching.setHit(pair[0], pair[1]) def composeKey(side, i): return side.lower() + str(i + 1).zfill(4) def extractKeyI(key): return int(key[1:]) - 1 def setMatchkeys(matching, path): job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 1 job.debug(verify, "getSimilarity " + path) if len(matching.matchkeys) > 0: return if (B.DATA_NODE_DDL in matching.comp.conf): job.debug(verify, "ddl " + path) a = path.split(":") ddl = matching.comp.conf[B.DATA_NODE_DDL] for x in a: if (len(x) < 2): continue if (x == B.DATA_NODE_DATA): break if x in ddl: ddl = ddl[x] job.debug(verify, "ddl " + json.dumps(ddl)) keys = {} for f in ddl: job.debug(verify, "ddl " + f) if (D.DDL_KEY in ddl[f]) and (len(ddl[f][D.DDL_KEY]) > 0): b = ddl[f][D.DDL_KEY].split(":") if (len(b) != 2): raise Exception("falsch formatierter Schluessel " + ddl[f][D.DDL_KEY]) if (not b[1].isnumeric()): raise Exception("falsch formatierter Schluessel " + ddl[f][D.DDL_KEY]) if (b[0] not in M.SIM_DEFAULT): raise Exception("falsch formatierter Schluessel " + ddl[f][D.DDL_KEY]) k = "k"+b[0]+""+b[1].zfill(2) job.debug(verify, "ddl " + f) keys[k] = {"ktyp": b[0], D.DDL_FNAME: ddl[f][D.DDL_FNAME], D.DDL_TYPE: ddl[f][D.DDL_TYPE], "rule": ddl[f][D.DDL_ACCEPTANCE]} matching.matchkeys = keys job.debug(verify, "ddl " + json.dumps(keys)) def getSimilarity(matching, rA, rB, i, simorder=M.SIM_DEFAULT): """ it calculates the similarity between both rows by: concat each criteria with single-similarity 00..99 and i with 999..000 """ job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 1 job.debug(verify, "getSimilarity ") mBsim = "" mTsim = "" topBsim = "" topTsim = "" for k in sorted(matching.matchkeys): if not D.DDL_FNAME in matching.matchkeys[k]: job.m.logError("technical Attribut is missing "+k ) continue if M.SIM_TECHNICAL in k: if matching.matchkeys[k][D.DDL_FNAME] in rA and matching.matchkeys[k][D.DDL_FNAME] in rB: mTsim += getStringSimilarity(job, str(rA[matching.matchkeys[k][D.DDL_FNAME]]), str(rB[matching.matchkeys[k][D.DDL_FNAME]])) else: mTsim += "00" topTsim += "99" if M.SIM_BUSINESS in k: if matching.matchkeys[k][D.DDL_FNAME] in rA and matching.matchkeys[k][D.DDL_FNAME] in rB: mBsim += getStringSimilarity(job, str(rA[matching.matchkeys[k][D.DDL_FNAME]]), str(rB[matching.matchkeys[k][D.DDL_FNAME]])) else: mBsim += "55" topBsim += "99" if mBsim == topBsim and mTsim == topTsim: job.debug(verify, "Treffer ") return "MATCH" elif simorder[0:1] == M.SIM_BUSINESS and mBsim == topBsim: job.debug(verify, "Treffer ") return "MATCH" elif simorder[0:1] == M.SIM_TECHNICAL and mTsim == topTsim: job.debug(verify, "Treffer ") return "MATCH" elif simorder[0:1] == M.SIM_TECHNICAL: return "S"+mTsim+mBsim+str(i).zfill(3) else: # if simorder[0:1] == M.SIM_BUSINESS: return "S" + mBsim + mTsim + str(i).zfill(3) def matchTree(matching): """ :param matching: :return: """ job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 4 job.debug(verify, "..>> start matching " + matching.mode) matchElement(matching, matching.sideA, matching.sideB, "") matching.setDiffFooter() job.debug(verify, "..>> ende matching " + matching.htmltext) return matching.htmltext def matchElement(matching, A, B, path): """ travers through the datatree """ job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 4 job.debug(verify, "matchElem " + path + " A " + str(type(A)) + " B " + str(type(B))) if ((A is not None) and (isinstance(A, list))) \ or ((B is not None) and (isinstance(B, list))): return matchArray(matching, A, B, path) elif ((A is not None) and (isinstance(A, dict))) \ or ((B is not None) and (isinstance(B, dict))): return matchDict(matching, A, B, path) else: return matching def getStringSimilarity(job, strA, strB): #job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 1 job.debug(verify, "getStringSimilarity " + strA + " ?= " + strB) if (strA == strB): return "99" if (strA.strip() == strB.strip()): return "77" if (strA.lower() == strB.lower()): return "66" if (strA.strip().lower() == strB.strip().lower()): return "55" return "00" def getEvaluation(matching, type, acceptance, sideA, sideB): job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 1 job.debug(verify, "getEvaluation " + str(sideA) + " ?= " + str(sideB)) match = getStringSimilarity(job, str(sideA), str(sideB)) classA = "novalue" classB = "novalue" result = "test" if match == "99": return ["MATCH", "novalue", "novalue", "novalue", "novalue"] if acceptance == "ignore": result = "ignore" if (matching.matchtype == M.MATCH_POSTCOND) and (result == "test"): result = "hard" classA = "diffA" classB = "diffB" else: classA = "acceptA" classB = "acceptB" return [result, classA, classB] def matchDict(matching, sideA, sideB, path): """ travers through the datatree """ job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 4 job.debug(verify, "matchDict " + path) if (sideA is not None): for k in sideA: job.debug(verify, "matchDict 400 " + k + ".") if k in ["_match", D.DATA_ATTR_COUNT, D.DATA_ATTR_DATE, B.DATA_NODE_HEADER]: continue if (sideB is not None) and (k in sideB): if (isinstance(sideA[k], dict)): sideA[k]["_match"] = "Y" if (isinstance(sideB[k], dict)): sideB[k]["_match"] = "Y" job.debug(verify, "matchDict 404 " + k + "." + path) matchElement(matching, sideA[k], sideB[k], path + ":" + k) else: if (isinstance(sideA[k], dict)): sideA[k]["_match"] = "N" job.debug(verify, "matchDict 408 " + path) matchElement(matching, sideA[k], None, path + ":" + k) if (sideB is not None): for k in sideB: job.debug(verify, "matchDict 412 " + k + ".") if k in ["_match", D.DATA_ATTR_COUNT, D.DATA_ATTR_DATE, B.DATA_NODE_HEADER]: continue if (sideA is not None) and (k in sideA): continue elif (sideA is None) or (k not in sideA): if (sideA is not None) and (isinstance(sideA[k], dict)): sideB[k]["_match"] = "N" job.debug(verify, "matchDict 418 " + k + "." + path) matchElement(matching, None, sideB[k], path + ":" + k) job.debug(verify, "matchDict 420 ...<<---") return matching def matchArray(matching, sideA, sideB, path): """ matches the datarows of the datatree """ job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 4 job.debug(verify, "matchArray " + path + "\n.." + matching.htmltext) matching.sideA = sideA matching.sideB = sideB setMatchkeys(matching, path) matchBestfit(matching, path) matchRestfit(matching) a = path.split(":") for x in a: if (x == "_data"): break table = x report = utils.report_tool.Report.getInstance(job) report.setPaths(getattr(job.par, B.PAR_TESTCASE), matching.comp.name, table, matching.matchtype, matching.matchfiles[M.M_SIDE_A], matching.matchfiles[M.M_SIDE_B]) # report.setMatchResult("TC0001", "comp01", "arte01", m, "result" + str(i), "" + str(i) + "
") htmltext = report.getTitle(getattr(job.par, B.PAR_TESTCASE), matching.comp.name, table, matching.matchtype) htmltext += report.getComponentHead(getattr(job.par, B.PAR_TESTCASE), matching.comp.name, table, matching.matchtype) htmltext += report.getArtefactBlock(getattr(job.par, B.PAR_TESTCASE), matching.comp.name, table, matching.matchtype) htmltext += compareRows(matching, path) matching.htmltext += htmltext def compareRows(matching, path): """ traverse through matched rows """ job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 1 ddl = matching.getTableDdl(path) report = utils.report_tool.Report.getInstance(job) table = "" header = [] # "

Tabelle : "+table+"

" htmltext = "" for f in ddl[B.DATA_NODE_HEADER]: job.debug(verify, "ddl " + f + " ") header.append({D.DDL_FNAME: f, D.DDL_TYPE: ddl[f][D.DDL_TYPE], D.DDL_ACCEPTANCE: ddl[f][D.DDL_ACCEPTANCE]}) htmltext += "" htmltext += "" matching.difftext = htmltext for k in sorted(matching.linksA): print(k) if (matching.isHitA(k)): htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))], matching.sideB[int(extractKeyI(matching.linksA[k]))]) else: htmltext += markRow(matching, header, matching.sideA[int(extractKeyI(k))], M.M_SIDE_A) matching.setCssClass("result3") for k in sorted(matching.linksB): if ignorePreid(matching, k): continue if (not matching.isHitB(k)): htmltext += markRow(matching, header, matching.sideB[int(extractKeyI(k))], M.M_SIDE_B) matching.setCssClass("result2") htmltext += "
" + f + "
" matching.difftext += "" return htmltext def markRow(matching, header, row, side): job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 4 text = "" cssClass = "" for f in header: if not f[D.DDL_FNAME] in row: val = "" cssClass = "novalue" elif side == M.M_SIDE_A: res = getEvaluation(matching, f[D.DDL_TYPE], f[D.DDL_ACCEPTANCE], row[f[D.DDL_FNAME]], "") val = str(row[f[D.DDL_FNAME]]) cssClass = res[1] else: res = getEvaluation(matching, f[D.DDL_TYPE], f[D.DDL_ACCEPTANCE], "", row[f[D.DDL_FNAME]]) val = str(row[f[D.DDL_FNAME]]) cssClass = res[2] text += "" + val + "" text = "" \ + M.MATCH[M.MATCH[matching.matchtype][side]]["short"] + "" + text + "" matching.difftext += text return text def compareRow(matching, header, rA, rB): """ traverse through matched rows """ job = matching.job # basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 4 allident = True textA = "" textB = "" text = "" valA = "" valB = "" classA = "novalue" classB = "novalue" for f in header: print(f) if f[D.DDL_FNAME] in rA and f[D.DDL_FNAME] in rB: res = getEvaluation(matching, f[D.DDL_TYPE], f[D.DDL_ACCEPTANCE], rA[f[D.DDL_FNAME]], rB[f[D.DDL_FNAME]]) match = res[0] classA = res[1] classB = res[2] valA = str(rA[f[D.DDL_FNAME]]) valB = str(rB[f[D.DDL_FNAME]]) elif f[D.DDL_FNAME] in rA: valA = str(rA[f[D.DDL_FNAME]]) match = "ddl" classA = "acceptA" classB = "acceptB" elif f[D.DDL_FNAME] in rB: valB = str(rB[f[D.DDL_FNAME]]) match = "ddl" classA = "acceptA" classB = "acceptB" else: match = "MATCH" classA = "acceptA" classB = "acceptB" if (match == "MATCH"): textA += "" + valA + "" textB += "" + valB + "" matching.setCssClass("result1") elif (match == "hard"): allident = False textA += "" + valA + "" textB += "" + valB + "" matching.setCssClass("result3") else: allident = False textA += "" + valA + " (" + match + ")" textB += "" + valB + " (" + match + ")" matching.setCssClass("result1") if allident: return "" + textA + "" text = "" + M.MATCH[M.MATCH[matching.matchtype][M.M_SIDE_A]]["short"] + "" + textA + "" text += "" + M.MATCH[M.MATCH[matching.matchtype][M.M_SIDE_B]]["short"] + "" + textB + "" matching.difftext += text return text # -------------------------------------------------------------------------- def matchLines(matching): pass