#!/usr/bin/python # -*- coding: utf-8 -*- # --------------------------------------------------------------------------------------------------------- # Author : Ulrich Carmesin # Source : gitea.ucarmesin.de # --------------------------------------------------------------------------------------------------------- import json import utils.css_tool import utils.report_tool import basic.program import basic.constants as B import utils.match_const as M # ------------------------------------------------------------ """ """ class Matching(): def __init__(self, comp): self.comp = comp self.elements = {} self.matchfiles = {} self.assignedFiles = {} self.linksA = {} self.linksB = {} self.nomatch = {} self.matchkeys = {} self.htmltext = "" self.sideA = [] self.sideB = [] self.mode = "" self.matchtype = "" self.cssClass = "result0" def setData(self, tdata, match): """ selects testdata for actual matching :param tdata: :param match: kind of actual match :return: """ sideA = M.MATCH[match]["A"] sideB = M.MATCH[match]["B"] self.sideA = tdata[sideA]["data"] self.sideB = tdata[sideB]["data"] self.matchfiles["A"] = tdata[sideA]["path"] self.matchfiles["B"] = tdata[sideB]["path"] self.matchtype = match self.mode = M.MATCH[match]["mode"] self.setDiffHeader() self.report = utils.report_tool.Report.getInstance() self.resetHits() def resetHits(self): self.linksA = {} self.linksB = {} self.nomatch = {} self.matchkeys = {} self.cssClass = "result0" def setCssClass(self, cssClass): if cssClass > self.cssClass: self.cssClass = cssClass def isHitA(self, key): return ((key in self.linksA) and (self.linksA[key] != B.SVAL_NULL)) def isHitB(self, key): return ((key in self.linksB) and (self.linksB[key] != B.SVAL_NULL)) def setHit(self, keyA, keyB): if (not self.isHitA(keyA)) and (not self.isHitB(keyB)): if (keyA != B.SVAL_NULL): self.linksA[keyA] = keyB if (keyB != B.SVAL_NULL): self.linksB[keyB] = keyA return "OK" raise Exception("one of the links are set") def setNohit(self, similarity, keyA, keyB): """ The similarity must be unique. Only a mismatch will be set. """ if similarity in self.nomatch: raise Exception("similarity "+similarity+" exists") if (self.isHitA(keyA) or self.isHitB(keyB)): return self.nomatch[similarity] = [keyA, keyB] def getTableDdl(self, path): a = path.split(":") ddl = self.comp.conf[B.DATA_NODE_DDL] for x in a: if (len(x) < 2): continue if (x == B.DATA_NODE_DATA): break if x in ddl: ddl = ddl[x] return ddl def setDiffHeader(matching): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-1 job.debug(verify, "getDiffHeader ") htmltxt = "" htmltxt += "" htmltxt += ""+M.MATCH[matching.matchtype]["title"]+"" htmltxt += utils.css_tool.getInternalStyle("diffFiles") htmltxt += "" htmltxt += "" htmltxt += "

"+M.MATCH[matching.matchtype]["title"]+"

" htmltxt += "

"+M.MATCH[M.MATCH[matching.matchtype]["A"]]["long"]+": "+matching.matchfiles["A"]+"

" htmltxt += "

"+M.MATCH[M.MATCH[matching.matchtype]["B"]]["long"]+": "+matching.matchfiles["B"]+"


" matching.htmltext = htmltxt def setDiffFooter(self): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool")) - 4 job.debug(verify, "getDiffFooter ") htmltext = self.htmltext htmltext += "" self.htmltext = htmltext def matchFiles(matching): """ post: :param matching: :return: """ def matchBestfit(matching, path): """ in this strategy the difference-score of all elements of both sides will be calculated. the best fit will assigned together until nothing is * the elements can be each kind of object * the count of elements should not be high :param matching: :return: """ # initialize all potential links with null i=0 if (matching.sideA is not None): for r in matching.sideA: k = composeKey("a", i) matching.setHit(k, B.SVAL_NULL) i += 1 i = 0 if (matching.sideB is not None): for r in matching.sideB: k = composeKey("b", i) matching.setHit(B.SVAL_NULL, k) i += 1 ia = 0 ix = 1 # CASE: one side is None if (matching.sideA is None) or (matching.sideB is None): return # CASE: to match for rA in matching.sideA: ib = 0 for rB in matching.sideB: if (matching.isHitA(composeKey("a", ia))): continue if (matching.isHitB(composeKey("b", ib))): ib += 1 continue similarity=getSimilarity(matching, path, rA, rB, ix) if (similarity == "MATCH"): matching.setHit(composeKey("a", ia), composeKey("b", ib)) continue else: matching.setNohit(similarity, composeKey("a", ia), composeKey("b", ib)) ib += 1 ix += 1 ia += 1 def matchRestfit(matching): """ """ job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-1 job.debug(verify, "matchRestfit ") for x in sorted(matching.nomatch, reverse=True): job.debug(verify, "matchRestfit " +x) pair = matching.nomatch[x] if (matching.isHitA(pair[0])): print("A "+pair[0]+" bereits zugeordnet mit "+matching.linksA[pair[0]]) if (matching.isHitB(pair[1])): print("B " + pair[1] + " bereits zugeordnet mit " + matching.linksB[pair[1]]) continue if (matching.isHitA(pair[0])): continue print("neues Matching "+pair[0]+" "+pair[1]) matching.setHit(pair[0], pair[1]) def composeKey(side, i): return side.lower()+str(i+1).zfill(4) def extractKeyI(key): return int(key[1:])-1 def getSimilarity(matching, path, rA, rB, i): """ it calculates the similarity between both rows by: concat each criteria with single-similarity 00..99 and i with 999..000 """ job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-1 job.debug(verify, "getSimilarity "+path+" "+str(i)) if len(matching.matchkeys) > 0: keys = matching.matchkeys elif (B.DATA_NODE_DDL in matching.comp.conf): job.debug(verify, "ddl " + path + " " + str(i)) a = path.split(":") ddl = matching.comp.conf[B.DATA_NODE_DDL] for x in a: if (len(x) < 2): continue if (x == B.DATA_NODE_DATA): break if x in ddl: ddl = ddl[x] job.debug(verify, "ddl " + json.dumps(ddl) + " " + str(i)) keys = {} for f in ddl: job.debug(verify, "ddl " + f + " " + str(i)) if ("key" in ddl[f]) and (len(ddl[f]["key"])>0): b = ddl[f]["key"].split(":") if (len(b)!=2): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"]) if (not b[1].isnumeric()): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"]) k = "k"+b[1].zfill(2) job.debug(verify, "ddl " + f + " " + str(i)) keys[k]={ "ktyp": b[0], "field": ddl[f]["feld"], "type": ddl[f]["type"], "rule": ddl[f]["acceptance"]} matching.matchkeys=keys job.debug(verify, "ddl " + json.dumps(keys) + " " + str(i)) msim="" topsim="" for k in sorted(matching.matchkeys): msim += getStringSimilarity(str(rA[matching.matchkeys[k]["field"]]), str(rB[matching.matchkeys[k]["field"]])) topsim += "99" if msim == topsim: job.debug(verify, "Treffer ") return "MATCH" else: job.debug(verify, "nomatch S"+msim+str(i).zfill(3)) return "S"+msim+str(i).zfill(3) pass def matchTree(matching): """ :param matching: :return: """ job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-4 job.debug(verify, "..>> start matching "+matching.mode) matchElement(matching, matching.sideA, matching.sideB, "") matching.setDiffFooter() job.debug(verify, "..>> ende matching "+matching.htmltext) return matching.htmltext def matchElement(matching, A, B, path): """ travers through the datatree """ job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-4 job.debug(verify, "matchElem "+path+" A "+str(type(A))+" B "+str(type(B))) if ((A is not None) and (isinstance(A, list))) \ or ((B is not None) and (isinstance(B, list))): return matchArray(matching, A, B, path) elif ((A is not None) and (isinstance(A, dict))) \ or ((B is not None) and (isinstance(B, dict))): return matchDict(matching, A, B, path) else: return matching def getStringSimilarity(strA, strB): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-1 job.debug(verify, "getStringSimilarity "+strA+" ?= "+strB) if (strA == strB): return "99" if (strA.strip() == strB.strip()): return "77" if (strA.lower() == strB.lower()): return "66" if (strA.strip().lower() == strB.strip().lower()): return "55" return "00" def getEvaluation(matching, type, acceptance, sideA, sideB): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-1 job.debug(verify, "getEvaluation "+str(sideA)+" ?= "+str(sideB)) match = getStringSimilarity(str(sideA), str(sideB)) classA = "novalue" classB = "novalue" result = "test" if match == "99": return ["MATCH", "novalue", "novalue", "novalue", "novalue"] if acceptance == "ignore": result = "ignore" if (matching.matchtype == M.MATCH_POSTCOND) and (result == "test"): result = "hard" classA = "diffA" classB = "diffB" else: classA = "acceptA" classB = "acceptB" return [result, classA, classB] def matchDict(matching, A, B, path): """ travers through the datatree """ job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-4 job.debug(verify, "matchDict "+path) if (A is not None): for k in A: job.debug(verify, "matchDict 400 " + k + ".") if k in ["_match"]: continue if (B is not None) and (k in B): if (isinstance(A[k], dict)): A[k]["_match"] = "Y" if (isinstance(B[k], dict)): B[k]["_match"] = "Y" job.debug(verify, "matchDict 404 " + k + "." + path) matchElement(matching, A[k], B[k], path+":"+k) else: if (isinstance(A[k], dict)): A[k]["_match"] = "N" job.debug(verify, "matchDict 408 " + path) matchElement(matching, A[k], None, path+":"+k) if (B is not None): for k in B: job.debug(verify, "matchDict 412 " + k + ".") if k in ["_match"]: continue if (A is not None) and (k in A): continue elif (A is None) or (k not in A): if (A is not None) and (isinstance(A[k], dict)): B[k]["_match"] = "N" job.debug(verify, "matchDict 418 " + k + "." + path) matchElement(matching, None, B[k], path+":"+k) job.debug(verify, "matchDict 420 ...<<---") return matching def matchArray(matching, A, B, path): """ matches the datarows of the datatree """ job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-4 job.debug(verify, "matchArray "+path+"\n.."+matching.htmltext) matching.sideA = A matching.sideB = B matchBestfit(matching, path) matchRestfit(matching) a = path.split(":") for x in a: if (x == "_data"): break table = x report = utils.report_tool.Report.getInstance() report.setPaths(getattr(job.par, "testcase"), matching.comp.name, table, matching.matchtype, matching.matchfiles["A"], matching.matchfiles["B"]) # report.setMatchResult("TC0001", "comp01", "arte01", m, "result" + str(i), "" + str(i) + "
") htmltext = report.getTitle(getattr(job.par, "testcase"), matching.comp.name, table, matching.matchtype) htmltext += report.getComponentHead(getattr(job.par, "testcase"), matching.comp.name, table, matching.matchtype) htmltext += report.getArtefactBlock(getattr(job.par, "testcase"), matching.comp.name, table, matching.matchtype) htmltext += compareRows(matching, path) matching.htmltext += htmltext def compareRows(matching, path): """ traverse through matched rows """ job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-1 ddl = matching.getTableDdl(path) report = utils.report_tool.Report.getInstance() table = "" header = [] # "

Tabelle : "+table+"

" htmltext = "" for f in ddl[B.DATA_NODE_HEADER]: job.debug(verify, "ddl " + f + " ") header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]}) htmltext += "" htmltext += "" matching.difftext = htmltext for k in sorted(matching.linksA): print(k) if (matching.isHitA(k)): htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))], matching.sideB[int(extractKeyI(matching.linksA[k]))]) else: htmltext += markRow(matching, header, matching.sideA[int(extractKeyI(k))], "A") matching.setCssClass("result3") for k in sorted(matching.linksB): if (not matching.isHitB(k)): htmltext += markRow(matching, header, matching.sideB[int(extractKeyI(k))], "B") matching.setCssClass("result2") htmltext += "
"+f+"
" matching.difftext += "" return htmltext def markRow(matching, header, row, side): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-4 text = "" cssClass = "" for f in header: if not f["field"] in row: val = "" cssClass = "novalue" elif side == "A": res = getEvaluation(matching, f["type"], f["acceptance"], row[f["field"]], "") val = str(row[f["field"]]) cssClass = res[1] else: res = getEvaluation(matching, f["type"], f["acceptance"], "", row[f["field"]]) val = str(row[f["field"]]) cssClass = res[2] text += ""+val+"" text = "" \ + M.MATCH[M.MATCH[matching.matchtype][side]]["short"] + ""+text+"" matching.difftext += text return text def compareRow(matching, header, rA, rB): """ traverse through matched rows """ job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("match_tool"))-4 allident = True textA = "" textB = "" text = "" valA = "" valB = "" classA = "novalue" classB = "novalue" for f in header: print(f) if f["field"] in rA and f["field"] in rB: res = getEvaluation(matching, f["type"], f["acceptance"], rA[f["field"]], rB[f["field"]]) match = res[0] classA = res[1] classB = res[2] valA = str(rA[f["field"]]) valB = str(rB[f["field"]]) elif f["field"] in rA : valA = str(rA[f["field"]]) match = "ddl" classA = "acceptA" classB = "acceptB" elif f["field"] in rB: valB = str(rB[f["field"]]) match = "ddl" classA = "acceptA" classB = "acceptB" else: match = "MATCH" classA = "acceptA" classB = "acceptB" if (match == "MATCH"): textA += ""+valA+"" textB += ""+valB+"" matching.setCssClass("result1") elif (match == "hard"): allident = False textA += ""+valA+"" textB += ""+valB+"" matching.setCssClass("result3") else: allident = False textA += ""+valA+" ("+match+")" textB += ""+valB+" ("+match+")" matching.setCssClass("result1") if allident: return ""+textA+"" text = ""+M.MATCH[M.MATCH[matching.matchtype]["A"]]["short"]+""+textA+"" text += ""+M.MATCH[matching.matchtype]["shortB"]+""+textB+"" matching.difftext += text return text # -------------------------------------------------------------------------- def matchLines(matching): pass