datestex/utils/match_tool.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import json
import utils.css_tool
import basic.program
import basic.constants as B
# ------------------------------------------------------------
"""

"""
MATCH = {
    "preconditions": {
        "A": "preReq",
        "B": "preAct",
        "shortA": "SV",
        "shortB": "IV",
        "longA": "Soll-Vorher",
        "longB": "Ist-Vorher",
        "mode": "info",
        "title": "Pruefung Vorbedingung (Soll-Vorher - Ist-Vorher)"
    },
    "postconditions": {
        "A": "postReq",
        "B": "postAct",
        "shortA": "SN",
        "shortB": "IN",
        "longA": "Soll-Nachher",
        "longB": "Ist-Nachher",
        "mode": "hard",
        "title": "Fachliche Auswertung (Soll-Nachher - Ist-Nachher)"
    },
    "prepost": {
        "A": "preAct",
        "B": "postAct",
        "shortA": "IV",
        "shortB": "IN",
        "longA": "Ist-Vorher",
        "longB": "Ist-Nachher",
        "mode": "action",
        "title": "Ablauf-Differenz (Ist-Vorher - Ist-Nachher)"
    },
    "prestep": {
        "A": "preStep",
        "B": "postAct",
        "shortA": "VS",
        "shortB": "IN",
        "longA": "Vor-Schritt",
        "longB": "Ist-Nachher",
        "mode": "action",
        "title": "Schritt-Differenz (Vorschritt-Nachher - Ist-Nachher)"
    }
}
MATCH_PREPOST = "prepost"
""" matches the action between pre- and postcondition of the actual testexecution """
MATCH_PRECOND = "preconditions"
""" matches the preconditions betwenn the required result the the actual testexecution 
 - just for info if the both executions have the same precondition """
MATCH_POSTCOND = "postconditions"
""" matches the postconditions betwenn the required result the the actual testexecution 
 - it is the main comparison """


class Matching():
    def __init__(self, comp):
        self.comp = comp
        self.elements = {}
        self.matchfiles = comp.files
        self.assignedFiles = {}
        self.linksA = {}
        self.linksB = {}
        self.nomatch = {}
        self.matchkeys = {}
        self.htmltext = ""
        self.sideA = []
        self.sideB = []
        self.mode = ""
        self.matchtype = ""

    def setData(self, tdata, match):
        """
        selects testdata for actual matching
        :param tdata:
        :param match: kind of actual match
        :return:
        """
        self.sideA = tdata[MATCH[match]["A"]]
        self.sideB = tdata[MATCH[match]["B"]]
        self.matchtype = match
        self.mode = MATCH[match]["mode"]
        self.setDiffHeader()

    def resetHits(self):
        self.linksA = {}
        self.linksB = {}
        self.nomatch = {}
        self.matchkeys = {}

    def isHitA(self, key):
        print("isHitA "+str(key))
        for k in self.linksA: print(k)
        return ((key in self.linksA) and (self.linksA[key] != "null"))
    def isHitB(self, key):
        return ((key in self.linksB) and (self.linksB[key] != "null"))
    def setHit(self, keyA, keyB):
        if (not self.isHitA(keyA)) and (not self.isHitB(keyB)):
            if (keyA != "null"): self.linksA[keyA] = keyB
            if (keyB != "null"): self.linksB[keyB] = keyA
            return "OK"
        raise Exception("one of the links are set")
    def setNohit(self, similarity, keyA, keyB):
        """ The similarity must be unique. Only a mismatch will be set. """
        if similarity in self.nomatch:
            raise Exception("similarity "+similarity+" exists")
        if (self.isHitA(keyA) or self.isHitB(keyB)):
            return
        self.nomatch[similarity] = [keyA, keyB]
    def getTableDdl(self, path):
        a = path.split(":")
        ddl = self.comp.conf["ddl"]
        for x in a:
            if (len(x) < 2): continue
            if (x == "_data"): break
            if x in ddl: ddl = ddl[x]
        return ddl
    def setDiffHeader(matching):
        job = basic.program.Job.getInstance()
        verify = int(job.getDebugLevel("match_tool"))-1
        job.debug(verify, "getDiffHeader ")
        htmltxt = "<!DOCTYPE html>"
        htmltxt += "<html><head>"
        htmltxt += "<title>"+MATCH[matching.matchtype]["title"]+"</title>"
        htmltxt += utils.css_tool.getInternalStyle("diffFiles")
        htmltxt += "</head>"
        htmltxt += "<body>"
        htmltxt += "<h1>"+MATCH[matching.matchtype]["title"]+"</h1>"
        htmltxt += "<h4>" + MATCH[matching.matchtype]["longA"] +": " + matching.matchfiles["A"] + "</h4>"
        htmltxt += "<h4>" + MATCH[matching.matchtype]["longB"] +": " + matching.matchfiles["B"] + "</h4><br>"
        matching.htmltext = htmltxt

    def setDiffFooter(self):
        job = basic.program.Job.getInstance()
        verify = int(job.getDebugLevel("match_tool")) - 4
        job.debug(verify, "getDiffFooter ")
        htmltext = self.htmltext
        htmltext += "</body></html>"
        self.htmltext = htmltext

def matchFiles(matching):
    """

    post:
    :param matching:
    :return:
    """

def matchBestfit(matching, path):
    """
    in this strategy the difference-score of all elements of both sides will be calculated.
    the best fit will assigned together until nothing is
    * the elements can be each kind of object
    * the count of elements should not be high
    :param matching:
    :return:
    """
    i=0
    if (matching.sideA is not None):
        for r in matching.sideA:
            k = composeKey("a", i)
            matching.setHit(k, "null")
            i += 1
    i = 0
    if (matching.sideB is not None):
        for r in matching.sideB:
            k = composeKey("b", i)
            matching.setHit("null", k)
            i += 1
    ia = 0
    ix = 1
    if (matching.sideA is None) or (matching.sideB is None):
        return
    for rA in matching.sideA:
        ib = 0
        for rB in matching.sideB:
            if (matching.isHitB(composeKey("b", ib))):
                ib += 1
                continue
            similarity=getSimilarity(matching, path, rA, rB, ix)
            if (similarity == "MATCH"):
                matching.setHit(composeKey("a", ia), composeKey("b", ib))
                continue
            else:
                matching.setNohit(similarity, composeKey("a", ia), composeKey("b", ib))
            ib += 1
            ix += 1
        ia += 1

def matchRestfit(matching):
    """ """
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-1
    job.debug(verify, "matchRestfit ")
    for x in sorted(matching.nomatch, reverse=True):
        job.debug(verify, "matchRestfit " +x)
        pair = matching.nomatch[x]
        if (matching.isHitA(pair[0])):
            print("A "+pair[0]+" bereits zugeordnet mit "+matching.linksA[pair[0]])
        if (matching.isHitB(pair[1])):
            print("B " + pair[1] + " bereits zugeordnet mit " + matching.linksB[pair[1]])
            continue
        if (matching.isHitA(pair[0])):
            continue
        print("neues Matching "+pair[0]+" "+pair[1])
        matching.setHit(pair[0], pair[1])

def composeKey(side, i):
    return side.lower()+str(i+1).zfill(4)
def extractKeyI(key):
    return int(key[1:])-1

def getSimilarity(matching, path, rA, rB, i):
    """ it calculates the similarity between both rows by:
    concat each criteria with single-similarity 00..99 and i with 999..000 """
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-1
    job.debug(verify, "getSimilarity "+path+" "+str(i))
    if len(matching.matchkeys) > 0:
        keys = matching.matchkeys
    elif ("ddl" in matching.comp.conf):
        job.debug(verify, "ddl " + path + " " + str(i))
        a = path.split(":")
        ddl = matching.comp.conf["ddl"]
        for x in a:
            if (len(x) < 2): continue
            if (x == "_data"): break
            if x in ddl: ddl = ddl[x]
        job.debug(verify, "ddl " + json.dumps(ddl) + " " + str(i))
        keys = {}
        for f in ddl:
            job.debug(verify, "ddl " + f + " " + str(i))
            if ("key" in ddl[f]) and (len(ddl[f]["key"])>0):
                b = ddl[f]["key"].split(":")
                if (len(b)!=2): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"])
                if (not b[1].isnumeric()): raise Exception("falsch formatierter Schluessel "+ddl[f]["key"])
                k = "k"+b[1].zfill(2)
                job.debug(verify, "ddl " + f + " " + str(i))
                keys[k]={ "ktyp": b[0], "field": ddl[f]["feld"], "type": ddl[f]["type"], "rule": ddl[f]["acceptance"]}
        matching.matchkeys=keys
        job.debug(verify, "ddl " + json.dumps(keys) + " " + str(i))
    msim=""
    topsim=""
    for k in sorted(matching.matchkeys):
        msim += getStringSimilarity(str(rA[matching.matchkeys[k]["field"]]), str(rB[matching.matchkeys[k]["field"]]))
        topsim += "99"
    if msim == topsim:
        job.debug(verify, "Treffer ")
        return "MATCH"
    else:
        job.debug(verify, "nomatch S"+msim+str(i).zfill(3))
        return "S"+msim+str(i).zfill(3)
    pass

def matchTree(matching):
    """
    :param matching:
    :return:
    """
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-4
    job.debug(verify, "matching "+matching.mode)
    matchElement(matching, matching.sideA, matching.sideB, "")
    matching.setDiffFooter()
    return matching.htmltext

def matchElement(matching, A, B, path):
    """ travers through the datatree """
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-4
    job.debug(verify, "matchElem "+path+" A "+str(type(A))+" B "+str(type(B)))
    if ((A is not None) and (isinstance(A, list))) \
        or ((B is not None) and (isinstance(B, list))):
        return matchArray(matching, A, B, path)
    elif ((A is not None) and (isinstance(A, dict))) \
        or ((B is not None) and (isinstance(B, dict))):
        return matchDict(matching, A, B, path)
    else:
        return matching

def getStringSimilarity(strA, strB):
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-1
    job.debug(verify, "getStringSimilarity "+strA+" ?= "+strB)
    if (strA == strB): return "99"
    if (strA.strip() == strB.strip()): return "77"
    if (strA.lower() == strB.lower()): return "66"
    if (strA.strip().lower() == strB.strip().lower()): return "55"
    return "00"

def getEvaluation(matching, type, acceptance, sideA, sideB):
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-1
    job.debug(verify, "getEvaluation "+str(sideA)+" ?= "+str(sideB))
    match = getStringSimilarity(str(sideA), str(sideB))
    classA = "black"
    classB = "black"
    result = "test"
    if match == "99": return ["MATCH", "black", "black", "black", "black"]
    if acceptance == "ignore": result = "ignore"
    if (matching.matchtype == MATCH_POSTCOND) and (result == "test"):
        result = "hard"
        classA = "diffA"
        classB = "diffB"
    else:
        classA = "acceptA"
        classB = "acceptB"
    return [result, classA, classB]

def matchDict(matching, A, B, path):
    """ travers through the datatree """
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-4
    job.debug(verify, "matchDict "+path)
    if (A is not None):
        for k in A:
            print(k)
            if k == "_match":
                continue
            if (B is not None) and (B[k]):
                if (isinstance(A[k], dict)): A[k]["_match"] = "Y"
                if (isinstance(B[k], dict)): B[k]["_match"] = "Y"
                matchElement(matching, A[k], B[k], path+":"+k)
            else:
                if (isinstance(A[k], dict)): A[k]["_march"] = "N"
                matchElement(matching, A[k], None, path+":"+k)
    if (B is not None):
        for k in B:
            if k == "_match":
                continue
            if (A is not None) and (A[k]):
                continue
            else:
                if (A is not None) and (isinstance(A[k], dict)): B[k]["_match"] = "N"
                matchElement(matching, None, B[k], path+":"+k)
    return matching

def matchArray(matching, A, B, path):
    """ matches the datarows of the datatree """
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-1
    job.debug(verify, "matchArray "+path)
    matching.sideA = A
    matching.sideB = B
    matchBestfit(matching, path)
    matchRestfit(matching)
    htmltext = compareRows(matching, path)
    matching.htmltext += htmltext

def compareRows(matching, path):
    """ traverse through matched rows """
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-1
    ddl = matching.getTableDdl(path)
    table = ""
    a = path.split(":")
    for x in a:
        if (x == "_data"): break
        table = x
    header = []
    htmltext = "<p>Tabelle : "+table+"</p>"
    htmltext += "<table>"
    htmltext += "<tr><th></th>"
    for f in ddl[B.DATA_NODE_HEADER]:
        job.debug(verify, "ddl " + f + " ")
        header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
        htmltext += "<th>"+f+"</th>"
    htmltext += "</tr>"
    matching.difftext = htmltext
    for k in sorted(matching.linksA):
        print(k)
        if (matching.isHitA(k)):
            htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))],
                   matching.sideB[int(extractKeyI(matching.linksA[k]))])
        else:
            htmltext += markRow(matching, header, matching.sideA[int(extractKeyI(k))], "A")
    for k in sorted(matching.linksB):
        if (not matching.isHitB(k)):
            htmltext += markRow(matching, header, matching.sideB[int(extractKeyI(k))], "B")
    htmltext += "</table>"
    matching.difftext += "</table>"
    return htmltext

def markRow(matching, header, row, side):
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-4
    text = ""
    cssClass = ""
    for f in header:
        if side == "A":
            res = getEvaluation(matching, f["type"], f["acceptance"], row[f["field"]], "")
            cssClass = res[1]
        else:
            res = getEvaluation(matching, f["type"], f["acceptance"], "", row[f["field"]])
            cssClass = res[2]
        text += "<td "+utils.css_tool.getInlineStyle("diffFiles", cssClass)+">" + str(row[f["field"]]) + "</td>"
    text = "<tr><td "+utils.css_tool.getInlineStyle("diffFiles", cssClass)+">" \
           + MATCH[matching.matchtype]["short"+side] + "</td>" + text + "</tr>"
    matching.difftext += text
    return text

def compareRow(matching, header, rA, rB):
    """ traverse through matched rows """
    job = basic.program.Job.getInstance()
    verify = int(job.getDebugLevel("match_tool"))-4
    allident = True
    textA = ""
    textB = ""
    text = ""
    for f in header:
        print(f)
        res = getEvaluation(matching, f["type"], f["acceptance"], rA[f["field"]], rB[f["field"]])
        match = res[0]
        classA = res[1]
        classB = res[2]
        if (match == "MATCH"):
            textA += "<td>"+str(rA[f["field"]])+"</td>"
            textB += "<td>"+str(rB[f["field"]])+"</td>"
        elif (match == "hard"):
            allident = False
            textA += "<td "+utils.css_tool.getInlineStyle("diffFiles", classA)+">" + str(rA[f["field"]]) + "</td>"
            textB += "<td "+utils.css_tool.getInlineStyle("diffFiles", classB)+">" + str(rB[f["field"]]) + "</td>"
        else:
            allident = False
            textA += "<td "+utils.css_tool.getInlineStyle("diffFiles", classA)+">"+str(rA[f["field"]])+" ("+match+")</td>"
            textB += "<td "+utils.css_tool.getInlineStyle("diffFiles", classB)+">"+str(rB[f["field"]])+" ("+match+")</td>"
    if allident:
        return "<tr><td/>"+textA+"</tr>"
    text = "<tr><td>"+MATCH[matching.matchtype]["shortA"]+"</td>"+textA+"</tr><tr><td>"+MATCH[matching.matchtype]["shortB"]+"</td>"+textB+"</tr>"
    matching.difftext += text
    return text

# --------------------------------------------------------------------------
def matchLines(matching):
    pass