diff --git a/test/test_compare.py b/test/test_compare.py
index e69de29..99646d3 100644
--- a/test/test_compare.py
+++ b/test/test_compare.py
@@ -0,0 +1,173 @@
+import json
+import unittest
+from basic.program import Job
+import utils.match_tool
+import components.component
+tdata = {
+ "postReq": {
+ "database": {
+ "scheme": {
+ "table": {
+ "_data": []
+ }
+ }
+ }
+ },
+ "preAct": {
+ "database": {
+ "scheme": {
+ "table": {
+ "_data": [
+ {"id": 1, "name": "Brecht", "year": 2014, "position": "top", "hobby": "meaning"},
+ {"id": 2, "name": "Meier", "year": 2015, "position": "first", "hobby": "reading" },
+ {"id": 3, "name": "Smith", "year": 2018, "position": "second", "hobby": "writing"},
+ {"id": 4, "name": "Smith", "year": 2019, "position": "third", "hobby": "executing"}
+ ]
+ }
+ }
+ }
+ },
+ "postAct": {
+ "database": {
+ "scheme": {
+ "table": {
+ "_data": [
+ {"id": 1, "name": "Brecht", "year": 2014, "position": "top", "hobby": "meaning"},
+ {"id": 2, "name": "Maier", "year": 2015, "position": "first", "hobby": "reading"},
+ {"id": 4, "name": "Smith", "year": 2018, "position": "second", "hobby": "writing"},
+ {"id": 3, "name": "Smith", "year": 2019, "position": "third", "hobby": "executing"}
+ ]
+ }
+ }
+ }
+ }
+}
+conf = {
+ "ddl": {
+ "database": {
+ "scheme": {
+ "table": {
+ "id": { "feld": "id", "type": "int", "acceptance": "ignore", "key": "T:3" },
+ "name": { "feld": "name", "type": "string", "acceptance": "must", "key": "F:1" },
+ "year": { "feld": "year", "type": "int", "acceptance": "must", "key": "F:2" },
+ "position": { "feld": "id", "type": "string", "acceptance": "must", "key": "" },
+ "hobby": { "feld": "id", "type": "string", "acceptance": "must", "key": "" },
+ "_header": ["id", "name", "year", "position", "hobby"]
+ }
+ }
+ }
+ }
+}
+
+class MyTestCase(unittest.TestCase):
+ def runTest(self):
+ self.test_matchstart()
+ self.test_hitmanage()
+ self.test_similarity()
+ self.test_bestfit()
+ self.test_compareRow()
+ self.test_compareRows()
+ self.test_match()
+
+ def test_matchstart(self):
+ job = Job("unit")
+ comp = components.component.Component()
+ comp.files = { "A": "/home/match/per.csv", "B": "/home/match/post.csv"}
+ comp.conf = conf
+ matching = utils.match_tool.Matching(comp)
+ matching.setData(tdata, utils.match_tool.MATCH_PREPOST)
+ print(matching.htmltext)
+
+ def test_hitmanage(self):
+ comp = components.component.Component()
+ comp.files = { "A": "/home/match/per.csv", "B": "/home/match/post.csv"}
+ comp.conf = conf
+ matching = utils.match_tool.Matching(comp)
+ matching.linksA = { "a0001": "null", "a0002": "null", "a0003": "null", "a0004": "null"}
+ matching.linksB = { "b0001": "null", "b0002": "null", "b0003": "null", "b0004": "null"}
+ self.assertEqual(matching.isHitA("a0001"), False, "with value null")
+ self.assertEqual(matching.isHitB("b0005"), False, "doesnt exist")
+ self.assertEqual(("b0005" not in matching.linksB), True, "doesnt exist")
+ matching.setHit("a0001", "b0005")
+ self.assertEqual(matching.isHitA("a0001"), True, "with value null")
+ self.assertEqual(matching.isHitB("b0005"), True, "doesnt exist")
+ self.assertEqual(("b0005" in matching.linksB), True, "doesnt exist")
+
+ def test_similarity(self):
+ matching = self.getMatching()
+ utils.match_tool.getSimilarity(matching, ":database:scheme:table:_data",
+ tdata["preAct"]["database"]["scheme"]["table"]["_data"][0],
+ tdata["postAct"]["database"]["scheme"]["table"]["_data"][0],1)
+
+ def test_bestfit(self):
+ job = Job("unit")
+ comp = components.component.Component()
+ comp.files = { "A": "/home/match/per.csv", "B": "/home/match/post.csv"}
+ comp.conf = conf
+ matching = utils.match_tool.Matching(comp)
+ matching.sideA = tdata["preAct"]["database"]["scheme"]["table"]["_data"]
+ matching.sideB = tdata["postAct"]["database"]["scheme"]["table"]["_data"]
+ utils.match_tool.matchBestfit(matching, ":database:scheme:table:_data")
+ print(json.dumps(matching.linksA))
+ print(json.dumps(matching.linksB))
+ print(json.dumps(matching.nomatch))
+ utils.match_tool.matchRestfit(matching)
+ print(json.dumps(matching.linksA))
+ print(json.dumps(matching.linksB))
+ print(json.dumps(matching.nomatch))
+
+ def test_compareRow(self):
+ job = Job("unit")
+ comp = components.component.Component()
+ comp.files = { "A": "/home/match/per.csv", "B": "/home/match/post.csv"}
+ comp.conf = conf
+ matching = self.getMatching()
+ matching.sideA = tdata["preAct"]["database"]["scheme"]["table"]["_data"]
+ matching.sideB = tdata["postAct"]["database"]["scheme"]["table"]["_data"]
+ ddl = conf["ddl"]["database"]["scheme"]["table"]
+ header = []
+ for f in ddl["_header"]:
+ header.append({"field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
+ i = 1
+ text = utils.match_tool.compareRow(matching, header, tdata["preAct"]["database"]["scheme"]["table"]["_data"][i], tdata["postAct"]["database"]["scheme"]["table"]["_data"][i])
+ print(text)
+
+ def test_compareRows(self):
+ job = Job("unit")
+ comp = components.component.Component()
+ comp.files = { "A": "/home/match/per.csv", "B": "/home/match/post.csv"}
+ comp.conf = conf
+ matching = self.getMatching()
+ matching.sideA = tdata["preAct"]["database"]["scheme"]["table"]["_data"]
+ matching.sideB = tdata["postAct"]["database"]["scheme"]["table"]["_data"]
+ linksA = {"a0001": "b0001", "a0002": "b0002" }
+ matching.linksA = linksA
+ text = utils.match_tool.compareRows(matching, ":database:scheme:table:_data")
+ print(text)
+
+ def test_match(self):
+ job = Job("unit")
+ comp = components.component.Component()
+ comp.files = {"A": "/home/match/per.csv", "B": "/home/match/post.csv"}
+ # tdata["postReq"] = tdata["preAct"]
+ comp.conf = conf
+ matching = utils.match_tool.Matching(comp)
+ matching.setData(tdata, utils.match_tool.MATCH_POSTCOND)
+ text = utils.match_tool.matchTree(matching)
+ print("\n-------------\n")
+ print(text)
+ print("\n-------------\n")
+ print(matching.difftext)
+
+ def getMatching(self):
+ job = Job("unit")
+ comp = components.component.Component()
+ comp.files = { "A": "/home/match/per.csv", "B": "/home/match/post.csv"}
+ comp.conf = conf
+ matching = utils.match_tool.Matching(comp)
+ matching.setData(tdata, utils.match_tool.MATCH_PREPOST)
+ matching.difftext = ""
+ return matching
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_config.py b/test/test_config.py
index 12eccdb..9001992 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -5,9 +5,13 @@ import test.constants
import os
HOME_PATH = test.constants.HOME_PATH
+VERIFY = False
class MyTestCase(unittest.TestCase):
- def test_something(self):
+ def runTest(self):
+ self.test_getConfig()
+
+ def test_getConfig(self):
job = Job("unit")
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
"modus": "unit"}
@@ -18,9 +22,10 @@ class MyTestCase(unittest.TestCase):
self.assertEqual(r, None)
r = t.getConfigPath("comp", "testA2")
r = t.getConfig("tool", "path")
- print("pattern " + r["pattern"]["log"])
- print("pattern " + r["pattern"]["precond"])
+ if VERIFY: print("pattern " + r["pattern"]["log"])
+ if VERIFY: print("pattern " + r["pattern"]["precond"])
if __name__ == '__main__':
+ VERIFY = True
unittest.main()
diff --git a/test/test_conn.py b/test/test_conn.py
index 492ab95..0eb52dc 100644
--- a/test/test_conn.py
+++ b/test/test_conn.py
@@ -3,7 +3,10 @@ import basic.program
import utils.conn_tool
class MyTestCase(unittest.TestCase):
- def test_something(self):
+ def runTest(self):
+ self.test_conn()
+
+ def test_conn(self):
print("# # # # tetsComponents # # # # #")
job = basic.program.Job("unit")
args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool"}
diff --git a/test/test_css.py b/test/test_css.py
index 8341ca8..01c6c4f 100644
--- a/test/test_css.py
+++ b/test/test_css.py
@@ -4,6 +4,9 @@ import basic.program
import json
class MyTestCase(unittest.TestCase):
+ def runTest(self):
+ self.test_css()
+
def test_css(self):
job = basic.program.Job("unit")
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug",
@@ -19,7 +22,7 @@ class MyTestCase(unittest.TestCase):
self.assertEqual(len(text), 37)
self.assertEqual(("style" in text), True)
text = utils.css_tool.getInternalStyle("diffFiles")
- self.assertEqual(len(text), 0)
+ self.assertEqual(len(text), 84)
text = utils.css_tool.getExternalStyle("diffFiles")
self.assertEqual(len(text), 0)
# ------- internal ---------------
@@ -29,7 +32,7 @@ class MyTestCase(unittest.TestCase):
self.assertEqual(("class" in text), True)
text = utils.css_tool.getInternalStyle("diffFiles")
print(text)
- self.assertEqual(len(text), 173)
+ self.assertEqual(len(text), 237)
self.assertEqual((""
+ out += " \n "
return out
def getExternalStyle(filetype):
job = basic.program.Job.getInstance()
- verify = int(job.getDebugLevel("match_tool")) - 4
- job.debug(verify, "getDiffHeader ")
+ verify = int(job.getDebugLevel("match_tool")) - 1
out = ""
if job.conf.confs.get("tools").get("csstyp") == "external":
for c in CSS_CLASS[filetype]:
diff --git a/utils/file_tool.py b/utils/file_tool.py
index 0d997df..7e80d71 100644
--- a/utils/file_tool.py
+++ b/utils/file_tool.py
@@ -90,8 +90,8 @@ def mkPaths(msg, pfad):
modus = job.conf.confs["paths"]["mode"]
os.makedirs(pfad, exist_ok=True)
-def getFileEncoding(path):
- print(path)
+def getFileEncoding(msg, path):
+ print("--- getFileEncoding "+path)
encodings = ['utf-8', 'iso-8859-1'] # add more
for e in encodings:
print(e)
@@ -106,24 +106,40 @@ def getFileEncoding(path):
else:
print('opening the file with encoding: %s ' % e)
return e
- break
+ return detectFileEncode(msg, path)
-def rest(path): # return ""
+def detectFileEncode(msg, path): # return ""
+ job = basic.program.Job.getInstance()
+ verify = int(job.getDebugLevel("file_tool"))
print(path)
cntIso = 0
cntUtf = 0
with open(path, 'rb') as file:
while (byte := file.read(1)):
i = int.from_bytes(byte, "little")
- print(str(byte)+" = "+str(i))
#byte = file.read(1)
if ((i == 196) or (i == 228) or (i == 214) or (i == 246) or (i == 220) or (i == 252) or (i == 191)):
cntIso += 1
- print("iso")
+ if (i == 160):
+ pass
elif (i > 127):
cntUtf += 1
- print("utf8")
if (cntIso > cntUtf):
return 'iso-8859-1'
return 'utf-8'
+def readFileLines(msg, path):
+ job = basic.program.Job.getInstance()
+ verify = int(job.getDebugLevel("file_tool"))
+ enc = detectFileEncode(msg, path)
+ with open(path, 'r', encoding=enc) as file:
+ lines = file.read().splitlines()
+ file.close()
+ return lines
+
+def writeFileText(msg, path, text, enc="utf-8"):
+ job = basic.program.Job.getInstance()
+ verify = int(job.getDebugLevel("file_tool"))
+ with open(path, 'w', encoding=enc) as file:
+ file.write(text)
+ file.close()
diff --git a/utils/match_tool.py b/utils/match_tool.py
index b1319f6..a9a6762 100644
--- a/utils/match_tool.py
+++ b/utils/match_tool.py
@@ -94,6 +94,8 @@ class Matching():
self.matchkeys = {}
def isHitA(self, key):
+ print("isHitA "+str(key))
+ for k in self.linksA: print(k)
return ((key in self.linksA) and (self.linksA[key] != "null"))
def isHitB(self, key):
return ((key in self.linksB) and (self.linksB[key] != "null"))
@@ -120,7 +122,7 @@ class Matching():
return ddl
def setDiffHeader(matching):
job = basic.program.Job.getInstance()
- verify = int(job.getDebugLevel("match_tool"))-4
+ verify = int(job.getDebugLevel("match_tool"))-1
job.debug(verify, "getDiffHeader ")
htmltxt = ""
htmltxt += ""
@@ -159,17 +161,21 @@ def matchBestfit(matching, path):
:return:
"""
i=0
- for r in matching.sideA:
- k = composeKey("a", i)
- matching.setHit(k, "null")
- i += 1
+ if (matching.sideA is not None):
+ for r in matching.sideA:
+ k = composeKey("a", i)
+ matching.setHit(k, "null")
+ i += 1
i = 0
- for r in matching.sideB:
- k = composeKey("b", i)
- matching.setHit("null", k)
- i += 1
+ if (matching.sideB is not None):
+ for r in matching.sideB:
+ k = composeKey("b", i)
+ matching.setHit("null", k)
+ i += 1
ia = 0
ix = 1
+ if (matching.sideA is None) or (matching.sideB is None):
+ return
for rA in matching.sideA:
ib = 0
for rB in matching.sideB:
@@ -253,7 +259,6 @@ def getSimilarity(matching, path, rA, rB, i):
def matchTree(matching):
"""
-
:param matching:
:return:
"""
@@ -293,8 +298,6 @@ def getEvaluation(matching, type, acceptance, sideA, sideB):
verify = int(job.getDebugLevel("match_tool"))-1
job.debug(verify, "getEvaluation "+str(sideA)+" ?= "+str(sideB))
match = getStringSimilarity(str(sideA), str(sideB))
- colorA = "black"
- colorB = "black"
classA = "black"
classB = "black"
result = "test"
@@ -302,41 +305,39 @@ def getEvaluation(matching, type, acceptance, sideA, sideB):
if acceptance == "ignore": result = "ignore"
if (matching.matchtype == MATCH_POSTCOND) and (result == "test"):
result = "hard"
- colorA = "green"
- colorB = "crimson"
classA = "diffA"
classB = "diffB"
else:
- colorA = "darkblue"
- colorB = "darkmagenta"
classA = "acceptA"
classB = "acceptB"
- return [result, colorA, colorB, classA, classB]
+ return [result, classA, classB]
def matchDict(matching, A, B, path):
""" travers through the datatree """
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("match_tool"))-4
job.debug(verify, "matchDict "+path)
- for k in A:
- print(k)
- if k == "_match":
- continue
- if (B is not None) and (B[k]):
- if (isinstance(A[k], dict)): A[k]["_match"] = "Y"
- if (isinstance(B[k], dict)): B[k]["_match"] = "Y"
- matchElement(matching, A[k], B[k], path+":"+k)
- else:
- if (isinstance(A[k], dict)): A[k]["_march"] = "N"
- matchElement(matching, A[k], None, path+":"+k)
- for k in B:
- if k == "_match":
- continue
- if (A is not None) and (A[k]):
- continue
- else:
- if (isinstance(A[k], dict)): B[k]["_match"] = "N"
- matchElement(matching, None, B[k], path+":"+k)
+ if (A is not None):
+ for k in A:
+ print(k)
+ if k == "_match":
+ continue
+ if (B is not None) and (B[k]):
+ if (isinstance(A[k], dict)): A[k]["_match"] = "Y"
+ if (isinstance(B[k], dict)): B[k]["_match"] = "Y"
+ matchElement(matching, A[k], B[k], path+":"+k)
+ else:
+ if (isinstance(A[k], dict)): A[k]["_march"] = "N"
+ matchElement(matching, A[k], None, path+":"+k)
+ if (B is not None):
+ for k in B:
+ if k == "_match":
+ continue
+ if (A is not None) and (A[k]):
+ continue
+ else:
+ if (A is not None) and (isinstance(A[k], dict)): B[k]["_match"] = "N"
+ matchElement(matching, None, B[k], path+":"+k)
return matching
def matchArray(matching, A, B, path):
@@ -370,12 +371,39 @@ def compareRows(matching, path):
header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
htmltext += "| "+f+" | "
htmltext += ""
+ matching.difftext = htmltext
for k in sorted(matching.linksA):
- htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))],
+ print(k)
+ if (matching.isHitA(k)):
+ htmltext += compareRow(matching, header, matching.sideA[int(extractKeyI(k))],
matching.sideB[int(extractKeyI(matching.linksA[k]))])
+ else:
+ htmltext += markRow(matching, header, matching.sideA[int(extractKeyI(k))], "A")
+ for k in sorted(matching.linksB):
+ if (not matching.isHitB(k)):
+ htmltext += markRow(matching, header, matching.sideB[int(extractKeyI(k))], "B")
htmltext += ""
+ matching.difftext += ""
return htmltext
+def markRow(matching, header, row, side):
+ job = basic.program.Job.getInstance()
+ verify = int(job.getDebugLevel("match_tool"))-4
+ text = ""
+ cssClass = ""
+ for f in header:
+ if side == "A":
+ res = getEvaluation(matching, f["type"], f["acceptance"], row[f["field"]], "")
+ cssClass = res[1]
+ else:
+ res = getEvaluation(matching, f["type"], f["acceptance"], "", row[f["field"]])
+ cssClass = res[2]
+ text += "" + str(row[f["field"]]) + " | "
+ text = "| " \
+ + MATCH[matching.matchtype]["short"+side] + " | " + text + "
"
+ matching.difftext += text
+ return text
+
def compareRow(matching, header, rA, rB):
""" traverse through matched rows """
job = basic.program.Job.getInstance()
@@ -383,14 +411,13 @@ def compareRow(matching, header, rA, rB):
allident = True
textA = ""
textB = ""
+ text = ""
for f in header:
print(f)
res = getEvaluation(matching, f["type"], f["acceptance"], rA[f["field"]], rB[f["field"]])
match = res[0]
- colorA = res[1]
- colorB = res[2]
- classA = res[3]
- classB = res[4]
+ classA = res[1]
+ classB = res[2]
if (match == "MATCH"):
textA += ""+str(rA[f["field"]])+" | "
textB += ""+str(rB[f["field"]])+" | "
@@ -398,28 +425,17 @@ def compareRow(matching, header, rA, rB):
allident = False
textA += "" + str(rA[f["field"]]) + " | "
textB += "" + str(rB[f["field"]]) + " | "
- #textB += "" + str(rB[f["field"]]) + " | "
else:
allident = False
textA += ""+str(rA[f["field"]])+" ("+match+") | "
textB += ""+str(rB[f["field"]])+" ("+match+") | "
- #textB += ""+str(rA[f["field"]])+" ("+match+") | "
if allident:
return " | "+textA+"
"
- return "| "+MATCH[matching.matchtype]["shortA"]+" | "+textA+"
| "+MATCH[matching.matchtype]["shortB"]+" | "+textB+"
"
+ text = "| "+MATCH[matching.matchtype]["shortA"]+" | "+textA+"
| "+MATCH[matching.matchtype]["shortB"]+" | "+textB+"
"
+ matching.difftext += text
+ return text
# --------------------------------------------------------------------------
def matchLines(matching):
pass
-# --------------------------------------------------------------------------
-
-def getScoreint(score):
- if score.is_integer():
- return score
- elif "h_" in score:
- return int(score[2:])
-def getHitscore(score):
- return "h_" + ':04d'.format(getScoreint(score))
-def getNextHitscore(score):
- return "h_" + ':04d'.format(getScoreint(score)+1)
diff --git a/utils/path_tool.py b/utils/path_tool.py
index c815d60..f97b6e7 100644
--- a/utils/path_tool.py
+++ b/utils/path_tool.py
@@ -12,18 +12,18 @@ def getKeyValue(key):
job = basic.program.Job.getInstance()
verify = job.getDebugLevel("path_tool")
pt = PathConf.getInstance()
- print("getKeyValue " + key)
+ job.debug(verify, "getKeyValue " + key)
if 'job.par' in key:
neu = job.getParameter(key[8:])
return neu
elif 'job.conf' in key:
neu = job.conf.confs["paths"][key[9:]]
- print(neu)
+ job.debug(verify, neu)
return neu
# return job.conf.confs["paths"][key[9:]]
elif (pt.pattern):
return pt.pattern[key]
- print("pt exists")
+ job.debug(verify, "pt exists")
else:
return "xx-"+key+"-xx"
@@ -31,36 +31,37 @@ def composePath(pathname, comp):
job = basic.program.Job.getInstance()
verify = job.getDebugLevel("path_tool")
pt = PathConf.getInstance()
- print("composePath " + pathname + " zu " + str(pt) + "mit ")
- print(pt.pattern)
+ job.debug(verify, "composePath " + pathname + " zu " + str(pt) + "mit ")
+ job.debug(verify, str(pt.pattern))
if pt.pattern[pathname]:
return composePatttern(pt.pattern[pathname], comp)
else:
- print("in Pattern nicht vorhanden: " + pathname)
+ job.debug(verify, "in Pattern nicht vorhanden: " + pathname)
def composePatttern(pattern, comp):
job = basic.program.Job.getInstance()
verify = job.getDebugLevel("path_tool")
- print("composePattern " + pattern)
+ job.debug(verify, "composePattern " + pattern)
max=5
l = re.findall('\{.*?\}', pattern)
- print(l)
+ job.debug(verify, l)
for pat in l:
pit = getKeyValue(pat[1:-1])
- print(str(max) + ": " + pattern + ": " + pat + ": " + pit)
+ job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
pattern = pattern.replace(pat, pit)
- print(str(max) + ": " + pattern + ": " + pat + ": " + pit)
+ job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
while ("{" in pattern):
max = max-1
- print(str(max) + ": " + pattern + ": " + pat + ": " + pit)
+ job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
pattern = composePatttern(pattern, comp)
- print(str(max) + ": " + pattern + ": " + pat + ": " + pit)
+ job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
if (max < 3) :
break
return pattern
def extractPattern(pathtyp):
job = basic.program.Job.getInstance()
+ verify = job.getDebugLevel("path_tool")
out = []
pt = PathConf.getInstance()
pattern = pt.pattern[pathtyp]
@@ -70,7 +71,7 @@ def extractPattern(pathtyp):
j = work.index("}")
pre = work[0:i]
pat = work[i+1:j]
- print (work + " von " + str(i) + "-" + str(j) + " pre " + pre + "pat " + pat)
+ job.debug(verify, work + " von " + str(i) + "-" + str(j) + " pre " + pre + "pat " + pat)
pit = getKeyValue(pat)
tup = (pre, pat, pit)
out.append(tup)
@@ -84,6 +85,7 @@ def extractPath(pathtyp, pathname):
i = 0
print("-- extractPatternList -- " + pathtyp + ":" + str(patterlist))
for p in patterlist:
+ if len(p) < 1 : continue
delim = p[0]
key = p[1]
val = p[2]
diff --git a/utils/tdata_tool.py b/utils/tdata_tool.py
index 306ed36..d598807 100644
--- a/utils/tdata_tool.py
+++ b/utils/tdata_tool.py
@@ -16,13 +16,32 @@ the testdata have several elements
the testdata itself which are written in different artifacts of modern applications are mostly stored as tree
- so as xml, json, always with plain data in the leaf. So the intern structure should be also a tree - in python: dictionary.
"""
+import os.path
+
import basic.program
-import csv
+import utils.file_tool
+import basic.constants as B
+
+DATA_SOURCE_DIR = "dir"
+DATA_SOURCE_CSV = "csv"
+
+CSV_HEADER_START = ["node", "table", "tabelle"]
+CSV_DELIMITER = ";"
+
+CSV_SPECTYPE_DATA = "data"
+CSV_SPECTYPE_TREE = "tree"
def getSourcefile(filename):
pass
def getTestdata():
+ """
+ get the testdata from one of the possible soources
+ * dir: each file in the specific testarchiv
+ * csv: specific file
+ * db: specific db with a testcase-catalogue
+ :return:
+ """
job = basic.program.Job.getInstance()
reftyp = getattr(job.par, "tdtyp")
source = getattr(job.par, "tdsrc")
@@ -36,11 +55,93 @@ def getTestdata():
elif reftyp == "csv":
# read file in testdata
job.m.logInfo("Test-Data readed from " + reftyp + " for " + criteria)
+ elif reftyp == "dir":
+ filename = os.path.join(job.conf.getJobConf("paths:testdata"), source, "testspec.csv")
+ data = getCsvSpec(job.m, filename, CSV_SPECTYPE_DATA)
+ for k in data:
+ tdata[k] = data[k]
else:
job.m.setFatal("test-Data: reftyp " + reftyp + " is not implemented")
return tdata
-def readCsv(filename):
+def getCsvSpec(msg, filename, type):
+ """
+ get data from a csv-file
+ a) data : like a table with data-array of key-value-pairs
+ b) tree : as a tree - the rows must be unique identified by the first column
+ """
+ data = {}
+ header = []
+ lines = utils.file_tool.readFileLines(msg, filename)
+ status = "start"
+ for l in lines:
+ fields = l.split(CSV_DELIMITER)
+ # check empty line, comment
+ if (len(l.strip().replace(CSV_DELIMITER,"")) < 1):
+ status = "start"
+ continue
+ if (fields[0][0:1] == "#"):
+ continue
+ a = fields[0].split(":")
+ # keywords option, step, table
+ if (a[0] not in data):
+ data[a[0]] = {}
+ if (a[0].lower() == "step"):
+ if (not "steps" in data):
+ data["steps"] = []
+ step = {}
+ step["comp"] = fields[1]
+ step["todo"] = fields[2]
+ step["args"] = fields[3]
+ data["steps"].append(step)
+ continue
+ elif (a[0].lower() == "option"):
+ data[a[0]][a[1]] = fields[1]
+ continue
+ elif (a[0].lower() in CSV_HEADER_START):
+ i = 0
+ for f in fields:
+ i += 1
+ if i == 1: continue
+ header.append(f)
+ status = CSV_SPECTYPE_DATA
+ continue
+ elif (status == CSV_SPECTYPE_DATA):
+ # check A-col for substructure
+ if (a[0] not in data):
+ data[a[0]] = {}
+ if len(a) == 1 and type == CSV_SPECTYPE_DATA:
+ data[a[0]]["_data"] = []
+ # its a component
+ if len(a) > 1 and a[1] not in data[a[0]]:
+ data[a[0]][a[1]] = {}
+ if len(a) == 2 and type == CSV_SPECTYPE_DATA:
+ data[a[0]][a[1]]["_data"] = []
+ if len(a) > 2 and a[2] not in data[a[0]][a[1]]:
+ data[a[0]][a[1]][a[2]] = {}
+ if len(a) == 3 and type == CSV_SPECTYPE_DATA:
+ data[a[0]][a[1]][a[1]]["_data"] = []
+ # fill data
+ row = {}
+ i = 1
+ for f in header:
+ row[f] = fields[i]
+ i += 1
+ if len(a) == 1 and type == CSV_SPECTYPE_DATA:
+ data[a[0]]["_data"].append(row)
+ elif len(a) == 1 and type == CSV_SPECTYPE_DATA:
+ data[a[0]] = {f: row}
+ elif len(a) == 2 and type == CSV_SPECTYPE_DATA:
+ data[a[0]][a[1]]["_data"].append(row)
+ elif len(a) == 1 and type == CSV_SPECTYPE_DATA:
+ data[a[0]][a[1]]["_data"] = {f: row}
+ elif len(a) == 3 and type == CSV_SPECTYPE_DATA:
+ data[a[0]][a[1]][a[2]] = row
+ elif len(a) == 1 and type == CSV_SPECTYPE_DATA:
+ data[a[0]][a[1]][a[2]] = {f: row}
+ return data
+
+def readCsv(msg, filename, comp):
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("tdata_tool")
job.debug(verify, "readCsv " + filename)
@@ -51,22 +152,24 @@ def readCsv(filename):
state = 0
data = []
cnt = 0
- with open(filename, 'r') as file:
- lines = file.read().splitlines()
+ lines = utils.file_tool.readFileLines(msg, filename)
+ startCols = 1
for line in lines:
- # while (line := file.readline().rstrip()):
fields = line.split(';')
testline = line.replace(";", "")
job.debug(verify, str(state) + " line " + line + " :" + str(len(fields)) + ": " + str(fields))
if len(testline) < 2 and state < 1:
state = 0
- elif fields[0].lower() == "node":
+ elif fields[0].lower() in CSV_HEADER_START:
state = 2
columns = []
cnt = len(fields)
job.debug(verify, str(state) + " cnt " + str(cnt))
j = 0
- for i in range(2, cnt-1):
+ for i in range(1, cnt-1):
+ if fields[0][0:1] == "_":
+ startCols += 1
+ continue
job.debug(verify, str(i) + " cnt " + str(fields[i]))
if len(fields[i]) > 0:
columns.append(fields[i])
@@ -79,7 +182,7 @@ def readCsv(filename):
job.debug(verify, str(state) + " nodes " + str(nodes))
state = 3
row = {}
- for i in range(2, cnt-1):
+ for i in range(startCols, cnt-1):
row[columns[i-2]] = fields[i]
job.debug(verify, str(state) + " row " + str(row))
data.append(row)
@@ -88,11 +191,7 @@ def readCsv(filename):
output = setSubnode(0, nodes, data, output)
data = []
state = 0
- print(str(output))
- print(str(output))
output = setSubnode(0, nodes, data, output)
- print(str(output))
- file.close()
return output
def setSubnode(i, nodes, data, tree):
@@ -121,7 +220,7 @@ def normalizeDataRow(dstruct, xpathtupel, row, referencedate):
verify = -1+job.getDebugLevel("tdata_tool")
job.debug(verify, "calcDataRow " + row)
-def writeDataTable(teststatus, tdata, comp):
+def writeDataTable(filename, tdata, comp):
"""
writes the testdata into a csv-file for documentation of the test-run
:param teststatus:
@@ -132,4 +231,15 @@ def writeDataTable(teststatus, tdata, comp):
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("tdata_tool")
job.debug(verify, "writeDataTable " + str(comp))
-
+ text = "table"
+ for f in tdata[B.DATA_NODE_HEADER]:
+ text += ";"+f
+ for r in tdata[B.DATA_NODE_DATA]:
+ text += "\n"
+ for f in tdata[B.DATA_NODE_HEADER]:
+ if f in r:
+ text += ";"+r[f]
+ else:
+ text += ";"
+ text += "\n"
+ utils.file_tool.writeFileText(comp.m, filename, text)
diff --git a/utils/xml_tool.py b/utils/xml_tool.py
index ce88539..20647c8 100644
--- a/utils/xml_tool.py
+++ b/utils/xml_tool.py
@@ -3,7 +3,7 @@ import json
import yaml
import re
import datetime
-#from tools.reader import fcts as reader
+import utils.tdata_tool
import os, sys, json
import xmltodict
import pprint
@@ -49,7 +49,7 @@ class register():
print (row)
satz = {}
satz = self.schreibeSegmentsatz(satz, content, row["_lfdNR"])
- # print("ausgabe vorher: "+str(ausgabe))
+ # print("ausgabe vorher: "+str(ausgabe))*
ausgabe["nachricht"]["datensegment"].append(satz["datensegment"])
return ausgabe
@@ -125,7 +125,7 @@ class Absender():
FELDER = ("ABSENDER_KENNUNG", "ABSENDER_IDENTITÄT", "ABSENDER_TELEFON", "ABSENDER_EMAIL", "ABSENDER_FAX", "ABSENDER_URL", "BERICHTSEINHEIT_ID", "REGNAME")
def __init__(self):
- content = reader.readCsv("testdaten/allgemein/absender.csv", "keys")
+ content = utils.tdata_tool.readCsv("testdaten/allgemein/absender.csv", "keys")
self.keys = content["tabelle"]["absender"]["_keys"]
class Vrfelder():