Data-Test-Executer Framework speziell zum Test von Datenverarbeitungen mit Datengenerierung, Systemvorbereitungen, Einspielungen, ganzheitlicher diversifizierender Vergleich
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

376 lines
13 KiB

#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import os
import re
import basic.program
import basic.catalog
import utils.config_tool
import basic.constants as B
import basic.toolHandling
import utils.data_const as D
import utils.file_tool
import utils.path_tool
import basic.catalog
ACT_ID = "actid"
ID_LIST = "idlist"
MAP_ID = "_id"
MAP_FOR = "_foreach"
MAP_ROW = "_row"
MAP_FCTS = [ MAP_FOR, MAP_ID, MAP_ROW ]
MODUL_NAME = "map_tool"
def mapTdata(job, mapping, tdata, step, comp):
"""
initialize the mapping from testdata into the mapping-structure
:param job:
:param mapping:
:param tdata:
:param step:
:param comp:
:return:
"""
verify = job.getDebugLevel(MODUL_NAME)
out = {}
path = ""
job.debug(verify, mapping)
args = {}
args[B.DATA_NODE_COMP] = comp
args[B.DATA_NODE_DATA] = tdata
args[B.DATA_NODE_STEPS] = step
args[ACT_ID] = {}
args[ID_LIST] = {}
out = mapElement(job, args, mapping, path, out)
job.debug(verify, ">>>>>>>>>>> \n"+str(out))
return out
def mapElement(job, args, elem, path, out):
"""
recursive mapping with building the dict of the testdata
:param job:
:param args:
:param elem:
:param path:
:param out:
:return:
"""
verify = job.getDebugLevel(MODUL_NAME)
job.debug(verify, "mapElem "+path+" id: "+str(args[ACT_ID]))
if isinstance(elem, dict):
job.debug(verify, "##### dict ")
nodes = []
attrNodes = []
leafNodes = []
objNodes = []
for k in elem:
if MAP_ID in elem[k] or MAP_FOR in elem[k]:
objNodes.append(k)
elif k[0:1] == '@' or k[0:1] == '#':
attrNodes.append(k)
else:
leafNodes.append(k)
job.debug(verify, "nodes "+str(attrNodes)+" - "+str(leafNodes)+" - "+str(objNodes))
nodes = attrNodes + leafNodes + objNodes
for k in nodes:
# iterating this elem is declared inside of the elem
# like foreach but only one element
job.debug(verify, "# # "+k)
if MAP_ID in elem[k] or MAP_FOR in elem[k]:
job.debug(verify, "# + k in obj : val "+k)
if MAP_ID in elem[k]:
key = elem[k][MAP_ID][0:elem[k][MAP_ID].find("=")]
idlist = getIds(job, args, elem[k][MAP_ID])
if len(idlist) > 1:
uniqueKeys = {}
for x in idlist:
uniqueKeys[x] = x
if len(uniqueKeys) > 1:
raise Exception("bei keyword _id there is only one element allowed "+str(idlist))
else:
idlist = uniqueKeys.keys()
elif MAP_FOR in elem[k]:
key = elem[k][MAP_FOR][0:elem[k][MAP_FOR].find("=")]
idlist = getIds(job, args, elem[k][MAP_FOR])
sublist = []
a = path.split(",")
a.append(k)
npath = ",".join(a)
for id in idlist:
args[ACT_ID][key] = str(id)
if MAP_ROW in elem[k]:
row = getRow(job, args, elem[k][MAP_ROW])
sublist.append(mapElement(job, args, elem[k], npath, {}))
out[k] = sublist
elif k == MAP_ID or k == MAP_FOR or k == MAP_ROW:
job.debug(verify, "# + k in MAP : continue "+k)
continue
else:
job.debug(verify, "# + k in leaf : val "+k)
a = path.split(",")
a.append(k)
npath = ",".join(a)
job.debug(verify, "mapElem - dict "+k)
out[k] = mapElement(job, args, elem[k], npath, {})
elif isinstance(elem, list):
out = []
i = 0
for k in elem:
job.debug(verify, "mapElem - list "+str(k))
a = path.split(",")
a.append(str(i))
npath = ",".join(a)
out.append(mapElement(job, args, elem[i], path, {}))
i += 1
else:
job.debug(verify, "mapElem - leaf " + elem)
if elem[0:1] == "{" and elem[-1:] == "}":
elem = elem[1:-1]
out = toSimpleType(job, getValue(job, args, elem))
return out
def toSimpleType(job, value):
if isinstance(value, (list, tuple)) and len(value) == 1:
return value[0]
return value
def extractIds(job, idval):
ids = []
if isinstance(idval, list) or isinstance(idval, tuple):
a = idval
else:
a = idval.split(",")
for k in a:
if "-" in k:
b = k.split("-")
for i in range(int(b[0].strip()), int(b[1].strip())+1):
ids.append(str(i).strip())
elif isinstance(k, str):
ids.append(k.strip())
else:
ids.append(k)
return ids
def getRow(job, args, fieldkey):
a = fieldkey.split(".")
row = getValue(job, args, fieldkey)
if B.DATA_NODE_ROW not in args: args[B.DATA_NODE_ROW] = {}
a[1] = a[1][0:a[1].find("(")]
args[B.DATA_NODE_ROW][a[1]] = row[0]
return row
def getIds(job, args, fieldkey):
"""
sets the id resp list of ids into args[idlist]
the fieldkey has a formula like id={_source.table.field(condition)}
:param job:
:param args:
:param fieldky:
:return:
"""
verify = job.getDebugLevel(MODUL_NAME)
job.debug(verify, "match = "+fieldkey)
out = []
idfield = fieldkey
if re.match(r"(.+)\=(.+)", fieldkey):
res = re.search(r"(.+)\=(.+)", fieldkey)
idfield = res.group(1)
fieldkey = res.group(2)
if fieldkey[0:1] == "{" and fieldkey[-1:] == "}":
fieldkey = fieldkey[1:-1]
if "temp" not in args: args["temp"] = {}
i = 0
while "(" in fieldkey and ")" in fieldkey:
innerCond = fieldkey[fieldkey.rfind("(")+1:fieldkey.find(")")]
if "." not in innerCond:
break
innerkey = "temp_"+str(i)
args["temp"][innerkey] = {}
args["temp"][innerkey]["idfield"] = idfield
args["temp"][innerkey]["fieldkey"] = fieldkey
innerlist = getIds(job, args, innerkey+"={"+innerCond+"}")
args[ACT_ID][innerkey] = ",".join(innerlist)
fieldkey = fieldkey.replace(innerCond, innerkey)
idfield = args["temp"][innerkey]["idfield"]
i += 1
if i > 3:
raise Exception("too much searches "+str(args["temp"]))
val = getValue(job, args, fieldkey)
idlist = extractIds(job, val)
args[ID_LIST][idfield] = idlist
job.debug(verify, "idlist " + str(args[ID_LIST]))
return idlist
def getConditions(job, args, fieldkey):
"""
gets a list of condition-value
:param job:
:param args:
:param fieldkey: in formula (c_1, c_2, ..)
:return: [v_1} ..]
"""
verify = job.getDebugLevel(MODUL_NAME)
while fieldkey[0:1] == "(" and fieldkey[-1:] == ")":
fieldkey = fieldkey[1:-1]
while fieldkey[0:1] == "{" and fieldkey[-1:] == "}":
fieldkey = fieldkey[1:-1]
out = []
a = fieldkey.split(",")
if len(a) > 1:
job.m.logError("map-condition should not have more parts - use tupel "+fieldkey)
for k in a:
if "." in k:
raise Exception("Formatfehler in " + fieldkey)
job.debug(verify, "actid " + str(args[ACT_ID]))
idelem = {}
idelem[k] = args[ACT_ID][k]
out.append(args[ACT_ID][k])
return out
def getValue(job, args, fieldkey):
"""
gets the value of the formula like {_source.table.field(condition)}
:param job:
:param args:
:param fieldkey:
:return:
"""
verify = job.getDebugLevel(MODUL_NAME)
job.debug(verify, "getValue "+fieldkey)
while fieldkey[0:1] == "{" and fieldkey[-1:] == "}":
fieldkey = fieldkey[1:-1]
val = ""
idfield = ""
source = ""
table = ""
field = ""
cond = ""
condIds = []
# special cases of id-management
# args[actid][xid] = actual id with name xid
# args[idlist][xid] = list of all ids with name xid
# a) declaration of the id : id={fielddefinition}
if re.match(r".+\=.+", fieldkey):
job.debug(verify, "getValue 222 " + fieldkey)
raise Exception("getIds sollte an passender Stelle direkt aufgerufen werden "+fieldkey)
return getIds(job, args, fieldkey)
# b) set of defined ids - neither fielddefinition nor a function
#elif "." not in fieldkey and "(" not in fieldkey or re.match(r"\(.+\)", fieldkey):
#print("getValue 226 " + fieldkey)
#raise Exception("getConditions sollte an passender Stelle direkt aufgerufen werden")
#return getConditions(job, args, fieldkey)
# fielddefinition with .-separated parts
b = fieldkey.split(".")
job.debug(verify, "match field "+fieldkey)
if re.match(r"(_.+)\..+\..+\(.+\)", fieldkey):
res = re.match(r"(_.+)\.(.+)\.(.+\(.+\))", fieldkey)
job.debug(verify, "match mit ()")
source = res.group(1)
table = res.group(2)
field = res.group(3)
#cond = res.group(4)
#condIds = getValue(job, args, cond)
elif len(b) == 1:
field = b[0]
elif len(b) == 2:
source = b[0]
field = b[1]
elif len(b) == 3:
source = b[0]
table = b[1]
field = b[2]
if re.match(r".+\(.+\)", field):
res = re.match(r"(.+)\((.+)\)", field)
field = res.group(1)
cond = res.group(2)
condIds = getConditions(job, args, cond)
job.debug(verify, source + " - " + table + " - " + field + " - " + cond + " : " + str(condIds))
if source == B.DATA_NODE_ROW:
if table not in args[B.DATA_NODE_ROW]:
raise Exception("row not initialiazed for table "+table+" "+str(args[B.DATA_NODE_ROW]))
row = args[B.DATA_NODE_ROW][table]
val = row[field]
elif source == B.DATA_NODE_DATA:
job.debug(verify, "data " + b[1])
if len(b) == 3:
job.debug(verify, table + ", " + field + ", " + cond + ", " + str(condIds))
val = toSimpleType(job, getFieldVal(job, args, table, field, condIds))
elif len(b) == 2:
job.debug(verify, table + ", " + field + ", " + cond + ", " + str(condIds))
val = getTdataRow(job, args[B.DATA_NODE_DATA], field, condIds)
elif source == B.DATA_NODE_STEPS:
job.debug(verify, "steps " + table+" - "+ field)
if hasattr(args[B.DATA_NODE_STEPS], field):
val = getattr(args[B.DATA_NODE_STEPS], field)
elif hasattr(args[B.DATA_NODE_STEPS], table):
row = getattr(args[B.DATA_NODE_STEPS], table)
if field in row:
val = row[field]
elif source[1:] == B.DATA_NODE_PAR:
job.debug(verify, "par " + b[1])
if getattr(job.par, b[1]):
val = getattr(job.par, b[1])
elif source == B.DATA_NODE_CATALOG:
job.debug(verify, "catalog " + table+", "+ field)
if len(b) != 3:
job.debug(verify, "catalog-Fehler")
return "Fehler-145"
row = basic.catalog.Catalog.getInstance().getValue(table, args[ACT_ID][cond], job)
if isinstance(row, dict) and field in row:
val = row[field]
elif source[1:] == B.DATA_NODE_COMP:
job.debug(verify, "comp "+table+", "+field)
comp = args[B.DATA_NODE_COMP]
if table == B.DATA_NODE_DDL:
fields = comp.conf[B.DATA_NODE_DDL][field][B.DATA_NODE_HEADER]
val = ",".join(fields)
else:
val = fieldkey
job.debug(verify, "return val "+str(val))
return val
def getFieldVal(job, args, table, field, condIds):
out = []
fields = field.split(",")
for row in getTdataRow(job, args[B.DATA_NODE_DATA], table, condIds):
if len(fields) == 1 and field in row:
out.append( str(row[field]).strip())
else:
tup = tuple()
for f in fields:
if f in row:
t = tuple( str(row[f]).strip() )
tup = tup + t
else:
raise Exception("field is missing in row "+f+", "+str(row))
out.append(tup)
return out
def getTdataRow(job, tdata, table, condIds):
verify = job.getDebugLevel(MODUL_NAME)
out = []
idFields = {}
job.debug(verify, "getTdata "+str(condIds))
for i in range(0, len(condIds)):
idFields[tdata[B.DATA_NODE_TABLES][table][B.DATA_NODE_HEADER][i]] = condIds[i]
job.debug(verify, "idFields "+str(idFields))
for row in tdata[B.DATA_NODE_TABLES][table][B.DATA_NODE_DATA]:
i = 0
for k in idFields:
for j in range(0, len(idFields[k])):
if row[k] == idFields[k][j]:
i += 1
if i == len(idFields):
out.append(row)
return out