Compare commits

...

2 Commits

  1. 23
      basic/catalog.py
  2. 19
      basic/componentHandling.py
  3. 2
      basic/constants.py
  4. 8
      basic/program.py
  5. 1
      test/constants.py
  6. 45
      test/test_component.py
  7. 8
      test/test_config.py
  8. 53
      test/test_tdata.py
  9. 6
      test/testtools.py
  10. 2
      utils/data_const.py
  11. 0
      utils/file_abstract.py
  12. 2
      utils/path_const.py
  13. 92
      utils/tdata_tool.py
  14. 0
      utils/xml1_tool.py

23
basic/catalog.py

@ -8,6 +8,9 @@ import os
import basic.program
import basic.constants as B
import utils.path_const as P
import utils.data_const as D
import utils.path_tool
import utils.tdata_tool
EXP_KEY_MISSING = "key is missing {}"
EXP_KEY_DOESNT_EXIST = "key doesnt exist in domain {}"
@ -20,21 +23,23 @@ class Catalog:
* initially the csv-file catalog.csv
* on demand other csv-files in the path
"""
def __init__(self):
self.catalog = {}
Catalog.__instance = self
pass
@staticmethod
def getInstance():
if Catalog.__instance == None:
return Catalog()
return Catalog.__instance
def getValue(self, domain, key):
"""
this function gets the value of the domain an key
:param domain:
:param key:
:return:
@ -51,12 +56,26 @@ class Catalog:
raise Exception(EXP_KEY_DOESNT_EXIST, (domain, key))
return self.catalog[domain][key]
def readDomain(self, domain):
"""
this function reads the domain-entries
:param domain:
:return:
"""
job = basic.program.Job.getInstance()
if not (isinstance(domain, str) and len(domain)):
raise Exception(EXP_KEY_MISSING, (domain))
if domain in self.catalog:
return
filename = utils.path_tool.rejoinPath(job.conf.getPath(P.ATTR_PATH_TDATA), "catalog", domain+".csv")
data = utils.tdata_tool.getCsvSpec(job.m, filename, D.CSV_SPECTYPE_KEYS)
self.catalog[domain] = data[B.DATA_NODE_TABLES][domain][B.DATA_NODE_KEYS]
def exportXSD(self, domain):
"""
this function exports the domain into xsd-declaration of simple types
:return:
"""
pass

19
basic/componentHandling.py

@ -81,10 +81,10 @@ class ComponentManager:
job = basic.program.Job.getInstance()
anw = job.par.application
job.m.logDebug("applicationscomponente -- " + str(type(job.par)))
if not job.conf.confs["applicationen"].get(anw):
if not job.conf.confs[B.SUBJECT_APPS].get(anw):
job.m.setFatal("application " + job.par.application + " is not configured")
return
for k in job.conf.confs["applicationen"].get(anw):
for k in job.conf.confs[B.SUBJECT_APPS].get(anw):
job.m.logDebug("applicationscomponente -- " + k + ":")
self.createComponent(k, 0, "")
@ -205,6 +205,8 @@ class ComponentManager:
conf = utils.config_tool.getConfig("DATASTRUCTURE", c.name, table)
if B.DATA_NODE_TABLES in conf and table in conf[B.DATA_NODE_TABLES]:
c.conf[B.DATA_NODE_DDL][table] = conf[B.DATA_NODE_TABLES][table]
elif table in conf:
c.conf[B.DATA_NODE_DDL][table] = conf[table]
else:
c.conf[B.DATA_NODE_DDL][table] = conf
comps[name] = c
@ -222,6 +224,19 @@ class ComponentManager:
self.createComponent(c, nr, suffix)
def getComponentDict(self):
job = basic.program.Job.getInstance()
verify = -2 + job.getDebugLevel("job_tool")
job.debug(verify, "getComponents ")
out = {}
for c in self.comps:
out[self.comps[c].name] = {}
for k in self.comps[c].conf.keys():
if isParameterSubnode(k): # B.SUBJECT_ARTS in k or "components" in k or "instance" in k:
out[self.comps[c].name][k] = copy.deepcopy(self.comps[c].conf[k])
return out
def getComponentFolder(comp):
return comp.lower()

2
basic/constants.py

@ -165,7 +165,7 @@ ATTR_PATH_TDATA = "testdata"
ATTR_PATH_PATTN = "pattern"
""" This constant defines the debug-folder in testing-filesystem """
#SUBJECT_APPS = "applications" # | x | | | | CompHandling
SUBJECT_APPS = "applications" # | x | | | | CompHandling
SUBJECT_INST = "instance" # | | | | x | CompHanlding
ATTR_INST_CNT = "count" # | | | | x | CompHanlding

8
basic/program.py

@ -368,11 +368,11 @@ class Parameter:
# args = str(sys.argv)
# print ("Job-Programm %s : " % args)
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--application', required=True, action='store')
parser.add_argument('-e', '--environment', required=True, action='store')
parser.add_argument('-a', '--'+B.SUBJECT_APPS, required=True, action='store')
parser.add_argument('-e', '--'+B.PAR_ENV, required=True, action='store')
parser.add_argument('-r', '--release', action='store')
parser.add_argument('-ts', '--tsdir', action='store')
parser.add_argument('-tc', '--tcdir', action='store')
parser.add_argument('-ts', '--'+B.PAR_TSDIR, action='store')
parser.add_argument('-tc', '--'+B.PAR_TCDIR, action='store')
parser.add_argument('-rs', '--rsdir', action='store')
parser.add_argument('-dt', '--tdtyp', action='store') # PAR_TDTYP
parser.add_argument('-ds', '--tdsrc', action='store') # PAR_TDSRC

1
test/constants.py

@ -1,3 +1,4 @@
HOME_PATH = "/home/ulrich/6_Projekte/Programme/datest"
DATA_PATH = "/home/ulrich/6_Projekte/Programme/data"
COMP_PATH = "/home/ulrich/6_Projekte/Programme/data/components"
OS_SYSTEM = "linux"

45
test/test_component.py

@ -18,7 +18,7 @@ conf = {}
# here you can select single testfunction for developping the tests
TEST_FUNCTIONS = ["test_10actHandler", "test_21createInstance", "test_22createComponent",
"test_23getComponents", "test_24getComponent"]
#TEST_FUNCTIONS = ["test_formatDbRows"]
TEST_FUNCTIONS = ["test_24getComponent"]
class MyTestCase(unittest.TestCase):
mymsg = "--------------------------------------------------------------"
@ -32,18 +32,18 @@ class MyTestCase(unittest.TestCase):
return
job = test.testtools.getJob()
cm = basic.componentHandling.ComponentManager()
cm.createComponent("testa21", 0, "")
self.assertIn("testa21", cm.comps, "component must be stored")
cm.createComponent("testpoldb", 0, "")
self.assertIn("testpoldb", cm.comps, "component must be stored")
self.assertEqual(len(cm.comps), 1, "component without subcomponents")
cnttest += 2
cm.createComponent("testa1", 0, "")
self.assertIn("testa1", cm.comps, "component must be stored")
self.assertIn("testa11", cm.comps, "subcomponent must be stored")
cm.createComponent("testprd", 0, "")
self.assertIn("testprd", cm.comps, "component must be stored")
self.assertIn("testprddb", cm.comps, "subcomponent must be stored")
self.assertEqual(len(cm.comps), 3, "2 component with 1 subcomponents")
cnttest += 2
cm.createComponent("testa", 0, "")
self.assertIn("testa_01", cm.comps, "multiple component must be stored")
self.assertIn("testa_02", cm.comps, "multiple component must be stored")
cm.createComponent("testcm", 0, "")
self.assertIn("testcm_01", cm.comps, "multiple component must be stored")
self.assertIn("testcm_02", cm.comps, "multiple component must be stored")
self.assertEqual(len(cm.comps), 6, "2 component with 2 subcomponents and 2 sub-subcomponents")
cnttest += 2
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
@ -57,23 +57,23 @@ class MyTestCase(unittest.TestCase):
return
job = test.testtools.getJob()
cm = basic.componentHandling.ComponentManager.getInstance("J")
componentName = "testa"
componentName = "testcm"
confs = utils.config_tool.getConfig("comp", componentName)
conns = utils.conn_tool.getConnections(componentName)
c = cm.createInstance(componentName, None, confs, conns, 1)
self.assertEqual(hasattr(c, "conf"), True, "cinfig-dict must be")
self.assertEqual(hasattr(c, "m"), True, "message-object must be")
self.assertEqual(c.name, "testa_01", "classname with number") # classname with number
self.assertEqual(c.name, "testcm_01", "classname with number") # classname with number
cnttest += 3
self.assertEqual(c.conf[B.SUBJECT_INST][B.ATTR_INST_CNT], 2, "conn-attribute overwrites config-attribute")
cnttest += 1 # it overwrites
self.assertEqual(c.conf[B.SUBJECT_INST][B.ATTR_INST_SGL], "n", "without conn-attribute the config-attribute keeps")
cnttest += 1 # it keep
componentName = "testa1"
componentName = "testprddb"
confs = utils.config_tool.getConfig("comp", componentName)
conns = utils.conn_tool.getConnections(componentName)
c = cm.createInstance(componentName, None, confs, conns, 0)
self.assertEqual(c.name, "testa1")
self.assertEqual(c.name, "testprddb")
self.assertIn(B.ATTR_DB_TYPE, c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB], "conn-attribute creates missing config-attribute")
cnttest += 2 # new attributes
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
@ -101,13 +101,14 @@ class MyTestCase(unittest.TestCase):
return
job = test.testtools.getJob()
cm = basic.componentHandling.ComponentManager()
cm.createComponent("testa", 0, "")
cm.createComponent("testcm", 0, "")
cm.createComponent("testcrm", 0, "")
prog = "init_testcase"
comps = cm.getComponents(prog)
self.assertEqual(len(comps), 5, "not all created comps for program "+prog)
prog = "finish_testcase"
self.assertEqual(len(comps), 7, "not all created comps for program "+prog)
prog = "check_environment"
comps = cm.getComponents(prog)
self.assertEqual(len(comps), 6, "all created comps for program "+prog)
self.assertEqual(len(comps), 8, "all created comps for program "+prog)
cnttest += 2
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
@ -120,15 +121,15 @@ class MyTestCase(unittest.TestCase):
return
job = test.testtools.getJob()
cm = basic.componentHandling.ComponentManager()
cm.createComponent("testa", 0, "")
cm.createComponent("testcm", 0, "")
self.assertEqual(len(cm.comps), 6, "6 components inside")
comp = cm.getComponent("testa1")
comp = cm.getComponent("testprd")
self.assertEqual(len(cm.comps), 6, "6 components inside")
self.assertEqual(comp.name, "testa1", "component found")
self.assertEqual(comp.name, "testprd", "component found")
cnttest += 2
comp = cm.getComponent("testb1")
comp = cm.getComponent("testcrm")
self.assertEqual(len(cm.comps), 6, "6 components inside and nothing added")
self.assertIsNotNone("terstb1")
self.assertIsNotNone("terstcrm")
cnttest += 2
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)

8
test/test_config.py

@ -33,8 +33,8 @@ class MyTestCase(unittest.TestCase):
self.assertRaises(Exception, utils.config_tool.getConfigPath, (P.KEY_COMP, "TestX2"))
# self.assertEqual(r, None)
cnttest += 1
r = utils.config_tool.getConfigPath(P.KEY_COMP, "testA2")
self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_COMPS, "testa2", "CONFIG"), r)
r = utils.config_tool.getConfigPath(P.KEY_COMP, "testcrm")
self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), "testcrm", "CONFIG"), r)
r = utils.config_tool.getConfig(P.KEY_TOOL, "path")
if VERIFY: print("pattern " + r["pattern"]["log"])
if VERIFY: print("pattern " + r["pattern"]["precond"])
@ -48,7 +48,7 @@ class MyTestCase(unittest.TestCase):
if actfunction not in TEST_FUNCTIONS:
return
job = test.testtools.getJob()
componentName = "testa"
componentName = "testcm"
confs = utils.config_tool.getConfig("comp", componentName)
conns = utils.conn_tool.getConnections(componentName)
self.assertEqual(confs["conf"][B.SUBJECT_INST][B.ATTR_INST_CNT], 1)
@ -59,7 +59,7 @@ class MyTestCase(unittest.TestCase):
cnttest += 1 # it overwrites
self.assertEqual(confs["conf"][B.SUBJECT_INST][B.ATTR_INST_SGL], "n")
cnttest += 1 # it keep
componentName = "testa1"
componentName = "testprddb"
confs = utils.config_tool.getConfig("comp", componentName)
conns = utils.conn_tool.getConnections(componentName)
self.assertNotIn(B.ATTR_DB_TYPE, confs["conf"][B.SUBJECT_ARTS][B.TOPIC_NODE_DB])

53
test/test_tdata.py

@ -3,9 +3,12 @@ import inspect
import utils.tdata_tool as t
import basic.constants as B
import utils.data_const as D
import utils.path_const as P
import utils.config_tool
import test.testtools
import test.constants
import basic.program
import utils.path_tool
import os
HOME_PATH = test.constants.HOME_PATH
@ -14,8 +17,8 @@ OS_SYSTEM = test.constants.OS_SYSTEM
# here you can select single testfunction for developping the tests
TEST_FUNCTIONS = ["test_tdata", "test_getCsvSpec_data", "test_getCsvSpec_tree", "test_getCsvSpec_key",
"test_getCsvSpec_conf", "test_extractPattern"]
#TEST_FUNCTIONS = ["test_getCsvSpec_key"]
"test_getCsvSpec_conf", "test_extractPattern", "test_parseCsv"]
TEST_FUNCTIONS = ["test_parseCsv"]
class MyTestCase(unittest.TestCase):
mymsg = "--------------------------------------------------------------"
@ -249,6 +252,52 @@ class MyTestCase(unittest.TestCase):
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
def test_parseCsv(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = test.testtools.getJob()
cm = basic.componentHandling.ComponentManager.getInstance("J")
componentName = "testcrmdb"
confs = utils.config_tool.getConfig("comp", componentName)
conns = utils.conn_tool.getConnections(componentName)
comp = cm.createInstance(componentName, None, confs, conns, 1)
fileLines = [
"table:person;_nr;famname;name;birth;sex",
"testcrmdb:person;1;Brecht;Bert;10.02.98;m",
"testcrmdb:person;2;Leon;Donna;28.09.42;f",
"#;;;;;;"
]
filename = utils.path_tool.composePath(P.P_TCBASE, "t_person.csv")
tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="")
print(str(tdata))
self.assertIn(B.DATA_NODE_TABLES, tdata)
self.assertIn("person", tdata[B.DATA_NODE_TABLES])
self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
cnttest += 3
fileLines = [
"date;27.06.2022",
"count;2",
"table:person;_nr;famname;name;birth;sex",
"testcrmdb:person;1;Brecht;Bert;10.02.98;m",
"testcrmdb:person;2;Leon;Donna;28.09.42;f",
"#;;;;;;"
]
tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="")
self.assertIn(B.DATA_NODE_TABLES, tdata)
self.assertIn("person", tdata[B.DATA_NODE_TABLES])
self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
cnttest += 3
text = ""
for k in tdata[B.DATA_NODE_TABLES]:
text += t.buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp)
text += "\n"
print(text)
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
def test_zzz(self):
print(MyTestCase.mymsg)

6
test/testtools.py

@ -2,11 +2,12 @@ import basic.program
import basic.constants as B
import basic.component
import utils.data_const as D
import test.constants as T
DEFAULT_GRAN = "tc"
DEFAULT_APP = "TEST"
DEFAULT_APP = "TESTAPP"
DEFAULT_ENV = "ENV01"
DEFAULT_DIR = "/home/ulrich/6_Projekte/Programme/datest/test/conf/lauf"
DEFAULT_DIR = T.DATA_PATH + "/tdata"
DEFAULT_TIME = "2022-03-19_12-09-09"
DEFAULT_MODE = "unit"
gran = ""
@ -71,6 +72,7 @@ def getJob(pgran="", papp="", penv="", ptstamp="", pmode=""):
elif gran == "ts":
path = DEFAULT_DIR + "/testlauf/TST001_" + tstamp
job = basic.program.Job("unit")
job.conf.confs[B.SUBJECT_PATH]["components"] = T.COMP_PATH
args = {"application": app, "environment": env, "modus": mode, gran+"time": tstamp,
gran+"dir": path,
"step": 2}

2
utils/data_const.py

@ -54,3 +54,5 @@ DEFAULT_DB_PARTITION = "n"
DEFAULT_DB_CONN_JAR = "n"
""" attribute for connection-jar-file instead of connection by ip, port """
ATTR_TABLE_DATE = "date"
ATTR_TABLE_CNT = "count"

0
utils/file_abstract.py

2
utils/path_const.py

@ -62,6 +62,8 @@ ATTR_PATH_ARCHIV = "archiv"
""" This constant defines the folder in testing-filesystem for results and log of execution """
ATTR_PATH_PROGRAM = "program"
""" This constant defines the program-folder in the workspace """
ATTR_PATH_COMPONENTS = "components"
""" This constant defines the program-folder in the workspace """
ATTR_PATH_ENV = "environment"
""" This constant defines the folder in testing-filesystem, used for configs related to environments """
ATTR_PATH_RELEASE = "release"

92
utils/tdata_tool.py

@ -237,33 +237,63 @@ def getTabContent(msg, data, path):
pass
def readCsv(msg, filename, comp, aliasNode=""):
lines = utils.file_tool.readFileLines(filename, msg)
return parseCsv(msg, filename, lines, comp, aliasNode)
def parseCsv(msg, filename, lines, comp, aliasNode=""):
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel(TOOL_NAME)
job.debug(verify, "readCsv " + filename)
verify = -4+job.getDebugLevel(TOOL_NAME)
job.debug(verify, "# # # # # # # # parseCsv " + filename + " :" + comp.name + ": " + str(lines))
fields = []
nodes = []
columns = []
output = {}
state = 0
data = []
data = {}
tableDict = {}
tableDate = ""
tableCnt = 0
cnt = 0
lines = utils.file_tool.readFileLines(filename, msg)
basename = os.path.basename(filename)[0:-4]
startCols = 1
for line in lines:
fields = line.split(';')
testline = line.replace(";", "")
a = fields[0].split(':')
job.debug(verify, str(state) + " line " + line + " :" + str(len(fields)) + ": " + str(fields))
if len(testline) < 2 and state < 1:
state = 0
elif fields[0].lower() in D.CSV_HEADER_START:
elif a[0].lower() == D.ATTR_TABLE_DATE:
tableDate = fields[1]
elif a[0].lower() == D.ATTR_TABLE_CNT:
tableCnt = fields[1]
elif a[0].lower() in D.CSV_HEADER_START:
state = 2
columns = []
h = a
cnt = len(fields)
job.debug(verify, str(state) + " cnt " + str(cnt))
data[B.DATA_NODE_TABLES] = {}
h[0] = B.DATA_NODE_TABLES
if not aliasNode.isspace() and len(aliasNode) > 3:
struct = aliasNode.split(":")
for x in struct:
if len(x) > 2:
nodes.append(x)
job.debug(verify, str(state) + " nodes " + str(nodes))
elif len(h) > 1:
for i in range(1, len(h)):
nodes.append(h[i])
job.debug(verify, str(state) + " nodes " + str(nodes))
tableDict = getTabContent(msg, data, h)
if len(tableDate) > 6:
tableDict[D.ATTR_TABLE_DATE] = tableDate
if int(tableCnt) > 0:
tableDict[D.ATTR_TABLE_CNT] = tableCnt
j = 0
for i in range(1, cnt):
if fields[0][0:1] == "_":
if fields[i][0:1] == "_":
startCols += 1
continue
job.debug(verify, str(i) + " cnt " + str(fields[i]))
@ -271,35 +301,27 @@ def readCsv(msg, filename, comp, aliasNode=""):
columns.append(fields[i])
j = j + 1
cnt = j
tableDict[B.DATA_NODE_HEADER] = columns
job.debug(verify, str(state) + " " + str(cnt) + " cols " + str(columns))
elif state >= 2 and len(testline) > 2:
if state == 2 and not aliasNode.isspace():
struct = aliasNode.split(":")
for x in struct:
if len(x) > 2:
nodes.append(x)
job.debug(verify, str(state) + " nodes " + str(nodes))
elif state == 2 and not fields[0].isspace():
struct = fields[0].split(":")
for x in struct:
if len(x) > 2:
nodes.append(x)
job.debug(verify, str(state) + " nodes " + str(nodes))
job.debug(verify, str(state) + " " + str(len(testline)))
tableDict = getTabContent(msg, data, h)
state = 3
row = {}
for i in range(startCols, cnt):
for i in range(startCols, cnt+startCols):
if i >= len(columns)+startCols:
break
row[columns[i-startCols]] = fields[i]
job.debug(verify, str(state) + " row " + str(row))
data.append(row)
if B.DATA_NODE_DATA not in tableDict:
tableDict[B.DATA_NODE_DATA] = []
tableDict[B.DATA_NODE_DATA].append(row)
setTabContent(msg, data, tableDict, h)
elif state == 3:
job.debug(verify, "structure " + str(state) + ": " + str(nodes))
output = setSubnode(0, nodes, data, output)
data = []
state = 0
if len(nodes) < 1:
nodes.append(basename)
output = setSubnode(0, nodes, data, output)
return output
return data
def setSubnode(i, nodes, data, tree):
print("setSubnode " + str(i) + ": " + ": " + str(tree))
@ -327,7 +349,8 @@ def normalizeDataRow(dstruct, xpathtupel, row, referencedate):
verify = -1+job.getDebugLevel(TOOL_NAME)
job.debug(verify, "calcDataRow " + row)
def writeCsvData(filename, tdata, comp):
def buildCsvData(filename, tdata, comp):
"""
writes the testdata into a csv-file for documentation of the test-run
:param teststatus:
@ -338,7 +361,11 @@ def writeCsvData(filename, tdata, comp):
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel(TOOL_NAME)
job.debug(verify, "writeDataTable " + str(comp))
text = "table"
text = ""
for k in [D.ATTR_TABLE_DATE, D.ATTR_TABLE_CNT]:
if k in tdata:
text += k+";"+tdata[k]+"\n"
text += "table"
for f in tdata[B.DATA_NODE_HEADER]:
text += ";"+f
for r in tdata[B.DATA_NODE_DATA]:
@ -349,4 +376,13 @@ def writeCsvData(filename, tdata, comp):
else:
text += ";"
text += "\n"
return text
def writeCsvData(filename, tdata, comp):
text = ""
if B.DATA_NODE_TABLES in tdata:
for k in tdata[B.DATA_NODE_TABLES]:
text += buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp)
text += "\n"
utils.file_tool.writeFileText(comp.m, filename, text)

0
utils/xml_tool.py → utils/xml1_tool.py

Loading…
Cancel
Save