diff --git a/test/test_db.py b/test/test_db.py index 3684eb6..e016748 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,20 +1,33 @@ -import unittest, os - +import unittest +import inspect import basic.program import utils.path_tool import basic.toolHandling import test.constants -import components.component +import basic.component import basic.constants as B import utils.db_abstract -import utils.data_const as D +import test.testtools +import utils.config_tool + HOME_PATH = test.constants.HOME_PATH conf = {} +# here you can select single testfunction for developping the tests +TEST_FUNCTIONS = ["test_parseSql", "test_toolhandling", "test_formatDbRows"] +#TEST_FUNCTIONS = ["test_formatDbRows"] class MyTestCase(unittest.TestCase): + mymsg = "--------------------------------------------------------------" + + def test_parseSql(self): - job = basic.program.Job("unit") + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return + job = test.testtools.getJob() args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", "modus": "unit"} job.par.setParameterArgs(args) @@ -30,21 +43,27 @@ class MyTestCase(unittest.TestCase): self.assertEqual(obj[1][3], "end") self.assertEqual(obj[1][1], "state") ddl = utils.config_tool.getConfig("DATASTRUCTURE", "testb1", "lofts") - dbwhere = utils.db_abstract.parseSQLwhere("family like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"]) + dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"]) self.assertIn("state", dbwhere) self.assertNotIn("family", dbwhere) dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"]) self.assertIn("state", dbwhere) self.assertIn("street", dbwhere) - pass + MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) - def xtest_toolhandling(self): + + def test_toolhandling(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return job = basic.program.Job("unit") args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", "modus": "unit"} job.par.setParameterArgs(args) #t = basic.toolHandling.ToolManager() - comp = components.component.Component() + comp = basic.component.Component() comp.name = "testb1" table = "lofts" comp.conf = {} @@ -86,6 +105,21 @@ class MyTestCase(unittest.TestCase): print(sqls) #tool.deleteRows("deltable") #tool.selectRows("deltable") + MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) + + + def test_formatDbRows(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return + job = test.testtools.getJob() + MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) + + + def test_zzz(self): + print(MyTestCase.mymsg) if __name__ == '__main__': diff --git a/test/test_job.py b/test/test_job.py index 31940bc..498c02e 100644 --- a/test/test_job.py +++ b/test/test_job.py @@ -1,20 +1,31 @@ import unittest import os +import inspect from basic.program import Job from basic.componentHandling import ComponentManager import init_testcase import test_executer import test.constants +import basic.constants as B +import test.constants as T HOME_PATH = test.constants.HOME_PATH PYTHON_CMD = "python" +TEST_FUNCTIONS = ["test_tdata", "test_getCsvSpec_data", "test_getCsvSpec_tree", "test_getCsvSpec_key", + "test_getCsvSpec_conf", "test_extractPattern", "test_parseCsv"] +TEST_FUNCTIONS = ["test_run"] + + class MyTestCase(unittest.TestCase): - def runTest(self): - #self.test_parameter() - #self.test_components() - self.test_run() + mymsg = "--------------------------------------------------------------" - def xtest_parameter(self): + + def test_parameter(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return job = Job("unit") args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool", "function": "reset_TData,load_TData" } @@ -30,17 +41,17 @@ class MyTestCase(unittest.TestCase): args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool", "tsdir": os.path.join(HOME_PATH, "test", "lauf", "V0.1", "startjob", "2021-08-21_18-ß2-01")} job.par.setParameterArgs(args) - def xtest_components(self): - print("# # # # tetsComponents # # # # #") - job = Job.resetInstance("unit") - args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool"} - job.par.setParameterArgs(args) - cm = ComponentManager() - cm.createComponent("testb", 0, "") - cm.createComponent("testa", 1, "") - def test_run(self): + def test_run(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return + programs = ["init_testcase"] + testcase = "TC0001" + timexec = "2022-06-28_21-23-34" # os.system("python "+os.path.join(HOME_PATH, "check_environment.py")+" -a TEST -e ENV01") # os.system("python "+os.path.join(HOME_PATH, "init_testsuite.py")+" -a TEST -e ENV01 " # "-ts "+os.path.join(HOME_PATH, "test","lauf","V0.1","implement_2021-08-28_23-50-51")+" -dt csv -ds implement -dn firstunit") @@ -50,17 +61,19 @@ class MyTestCase(unittest.TestCase): # "tool": "job_tool", "tsdir": os.path.join(HOME_PATH,"test","conf","lauf","V0.1","TC0001_2021-08-28_23-50-51")} #"loglevel": "debug", "tdtyp": "dir", # "tdsrc": "TC0001", "tdname": "xxx", - job = Job("unit") - args = { "application": "TEST", "environment": "ENV01", "modus": "unit", "tstime": "2022-03-19_12-09-09", - "tsdir": '/home/ulrich/6_Projekte/Programme/datest/test/conf/lauf/testlauf/TST001_2022-03-19_12-09-09', - "step": 2 } - # "usecase": "TST001", "tstime": "2022-03-17_17-28"} - job.par.setParameterArgs(args) - job.setProgram("test_executer") - # init_testcase.start(job) - job.startJob() - test_executer.start(job) - job.stopJob(1) + if "init_testcase" in programs: + program = "init_testcase" + job = Job("unit") + args = { B.PAR_APP: "TESTAPP", B.PAR_ENV: "ENV01", "modus": "unit", + B.PAR_TCDIR: os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ARCHIV], testcase, timexec), + "step": 1 } + # "usecase": "TST001", "tstime": "2022-03-17_17-28"} + job.par.setParameterArgs(args) + job.setProgram(program) + init_testcase.startPyJob(job) + job.startJob() + #test_executer.start(job) + job.stopJob(1) if __name__ == '__main__': unittest.main() diff --git a/test/test_tdata.py b/test/test_tdata.py index b8637cb..bb4824f 100644 --- a/test/test_tdata.py +++ b/test/test_tdata.py @@ -9,6 +9,7 @@ import test.testtools import test.constants import basic.program import utils.path_tool +import utils.file_tool import os HOME_PATH = test.constants.HOME_PATH @@ -270,7 +271,7 @@ class MyTestCase(unittest.TestCase): "testcrmdb:person;2;Leon;Donna;28.09.42;f", "#;;;;;;" ] - filename = utils.path_tool.composePath(P.P_TCBASE, "t_person.csv") + filename = utils.path_tool.rejoinPath(utils.path_tool.composePath(P.P_TCBASE, comp), "t_person.csv") tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="") print(str(tdata)) self.assertIn(B.DATA_NODE_TABLES, tdata) @@ -278,8 +279,8 @@ class MyTestCase(unittest.TestCase): self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA])) cnttest += 3 fileLines = [ - "date;27.06.2022", - "count;2", + "_date;27.06.2022", + "_count;2", "table:person;_nr;famname;name;birth;sex", "testcrmdb:person;1;Brecht;Bert;10.02.98;m", "testcrmdb:person;2;Leon;Donna;28.09.42;f", @@ -290,6 +291,20 @@ class MyTestCase(unittest.TestCase): self.assertIn("person", tdata[B.DATA_NODE_TABLES]) self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA])) cnttest += 3 + filename = utils.path_tool.rejoinPath(utils.path_tool.composePath(P.P_TCRESULT, comp), "person.csv") + fileLines = [ + "_date;27.06.2022", + "_count;2", + "persid;famname;name;birth;sex", + "1;Brecht;Bert;10.02.98;m", + "2;Leon;Donna;28.09.42;f", + "#;;;;;;" + ] + tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="") + self.assertIn(B.DATA_NODE_TABLES, tdata) + self.assertIn("person", tdata[B.DATA_NODE_TABLES]) + self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA])) + cnttest += 3 text = "" for k in tdata[B.DATA_NODE_TABLES]: text += t.buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp) diff --git a/test/test_xml.py b/test/test_xml.py index ce1acb1..36ec229 100644 --- a/test/test_xml.py +++ b/test/test_xml.py @@ -1,7 +1,7 @@ import unittest #import basic.program -import utils.xml_tool -import utils.xml_tool +import utils.xml1_tool +import utils.xml1_tool class MyTestCase(unittest.TestCase): def xtest_xmlTool(self): @@ -12,17 +12,17 @@ class MyTestCase(unittest.TestCase): beispiel_json = {'root': {'@attr': 'xyz', '$': 'inhalt', "b": "bold"}} tree = {} tree["root"] = args - xml = utils.xml_tool.dict2xml(tree) + xml = utils.xml1_tool.dict2xml(tree) print(xml) - xml = utils.xml_tool.dict2xml(beispiel_json) + xml = utils.xml1_tool.dict2xml(beispiel_json) print(xml) self.assertEqual(1, 1) - f = utils.xml_tool.fcts() + f = utils.xml1_tool.fcts() def xtest_addSingle(self): tree = {} # tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', 2, "abc") - tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc") + tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc") self.assertEqual(tree["kratz"]["mm"][0]["wert"], "abc") self.assertEqual(tree["kratz"]["mm"][0]["@name"], "NAME") @@ -31,9 +31,9 @@ class MyTestCase(unittest.TestCase): # c-a-a c-a-b tree = {} print("--------------------------------------------------------------------------------") - tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc") + tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc") print("--------------------------------------------------------------------------------") - tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="LAND"]/wert', 2, "xyz") + tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="LAND"]/wert', 2, "xyz") baum = {} baum["root"] = tree print("<------"+str(baum["root"])) @@ -45,7 +45,7 @@ class MyTestCase(unittest.TestCase): def xtest_addOnePaths(self): tree = {} print("--------------------------------------------------------------------------------") - tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc") + tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc") baum = {} baum["root"] = tree print("<------"+str(baum["root"])) @@ -56,9 +56,9 @@ class MyTestCase(unittest.TestCase): def xtest_addTwoPaths(self): tree = {} print("--------------------------------------------------------------------------------") - tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc") + tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc") print("--------------------------------------------------------------------------------") - tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="LAND"]/wert', "xyz") + tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="LAND"]/wert', "xyz") baum = {} baum["root"] = tree print("<------"+str(baum["root"])) @@ -74,7 +74,7 @@ class MyTestCase(unittest.TestCase): tree_attrx = { "eins": [ {"zwei": 2, "drei": 3, "vier": { "@name": "attr", "fuenf": 5} } ] } tree_attr2 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "fuenf": 5} ] } ] } tree_attr3 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "#text": "text" } ] } ] } - c = utils.xml_tool.fcts() + c = utils.xml1_tool.fcts() xml = c.tidy(tree_dict, 0) print("-------------------------------------") print(xml) diff --git a/utils/db_abstract.py b/utils/db_abstract.py index 89d5a3f..64f39d9 100644 --- a/utils/db_abstract.py +++ b/utils/db_abstract.py @@ -38,7 +38,10 @@ SPECIAL CASES: * If the table is partitioned tables the functions delete/insert/select calls the callback-functions COMP.nextTable() resp. COMP.nextTdata(). """ +import re + import basic.program +import basic.catalog import utils.config_tool import basic.constants as B import utils.data_const as D @@ -86,6 +89,7 @@ def getDbAttributes(comp, table): print("f " + attr + " " + out[attr]) return out + def getStringIndex(text, intern): if intern in text: return text.index(intern) @@ -142,12 +146,104 @@ def parseSQLwhere(condition, ddl=None): conjunctor = "" dbwhere = "" for cond in parts: + print("exp "+cond[1]+" "+str(ddl)) if cond[1] in ddl[B.DATA_NODE_HEADER]: dbwhere += " "+conjunctor+" "+cond[1]+" "+cond[0]+" "+cond[2] conjunctor = cond[3] + print("exp ok") return "WHERE "+dbwhere.strip() +def getSqlTable(comp, table): + """ + the function gets the technical tablename inclusive necessary schema information + :param comp: + :param table: + :return: + """ + attr = getDbAttributes(comp, table) + if attr[B.ATTR_DB_TABNAME] != "": + sqltable = attr[B.ATTR_DB_TABNAME] + else: + sqltable = table + if attr[B.ATTR_DB_SCHEMA] != "": + sqltable = attr[B.ATTR_DB_SCHEMA] + "." + sqltable + return sqltable + + +def getTechnicalIDFields(ddl): + ids = [] + keys = {} + for f in ddl: + if f[0:1] == "_": + continue + if "T" in ddl[f][D.DDL_KEY]: + keys[ddl[f][D.DDL_KEY]] = f + for k in keys: + ids.append(k) + return ids + + +def formatDbRows(table, comp, rows): + out = [] + fields = comp.conf[B.DATA_NODE_DDL][table] + header = comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER] + for r in rows: + ro = {} + for f in header: + if f in r: + val = formatDbField(comp, r[f], fields[f]) + ro[f] = val + else: + val = formatDbField(comp, B.SVAL_NULL, fields[f]) + ro[f] = val + out.append(ro) + return out + + +def formatDbField(comp, val, field): + if val == B.SVAL_NULL: + if field[D.DDL_FNULLABLE] != B.SVAL_YES: + comp.m.logError("must-field is null "+ field[D.DDL_FNAME]) + return None + print("formatDbField "+str(comp)) + print("formatDbField "+str(field)+" "+str(val)) + return formatDbVal(comp.m, val, field[D.DDL_TYPE]) + + +def formatDbVal(msg, val, dtyp): + ctlg = basic.catalog.Catalog.getInstance() + if dtyp == D.TYPE_STRING: + if not isinstance(val, str): + msg.logError("field must be " + dtyp + ", " + str(val)) + return str(val) + if dtyp == D.TYPE_INT: + if not (isinstance(val, int) or re.match(r"^\d+$", val)): + msg.logError("field must be " + dtyp + ", " + str(val)) + return 0 + return int(val) + if dtyp == D.TYPE_FLOAT: + if not (isinstance(val, float) or re.match(r"^\d+[.,]\d+$", val)): + msg.logError("field must be " + dtyp + ", " + str(val)) + return 0 + return float(val) + else: + pass + + +def isCompTable(comp, data, table): + """ checks if the table in data relates to the component """ + print(str(data)) + return isCompRow(comp, data[B.DATA_NODE_TABLES][table]) + +def isCompRow(comp, row): + """ checks if the table in data relates to the component """ + print("isCompRow "+comp.name+" "+str(row)) + if comp.name in row[B.ATTR_DATA_COMP] \ + and row[B.ATTR_DATA_COMP][comp.name] in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]: + return True + return False + # --------------------------------------------------------------------------------------------------------------- class DbFcts(): @@ -184,6 +280,7 @@ class DbFcts(): raise Exception("missing data node in table") tdata[subdir][t][D.DATA_ATTR_COUNT] = len(tdata[subdir][t][B.DATA_NODE_DATA]) tdata[subdir][t][D.DATA_ATTR_DATE] = utils.date_tool.getActdate(utils.date_tool.F_DE) + self.comp.m.logMsg("Tabelle {} mit {} Zeilen gelesen".format(t, len(tdata[subdir][t][B.DATA_NODE_DATA]))) return tdata def selectRows(self, statement): @@ -215,14 +312,19 @@ class DbFcts(): raise Exception(B.EXCEPT_NOT_IMPLEMENT) def insertTables(self, tdata): - """ method to insert rows into a database - statement written in sql """ + """ + method to insert rows into the database of the component + """ + # TODO wird der Tabellenname/DB/Schema unter tdata gespeichert? plainname = basic.componentHandling.getPlainCompname(self.comp.name) self.loadDdl() - for t in self.comp.conf[B.DATA_NODE_DDL]: - print("einzufuegende Tabelle "+t) - if (t in tdata[plainname]): - self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA]) + for t in tdata[B.DATA_NODE_TABLES]: + print("einzufuegende Tabelle "+self.comp.name+" "+t) + if isCompTable(self.comp, tdata, t): + self.insertRows(t, tdata[B.DATA_NODE_TABLES][t][B.DATA_NODE_DATA]) + self.comp.m.logMsg("in Tabelle {} {} Zeilen eingefuegt".format( + t, len(tdata[B.DATA_NODE_TABLES][t][B.DATA_NODE_DATA]))) + def insertRows(self, rows): """ method to insert rows into a database @@ -253,6 +355,8 @@ class DbFcts(): return "" def getDbValue(self, fo, value): + # TODO Untersceidung csv und echte DB + return formatDbField(self.comp, value, fo) if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES: return self.getDbNull() if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING: diff --git a/utils/dbcsv_tool.py b/utils/dbcsv_tool.py index a5f95fa..c552c01 100644 --- a/utils/dbcsv_tool.py +++ b/utils/dbcsv_tool.py @@ -8,7 +8,9 @@ import basic.program import utils.config_tool import utils.db_abstract import basic.constants as B - +import utils.path_tool +import utils.file_tool +import utils.tdata_tool class DbFcts(utils.db_abstract.DbFcts): """ @@ -22,51 +24,87 @@ class DbFcts(utils.db_abstract.DbFcts): def selectRows(self, table): """ method to select rows from a database statement written in sql """ + sqlTable = utils.db_abstract.getSqlTable(self.comp, table) + header = "" + path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp) + print(path) tdata = {} + data = utils.tdata_tool.readCsv(self.comp.m, path, self.comp) + tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER] + if B.DATA_NODE_TABLES in data \ + and table in data[B.DATA_NODE_TABLES]\ + and B.DATA_NODE_DATA in data[B.DATA_NODE_TABLES][table]: + tdata[B.DATA_NODE_DATA] = data[B.DATA_NODE_TABLES][table][B.DATA_NODE_DATA] + else: + tdata[B.DATA_NODE_DATA] = {} return tdata + def deleteRows(self, table): """ method to delete rows from a database statement written in sql """ job = basic.program.Job.getInstance() verify = -1+job.getDebugLevel("db_tool") - cmd = "DELETE FROM "+table+";" + sqlTable = utils.db_abstract.getSqlTable(self.comp, table) + header = "" + path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp) + for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: + print(h) + header += ";"+h + cmd = header[1:]+"\n" + utils.file_tool.writeFileText(self.comp.m, path, cmd) self.comp.m.logInfo(cmd) + def updateRows(self, statement): """ method to delete rows from a database statement written in sql """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) + def insertRows(self, table, rows): """ method to insert rows into a database the rows will be interpreted by the ddl of the component """ job = basic.program.Job.getInstance() verify = -1+job.getDebugLevel("db_tool") - cmd = "INSERT INTO "+table+";" + sqlTable = utils.db_abstract.getSqlTable(self.comp, table) header = "" + path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp) + if len(rows) == 0: + return "" + else: + pass for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: print(h) - header += ", "+h - cmd += " (" + header[1:]+" ) " + header += ";"+h + cmd = "table:" + table + header+"\n" + print("HEADER : "+cmd) rowvalues = "" for r in rows: + if not utils.db_abstract.isCompRow(self.comp, r): + continue print("r-----------------") print(r) - rowvalues = "" - cmd += "\n ( " + rowvalues = self.comp.name+":"+table for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: + print("head "+h) + if h in B.LIST_DB_ATTR: + continue + if B.DATA_NODE_DATA not in self.comp.conf[B.DATA_NODE_DDL][table]: + rowvalues = "" + break print("h "+h) if (h in r): - rowvalues += ", "+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], r[h]) + rowvalues += ";"+str(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], r[h])) else: - rowvalues += ", "+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], "") + rowvalues += ";"+str(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], "")) print("rv " + rowvalues) - cmd += rowvalues[1:]+" )," - cmd = cmd[0:-1]+";" + cmd += rowvalues+"\n" + utils.file_tool.writeFileText(self.comp.m, path, cmd) self.comp.m.logInfo(cmd) + def getConnector(self): """ add-on-method to get the connector this method should only called by the class itself """ diff --git a/utils/tdata_tool.py b/utils/tdata_tool.py index 6539a8d..ad7017c 100644 --- a/utils/tdata_tool.py +++ b/utils/tdata_tool.py @@ -26,6 +26,7 @@ import basic.program import utils.file_tool import basic.constants as B import utils.data_const as D +import utils.date_tool TOOL_NAME = "tdata_tool" """ name of the tool in order to switch debug-info on """ @@ -71,13 +72,24 @@ def getTestdata(): # read file in testdata job.m.logInfo("Test-Data readed from " + tdata[D.ATTR_SRC_TYPE] + " for " + tdata[D.ATTR_SRC_NAME]) elif tdata[D.ATTR_SRC_TYPE] == D.DATA_SRC_DIR: - filename = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.D.ATTR_PATH_TDATA), tdata[D.ATTR_SRC_NAME], "testspec.csv") + path = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.ATTR_PATH_TDATA), tdata[D.ATTR_SRC_NAME]) + filename = os.path.join(path , "testspec.csv") data = getCsvSpec(job.m, filename, D.CSV_SPECTYPE_DATA) for k in data: tdata[k] = data[k] if (k == D.CSV_BLOCK_OPTION): for p in data[k]: setattr(job.par, p, data[k][p]) + files = utils.file_tool.getFiles(job.m, path, "table_", None) + for f in files: + print(f) + filename = os.path.join(path, f) + data = readCsv(job.m, filename, None) + table = f[6:-4] + print(filename+" "+table) + if B.DATA_NODE_TABLES not in tdata: + tdata[B.DATA_NODE_TABLES] = {} + tdata[B.DATA_NODE_TABLES][table] = data[B.DATA_NODE_TABLES][table] else: job.m.setFatal("test-Data: reftyp " + tdata[D.ATTR_SRC_TYPE] + " is not implemented") return tdata @@ -110,7 +122,7 @@ def parseCsvSpec(msg, lines, type): header = [] h = [] # from a[] status = "start" - + tableDate = utils.date_tool.getActdate(utils.date_tool.F_DE) tableDict = {} for l in lines: print("lines "+l) @@ -145,12 +157,15 @@ def parseCsvSpec(msg, lines, type): raise Exception(D.EXCP_MALFORMAT+""+l) data[a[0]][a[1]] = fields[1] continue + elif a[0].lower() == D.DATA_ATTR_DATE: + tableDate = fields[1] elif (a[0].lower() in D.CSV_HEADER_START): # create deep structure a_0 ... a_n print("tdata 136 CSV_HEADER_START "+str(len(a))) h = a data[B.DATA_NODE_TABLES] = {} h[0] = B.DATA_NODE_TABLES + comps = {} tableDict = getTabContent(msg, data, h) i = 0 for f in fields: @@ -171,6 +186,7 @@ def parseCsvSpec(msg, lines, type): headerFields = [] else: tableDict[B.DATA_NODE_DATA] = [] + tableDict[D.DATA_ATTR_DATE] = tableDate setTabContent(msg, data, tableDict, h) status = D.CSV_SPECTYPE_DATA continue @@ -187,6 +203,15 @@ def parseCsvSpec(msg, lines, type): tableDict[B.DATA_NODE_DATA][f] = fields[i] i += 1 if type == D.CSV_SPECTYPE_DATA: + print("parseSpec "+ str(fields[0])) + for c in fields[0].split(","): + a = c.split(":") + print("parseSpec " + str(a)) + comps[a[0]] = a[1] + row[B.ATTR_DATA_COMP] = {} + row[B.ATTR_DATA_COMP][a[0]] = a[1] + #row[B.ATTR_DATA_COMP] = fields[0].split(",") + tableDict[B.ATTR_DATA_COMP] = comps tableDict[B.DATA_NODE_DATA].append(row) elif type == D.CSV_SPECTYPE_KEYS: tableDict[D.CSV_NODETYPE_KEYS][fields[1]] = row @@ -205,6 +230,13 @@ def parseCsvSpec(msg, lines, type): return data +def mergeTableComponents(comps, rowComps): + for c in rowComps.split(","): + a = c.split(":") + comps[a[0]] = a[1] + return comps + + def setTabContent(msg, data, tabledata, path): if len(path) >= 2 and path[1] not in data[path[0]]: data[path[0]][path[1]] = {} @@ -238,13 +270,15 @@ def getTabContent(msg, data, path): def readCsv(msg, filename, comp, aliasNode=""): lines = utils.file_tool.readFileLines(filename, msg) + print("readCsv "+filename) + print(lines) return parseCsv(msg, filename, lines, comp, aliasNode) def parseCsv(msg, filename, lines, comp, aliasNode=""): job = basic.program.Job.getInstance() verify = -4+job.getDebugLevel(TOOL_NAME) - job.debug(verify, "# # # # # # # # parseCsv " + filename + " :" + comp.name + ": " + str(lines)) + job.debug(verify, "# # # # # # # # parseCsv " + filename + " :" + str(lines)) fields = [] nodes = [] columns = [] @@ -264,14 +298,22 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""): job.debug(verify, str(state) + " line " + line + " :" + str(len(fields)) + ": " + str(fields)) if len(testline) < 2 and state < 1: state = 0 - elif a[0].lower() == D.ATTR_TABLE_DATE: + elif a[0].lower() == D.DATA_ATTR_DATE: tableDate = fields[1] - elif a[0].lower() == D.ATTR_TABLE_CNT: + state = 1 + elif a[0].lower() == D.DATA_ATTR_COUNT: tableCnt = fields[1] - elif a[0].lower() in D.CSV_HEADER_START: + state = 1 + elif a[0].lower() in D.CSV_HEADER_START or \ + (comp is not None and state == 1 + and isCompTableFile(comp, filename)): state = 2 columns = [] h = a + if len(h) < 2 and comp is not None: + a = ["table", basename] + h = a + startCols = 0 cnt = len(fields) job.debug(verify, str(state) + " cnt " + str(cnt)) data[B.DATA_NODE_TABLES] = {} @@ -287,10 +329,11 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""): nodes.append(h[i]) job.debug(verify, str(state) + " nodes " + str(nodes)) tableDict = getTabContent(msg, data, h) + tableDict[B.ATTR_DATA_COMP] = {} if len(tableDate) > 6: - tableDict[D.ATTR_TABLE_DATE] = tableDate + tableDict[D.DATA_ATTR_DATE] = tableDate if int(tableCnt) > 0: - tableDict[D.ATTR_TABLE_CNT] = tableCnt + tableDict[D.DATA_ATTR_COUNT] = tableCnt j = 0 for i in range(1, cnt): if fields[i][0:1] == "_": @@ -308,7 +351,14 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""): tableDict = getTabContent(msg, data, h) state = 3 row = {} + print(line) + if startCols > 0: + row[B.ATTR_DATA_COMP] = {} + row[B.ATTR_DATA_COMP][a[0]] = a[1] + tableDict[B.ATTR_DATA_COMP][a[0]] = a[1] for i in range(startCols, cnt+startCols): + print("for "+str(i)+" "+str(len(row))+" "+str(startCols)+" "+str(len(fields))) + print(str(fields[i])) if i >= len(columns)+startCols: break row[columns[i-startCols]] = fields[i] @@ -358,25 +408,38 @@ def buildCsvData(filename, tdata, comp): :param comp: if specific else None :return: """ + compColumn = not isCompTableFile(comp, filename) job = basic.program.Job.getInstance() verify = -1+job.getDebugLevel(TOOL_NAME) job.debug(verify, "writeDataTable " + str(comp)) text = "" - for k in [D.ATTR_TABLE_DATE, D.ATTR_TABLE_CNT]: + for k in [D.DATA_ATTR_DATE, D.DATA_ATTR_COUNT]: if k in tdata: - text += k+";"+tdata[k]+"\n" - text += "table" + text += k+";"+str(tdata[k])+"\n" + header = "table" for f in tdata[B.DATA_NODE_HEADER]: - text += ";"+f + header += ";"+f + if compColumn: + text += header + else: + #text += "_nr;" + header[6:] + "\n" + text += header[6:] + "\n" + i = 0 for r in tdata[B.DATA_NODE_DATA]: - text += "\n" + row = "" + i += 1 for f in tdata[B.DATA_NODE_HEADER]: if f in r: - text += ";"+str(r[f]) + row += ";"+str(r[f]) else: - text += ";" - text += "\n" - return text + row += ";" + if compColumn: + text += row + else: + text += row[1:] + #text += str(i) + row + text += "\n" + return text def writeCsvData(filename, tdata, comp): @@ -386,3 +449,14 @@ def writeCsvData(filename, tdata, comp): text += buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp) text += "\n" utils.file_tool.writeFileText(comp.m, filename, text) + + +def isCompTableFile(comp, filename): + """ check if the filename belongs to the component """ + basetable = os.path.basename(filename)[0:-4] + if comp is None: + return False + if B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_ARTS] and basetable in comp.conf[B.DATA_NODE_DDL] \ + and comp.name in filename: + return True + return False \ No newline at end of file