tests arround dbcsv

test/test_db.py

@@ -1,20 +1,33 @@
-import unittest, os
+import unittest
-
+import inspect
 import basic.program
 import utils.path_tool
 import basic.toolHandling
 import test.constants
-import components.component
+import basic.component
 import basic.constants as B
 import utils.db_abstract
-import utils.data_const as D
+import test.testtools
+import utils.config_tool
+
 HOME_PATH = test.constants.HOME_PATH
 
 conf = {}
+# here you can select single testfunction for developping the tests
+TEST_FUNCTIONS = ["test_parseSql", "test_toolhandling", "test_formatDbRows"]
+#TEST_FUNCTIONS = ["test_formatDbRows"]
 
 class MyTestCase(unittest.TestCase):
+    mymsg = "--------------------------------------------------------------"
+
+
     def test_parseSql(self):
-        job = basic.program.Job("unit")
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
+        job = test.testtools.getJob()
         args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
                 "modus": "unit"}
         job.par.setParameterArgs(args)
@@ -30,21 +43,27 @@ class MyTestCase(unittest.TestCase):
         self.assertEqual(obj[1][3], "end")
         self.assertEqual(obj[1][1], "state")
         ddl = utils.config_tool.getConfig("DATASTRUCTURE", "testb1", "lofts")
-        dbwhere = utils.db_abstract.parseSQLwhere("family like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"])
+        dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"])
         self.assertIn("state", dbwhere)
         self.assertNotIn("family", dbwhere)
         dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"])
         self.assertIn("state", dbwhere)
         self.assertIn("street", dbwhere)
-        pass
+        MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
 
-    def xtest_toolhandling(self):
+
+    def test_toolhandling(self):
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
         job = basic.program.Job("unit")
         args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
                 "modus": "unit"}
         job.par.setParameterArgs(args)
         #t = basic.toolHandling.ToolManager()
-        comp = components.component.Component()
+        comp = basic.component.Component()
         comp.name = "testb1"
         table = "lofts"
         comp.conf = {}
@@ -86,6 +105,21 @@ class MyTestCase(unittest.TestCase):
         print(sqls)
         #tool.deleteRows("deltable")
         #tool.selectRows("deltable")
+        MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
+
+
+    def test_formatDbRows(self):
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
+        job = test.testtools.getJob()
+        MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
+
+
+    def test_zzz(self):
+        print(MyTestCase.mymsg)
 
 
 if __name__ == '__main__':

test/test_job.py

@@ -1,20 +1,31 @@
 import unittest
 import os
+import inspect
 from basic.program import Job
 from basic.componentHandling import ComponentManager
 import init_testcase
 import test_executer
 import test.constants
+import basic.constants as B
+import test.constants as T
 
 HOME_PATH = test.constants.HOME_PATH
 PYTHON_CMD = "python"
+TEST_FUNCTIONS = ["test_tdata", "test_getCsvSpec_data", "test_getCsvSpec_tree", "test_getCsvSpec_key",
+                  "test_getCsvSpec_conf",  "test_extractPattern", "test_parseCsv"]
+TEST_FUNCTIONS = ["test_run"]
+
+
 class MyTestCase(unittest.TestCase):
-    def runTest(self):
+    mymsg = "--------------------------------------------------------------"
-        #self.test_parameter()
-        #self.test_components()
-        self.test_run()
 
-    def xtest_parameter(self):
+
+    def test_parameter(self):
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
         job = Job("unit")
         args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug",
                  "tool" : "job_tool", "function": "reset_TData,load_TData" }
@@ -30,17 +41,17 @@ class MyTestCase(unittest.TestCase):
         args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug",
                  "tool" : "job_tool", "tsdir": os.path.join(HOME_PATH, "test", "lauf", "V0.1", "startjob", "2021-08-21_18-ß2-01")}
         job.par.setParameterArgs(args)
-    def xtest_components(self):
-        print("# # # # tetsComponents # # # # #")
-        job = Job.resetInstance("unit")
-        args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool"}
-        job.par.setParameterArgs(args)
-        cm = ComponentManager()
-        cm.createComponent("testb", 0, "")
-        cm.createComponent("testa", 1, "")
 
-    def test_run(self):
 
+    def test_run(self):
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
+        programs = ["init_testcase"]
+        testcase = "TC0001"
+        timexec = "2022-06-28_21-23-34"
         # os.system("python "+os.path.join(HOME_PATH, "check_environment.py")+" -a TEST -e ENV01")
         # os.system("python "+os.path.join(HOME_PATH, "init_testsuite.py")+" -a TEST -e ENV01 "
         #    "-ts "+os.path.join(HOME_PATH, "test","lauf","V0.1","implement_2021-08-28_23-50-51")+" -dt csv -ds implement -dn firstunit")
@@ -50,17 +61,19 @@ class MyTestCase(unittest.TestCase):
         #         "tool": "job_tool", "tsdir": os.path.join(HOME_PATH,"test","conf","lauf","V0.1","TC0001_2021-08-28_23-50-51")}
         #"loglevel": "debug", "tdtyp": "dir",
         #         "tdsrc": "TC0001", "tdname": "xxx",
-        job = Job("unit")
+        if "init_testcase" in programs:
-        args = { "application": "TEST", "environment": "ENV01", "modus": "unit", "tstime": "2022-03-19_12-09-09",
+            program = "init_testcase"
-                 "tsdir": '/home/ulrich/6_Projekte/Programme/datest/test/conf/lauf/testlauf/TST001_2022-03-19_12-09-09',
+            job = Job("unit")
-                 "step": 2 }
+            args = { B.PAR_APP: "TESTAPP", B.PAR_ENV: "ENV01", "modus": "unit",
-        #         "usecase": "TST001", "tstime": "2022-03-17_17-28"}
+                     B.PAR_TCDIR: os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ARCHIV], testcase, timexec),
-        job.par.setParameterArgs(args)
+                     "step": 1 }
-        job.setProgram("test_executer")
+            #         "usecase": "TST001", "tstime": "2022-03-17_17-28"}
-        # init_testcase.start(job)
+            job.par.setParameterArgs(args)
-        job.startJob()
+            job.setProgram(program)
-        test_executer.start(job)
+            init_testcase.startPyJob(job)
-        job.stopJob(1)
+            job.startJob()
+            #test_executer.start(job)
+            job.stopJob(1)
 
 if __name__ == '__main__':
     unittest.main()

test/test_tdata.py

@@ -9,6 +9,7 @@ import test.testtools
 import test.constants
 import basic.program
 import utils.path_tool
+import utils.file_tool
 import os
 
 HOME_PATH = test.constants.HOME_PATH
@@ -270,7 +271,7 @@ class MyTestCase(unittest.TestCase):
             "testcrmdb:person;2;Leon;Donna;28.09.42;f",
             "#;;;;;;"
         ]
-        filename = utils.path_tool.composePath(P.P_TCBASE, "t_person.csv")
+        filename = utils.path_tool.rejoinPath(utils.path_tool.composePath(P.P_TCBASE, comp), "t_person.csv")
         tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="")
         print(str(tdata))
         self.assertIn(B.DATA_NODE_TABLES, tdata)
@@ -278,8 +279,8 @@ class MyTestCase(unittest.TestCase):
         self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
         cnttest += 3
         fileLines = [
-            "date;27.06.2022",
+            "_date;27.06.2022",
-            "count;2",
+            "_count;2",
             "table:person;_nr;famname;name;birth;sex",
             "testcrmdb:person;1;Brecht;Bert;10.02.98;m",
             "testcrmdb:person;2;Leon;Donna;28.09.42;f",
@@ -290,6 +291,20 @@ class MyTestCase(unittest.TestCase):
         self.assertIn("person", tdata[B.DATA_NODE_TABLES])
         self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
         cnttest += 3
+        filename = utils.path_tool.rejoinPath(utils.path_tool.composePath(P.P_TCRESULT, comp), "person.csv")
+        fileLines = [
+            "_date;27.06.2022",
+            "_count;2",
+            "persid;famname;name;birth;sex",
+            "1;Brecht;Bert;10.02.98;m",
+            "2;Leon;Donna;28.09.42;f",
+            "#;;;;;;"
+        ]
+        tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="")
+        self.assertIn(B.DATA_NODE_TABLES, tdata)
+        self.assertIn("person", tdata[B.DATA_NODE_TABLES])
+        self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
+        cnttest += 3
         text = ""
         for k in tdata[B.DATA_NODE_TABLES]:
             text += t.buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp)

test/test_xml.py

@@ -1,7 +1,7 @@
 import unittest
 #import basic.program
-import utils.xml_tool
+import utils.xml1_tool
-import utils.xml_tool
+import utils.xml1_tool
 
 class MyTestCase(unittest.TestCase):
     def xtest_xmlTool(self):
@@ -12,17 +12,17 @@ class MyTestCase(unittest.TestCase):
         beispiel_json = {'root': {'@attr': 'xyz', '$': 'inhalt', "b": "bold"}}
         tree = {}
         tree["root"] = args
-        xml = utils.xml_tool.dict2xml(tree)
+        xml = utils.xml1_tool.dict2xml(tree)
         print(xml)
-        xml = utils.xml_tool.dict2xml(beispiel_json)
+        xml = utils.xml1_tool.dict2xml(beispiel_json)
         print(xml)
         self.assertEqual(1, 1)
-        f = utils.xml_tool.fcts()
+        f = utils.xml1_tool.fcts()
 
     def xtest_addSingle(self):
         tree = {}
         # tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', 2, "abc")
-        tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc")
+        tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc")
         self.assertEqual(tree["kratz"]["mm"][0]["wert"], "abc")
         self.assertEqual(tree["kratz"]["mm"][0]["@name"], "NAME")
 
@@ -31,9 +31,9 @@ class MyTestCase(unittest.TestCase):
         # c-a-a c-a-b
         tree = {}
         print("--------------------------------------------------------------------------------")
-        tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc")
+        tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc")
         print("--------------------------------------------------------------------------------")
-        tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="LAND"]/wert', 2, "xyz")
+        tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="LAND"]/wert', 2, "xyz")
         baum = {}
         baum["root"] = tree
         print("<------"+str(baum["root"]))
@@ -45,7 +45,7 @@ class MyTestCase(unittest.TestCase):
     def xtest_addOnePaths(self):
         tree = {}
         print("--------------------------------------------------------------------------------")
-        tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc")
+        tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc")
         baum = {}
         baum["root"] = tree
         print("<------"+str(baum["root"]))
@@ -56,9 +56,9 @@ class MyTestCase(unittest.TestCase):
     def xtest_addTwoPaths(self):
         tree = {}
         print("--------------------------------------------------------------------------------")
-        tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc")
+        tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc")
         print("--------------------------------------------------------------------------------")
-        tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="LAND"]/wert', "xyz")
+        tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="LAND"]/wert', "xyz")
         baum = {}
         baum["root"] = tree
         print("<------"+str(baum["root"]))
@@ -74,7 +74,7 @@ class MyTestCase(unittest.TestCase):
         tree_attrx = { "eins": [ {"zwei": 2, "drei": 3, "vier": { "@name": "attr", "fuenf": 5} } ] }
         tree_attr2 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "fuenf": 5} ] } ] }
         tree_attr3 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "#text": "text" } ] } ] }
-        c = utils.xml_tool.fcts()
+        c = utils.xml1_tool.fcts()
         xml = c.tidy(tree_dict, 0)
         print("-------------------------------------")
         print(xml)

utils/db_abstract.py

@@ -38,7 +38,10 @@ SPECIAL CASES:
    * If the table is partitioned tables the functions delete/insert/select calls the callback-functions
      COMP.nextTable() resp. COMP.nextTdata().
 """
+import re
+
 import basic.program
+import basic.catalog
 import utils.config_tool
 import basic.constants as B
 import utils.data_const as D
@@ -86,6 +89,7 @@ def getDbAttributes(comp, table):
             print("f " + attr + " " + out[attr])
     return out
 
+
 def getStringIndex(text, intern):
     if intern in text:
         return text.index(intern)
@@ -142,12 +146,104 @@ def parseSQLwhere(condition, ddl=None):
     conjunctor = ""
     dbwhere = ""
     for cond in parts:
+        print("exp "+cond[1]+" "+str(ddl))
         if cond[1] in ddl[B.DATA_NODE_HEADER]:
             dbwhere += " "+conjunctor+" "+cond[1]+" "+cond[0]+" "+cond[2]
             conjunctor = cond[3]
+        print("exp ok")
     return "WHERE "+dbwhere.strip()
 
 
+def getSqlTable(comp, table):
+    """
+    the function gets the technical tablename inclusive necessary schema information
+    :param comp:
+    :param table:
+    :return:
+    """
+    attr = getDbAttributes(comp, table)
+    if attr[B.ATTR_DB_TABNAME] != "":
+        sqltable = attr[B.ATTR_DB_TABNAME]
+    else:
+        sqltable = table
+    if attr[B.ATTR_DB_SCHEMA] != "":
+        sqltable = attr[B.ATTR_DB_SCHEMA] + "." + sqltable
+    return sqltable
+
+
+def getTechnicalIDFields(ddl):
+    ids = []
+    keys = {}
+    for f in ddl:
+        if f[0:1] == "_":
+            continue
+        if "T" in ddl[f][D.DDL_KEY]:
+            keys[ddl[f][D.DDL_KEY]] = f
+    for k in keys:
+        ids.append(k)
+    return  ids
+
+
+def formatDbRows(table, comp, rows):
+    out = []
+    fields = comp.conf[B.DATA_NODE_DDL][table]
+    header = comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
+    for r in rows:
+        ro = {}
+        for f in header:
+            if f in r:
+                val = formatDbField(comp, r[f], fields[f])
+                ro[f] = val
+            else:
+                val = formatDbField(comp, B.SVAL_NULL, fields[f])
+                ro[f] = val
+        out.append(ro)
+    return out
+
+
+def formatDbField(comp, val, field):
+    if val == B.SVAL_NULL:
+        if field[D.DDL_FNULLABLE] != B.SVAL_YES:
+            comp.m.logError("must-field is null "+ field[D.DDL_FNAME])
+        return None
+    print("formatDbField "+str(comp))
+    print("formatDbField "+str(field)+" "+str(val))
+    return formatDbVal(comp.m, val, field[D.DDL_TYPE])
+
+
+def formatDbVal(msg, val, dtyp):
+    ctlg = basic.catalog.Catalog.getInstance()
+    if dtyp == D.TYPE_STRING:
+        if not isinstance(val, str):
+            msg.logError("field must be " + dtyp + ", " + str(val))
+        return str(val)
+    if dtyp == D.TYPE_INT:
+        if not (isinstance(val, int) or re.match(r"^\d+$", val)):
+            msg.logError("field must be " + dtyp + ", " + str(val))
+            return 0
+        return int(val)
+    if dtyp == D.TYPE_FLOAT:
+        if not (isinstance(val, float) or re.match(r"^\d+[.,]\d+$", val)):
+            msg.logError("field must be " + dtyp + ", " + str(val))
+            return 0
+        return float(val)
+    else:
+        pass
+
+
+def isCompTable(comp, data, table):
+    """ checks if the table in data relates to the component """
+    print(str(data))
+    return isCompRow(comp, data[B.DATA_NODE_TABLES][table])
+
+def isCompRow(comp, row):
+    """ checks if the table in data relates to the component """
+    print("isCompRow "+comp.name+" "+str(row))
+    if comp.name in row[B.ATTR_DATA_COMP] \
+    and row[B.ATTR_DATA_COMP][comp.name] in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]:
+        return True
+    return False
+
 # ---------------------------------------------------------------------------------------------------------------
 
 class DbFcts():
@@ -184,6 +280,7 @@ class DbFcts():
                 raise Exception("missing data node in table")
             tdata[subdir][t][D.DATA_ATTR_COUNT] = len(tdata[subdir][t][B.DATA_NODE_DATA])
             tdata[subdir][t][D.DATA_ATTR_DATE] = utils.date_tool.getActdate(utils.date_tool.F_DE)
+            self.comp.m.logMsg("Tabelle {} mit {} Zeilen gelesen".format(t, len(tdata[subdir][t][B.DATA_NODE_DATA])))
         return tdata
 
     def selectRows(self, statement):
@@ -215,14 +312,19 @@ class DbFcts():
         raise Exception(B.EXCEPT_NOT_IMPLEMENT)
 
     def insertTables(self, tdata):
-        """ method to insert rows into a database
+        """
-        statement written in sql """
+            method to insert rows into the database of the component
+        """
+        # TODO wird der Tabellenname/DB/Schema unter tdata gespeichert?
         plainname = basic.componentHandling.getPlainCompname(self.comp.name)
         self.loadDdl()
-        for t in self.comp.conf[B.DATA_NODE_DDL]:
+        for t in tdata[B.DATA_NODE_TABLES]:
-            print("einzufuegende Tabelle "+t)
+            print("einzufuegende Tabelle "+self.comp.name+" "+t)
-            if (t in tdata[plainname]):
+            if isCompTable(self.comp, tdata, t):
-                self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA])
+                self.insertRows(t, tdata[B.DATA_NODE_TABLES][t][B.DATA_NODE_DATA])
+                self.comp.m.logMsg("in Tabelle {} {} Zeilen eingefuegt".format(
+                    t, len(tdata[B.DATA_NODE_TABLES][t][B.DATA_NODE_DATA])))
+
 
     def insertRows(self, rows):
         """ method to insert rows into a database
@@ -253,6 +355,8 @@ class DbFcts():
         return ""
 
     def getDbValue(self, fo, value):
+        # TODO Untersceidung csv und echte DB
+        return formatDbField(self.comp, value, fo)
         if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES:
             return self.getDbNull()
         if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING:

utils/dbcsv_tool.py

@@ -8,7 +8,9 @@ import basic.program
 import utils.config_tool
 import utils.db_abstract
 import basic.constants as B
-
+import utils.path_tool
+import utils.file_tool
+import utils.tdata_tool
 
 class DbFcts(utils.db_abstract.DbFcts):
   """
@@ -22,51 +24,87 @@ class DbFcts(utils.db_abstract.DbFcts):
   def selectRows(self, table):
     """ method to select rows from a database
     statement written in sql """
+    sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
+    header = ""
+    path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
+    print(path)
     tdata = {}
+    data = utils.tdata_tool.readCsv(self.comp.m, path, self.comp)
+    tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
+    if B.DATA_NODE_TABLES in data \
+            and table in data[B.DATA_NODE_TABLES]\
+            and B.DATA_NODE_DATA in data[B.DATA_NODE_TABLES][table]:
+      tdata[B.DATA_NODE_DATA] = data[B.DATA_NODE_TABLES][table][B.DATA_NODE_DATA]
+    else:
+      tdata[B.DATA_NODE_DATA] = {}
     return tdata
 
+
   def deleteRows(self, table):
     """ method to delete rows from a database
     statement written in sql """
     job = basic.program.Job.getInstance()
     verify = -1+job.getDebugLevel("db_tool")
-    cmd = "DELETE FROM "+table+";"
+    sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
+    header = ""
+    path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
+    for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
+      print(h)
+      header += ";"+h
+    cmd = header[1:]+"\n"
+    utils.file_tool.writeFileText(self.comp.m, path, cmd)
     self.comp.m.logInfo(cmd)
 
+
   def updateRows(self, statement):
     """ method to delete rows from a database
     statement written in sql """
     raise Exception(B.EXCEPT_NOT_IMPLEMENT)
 
+
   def insertRows(self, table, rows):
     """ method to insert rows into a database
     the rows will be interpreted by the ddl of the component
     """
     job = basic.program.Job.getInstance()
     verify = -1+job.getDebugLevel("db_tool")
-    cmd = "INSERT INTO "+table+";"
+    sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
     header = ""
+    path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
+    if len(rows) == 0:
+      return ""
+    else:
+      pass
     for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
       print(h)
-      header += ", "+h
+      header += ";"+h
-    cmd += " (" + header[1:]+" ) "
+    cmd = "table:" + table + header+"\n"
+    print("HEADER : "+cmd)
     rowvalues = ""
     for r in rows:
+      if not utils.db_abstract.isCompRow(self.comp, r):
+        continue
       print("r-----------------")
       print(r)
-      rowvalues = ""
+      rowvalues = self.comp.name+":"+table
-      cmd += "\n ( "
       for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
+        print("head "+h)
+        if h in B.LIST_DB_ATTR:
+          continue
+        if B.DATA_NODE_DATA not in self.comp.conf[B.DATA_NODE_DDL][table]:
+          rowvalues = ""
+          break
         print("h "+h)
         if (h in r):
-          rowvalues += ", "+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], r[h])
+          rowvalues += ";"+str(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], r[h]))
         else:
-          rowvalues += ", "+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], "")
+          rowvalues += ";"+str(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], ""))
         print("rv " + rowvalues)
-      cmd += rowvalues[1:]+" ),"
+      cmd += rowvalues+"\n"
-    cmd = cmd[0:-1]+";"
+    utils.file_tool.writeFileText(self.comp.m, path, cmd)
     self.comp.m.logInfo(cmd)
 
+
   def getConnector(self):
     """ add-on-method to get the connector
     this method should only called by the class itself """

utils/tdata_tool.py

@@ -26,6 +26,7 @@ import basic.program
 import utils.file_tool
 import basic.constants as B
 import utils.data_const as D
+import utils.date_tool
 
 TOOL_NAME = "tdata_tool"
 """ name of the tool in order to switch debug-info on """
@@ -71,13 +72,24 @@ def getTestdata():
         # read file in testdata
         job.m.logInfo("Test-Data readed from " + tdata[D.ATTR_SRC_TYPE] + " for " + tdata[D.ATTR_SRC_NAME])
     elif tdata[D.ATTR_SRC_TYPE] == D.DATA_SRC_DIR:
-        filename = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.D.ATTR_PATH_TDATA), tdata[D.ATTR_SRC_NAME], "testspec.csv")
+        path = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.ATTR_PATH_TDATA), tdata[D.ATTR_SRC_NAME])
+        filename = os.path.join(path , "testspec.csv")
         data = getCsvSpec(job.m, filename, D.CSV_SPECTYPE_DATA)
         for k in data:
             tdata[k] = data[k]
             if (k == D.CSV_BLOCK_OPTION):
                 for p in data[k]:
                     setattr(job.par, p, data[k][p])
+        files = utils.file_tool.getFiles(job.m, path, "table_", None)
+        for f in files:
+            print(f)
+            filename = os.path.join(path, f)
+            data = readCsv(job.m, filename, None)
+            table = f[6:-4]
+            print(filename+" "+table)
+            if B.DATA_NODE_TABLES not in tdata:
+                tdata[B.DATA_NODE_TABLES] = {}
+            tdata[B.DATA_NODE_TABLES][table] = data[B.DATA_NODE_TABLES][table]
     else:
         job.m.setFatal("test-Data: reftyp " + tdata[D.ATTR_SRC_TYPE] + " is not implemented")
     return tdata
@@ -110,7 +122,7 @@ def parseCsvSpec(msg, lines, type):
     header = []
     h = []  # from a[]
     status = "start"
-
+    tableDate = utils.date_tool.getActdate(utils.date_tool.F_DE)
     tableDict = {}
     for l in lines:
         print("lines "+l)
@@ -145,12 +157,15 @@ def parseCsvSpec(msg, lines, type):
                 raise Exception(D.EXCP_MALFORMAT+""+l)
             data[a[0]][a[1]] = fields[1]
             continue
+        elif a[0].lower() == D.DATA_ATTR_DATE:
+            tableDate = fields[1]
         elif (a[0].lower() in D.CSV_HEADER_START):
             # create deep structure a_0 ... a_n
             print("tdata 136 CSV_HEADER_START "+str(len(a)))
             h = a
             data[B.DATA_NODE_TABLES] = {}
             h[0] = B.DATA_NODE_TABLES
+            comps = {}
             tableDict = getTabContent(msg, data, h)
             i = 0
             for f in fields:
@@ -171,6 +186,7 @@ def parseCsvSpec(msg, lines, type):
                 headerFields = []
             else:
                 tableDict[B.DATA_NODE_DATA] = []
+            tableDict[D.DATA_ATTR_DATE] = tableDate
             setTabContent(msg, data, tableDict, h)
             status = D.CSV_SPECTYPE_DATA
             continue
@@ -187,6 +203,15 @@ def parseCsvSpec(msg, lines, type):
                     tableDict[B.DATA_NODE_DATA][f] = fields[i]
                 i += 1
             if type == D.CSV_SPECTYPE_DATA:
+                print("parseSpec "+ str(fields[0]))
+                for c in fields[0].split(","):
+                    a = c.split(":")
+                    print("parseSpec " + str(a))
+                    comps[a[0]] = a[1]
+                    row[B.ATTR_DATA_COMP] = {}
+                    row[B.ATTR_DATA_COMP][a[0]] = a[1]
+                #row[B.ATTR_DATA_COMP] = fields[0].split(",")
+                tableDict[B.ATTR_DATA_COMP] = comps
                 tableDict[B.DATA_NODE_DATA].append(row)
             elif type == D.CSV_SPECTYPE_KEYS:
                 tableDict[D.CSV_NODETYPE_KEYS][fields[1]] = row
@@ -205,6 +230,13 @@ def parseCsvSpec(msg, lines, type):
     return data
 
 
+def mergeTableComponents(comps, rowComps):
+    for c in rowComps.split(","):
+        a = c.split(":")
+        comps[a[0]] = a[1]
+    return comps
+
+
 def setTabContent(msg, data, tabledata, path):
     if len(path) >= 2 and path[1] not in data[path[0]]:
         data[path[0]][path[1]] = {}
@@ -238,13 +270,15 @@ def getTabContent(msg, data, path):
 
 def readCsv(msg, filename, comp, aliasNode=""):
     lines = utils.file_tool.readFileLines(filename, msg)
+    print("readCsv "+filename)
+    print(lines)
     return parseCsv(msg, filename, lines, comp, aliasNode)
 
 
 def parseCsv(msg, filename, lines, comp, aliasNode=""):
     job = basic.program.Job.getInstance()
     verify = -4+job.getDebugLevel(TOOL_NAME)
-    job.debug(verify, "# # # # # # # #  parseCsv " + filename + " :" + comp.name + ": " + str(lines))
+    job.debug(verify, "# # # # # # # #  parseCsv " + filename + " :" + str(lines))
     fields = []
     nodes = []
     columns = []
@@ -264,14 +298,22 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
         job.debug(verify, str(state) + " line " + line + " :" + str(len(fields)) + ": " + str(fields))
         if len(testline) < 2 and state < 1:
             state = 0
-        elif a[0].lower() == D.ATTR_TABLE_DATE:
+        elif a[0].lower() == D.DATA_ATTR_DATE:
             tableDate = fields[1]
-        elif a[0].lower() == D.ATTR_TABLE_CNT:
+            state = 1
+        elif a[0].lower() == D.DATA_ATTR_COUNT:
             tableCnt = fields[1]
-        elif a[0].lower() in D.CSV_HEADER_START:
+            state = 1
+        elif a[0].lower() in D.CSV_HEADER_START or \
+            (comp is not None and state == 1
+            and isCompTableFile(comp, filename)):
             state = 2
             columns = []
             h = a
+            if len(h) < 2 and comp is not None:
+                a = ["table", basename]
+                h = a
+                startCols = 0
             cnt = len(fields)
             job.debug(verify, str(state) + " cnt " + str(cnt))
             data[B.DATA_NODE_TABLES] = {}
@@ -287,10 +329,11 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
                     nodes.append(h[i])
                 job.debug(verify, str(state) + " nodes " + str(nodes))
             tableDict = getTabContent(msg, data, h)
+            tableDict[B.ATTR_DATA_COMP] = {}
             if len(tableDate) > 6:
-                tableDict[D.ATTR_TABLE_DATE] = tableDate
+                tableDict[D.DATA_ATTR_DATE] = tableDate
             if int(tableCnt) > 0:
-                tableDict[D.ATTR_TABLE_CNT] = tableCnt
+                tableDict[D.DATA_ATTR_COUNT] = tableCnt
             j = 0
             for i in range(1, cnt):
                 if fields[i][0:1] == "_":
@@ -308,7 +351,14 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
             tableDict = getTabContent(msg, data, h)
             state = 3
             row = {}
+            print(line)
+            if startCols > 0:
+                row[B.ATTR_DATA_COMP] = {}
+                row[B.ATTR_DATA_COMP][a[0]] = a[1]
+                tableDict[B.ATTR_DATA_COMP][a[0]] = a[1]
             for i in range(startCols, cnt+startCols):
+                print("for "+str(i)+" "+str(len(row))+" "+str(startCols)+" "+str(len(fields)))
+                print(str(fields[i]))
                 if i >= len(columns)+startCols:
                     break
                 row[columns[i-startCols]] = fields[i]
@@ -358,25 +408,38 @@ def buildCsvData(filename, tdata, comp):
     :param comp: if specific else None
     :return:
     """
+    compColumn = not isCompTableFile(comp, filename)
     job = basic.program.Job.getInstance()
     verify = -1+job.getDebugLevel(TOOL_NAME)
     job.debug(verify, "writeDataTable " + str(comp))
     text = ""
-    for k in [D.ATTR_TABLE_DATE, D.ATTR_TABLE_CNT]:
+    for k in [D.DATA_ATTR_DATE, D.DATA_ATTR_COUNT]:
         if k in tdata:
-            text += k+";"+tdata[k]+"\n"
+            text += k+";"+str(tdata[k])+"\n"
-    text += "table"
+    header = "table"
     for f in tdata[B.DATA_NODE_HEADER]:
-        text += ";"+f
+        header += ";"+f
+    if compColumn:
+        text += header
+    else:
+        #text += "_nr;" + header[6:] + "\n"
+        text += header[6:] + "\n"
+    i = 0
     for r in tdata[B.DATA_NODE_DATA]:
-        text += "\n"
+        row = ""
+        i += 1
         for f in tdata[B.DATA_NODE_HEADER]:
             if f in r:
-                text += ";"+str(r[f])
+                row += ";"+str(r[f])
             else:
-                text += ";"
+                row += ";"
-    text += "\n"
+        if compColumn:
-    return  text
+            text += row
+        else:
+            text += row[1:]
+            #text += str(i) + row
+        text += "\n"
+    return text
 
 
 def writeCsvData(filename, tdata, comp):
@@ -386,3 +449,14 @@ def writeCsvData(filename, tdata, comp):
             text += buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp)
             text += "\n"
     utils.file_tool.writeFileText(comp.m, filename, text)
+
+
+def isCompTableFile(comp, filename):
+    """ check if the filename belongs to the component """
+    basetable = os.path.basename(filename)[0:-4]
+    if comp is None:
+        return False
+    if B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_ARTS] and basetable in comp.conf[B.DATA_NODE_DDL] \
+            and comp.name in filename:
+        return True
+    return False