Browse Source

tests arround dbcsv

master
Ulrich Carmesin 2 years ago
parent
commit
30f0c2ae96
  1. 52
      test/test_db.py
  2. 63
      test/test_job.py
  3. 21
      test/test_tdata.py
  4. 24
      test/test_xml.py
  5. 116
      utils/db_abstract.py
  6. 60
      utils/dbcsv_tool.py
  7. 108
      utils/tdata_tool.py

52
test/test_db.py

@ -1,20 +1,33 @@
import unittest, os import unittest
import inspect
import basic.program import basic.program
import utils.path_tool import utils.path_tool
import basic.toolHandling import basic.toolHandling
import test.constants import test.constants
import components.component import basic.component
import basic.constants as B import basic.constants as B
import utils.db_abstract import utils.db_abstract
import utils.data_const as D import test.testtools
import utils.config_tool
HOME_PATH = test.constants.HOME_PATH HOME_PATH = test.constants.HOME_PATH
conf = {} conf = {}
# here you can select single testfunction for developping the tests
TEST_FUNCTIONS = ["test_parseSql", "test_toolhandling", "test_formatDbRows"]
#TEST_FUNCTIONS = ["test_formatDbRows"]
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
mymsg = "--------------------------------------------------------------"
def test_parseSql(self): def test_parseSql(self):
job = basic.program.Job("unit") global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = test.testtools.getJob()
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
"modus": "unit"} "modus": "unit"}
job.par.setParameterArgs(args) job.par.setParameterArgs(args)
@ -30,21 +43,27 @@ class MyTestCase(unittest.TestCase):
self.assertEqual(obj[1][3], "end") self.assertEqual(obj[1][3], "end")
self.assertEqual(obj[1][1], "state") self.assertEqual(obj[1][1], "state")
ddl = utils.config_tool.getConfig("DATASTRUCTURE", "testb1", "lofts") ddl = utils.config_tool.getConfig("DATASTRUCTURE", "testb1", "lofts")
dbwhere = utils.db_abstract.parseSQLwhere("family like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"]) dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"])
self.assertIn("state", dbwhere) self.assertIn("state", dbwhere)
self.assertNotIn("family", dbwhere) self.assertNotIn("family", dbwhere)
dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"]) dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"])
self.assertIn("state", dbwhere) self.assertIn("state", dbwhere)
self.assertIn("street", dbwhere) self.assertIn("street", dbwhere)
pass MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
def xtest_toolhandling(self):
def test_toolhandling(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = basic.program.Job("unit") job = basic.program.Job("unit")
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
"modus": "unit"} "modus": "unit"}
job.par.setParameterArgs(args) job.par.setParameterArgs(args)
#t = basic.toolHandling.ToolManager() #t = basic.toolHandling.ToolManager()
comp = components.component.Component() comp = basic.component.Component()
comp.name = "testb1" comp.name = "testb1"
table = "lofts" table = "lofts"
comp.conf = {} comp.conf = {}
@ -86,6 +105,21 @@ class MyTestCase(unittest.TestCase):
print(sqls) print(sqls)
#tool.deleteRows("deltable") #tool.deleteRows("deltable")
#tool.selectRows("deltable") #tool.selectRows("deltable")
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
def test_formatDbRows(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = test.testtools.getJob()
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
def test_zzz(self):
print(MyTestCase.mymsg)
if __name__ == '__main__': if __name__ == '__main__':

63
test/test_job.py

@ -1,20 +1,31 @@
import unittest import unittest
import os import os
import inspect
from basic.program import Job from basic.program import Job
from basic.componentHandling import ComponentManager from basic.componentHandling import ComponentManager
import init_testcase import init_testcase
import test_executer import test_executer
import test.constants import test.constants
import basic.constants as B
import test.constants as T
HOME_PATH = test.constants.HOME_PATH HOME_PATH = test.constants.HOME_PATH
PYTHON_CMD = "python" PYTHON_CMD = "python"
TEST_FUNCTIONS = ["test_tdata", "test_getCsvSpec_data", "test_getCsvSpec_tree", "test_getCsvSpec_key",
"test_getCsvSpec_conf", "test_extractPattern", "test_parseCsv"]
TEST_FUNCTIONS = ["test_run"]
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
def runTest(self): mymsg = "--------------------------------------------------------------"
#self.test_parameter()
#self.test_components()
self.test_run()
def xtest_parameter(self):
def test_parameter(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = Job("unit") job = Job("unit")
args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug",
"tool" : "job_tool", "function": "reset_TData,load_TData" } "tool" : "job_tool", "function": "reset_TData,load_TData" }
@ -30,17 +41,17 @@ class MyTestCase(unittest.TestCase):
args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug",
"tool" : "job_tool", "tsdir": os.path.join(HOME_PATH, "test", "lauf", "V0.1", "startjob", "2021-08-21_18-ß2-01")} "tool" : "job_tool", "tsdir": os.path.join(HOME_PATH, "test", "lauf", "V0.1", "startjob", "2021-08-21_18-ß2-01")}
job.par.setParameterArgs(args) job.par.setParameterArgs(args)
def xtest_components(self):
print("# # # # tetsComponents # # # # #")
job = Job.resetInstance("unit")
args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool"}
job.par.setParameterArgs(args)
cm = ComponentManager()
cm.createComponent("testb", 0, "")
cm.createComponent("testa", 1, "")
def test_run(self):
def test_run(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
programs = ["init_testcase"]
testcase = "TC0001"
timexec = "2022-06-28_21-23-34"
# os.system("python "+os.path.join(HOME_PATH, "check_environment.py")+" -a TEST -e ENV01") # os.system("python "+os.path.join(HOME_PATH, "check_environment.py")+" -a TEST -e ENV01")
# os.system("python "+os.path.join(HOME_PATH, "init_testsuite.py")+" -a TEST -e ENV01 " # os.system("python "+os.path.join(HOME_PATH, "init_testsuite.py")+" -a TEST -e ENV01 "
# "-ts "+os.path.join(HOME_PATH, "test","lauf","V0.1","implement_2021-08-28_23-50-51")+" -dt csv -ds implement -dn firstunit") # "-ts "+os.path.join(HOME_PATH, "test","lauf","V0.1","implement_2021-08-28_23-50-51")+" -dt csv -ds implement -dn firstunit")
@ -50,17 +61,19 @@ class MyTestCase(unittest.TestCase):
# "tool": "job_tool", "tsdir": os.path.join(HOME_PATH,"test","conf","lauf","V0.1","TC0001_2021-08-28_23-50-51")} # "tool": "job_tool", "tsdir": os.path.join(HOME_PATH,"test","conf","lauf","V0.1","TC0001_2021-08-28_23-50-51")}
#"loglevel": "debug", "tdtyp": "dir", #"loglevel": "debug", "tdtyp": "dir",
# "tdsrc": "TC0001", "tdname": "xxx", # "tdsrc": "TC0001", "tdname": "xxx",
job = Job("unit") if "init_testcase" in programs:
args = { "application": "TEST", "environment": "ENV01", "modus": "unit", "tstime": "2022-03-19_12-09-09", program = "init_testcase"
"tsdir": '/home/ulrich/6_Projekte/Programme/datest/test/conf/lauf/testlauf/TST001_2022-03-19_12-09-09', job = Job("unit")
"step": 2 } args = { B.PAR_APP: "TESTAPP", B.PAR_ENV: "ENV01", "modus": "unit",
# "usecase": "TST001", "tstime": "2022-03-17_17-28"} B.PAR_TCDIR: os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ARCHIV], testcase, timexec),
job.par.setParameterArgs(args) "step": 1 }
job.setProgram("test_executer") # "usecase": "TST001", "tstime": "2022-03-17_17-28"}
# init_testcase.start(job) job.par.setParameterArgs(args)
job.startJob() job.setProgram(program)
test_executer.start(job) init_testcase.startPyJob(job)
job.stopJob(1) job.startJob()
#test_executer.start(job)
job.stopJob(1)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

21
test/test_tdata.py

@ -9,6 +9,7 @@ import test.testtools
import test.constants import test.constants
import basic.program import basic.program
import utils.path_tool import utils.path_tool
import utils.file_tool
import os import os
HOME_PATH = test.constants.HOME_PATH HOME_PATH = test.constants.HOME_PATH
@ -270,7 +271,7 @@ class MyTestCase(unittest.TestCase):
"testcrmdb:person;2;Leon;Donna;28.09.42;f", "testcrmdb:person;2;Leon;Donna;28.09.42;f",
"#;;;;;;" "#;;;;;;"
] ]
filename = utils.path_tool.composePath(P.P_TCBASE, "t_person.csv") filename = utils.path_tool.rejoinPath(utils.path_tool.composePath(P.P_TCBASE, comp), "t_person.csv")
tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="") tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="")
print(str(tdata)) print(str(tdata))
self.assertIn(B.DATA_NODE_TABLES, tdata) self.assertIn(B.DATA_NODE_TABLES, tdata)
@ -278,8 +279,8 @@ class MyTestCase(unittest.TestCase):
self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA])) self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
cnttest += 3 cnttest += 3
fileLines = [ fileLines = [
"date;27.06.2022", "_date;27.06.2022",
"count;2", "_count;2",
"table:person;_nr;famname;name;birth;sex", "table:person;_nr;famname;name;birth;sex",
"testcrmdb:person;1;Brecht;Bert;10.02.98;m", "testcrmdb:person;1;Brecht;Bert;10.02.98;m",
"testcrmdb:person;2;Leon;Donna;28.09.42;f", "testcrmdb:person;2;Leon;Donna;28.09.42;f",
@ -290,6 +291,20 @@ class MyTestCase(unittest.TestCase):
self.assertIn("person", tdata[B.DATA_NODE_TABLES]) self.assertIn("person", tdata[B.DATA_NODE_TABLES])
self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA])) self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
cnttest += 3 cnttest += 3
filename = utils.path_tool.rejoinPath(utils.path_tool.composePath(P.P_TCRESULT, comp), "person.csv")
fileLines = [
"_date;27.06.2022",
"_count;2",
"persid;famname;name;birth;sex",
"1;Brecht;Bert;10.02.98;m",
"2;Leon;Donna;28.09.42;f",
"#;;;;;;"
]
tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="")
self.assertIn(B.DATA_NODE_TABLES, tdata)
self.assertIn("person", tdata[B.DATA_NODE_TABLES])
self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
cnttest += 3
text = "" text = ""
for k in tdata[B.DATA_NODE_TABLES]: for k in tdata[B.DATA_NODE_TABLES]:
text += t.buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp) text += t.buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp)

24
test/test_xml.py

@ -1,7 +1,7 @@
import unittest import unittest
#import basic.program #import basic.program
import utils.xml_tool import utils.xml1_tool
import utils.xml_tool import utils.xml1_tool
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
def xtest_xmlTool(self): def xtest_xmlTool(self):
@ -12,17 +12,17 @@ class MyTestCase(unittest.TestCase):
beispiel_json = {'root': {'@attr': 'xyz', '$': 'inhalt', "b": "bold"}} beispiel_json = {'root': {'@attr': 'xyz', '$': 'inhalt', "b": "bold"}}
tree = {} tree = {}
tree["root"] = args tree["root"] = args
xml = utils.xml_tool.dict2xml(tree) xml = utils.xml1_tool.dict2xml(tree)
print(xml) print(xml)
xml = utils.xml_tool.dict2xml(beispiel_json) xml = utils.xml1_tool.dict2xml(beispiel_json)
print(xml) print(xml)
self.assertEqual(1, 1) self.assertEqual(1, 1)
f = utils.xml_tool.fcts() f = utils.xml1_tool.fcts()
def xtest_addSingle(self): def xtest_addSingle(self):
tree = {} tree = {}
# tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', 2, "abc") # tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', 2, "abc")
tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc") tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc")
self.assertEqual(tree["kratz"]["mm"][0]["wert"], "abc") self.assertEqual(tree["kratz"]["mm"][0]["wert"], "abc")
self.assertEqual(tree["kratz"]["mm"][0]["@name"], "NAME") self.assertEqual(tree["kratz"]["mm"][0]["@name"], "NAME")
@ -31,9 +31,9 @@ class MyTestCase(unittest.TestCase):
# c-a-a c-a-b # c-a-a c-a-b
tree = {} tree = {}
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc") tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc")
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="LAND"]/wert', 2, "xyz") tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="LAND"]/wert', 2, "xyz")
baum = {} baum = {}
baum["root"] = tree baum["root"] = tree
print("<------"+str(baum["root"])) print("<------"+str(baum["root"]))
@ -45,7 +45,7 @@ class MyTestCase(unittest.TestCase):
def xtest_addOnePaths(self): def xtest_addOnePaths(self):
tree = {} tree = {}
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc") tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc")
baum = {} baum = {}
baum["root"] = tree baum["root"] = tree
print("<------"+str(baum["root"])) print("<------"+str(baum["root"]))
@ -56,9 +56,9 @@ class MyTestCase(unittest.TestCase):
def xtest_addTwoPaths(self): def xtest_addTwoPaths(self):
tree = {} tree = {}
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc") tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc")
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="LAND"]/wert', "xyz") tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="LAND"]/wert', "xyz")
baum = {} baum = {}
baum["root"] = tree baum["root"] = tree
print("<------"+str(baum["root"])) print("<------"+str(baum["root"]))
@ -74,7 +74,7 @@ class MyTestCase(unittest.TestCase):
tree_attrx = { "eins": [ {"zwei": 2, "drei": 3, "vier": { "@name": "attr", "fuenf": 5} } ] } tree_attrx = { "eins": [ {"zwei": 2, "drei": 3, "vier": { "@name": "attr", "fuenf": 5} } ] }
tree_attr2 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "fuenf": 5} ] } ] } tree_attr2 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "fuenf": 5} ] } ] }
tree_attr3 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "#text": "text" } ] } ] } tree_attr3 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "#text": "text" } ] } ] }
c = utils.xml_tool.fcts() c = utils.xml1_tool.fcts()
xml = c.tidy(tree_dict, 0) xml = c.tidy(tree_dict, 0)
print("-------------------------------------") print("-------------------------------------")
print(xml) print(xml)

116
utils/db_abstract.py

@ -38,7 +38,10 @@ SPECIAL CASES:
* If the table is partitioned tables the functions delete/insert/select calls the callback-functions * If the table is partitioned tables the functions delete/insert/select calls the callback-functions
COMP.nextTable() resp. COMP.nextTdata(). COMP.nextTable() resp. COMP.nextTdata().
""" """
import re
import basic.program import basic.program
import basic.catalog
import utils.config_tool import utils.config_tool
import basic.constants as B import basic.constants as B
import utils.data_const as D import utils.data_const as D
@ -86,6 +89,7 @@ def getDbAttributes(comp, table):
print("f " + attr + " " + out[attr]) print("f " + attr + " " + out[attr])
return out return out
def getStringIndex(text, intern): def getStringIndex(text, intern):
if intern in text: if intern in text:
return text.index(intern) return text.index(intern)
@ -142,12 +146,104 @@ def parseSQLwhere(condition, ddl=None):
conjunctor = "" conjunctor = ""
dbwhere = "" dbwhere = ""
for cond in parts: for cond in parts:
print("exp "+cond[1]+" "+str(ddl))
if cond[1] in ddl[B.DATA_NODE_HEADER]: if cond[1] in ddl[B.DATA_NODE_HEADER]:
dbwhere += " "+conjunctor+" "+cond[1]+" "+cond[0]+" "+cond[2] dbwhere += " "+conjunctor+" "+cond[1]+" "+cond[0]+" "+cond[2]
conjunctor = cond[3] conjunctor = cond[3]
print("exp ok")
return "WHERE "+dbwhere.strip() return "WHERE "+dbwhere.strip()
def getSqlTable(comp, table):
"""
the function gets the technical tablename inclusive necessary schema information
:param comp:
:param table:
:return:
"""
attr = getDbAttributes(comp, table)
if attr[B.ATTR_DB_TABNAME] != "":
sqltable = attr[B.ATTR_DB_TABNAME]
else:
sqltable = table
if attr[B.ATTR_DB_SCHEMA] != "":
sqltable = attr[B.ATTR_DB_SCHEMA] + "." + sqltable
return sqltable
def getTechnicalIDFields(ddl):
ids = []
keys = {}
for f in ddl:
if f[0:1] == "_":
continue
if "T" in ddl[f][D.DDL_KEY]:
keys[ddl[f][D.DDL_KEY]] = f
for k in keys:
ids.append(k)
return ids
def formatDbRows(table, comp, rows):
out = []
fields = comp.conf[B.DATA_NODE_DDL][table]
header = comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
for r in rows:
ro = {}
for f in header:
if f in r:
val = formatDbField(comp, r[f], fields[f])
ro[f] = val
else:
val = formatDbField(comp, B.SVAL_NULL, fields[f])
ro[f] = val
out.append(ro)
return out
def formatDbField(comp, val, field):
if val == B.SVAL_NULL:
if field[D.DDL_FNULLABLE] != B.SVAL_YES:
comp.m.logError("must-field is null "+ field[D.DDL_FNAME])
return None
print("formatDbField "+str(comp))
print("formatDbField "+str(field)+" "+str(val))
return formatDbVal(comp.m, val, field[D.DDL_TYPE])
def formatDbVal(msg, val, dtyp):
ctlg = basic.catalog.Catalog.getInstance()
if dtyp == D.TYPE_STRING:
if not isinstance(val, str):
msg.logError("field must be " + dtyp + ", " + str(val))
return str(val)
if dtyp == D.TYPE_INT:
if not (isinstance(val, int) or re.match(r"^\d+$", val)):
msg.logError("field must be " + dtyp + ", " + str(val))
return 0
return int(val)
if dtyp == D.TYPE_FLOAT:
if not (isinstance(val, float) or re.match(r"^\d+[.,]\d+$", val)):
msg.logError("field must be " + dtyp + ", " + str(val))
return 0
return float(val)
else:
pass
def isCompTable(comp, data, table):
""" checks if the table in data relates to the component """
print(str(data))
return isCompRow(comp, data[B.DATA_NODE_TABLES][table])
def isCompRow(comp, row):
""" checks if the table in data relates to the component """
print("isCompRow "+comp.name+" "+str(row))
if comp.name in row[B.ATTR_DATA_COMP] \
and row[B.ATTR_DATA_COMP][comp.name] in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]:
return True
return False
# --------------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------
class DbFcts(): class DbFcts():
@ -184,6 +280,7 @@ class DbFcts():
raise Exception("missing data node in table") raise Exception("missing data node in table")
tdata[subdir][t][D.DATA_ATTR_COUNT] = len(tdata[subdir][t][B.DATA_NODE_DATA]) tdata[subdir][t][D.DATA_ATTR_COUNT] = len(tdata[subdir][t][B.DATA_NODE_DATA])
tdata[subdir][t][D.DATA_ATTR_DATE] = utils.date_tool.getActdate(utils.date_tool.F_DE) tdata[subdir][t][D.DATA_ATTR_DATE] = utils.date_tool.getActdate(utils.date_tool.F_DE)
self.comp.m.logMsg("Tabelle {} mit {} Zeilen gelesen".format(t, len(tdata[subdir][t][B.DATA_NODE_DATA])))
return tdata return tdata
def selectRows(self, statement): def selectRows(self, statement):
@ -215,14 +312,19 @@ class DbFcts():
raise Exception(B.EXCEPT_NOT_IMPLEMENT) raise Exception(B.EXCEPT_NOT_IMPLEMENT)
def insertTables(self, tdata): def insertTables(self, tdata):
""" method to insert rows into a database """
statement written in sql """ method to insert rows into the database of the component
"""
# TODO wird der Tabellenname/DB/Schema unter tdata gespeichert?
plainname = basic.componentHandling.getPlainCompname(self.comp.name) plainname = basic.componentHandling.getPlainCompname(self.comp.name)
self.loadDdl() self.loadDdl()
for t in self.comp.conf[B.DATA_NODE_DDL]: for t in tdata[B.DATA_NODE_TABLES]:
print("einzufuegende Tabelle "+t) print("einzufuegende Tabelle "+self.comp.name+" "+t)
if (t in tdata[plainname]): if isCompTable(self.comp, tdata, t):
self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA]) self.insertRows(t, tdata[B.DATA_NODE_TABLES][t][B.DATA_NODE_DATA])
self.comp.m.logMsg("in Tabelle {} {} Zeilen eingefuegt".format(
t, len(tdata[B.DATA_NODE_TABLES][t][B.DATA_NODE_DATA])))
def insertRows(self, rows): def insertRows(self, rows):
""" method to insert rows into a database """ method to insert rows into a database
@ -253,6 +355,8 @@ class DbFcts():
return "" return ""
def getDbValue(self, fo, value): def getDbValue(self, fo, value):
# TODO Untersceidung csv und echte DB
return formatDbField(self.comp, value, fo)
if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES: if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES:
return self.getDbNull() return self.getDbNull()
if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING: if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING:

60
utils/dbcsv_tool.py

@ -8,7 +8,9 @@ import basic.program
import utils.config_tool import utils.config_tool
import utils.db_abstract import utils.db_abstract
import basic.constants as B import basic.constants as B
import utils.path_tool
import utils.file_tool
import utils.tdata_tool
class DbFcts(utils.db_abstract.DbFcts): class DbFcts(utils.db_abstract.DbFcts):
""" """
@ -22,51 +24,87 @@ class DbFcts(utils.db_abstract.DbFcts):
def selectRows(self, table): def selectRows(self, table):
""" method to select rows from a database """ method to select rows from a database
statement written in sql """ statement written in sql """
sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
header = ""
path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
print(path)
tdata = {} tdata = {}
data = utils.tdata_tool.readCsv(self.comp.m, path, self.comp)
tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
if B.DATA_NODE_TABLES in data \
and table in data[B.DATA_NODE_TABLES]\
and B.DATA_NODE_DATA in data[B.DATA_NODE_TABLES][table]:
tdata[B.DATA_NODE_DATA] = data[B.DATA_NODE_TABLES][table][B.DATA_NODE_DATA]
else:
tdata[B.DATA_NODE_DATA] = {}
return tdata return tdata
def deleteRows(self, table): def deleteRows(self, table):
""" method to delete rows from a database """ method to delete rows from a database
statement written in sql """ statement written in sql """
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool") verify = -1+job.getDebugLevel("db_tool")
cmd = "DELETE FROM "+table+";" sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
header = ""
path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
print(h)
header += ";"+h
cmd = header[1:]+"\n"
utils.file_tool.writeFileText(self.comp.m, path, cmd)
self.comp.m.logInfo(cmd) self.comp.m.logInfo(cmd)
def updateRows(self, statement): def updateRows(self, statement):
""" method to delete rows from a database """ method to delete rows from a database
statement written in sql """ statement written in sql """
raise Exception(B.EXCEPT_NOT_IMPLEMENT) raise Exception(B.EXCEPT_NOT_IMPLEMENT)
def insertRows(self, table, rows): def insertRows(self, table, rows):
""" method to insert rows into a database """ method to insert rows into a database
the rows will be interpreted by the ddl of the component the rows will be interpreted by the ddl of the component
""" """
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool") verify = -1+job.getDebugLevel("db_tool")
cmd = "INSERT INTO "+table+";" sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
header = "" header = ""
path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
if len(rows) == 0:
return ""
else:
pass
for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
print(h) print(h)
header += ", "+h header += ";"+h
cmd += " (" + header[1:]+" ) " cmd = "table:" + table + header+"\n"
print("HEADER : "+cmd)
rowvalues = "" rowvalues = ""
for r in rows: for r in rows:
if not utils.db_abstract.isCompRow(self.comp, r):
continue
print("r-----------------") print("r-----------------")
print(r) print(r)
rowvalues = "" rowvalues = self.comp.name+":"+table
cmd += "\n ( "
for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
print("head "+h)
if h in B.LIST_DB_ATTR:
continue
if B.DATA_NODE_DATA not in self.comp.conf[B.DATA_NODE_DDL][table]:
rowvalues = ""
break
print("h "+h) print("h "+h)
if (h in r): if (h in r):
rowvalues += ", "+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], r[h]) rowvalues += ";"+str(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], r[h]))
else: else:
rowvalues += ", "+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], "") rowvalues += ";"+str(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], ""))
print("rv " + rowvalues) print("rv " + rowvalues)
cmd += rowvalues[1:]+" )," cmd += rowvalues+"\n"
cmd = cmd[0:-1]+";" utils.file_tool.writeFileText(self.comp.m, path, cmd)
self.comp.m.logInfo(cmd) self.comp.m.logInfo(cmd)
def getConnector(self): def getConnector(self):
""" add-on-method to get the connector """ add-on-method to get the connector
this method should only called by the class itself """ this method should only called by the class itself """

108
utils/tdata_tool.py

@ -26,6 +26,7 @@ import basic.program
import utils.file_tool import utils.file_tool
import basic.constants as B import basic.constants as B
import utils.data_const as D import utils.data_const as D
import utils.date_tool
TOOL_NAME = "tdata_tool" TOOL_NAME = "tdata_tool"
""" name of the tool in order to switch debug-info on """ """ name of the tool in order to switch debug-info on """
@ -71,13 +72,24 @@ def getTestdata():
# read file in testdata # read file in testdata
job.m.logInfo("Test-Data readed from " + tdata[D.ATTR_SRC_TYPE] + " for " + tdata[D.ATTR_SRC_NAME]) job.m.logInfo("Test-Data readed from " + tdata[D.ATTR_SRC_TYPE] + " for " + tdata[D.ATTR_SRC_NAME])
elif tdata[D.ATTR_SRC_TYPE] == D.DATA_SRC_DIR: elif tdata[D.ATTR_SRC_TYPE] == D.DATA_SRC_DIR:
filename = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.D.ATTR_PATH_TDATA), tdata[D.ATTR_SRC_NAME], "testspec.csv") path = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.ATTR_PATH_TDATA), tdata[D.ATTR_SRC_NAME])
filename = os.path.join(path , "testspec.csv")
data = getCsvSpec(job.m, filename, D.CSV_SPECTYPE_DATA) data = getCsvSpec(job.m, filename, D.CSV_SPECTYPE_DATA)
for k in data: for k in data:
tdata[k] = data[k] tdata[k] = data[k]
if (k == D.CSV_BLOCK_OPTION): if (k == D.CSV_BLOCK_OPTION):
for p in data[k]: for p in data[k]:
setattr(job.par, p, data[k][p]) setattr(job.par, p, data[k][p])
files = utils.file_tool.getFiles(job.m, path, "table_", None)
for f in files:
print(f)
filename = os.path.join(path, f)
data = readCsv(job.m, filename, None)
table = f[6:-4]
print(filename+" "+table)
if B.DATA_NODE_TABLES not in tdata:
tdata[B.DATA_NODE_TABLES] = {}
tdata[B.DATA_NODE_TABLES][table] = data[B.DATA_NODE_TABLES][table]
else: else:
job.m.setFatal("test-Data: reftyp " + tdata[D.ATTR_SRC_TYPE] + " is not implemented") job.m.setFatal("test-Data: reftyp " + tdata[D.ATTR_SRC_TYPE] + " is not implemented")
return tdata return tdata
@ -110,7 +122,7 @@ def parseCsvSpec(msg, lines, type):
header = [] header = []
h = [] # from a[] h = [] # from a[]
status = "start" status = "start"
tableDate = utils.date_tool.getActdate(utils.date_tool.F_DE)
tableDict = {} tableDict = {}
for l in lines: for l in lines:
print("lines "+l) print("lines "+l)
@ -145,12 +157,15 @@ def parseCsvSpec(msg, lines, type):
raise Exception(D.EXCP_MALFORMAT+""+l) raise Exception(D.EXCP_MALFORMAT+""+l)
data[a[0]][a[1]] = fields[1] data[a[0]][a[1]] = fields[1]
continue continue
elif a[0].lower() == D.DATA_ATTR_DATE:
tableDate = fields[1]
elif (a[0].lower() in D.CSV_HEADER_START): elif (a[0].lower() in D.CSV_HEADER_START):
# create deep structure a_0 ... a_n # create deep structure a_0 ... a_n
print("tdata 136 CSV_HEADER_START "+str(len(a))) print("tdata 136 CSV_HEADER_START "+str(len(a)))
h = a h = a
data[B.DATA_NODE_TABLES] = {} data[B.DATA_NODE_TABLES] = {}
h[0] = B.DATA_NODE_TABLES h[0] = B.DATA_NODE_TABLES
comps = {}
tableDict = getTabContent(msg, data, h) tableDict = getTabContent(msg, data, h)
i = 0 i = 0
for f in fields: for f in fields:
@ -171,6 +186,7 @@ def parseCsvSpec(msg, lines, type):
headerFields = [] headerFields = []
else: else:
tableDict[B.DATA_NODE_DATA] = [] tableDict[B.DATA_NODE_DATA] = []
tableDict[D.DATA_ATTR_DATE] = tableDate
setTabContent(msg, data, tableDict, h) setTabContent(msg, data, tableDict, h)
status = D.CSV_SPECTYPE_DATA status = D.CSV_SPECTYPE_DATA
continue continue
@ -187,6 +203,15 @@ def parseCsvSpec(msg, lines, type):
tableDict[B.DATA_NODE_DATA][f] = fields[i] tableDict[B.DATA_NODE_DATA][f] = fields[i]
i += 1 i += 1
if type == D.CSV_SPECTYPE_DATA: if type == D.CSV_SPECTYPE_DATA:
print("parseSpec "+ str(fields[0]))
for c in fields[0].split(","):
a = c.split(":")
print("parseSpec " + str(a))
comps[a[0]] = a[1]
row[B.ATTR_DATA_COMP] = {}
row[B.ATTR_DATA_COMP][a[0]] = a[1]
#row[B.ATTR_DATA_COMP] = fields[0].split(",")
tableDict[B.ATTR_DATA_COMP] = comps
tableDict[B.DATA_NODE_DATA].append(row) tableDict[B.DATA_NODE_DATA].append(row)
elif type == D.CSV_SPECTYPE_KEYS: elif type == D.CSV_SPECTYPE_KEYS:
tableDict[D.CSV_NODETYPE_KEYS][fields[1]] = row tableDict[D.CSV_NODETYPE_KEYS][fields[1]] = row
@ -205,6 +230,13 @@ def parseCsvSpec(msg, lines, type):
return data return data
def mergeTableComponents(comps, rowComps):
for c in rowComps.split(","):
a = c.split(":")
comps[a[0]] = a[1]
return comps
def setTabContent(msg, data, tabledata, path): def setTabContent(msg, data, tabledata, path):
if len(path) >= 2 and path[1] not in data[path[0]]: if len(path) >= 2 and path[1] not in data[path[0]]:
data[path[0]][path[1]] = {} data[path[0]][path[1]] = {}
@ -238,13 +270,15 @@ def getTabContent(msg, data, path):
def readCsv(msg, filename, comp, aliasNode=""): def readCsv(msg, filename, comp, aliasNode=""):
lines = utils.file_tool.readFileLines(filename, msg) lines = utils.file_tool.readFileLines(filename, msg)
print("readCsv "+filename)
print(lines)
return parseCsv(msg, filename, lines, comp, aliasNode) return parseCsv(msg, filename, lines, comp, aliasNode)
def parseCsv(msg, filename, lines, comp, aliasNode=""): def parseCsv(msg, filename, lines, comp, aliasNode=""):
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -4+job.getDebugLevel(TOOL_NAME) verify = -4+job.getDebugLevel(TOOL_NAME)
job.debug(verify, "# # # # # # # # parseCsv " + filename + " :" + comp.name + ": " + str(lines)) job.debug(verify, "# # # # # # # # parseCsv " + filename + " :" + str(lines))
fields = [] fields = []
nodes = [] nodes = []
columns = [] columns = []
@ -264,14 +298,22 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
job.debug(verify, str(state) + " line " + line + " :" + str(len(fields)) + ": " + str(fields)) job.debug(verify, str(state) + " line " + line + " :" + str(len(fields)) + ": " + str(fields))
if len(testline) < 2 and state < 1: if len(testline) < 2 and state < 1:
state = 0 state = 0
elif a[0].lower() == D.ATTR_TABLE_DATE: elif a[0].lower() == D.DATA_ATTR_DATE:
tableDate = fields[1] tableDate = fields[1]
elif a[0].lower() == D.ATTR_TABLE_CNT: state = 1
elif a[0].lower() == D.DATA_ATTR_COUNT:
tableCnt = fields[1] tableCnt = fields[1]
elif a[0].lower() in D.CSV_HEADER_START: state = 1
elif a[0].lower() in D.CSV_HEADER_START or \
(comp is not None and state == 1
and isCompTableFile(comp, filename)):
state = 2 state = 2
columns = [] columns = []
h = a h = a
if len(h) < 2 and comp is not None:
a = ["table", basename]
h = a
startCols = 0
cnt = len(fields) cnt = len(fields)
job.debug(verify, str(state) + " cnt " + str(cnt)) job.debug(verify, str(state) + " cnt " + str(cnt))
data[B.DATA_NODE_TABLES] = {} data[B.DATA_NODE_TABLES] = {}
@ -287,10 +329,11 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
nodes.append(h[i]) nodes.append(h[i])
job.debug(verify, str(state) + " nodes " + str(nodes)) job.debug(verify, str(state) + " nodes " + str(nodes))
tableDict = getTabContent(msg, data, h) tableDict = getTabContent(msg, data, h)
tableDict[B.ATTR_DATA_COMP] = {}
if len(tableDate) > 6: if len(tableDate) > 6:
tableDict[D.ATTR_TABLE_DATE] = tableDate tableDict[D.DATA_ATTR_DATE] = tableDate
if int(tableCnt) > 0: if int(tableCnt) > 0:
tableDict[D.ATTR_TABLE_CNT] = tableCnt tableDict[D.DATA_ATTR_COUNT] = tableCnt
j = 0 j = 0
for i in range(1, cnt): for i in range(1, cnt):
if fields[i][0:1] == "_": if fields[i][0:1] == "_":
@ -308,7 +351,14 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
tableDict = getTabContent(msg, data, h) tableDict = getTabContent(msg, data, h)
state = 3 state = 3
row = {} row = {}
print(line)
if startCols > 0:
row[B.ATTR_DATA_COMP] = {}
row[B.ATTR_DATA_COMP][a[0]] = a[1]
tableDict[B.ATTR_DATA_COMP][a[0]] = a[1]
for i in range(startCols, cnt+startCols): for i in range(startCols, cnt+startCols):
print("for "+str(i)+" "+str(len(row))+" "+str(startCols)+" "+str(len(fields)))
print(str(fields[i]))
if i >= len(columns)+startCols: if i >= len(columns)+startCols:
break break
row[columns[i-startCols]] = fields[i] row[columns[i-startCols]] = fields[i]
@ -358,25 +408,38 @@ def buildCsvData(filename, tdata, comp):
:param comp: if specific else None :param comp: if specific else None
:return: :return:
""" """
compColumn = not isCompTableFile(comp, filename)
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel(TOOL_NAME) verify = -1+job.getDebugLevel(TOOL_NAME)
job.debug(verify, "writeDataTable " + str(comp)) job.debug(verify, "writeDataTable " + str(comp))
text = "" text = ""
for k in [D.ATTR_TABLE_DATE, D.ATTR_TABLE_CNT]: for k in [D.DATA_ATTR_DATE, D.DATA_ATTR_COUNT]:
if k in tdata: if k in tdata:
text += k+";"+tdata[k]+"\n" text += k+";"+str(tdata[k])+"\n"
text += "table" header = "table"
for f in tdata[B.DATA_NODE_HEADER]: for f in tdata[B.DATA_NODE_HEADER]:
text += ";"+f header += ";"+f
if compColumn:
text += header
else:
#text += "_nr;" + header[6:] + "\n"
text += header[6:] + "\n"
i = 0
for r in tdata[B.DATA_NODE_DATA]: for r in tdata[B.DATA_NODE_DATA]:
text += "\n" row = ""
i += 1
for f in tdata[B.DATA_NODE_HEADER]: for f in tdata[B.DATA_NODE_HEADER]:
if f in r: if f in r:
text += ";"+str(r[f]) row += ";"+str(r[f])
else: else:
text += ";" row += ";"
text += "\n" if compColumn:
return text text += row
else:
text += row[1:]
#text += str(i) + row
text += "\n"
return text
def writeCsvData(filename, tdata, comp): def writeCsvData(filename, tdata, comp):
@ -386,3 +449,14 @@ def writeCsvData(filename, tdata, comp):
text += buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp) text += buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp)
text += "\n" text += "\n"
utils.file_tool.writeFileText(comp.m, filename, text) utils.file_tool.writeFileText(comp.m, filename, text)
def isCompTableFile(comp, filename):
""" check if the filename belongs to the component """
basetable = os.path.basename(filename)[0:-4]
if comp is None:
return False
if B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_ARTS] and basetable in comp.conf[B.DATA_NODE_DDL] \
and comp.name in filename:
return True
return False
Loading…
Cancel
Save