Compare commits

...

3 Commits

  1. 14
      basic/componentHandling.py
  2. 13
      basic/constants.py
  3. 13
      basic/program.py
  4. 41
      basic/testexec.py
  5. 1
      requirements.txt
  6. 52
      test/test_db.py
  7. 63
      test/test_job.py
  8. 42
      test/test_tdata.py
  9. 2
      test/test_toolhandling.py
  10. 24
      test/test_xml.py
  11. 7
      test/testtools.py
  12. 44
      utils/config/path.yml
  13. 29
      utils/config_tool.py
  14. 4
      utils/conn_tool.py
  15. 8
      utils/data_const.py
  16. 19
      utils/date_tool.py
  17. 120
      utils/db_abstract.py
  18. 57
      utils/dbcsv_tool.py
  19. 31
      utils/file_abstract.py
  20. 1
      utils/file_tool.py
  21. 3
      utils/path_tool.py
  22. 133
      utils/tdata_tool.py

14
basic/componentHandling.py

@ -23,6 +23,7 @@ import basic.component
import importlib import importlib
import copy import copy
import basic.constants as B import basic.constants as B
import utils.data_const as D
comps = {} comps = {}
PARAM_NOSUBNODE = [B.SUBJECT_ARTS, "components", "instance"] PARAM_NOSUBNODE = [B.SUBJECT_ARTS, "components", "instance"]
@ -86,6 +87,7 @@ class ComponentManager:
return return
for k in job.conf.confs[B.SUBJECT_APPS].get(anw): for k in job.conf.confs[B.SUBJECT_APPS].get(anw):
job.m.logDebug("applicationscomponente -- " + k + ":") job.m.logDebug("applicationscomponente -- " + k + ":")
print("applicationscomponente -- " + k + ":")
self.createComponent(k, 0, "") self.createComponent(k, 0, "")
@ -192,17 +194,17 @@ class ComponentManager:
c.conf[B.SUBJECT_CONN] = conns[i] c.conf[B.SUBJECT_CONN] = conns[i]
c.init() c.init()
if parContent is not None: if parContent is not None:
print("createComponent 5 a " + name + " : " + str(parContent)) print("createComponent 5 a " + compName + " : " + str(parContent))
if name in parContent["comps"]: if B.SUBJECT_COMPS in parContent and compName in parContent[B.SUBJECT_COMPS]:
for k in parContent["comps"][name].keys(): for k in parContent[B.SUBJECT_COMPS][compName].keys():
c.conf[k] = parContent["comps"][name][k] c.conf[k] = parContent[B.SUBJECT_COMPS][compName][k]
if B.SUBJECT_ARTS in c.conf and B.TOPIC_NODE_DB in c.conf[B.SUBJECT_ARTS]: if B.SUBJECT_ARTS in c.conf and B.TOPIC_NODE_DB in c.conf[B.SUBJECT_ARTS]:
if not B.DATA_NODE_DDL in c.conf: if not B.DATA_NODE_DDL in c.conf:
c.conf[B.DATA_NODE_DDL] = {} c.conf[B.DATA_NODE_DDL] = {}
for table in c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]: for table in c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]:
if table in ["type"]: if table in B.LIST_DB_ATTR:
continue continue
conf = utils.config_tool.getConfig("DATASTRUCTURE", c.name, table) conf = utils.config_tool.getConfig(D.DDL_FILENAME, compName, table)
if B.DATA_NODE_TABLES in conf and table in conf[B.DATA_NODE_TABLES]: if B.DATA_NODE_TABLES in conf and table in conf[B.DATA_NODE_TABLES]:
c.conf[B.DATA_NODE_DDL][table] = conf[B.DATA_NODE_TABLES][table] c.conf[B.DATA_NODE_DDL][table] = conf[B.DATA_NODE_TABLES][table]
elif table in conf: elif table in conf:

13
basic/constants.py

@ -102,6 +102,9 @@ DATA_NODE_PAR = "par"
TOPIC_NODE_DB = "db" TOPIC_NODE_DB = "db"
# testexec, db_abstr # testexec, db_abstr
ATTR_DB_TYPE = "type" ATTR_DB_TYPE = "type"
""" must attribute for the type of the database """
ATTR_DB_RESET = "reset"
""" optional attribute in order to use a different technical name for the db-table """
ATTR_DB_PARTITION = "partitioned" ATTR_DB_PARTITION = "partitioned"
""" optional attribute if table is partitioned """ optional attribute if table is partitioned
- this keyword delimited by "+" will be replaced by partition-names which are parametrized """ - this keyword delimited by "+" will be replaced by partition-names which are parametrized """
@ -111,7 +114,10 @@ ATTR_DB_SCHEMA = "schema"
""" optional attribute for technical name of the schema """ """ optional attribute for technical name of the schema """
ATTR_DB_TABNAME = "tabname" ATTR_DB_TABNAME = "tabname"
""" optional attribute in order to use a different technical name for the db-table """ """ optional attribute in order to use a different technical name for the db-table """
LIST_DB_ATTR = [ATTR_DB_TYPE, ATTR_DB_PARTITION, ATTR_DB_DATABASE, ATTR_DB_SCHEMA, ATTR_DB_TABNAME] ATTR_DB_PRESTEP = "prestep"
""" optional attribute to define a source-table for this table """
LIST_DB_ATTR = [ATTR_DB_TYPE, ATTR_DB_RESET, ATTR_DB_PARTITION,
ATTR_DB_DATABASE, ATTR_DB_SCHEMA, ATTR_DB_TABNAME, ATTR_DB_PRESTEP]
ATTR_DB_CONN_JAR = "conn_jar_name" ATTR_DB_CONN_JAR = "conn_jar_name"
""" optional attribute for connection-jar-file instead of connection by ip, port """ """ optional attribute for connection-jar-file instead of connection by ip, port """
ATTR_CONN_HOST = "hostname" ATTR_CONN_HOST = "hostname"
@ -171,7 +177,8 @@ SUBJECT_INST = "instance" # | | | | x | CompHanldin
ATTR_INST_CNT = "count" # | | | | x | CompHanlding ATTR_INST_CNT = "count" # | | | | x | CompHanlding
ATTR_INST_SGL = "single" ATTR_INST_SGL = "single"
LIST_INST_ATTR = [ATTR_INST_CNT, ATTR_INST_SGL] LIST_INST_ATTR = [ATTR_INST_CNT, ATTR_INST_SGL]
ATTR_INST_SUBCOMP = "components" SUBJECT_COMPS = "components"
ATTR_INST_SUBCOMP = SUBJECT_COMPS
#SUBJECT_FCT = "function" # | | | | x | main-programs #SUBJECT_FCT = "function" # | | | | x | main-programs
SUBJECT_ARTS = "artifact" # | | | | x | Component SUBJECT_ARTS = "artifact" # | | | | x | Component
@ -194,7 +201,9 @@ ATTR_TYPE = "type" # | x | x | | x | conn_tool, toolHandl
RULE_ACCEPTANCE = "acceptance" # | | | | x | tools_match RULE_ACCEPTANCE = "acceptance" # | | | | x | tools_match
ATTR_STEP_ARGS = "args" ATTR_STEP_ARGS = "args"
ATTR_EXEC_REF = "_exec"
ATTR_DATA_REF = "_nr" ATTR_DATA_REF = "_nr"
ATTR_DATA_COMP = "_comp"
# ------------------------------------------------------------- # -------------------------------------------------------------
# exception texts # exception texts

13
basic/program.py

@ -245,13 +245,13 @@ class Job:
if len(cconf) < 1: if len(cconf) < 1:
utils.file_tool.writeFileDict(self.m, parpath, output) utils.file_tool.writeFileDict(self.m, parpath, output)
return return
output["comps"] = {} output[B.SUBJECT_COMPS] = {}
for c in cconf: for c in cconf:
output["comps"][c] = {} output[B.SUBJECT_COMPS][c] = {}
for x in ["function", "conn"]: for x in ["function", "conn"]:
output["comps"][c][x] = cconf[c][x] output[B.SUBJECT_COMPS][c][x] = cconf[c][x]
if x == "conn" and "passwd" in cconf[c][x]: if x == B.SUBJECT_CONN and "passwd" in cconf[c][x]:
cconf["comps"][c][x]["passwd"] = "xxxxx" cconf[B.SUBJECT_COMPS][c][x]["passwd"] = "xxxxx"
utils.file_tool.writeFileDict(self.m, parpath, output) utils.file_tool.writeFileDict(self.m, parpath, output)
@ -441,6 +441,9 @@ class Configuration:
self.program = program self.program = program
print (f"job initialisiert {self.program}") print (f"job initialisiert {self.program}")
if program == "unit": if program == "unit":
if (os.path.exists(utils.path_tool.rejoinPath("..", "..", "config", B.BASIS_FILE))):
self.setConfiguration(utils.path_tool.rejoinPath("..", "..", "config", B.BASIS_FILE))
return
if (os.path.exists(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))): if (os.path.exists(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))):
self.setConfiguration(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE)) self.setConfiguration(utils.path_tool.rejoinPath("..", "config", B.BASIS_FILE))
return return

41
basic/testexec.py

@ -74,7 +74,7 @@ class Testexecuter():
+ datetime.now().strftime("%Y%m%d_%H%M%S")+" for " + str(self.name).upper()) + datetime.now().strftime("%Y%m%d_%H%M%S")+" for " + str(self.name).upper())
if B.ATTR_ARTS_LOG in self.conf[B.SUBJECT_ARTS]: if B.ATTR_ARTS_LOG in self.conf[B.SUBJECT_ARTS]:
self.m.logInfo("log rotate in "+ self.name) self.m.logInfo("log rotate in "+ self.name)
if "db" in self.conf[B.SUBJECT_ARTS]: if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS]:
self.m.logInfo("delete content "+ self.name) self.m.logInfo("delete content "+ self.name)
dbi = basic.toolHandling.getDbTool(self) dbi = basic.toolHandling.getDbTool(self)
dbi.deleteTables() dbi.deleteTables()
@ -85,7 +85,7 @@ class Testexecuter():
self.m.setMsg("resetInstance for " + self.name + " is OK") self.m.setMsg("resetInstance for " + self.name + " is OK")
self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper())
def load_TData(self, granularity, testdata): def load_TData(self, granularity, tdata):
""" """
the testdata will be loaded into the componend especially into databses the testdata will be loaded into the componend especially into databses
or with import-functions of the component or with import-functions of the component
@ -96,13 +96,14 @@ class Testexecuter():
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel(self.name) verify = -1+job.getDebugLevel(self.name)
self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper())
plainname = basic.componentHandling.getPlainCompname(self.name) if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS] and B.DATA_NODE_TABLES in tdata:
if plainname in testdata: for t in tdata[B.DATA_NODE_TABLES]:
print("plainname in testdata "+plainname) print (t)
if "db" in self.conf[B.SUBJECT_ARTS]: if utils.db_abstract.isCompTable(self, tdata, t):
self.m.logInfo("delete content "+ self.name) self.m.logInfo("insert content "+ self.name)
dbi = basic.toolHandling.getDbTool(self) dbi = basic.toolHandling.getDbTool(self)
dbi.insertTables(testdata) dbi.insertTables(tdata)
break
self.m.setMsg("data loaded for " + self.name + " is OK") self.m.setMsg("data loaded for " + self.name + " is OK")
self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper())
@ -125,11 +126,17 @@ class Testexecuter():
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel(self.name) verify = -1+job.getDebugLevel(self.name)
self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper())
if "db" in self.conf[B.SUBJECT_ARTS]: if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS]:
self.m.logInfo("select db-content "+ self.name) self.m.logInfo("select db-content "+ self.name)
dbi = basic.toolHandling.getDbTool(self) dbi = basic.toolHandling.getDbTool(self)
tdata = dbi.selectTables() data = dbi.selectTables(subdir)
utils.tdata_tool.writeCsvData("", tdata, self) print("ppp")
#data = {}
for t in data[subdir]:
data[B.DATA_NODE_TABLES] = {}
data[B.DATA_NODE_TABLES][t] = data[subdir][t]
utils.tdata_tool.writeCsvData(utils.path_tool.rejoinPath(
utils.path_tool.composePattern("{tcresult}", self), subdir, t+".csv"), data, self)
if B.ATTR_ARTS_LOB in self.conf[B.SUBJECT_ARTS]: if B.ATTR_ARTS_LOB in self.conf[B.SUBJECT_ARTS]:
self.m.logInfo("check lob if is deleted with flaskdb "+ self.name) self.m.logInfo("check lob if is deleted with flaskdb "+ self.name)
self.m.setMsg("readInstance for " + self.name + " is OK") self.m.setMsg("readInstance for " + self.name + " is OK")
@ -305,15 +312,15 @@ class Testexecuter():
cm = basic.componentHandling.ComponentManager.getInstance() cm = basic.componentHandling.ComponentManager.getInstance()
data = {} data = {}
matching = utils.match_tool.Matching() matching = utils.match_tool.Matching()
if "db" in self.conf[B.SUBJECT_ARTS]: if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS]:
for t in self.conf[B.SUBJECT_ARTS]["db"]: for t in self.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]:
if t in ["type"]: if t in B.LIST_DB_ATTR:
continue continue
# fill each data into matching-object # fill each data into matching-object
for side in M.MATCH_SIDES: for side in M.MATCH_SIDES:
if side == M.MATCH_SIDE_PRESTEP: if side == M.MATCH_SIDE_PRESTEP:
if "prestep" in self.conf[B.SUBJECT_ARTS]["db"][t]: if B.ATTR_DB_PRESTEP in self.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][t]:
a = self.conf[B.SUBJECT_ARTS]["db"][t]["prestep"].split(":") a = self.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][t][B.ATTR_DB_PRESTEP].split(":")
if a[0] != self.name: if a[0] != self.name:
comp = cm.getComponent(a[0]) comp = cm.getComponent(a[0])
else: else:

1
requirements.txt

@ -1 +1,2 @@
pyyaml pyyaml
paramiko

52
test/test_db.py

@ -1,20 +1,33 @@
import unittest, os import unittest
import inspect
import basic.program import basic.program
import utils.path_tool import utils.path_tool
import basic.toolHandling import basic.toolHandling
import test.constants import test.constants
import components.component import basic.component
import basic.constants as B import basic.constants as B
import utils.db_abstract import utils.db_abstract
import utils.data_const as D import test.testtools
import utils.config_tool
HOME_PATH = test.constants.HOME_PATH HOME_PATH = test.constants.HOME_PATH
conf = {} conf = {}
# here you can select single testfunction for developping the tests
TEST_FUNCTIONS = ["test_parseSql", "test_toolhandling", "test_formatDbRows"]
#TEST_FUNCTIONS = ["test_formatDbRows"]
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
mymsg = "--------------------------------------------------------------"
def test_parseSql(self): def test_parseSql(self):
job = basic.program.Job("unit") global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = test.testtools.getJob()
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
"modus": "unit"} "modus": "unit"}
job.par.setParameterArgs(args) job.par.setParameterArgs(args)
@ -30,21 +43,27 @@ class MyTestCase(unittest.TestCase):
self.assertEqual(obj[1][3], "end") self.assertEqual(obj[1][3], "end")
self.assertEqual(obj[1][1], "state") self.assertEqual(obj[1][1], "state")
ddl = utils.config_tool.getConfig("DATASTRUCTURE", "testb1", "lofts") ddl = utils.config_tool.getConfig("DATASTRUCTURE", "testb1", "lofts")
dbwhere = utils.db_abstract.parseSQLwhere("family like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"]) dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"])
self.assertIn("state", dbwhere) self.assertIn("state", dbwhere)
self.assertNotIn("family", dbwhere) self.assertNotIn("family", dbwhere)
dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"]) dbwhere = utils.db_abstract.parseSQLwhere("street like !%utz%! and state = !+reg+!", ddl["testb1"]["lofts"])
self.assertIn("state", dbwhere) self.assertIn("state", dbwhere)
self.assertIn("street", dbwhere) self.assertIn("street", dbwhere)
pass MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
def xtest_toolhandling(self):
def test_toolhandling(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = basic.program.Job("unit") job = basic.program.Job("unit")
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
"modus": "unit"} "modus": "unit"}
job.par.setParameterArgs(args) job.par.setParameterArgs(args)
#t = basic.toolHandling.ToolManager() #t = basic.toolHandling.ToolManager()
comp = components.component.Component() comp = basic.component.Component()
comp.name = "testb1" comp.name = "testb1"
table = "lofts" table = "lofts"
comp.conf = {} comp.conf = {}
@ -86,6 +105,21 @@ class MyTestCase(unittest.TestCase):
print(sqls) print(sqls)
#tool.deleteRows("deltable") #tool.deleteRows("deltable")
#tool.selectRows("deltable") #tool.selectRows("deltable")
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
def test_formatDbRows(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = test.testtools.getJob()
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
def test_zzz(self):
print(MyTestCase.mymsg)
if __name__ == '__main__': if __name__ == '__main__':

63
test/test_job.py

@ -1,20 +1,31 @@
import unittest import unittest
import os import os
import inspect
from basic.program import Job from basic.program import Job
from basic.componentHandling import ComponentManager from basic.componentHandling import ComponentManager
import init_testcase import init_testcase
import test_executer import test_executer
import test.constants import test.constants
import basic.constants as B
import test.constants as T
HOME_PATH = test.constants.HOME_PATH HOME_PATH = test.constants.HOME_PATH
PYTHON_CMD = "python" PYTHON_CMD = "python"
TEST_FUNCTIONS = ["test_tdata", "test_getCsvSpec_data", "test_getCsvSpec_tree", "test_getCsvSpec_key",
"test_getCsvSpec_conf", "test_extractPattern", "test_parseCsv"]
TEST_FUNCTIONS = ["test_run"]
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
def runTest(self): mymsg = "--------------------------------------------------------------"
#self.test_parameter()
#self.test_components()
self.test_run()
def xtest_parameter(self):
def test_parameter(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
job = Job("unit") job = Job("unit")
args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug",
"tool" : "job_tool", "function": "reset_TData,load_TData" } "tool" : "job_tool", "function": "reset_TData,load_TData" }
@ -30,17 +41,17 @@ class MyTestCase(unittest.TestCase):
args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug",
"tool" : "job_tool", "tsdir": os.path.join(HOME_PATH, "test", "lauf", "V0.1", "startjob", "2021-08-21_18-ß2-01")} "tool" : "job_tool", "tsdir": os.path.join(HOME_PATH, "test", "lauf", "V0.1", "startjob", "2021-08-21_18-ß2-01")}
job.par.setParameterArgs(args) job.par.setParameterArgs(args)
def xtest_components(self):
print("# # # # tetsComponents # # # # #")
job = Job.resetInstance("unit")
args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool"}
job.par.setParameterArgs(args)
cm = ComponentManager()
cm.createComponent("testb", 0, "")
cm.createComponent("testa", 1, "")
def test_run(self):
def test_run(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
programs = ["init_testcase"]
testcase = "TC0001"
timexec = "2022-06-28_21-23-34"
# os.system("python "+os.path.join(HOME_PATH, "check_environment.py")+" -a TEST -e ENV01") # os.system("python "+os.path.join(HOME_PATH, "check_environment.py")+" -a TEST -e ENV01")
# os.system("python "+os.path.join(HOME_PATH, "init_testsuite.py")+" -a TEST -e ENV01 " # os.system("python "+os.path.join(HOME_PATH, "init_testsuite.py")+" -a TEST -e ENV01 "
# "-ts "+os.path.join(HOME_PATH, "test","lauf","V0.1","implement_2021-08-28_23-50-51")+" -dt csv -ds implement -dn firstunit") # "-ts "+os.path.join(HOME_PATH, "test","lauf","V0.1","implement_2021-08-28_23-50-51")+" -dt csv -ds implement -dn firstunit")
@ -50,17 +61,19 @@ class MyTestCase(unittest.TestCase):
# "tool": "job_tool", "tsdir": os.path.join(HOME_PATH,"test","conf","lauf","V0.1","TC0001_2021-08-28_23-50-51")} # "tool": "job_tool", "tsdir": os.path.join(HOME_PATH,"test","conf","lauf","V0.1","TC0001_2021-08-28_23-50-51")}
#"loglevel": "debug", "tdtyp": "dir", #"loglevel": "debug", "tdtyp": "dir",
# "tdsrc": "TC0001", "tdname": "xxx", # "tdsrc": "TC0001", "tdname": "xxx",
job = Job("unit") if "init_testcase" in programs:
args = { "application": "TEST", "environment": "ENV01", "modus": "unit", "tstime": "2022-03-19_12-09-09", program = "init_testcase"
"tsdir": '/home/ulrich/6_Projekte/Programme/datest/test/conf/lauf/testlauf/TST001_2022-03-19_12-09-09', job = Job("unit")
"step": 2 } args = { B.PAR_APP: "TESTAPP", B.PAR_ENV: "ENV01", "modus": "unit",
# "usecase": "TST001", "tstime": "2022-03-17_17-28"} B.PAR_TCDIR: os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ARCHIV], testcase, timexec),
job.par.setParameterArgs(args) "step": 1 }
job.setProgram("test_executer") # "usecase": "TST001", "tstime": "2022-03-17_17-28"}
# init_testcase.start(job) job.par.setParameterArgs(args)
job.startJob() job.setProgram(program)
test_executer.start(job) init_testcase.startPyJob(job)
job.stopJob(1) job.startJob()
#test_executer.start(job)
job.stopJob(1)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

42
test/test_tdata.py

@ -9,6 +9,7 @@ import test.testtools
import test.constants import test.constants
import basic.program import basic.program
import utils.path_tool import utils.path_tool
import utils.file_tool
import os import os
HOME_PATH = test.constants.HOME_PATH HOME_PATH = test.constants.HOME_PATH
@ -18,7 +19,7 @@ OS_SYSTEM = test.constants.OS_SYSTEM
# here you can select single testfunction for developping the tests # here you can select single testfunction for developping the tests
TEST_FUNCTIONS = ["test_tdata", "test_getCsvSpec_data", "test_getCsvSpec_tree", "test_getCsvSpec_key", TEST_FUNCTIONS = ["test_tdata", "test_getCsvSpec_data", "test_getCsvSpec_tree", "test_getCsvSpec_key",
"test_getCsvSpec_conf", "test_extractPattern", "test_parseCsv"] "test_getCsvSpec_conf", "test_extractPattern", "test_parseCsv"]
TEST_FUNCTIONS = ["test_parseCsv"] TEST_FUNCTIONS = ["test_getCsvSpec_data"]
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
mymsg = "--------------------------------------------------------------" mymsg = "--------------------------------------------------------------"
@ -107,7 +108,7 @@ class MyTestCase(unittest.TestCase):
cnttest += 2 cnttest += 2
if D.CSV_BLOCK_STEP in tests: if D.CSV_BLOCK_STEP in tests:
specLines = [ specLines = [
"step:1;testa;1;table:_lofts,action:import;;;;;", "step:1;testa;1;1;table:_lofts,action:import;;;;;",
"#;;;;;;" "#;;;;;;"
] ]
tdata = t.parseCsvSpec(job.m, specLines, D.CSV_SPECTYPE_DATA) tdata = t.parseCsvSpec(job.m, specLines, D.CSV_SPECTYPE_DATA)
@ -139,6 +140,23 @@ class MyTestCase(unittest.TestCase):
self.assertIn(B.DATA_NODE_HEADER, table) self.assertIn(B.DATA_NODE_HEADER, table)
self.assertIn(B.DATA_NODE_DATA, table) self.assertIn(B.DATA_NODE_DATA, table)
cnttest += 2 cnttest += 2
if B.DATA_NODE_TABLES in tests:
specLines = [
"option:description;create 2 new contracts;;;;",
"# ;;;;;",
"# ;component;exec;_nr;action;args;;",
"step:1;testrest;2;1;function:xml-rest;action:new;;",
"step:2;testrest;3;1,2;function:json-rest;action:new;;",
"# ;;;;;",
"# testdate only here specified;expect:row 2 is inserted as precond;;;;",
"_date;01.07.2022;;;;",
"table:person;_nr;famname;name;birth;sex",
"testrest:person;1;Brecht;Bert;10.02.98;m",
"testrest:person,testcrmdb:person;2;Leon;Donna;28.09.42;f"
]
tdata = t.parseCsvSpec(job.m, specLines, D.CSV_SPECTYPE_DATA)
print(tdata)
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
@ -270,7 +288,7 @@ class MyTestCase(unittest.TestCase):
"testcrmdb:person;2;Leon;Donna;28.09.42;f", "testcrmdb:person;2;Leon;Donna;28.09.42;f",
"#;;;;;;" "#;;;;;;"
] ]
filename = utils.path_tool.composePath(P.P_TCBASE, "t_person.csv") filename = utils.path_tool.rejoinPath(utils.path_tool.composePath(P.P_TCBASE, comp), "t_person.csv")
tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="") tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="")
print(str(tdata)) print(str(tdata))
self.assertIn(B.DATA_NODE_TABLES, tdata) self.assertIn(B.DATA_NODE_TABLES, tdata)
@ -278,8 +296,8 @@ class MyTestCase(unittest.TestCase):
self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA])) self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
cnttest += 3 cnttest += 3
fileLines = [ fileLines = [
"date;27.06.2022", "_date;27.06.2022",
"count;2", "_count;2",
"table:person;_nr;famname;name;birth;sex", "table:person;_nr;famname;name;birth;sex",
"testcrmdb:person;1;Brecht;Bert;10.02.98;m", "testcrmdb:person;1;Brecht;Bert;10.02.98;m",
"testcrmdb:person;2;Leon;Donna;28.09.42;f", "testcrmdb:person;2;Leon;Donna;28.09.42;f",
@ -290,6 +308,20 @@ class MyTestCase(unittest.TestCase):
self.assertIn("person", tdata[B.DATA_NODE_TABLES]) self.assertIn("person", tdata[B.DATA_NODE_TABLES])
self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA])) self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
cnttest += 3 cnttest += 3
filename = utils.path_tool.rejoinPath(utils.path_tool.composePath(P.P_TCRESULT, comp), "person.csv")
fileLines = [
"_date;27.06.2022",
"_count;2",
"persid;famname;name;birth;sex",
"1;Brecht;Bert;10.02.98;m",
"2;Leon;Donna;28.09.42;f",
"#;;;;;;"
]
tdata = t.parseCsv(comp.m, filename, fileLines, comp, aliasNode="")
self.assertIn(B.DATA_NODE_TABLES, tdata)
self.assertIn("person", tdata[B.DATA_NODE_TABLES])
self.assertEqual(2, len(tdata[B.DATA_NODE_TABLES]["person"][B.DATA_NODE_DATA]))
cnttest += 3
text = "" text = ""
for k in tdata[B.DATA_NODE_TABLES]: for k in tdata[B.DATA_NODE_TABLES]:
text += t.buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp) text += t.buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp)

2
test/test_toolhandling.py

@ -15,7 +15,7 @@ class MyTestCase(unittest.TestCase):
def test_toolhandling(self): def test_toolhandling(self):
job = basic.program.Job("unit") job = basic.program.Job("unit")
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", args = {B.PAR_APP: "TEST", B.PAR_ENV: "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
"modus": "unit"} "modus": "unit"}
job.par.setParameterArgs(args) job.par.setParameterArgs(args)
#t = basic.toolHandling.ToolManager() #t = basic.toolHandling.ToolManager()

24
test/test_xml.py

@ -1,7 +1,7 @@
import unittest import unittest
#import basic.program #import basic.program
import utils.xml_tool import utils.xml1_tool
import utils.xml_tool import utils.xml1_tool
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
def xtest_xmlTool(self): def xtest_xmlTool(self):
@ -12,17 +12,17 @@ class MyTestCase(unittest.TestCase):
beispiel_json = {'root': {'@attr': 'xyz', '$': 'inhalt', "b": "bold"}} beispiel_json = {'root': {'@attr': 'xyz', '$': 'inhalt', "b": "bold"}}
tree = {} tree = {}
tree["root"] = args tree["root"] = args
xml = utils.xml_tool.dict2xml(tree) xml = utils.xml1_tool.dict2xml(tree)
print(xml) print(xml)
xml = utils.xml_tool.dict2xml(beispiel_json) xml = utils.xml1_tool.dict2xml(beispiel_json)
print(xml) print(xml)
self.assertEqual(1, 1) self.assertEqual(1, 1)
f = utils.xml_tool.fcts() f = utils.xml1_tool.fcts()
def xtest_addSingle(self): def xtest_addSingle(self):
tree = {} tree = {}
# tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', 2, "abc") # tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', 2, "abc")
tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc") tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc")
self.assertEqual(tree["kratz"]["mm"][0]["wert"], "abc") self.assertEqual(tree["kratz"]["mm"][0]["wert"], "abc")
self.assertEqual(tree["kratz"]["mm"][0]["@name"], "NAME") self.assertEqual(tree["kratz"]["mm"][0]["@name"], "NAME")
@ -31,9 +31,9 @@ class MyTestCase(unittest.TestCase):
# c-a-a c-a-b # c-a-a c-a-b
tree = {} tree = {}
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc") tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="NAME"]/wert', 2, "abc")
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="LAND"]/wert', 2, "xyz") tree = utils.xml1_tool.fcts.addMerkmal(tree, '/root/datensegment/kratz/mm[@name="LAND"]/wert', 2, "xyz")
baum = {} baum = {}
baum["root"] = tree baum["root"] = tree
print("<------"+str(baum["root"])) print("<------"+str(baum["root"]))
@ -45,7 +45,7 @@ class MyTestCase(unittest.TestCase):
def xtest_addOnePaths(self): def xtest_addOnePaths(self):
tree = {} tree = {}
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc") tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc")
baum = {} baum = {}
baum["root"] = tree baum["root"] = tree
print("<------"+str(baum["root"])) print("<------"+str(baum["root"]))
@ -56,9 +56,9 @@ class MyTestCase(unittest.TestCase):
def xtest_addTwoPaths(self): def xtest_addTwoPaths(self):
tree = {} tree = {}
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc") tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="NAME"]/wert', "abc")
print("--------------------------------------------------------------------------------") print("--------------------------------------------------------------------------------")
tree = utils.xml_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="LAND"]/wert', "xyz") tree = utils.xml1_tool.fcts.setMerkmal(tree, '/root/datensegment/satz[@klasse="4711x"]/mm[@name="LAND"]/wert', "xyz")
baum = {} baum = {}
baum["root"] = tree baum["root"] = tree
print("<------"+str(baum["root"])) print("<------"+str(baum["root"]))
@ -74,7 +74,7 @@ class MyTestCase(unittest.TestCase):
tree_attrx = { "eins": [ {"zwei": 2, "drei": 3, "vier": { "@name": "attr", "fuenf": 5} } ] } tree_attrx = { "eins": [ {"zwei": 2, "drei": 3, "vier": { "@name": "attr", "fuenf": 5} } ] }
tree_attr2 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "fuenf": 5} ] } ] } tree_attr2 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "fuenf": 5} ] } ] }
tree_attr3 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "#text": "text" } ] } ] } tree_attr3 = { "eins": [ {"zwei": 2, "drei": 3, "vier": [ { "@name": "attr", "#text": "text" } ] } ] }
c = utils.xml_tool.fcts() c = utils.xml1_tool.fcts()
xml = c.tidy(tree_dict, 0) xml = c.tidy(tree_dict, 0)
print("-------------------------------------") print("-------------------------------------")
print(xml) print(xml)

7
test/testtools.py

@ -7,7 +7,8 @@ import test.constants as T
DEFAULT_GRAN = "tc" DEFAULT_GRAN = "tc"
DEFAULT_APP = "TESTAPP" DEFAULT_APP = "TESTAPP"
DEFAULT_ENV = "ENV01" DEFAULT_ENV = "ENV01"
DEFAULT_DIR = T.DATA_PATH + "/tdata" DEFAULT_DATA_DIR = T.DATA_PATH + "/tdata"
DEFAULT_ARCHIV_DIR = T.DATA_PATH + "/lauf"
DEFAULT_TIME = "2022-03-19_12-09-09" DEFAULT_TIME = "2022-03-19_12-09-09"
DEFAULT_MODE = "unit" DEFAULT_MODE = "unit"
gran = "" gran = ""
@ -68,9 +69,9 @@ def getJob(pgran="", papp="", penv="", ptstamp="", pmode=""):
else: else:
mode = pmode mode = pmode
if gran == "tc": if gran == "tc":
path = DEFAULT_DIR+"/TC0001/"+tstamp path = DEFAULT_ARCHIV_DIR + "/TC0001/" + tstamp
elif gran == "ts": elif gran == "ts":
path = DEFAULT_DIR + "/testlauf/TST001_" + tstamp path = DEFAULT_ARCHIV_DIR + "/testlauf/TST001_" + tstamp
job = basic.program.Job("unit") job = basic.program.Job("unit")
job.conf.confs[B.SUBJECT_PATH]["components"] = T.COMP_PATH job.conf.confs[B.SUBJECT_PATH]["components"] = T.COMP_PATH
args = {"application": app, "environment": env, "modus": mode, gran+"time": tstamp, args = {"application": app, "environment": env, "modus": mode, gran+"time": tstamp,

44
utils/config/path.yml

@ -0,0 +1,44 @@
#
pattern:
# Keywords
log: log
parfile: PARAMETER_{job.par.application}_{job.par.environment}.yml
precond: vorher
postcond: nachher
diff: diff_fach
prediff: diff_init
rundiff: diff_ablauf
result: Ergebnisse/{comp.name}
origin: original
parts: teilergebnisse
sumfile: xxx
backup: backup
reffile: Herkunft.txt
tc: testfall
ts: testlauf
debugname: debug
logname: log
debugs: "{job.conf.home}/test/log"
# environment
envbase: "{job.conf.environment}/{job.par.environment}"
envlog: "{envbase}/{log}"
envparfile: "{envbase}/{parfile}"
# testcase
tcbase: "{job.conf.archiv}/{job.par.testcase}/{job.par.tctime}"
tclog: "{tcbase}/{log}"
tcresult: "{tcbase}/{result}"
tcparfile: "{tcbase}/{parfile}"
tcdiff: "{tcresult}/{diff}"
tcprediff: "{tcresult}/{prediff}"
tcrundiff: "{tcresult}/{rundiff}"
tcprecond: "{tcresult}/{precond}"
tcpostcond: "{tcresult}/{postcond}"
# testset
tsbase: "{job.conf.archiv}/{ts}/{job.par.usecase}_{job.par.tstime}"
tslog: "{tsbase}/{log}"
tsparfile: "{tsbase}/{parfile}"
tssum: "{tsbase}/Ergebnis"
# expectation-result rs
xpbase: "{job.conf.expect}/{job.par.branch}"
xpresult: "{xpbase}/{result}"
xpbackup: "{xpbase}/{result}"

29
utils/config_tool.py

@ -72,8 +72,8 @@ def getConfigPath(modul, name, subname=""):
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
P.VAL_COMPS, basic.componentHandling.getComponentFolder(name), "CONFIG." + format) basic.componentHandling.getComponentFolder(name), "CONFIG." + format)
job.debug(verify, "5 " + pathname) job.debug(verify, "5 " + pathname)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
@ -82,13 +82,13 @@ def getConfigPath(modul, name, subname=""):
elif modul in COMP_FILES: elif modul in COMP_FILES:
# for example DATASTRUCURE or the table # for example DATASTRUCURE or the table
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_COMPS, pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
basic.componentHandling.getComponentFolder(name), modul+"."+format) basic.componentHandling.getComponentFolder(name), modul+"."+format)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
if len(subname) > 1: if len(subname) > 1:
pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_COMPS, pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
basic.componentHandling.getComponentFolder(name), subname+"."+format) basic.componentHandling.getComponentFolder(name), subname+"."+format)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
@ -184,6 +184,17 @@ def getConfig(modul, name, subname=""):
if len(pathname) < 1: if len(pathname) < 1:
return confs return confs
doc = utils.file_tool.readFileDict(pathname, msg) doc = utils.file_tool.readFileDict(pathname, msg)
if modul == D.DDL_FILENAME:
# in csv the root is the subname
# from the Dict-structure of DDL_FILENAME pick the substructure of the subname
keys = list(doc.keys())
if subname not in keys and len(keys) == 1:
doc0 = doc[keys[0]]
doc = doc0
keys = list(doc.keys())
if subname in keys:
doc0 = doc[subname]
doc = doc0
for i, v in doc.items(): for i, v in doc.items():
confs[i] = v confs[i] = v
return confs return confs
@ -197,6 +208,8 @@ def mergeConn(msg, conf, conn):
:param conn: :param conn:
:return: :return:
""" """
if B.SUBJECT_INST not in conf:
conf[B.SUBJECT_INST] = {}
for a in conn[B.SUBJECT_INST]: for a in conn[B.SUBJECT_INST]:
conf[B.SUBJECT_INST][a] = conn[B.SUBJECT_INST][a] conf[B.SUBJECT_INST][a] = conn[B.SUBJECT_INST][a]
for topic in [B.TOPIC_NODE_DB, B.TOPIC_NODE_CLI, B.TOPIC_NODE_API, B.TOPIC_NODE_FILE]: for topic in [B.TOPIC_NODE_DB, B.TOPIC_NODE_CLI, B.TOPIC_NODE_API, B.TOPIC_NODE_FILE]:
@ -210,20 +223,28 @@ def mergeConn(msg, conf, conn):
list = B.LIST_API_ATTR list = B.LIST_API_ATTR
if topic == B.TOPIC_NODE_FILE: if topic == B.TOPIC_NODE_FILE:
list = B.LIST_FILE_ATTR list = B.LIST_FILE_ATTR
print(" --- merge-conn " + topic + " " + str(list))
for a in conf[B.SUBJECT_ARTS][topic]: for a in conf[B.SUBJECT_ARTS][topic]:
if topic not in conn:
continue
if a in list: if a in list:
if a in conn[topic]: if a in conn[topic]:
conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a]
else: else:
for b in conf[B.SUBJECT_ARTS][topic][a]: for b in conf[B.SUBJECT_ARTS][topic][a]:
print(" --- merge-conn b " + topic + " " + a+" "+b)
if b not in list: if b not in list:
msg.logError("not-topic-attribute in topic-connection: "+topic+", "+b) msg.logError("not-topic-attribute in topic-connection: "+topic+", "+b)
continue continue
if a not in conn[topic]:
continue
if b in conn[topic][a]: if b in conn[topic][a]:
conf[B.SUBJECT_ARTS][topic][a][b] = conn[topic][a][b] conf[B.SUBJECT_ARTS][topic][a][b] = conn[topic][a][b]
for a in list: for a in list:
if topic not in conn: if topic not in conn:
break break
if topic not in conn:
continue
if a in conn[topic]: if a in conn[topic]:
conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a]
return conf return conf

4
utils/conn_tool.py

@ -14,7 +14,7 @@ def getConnection(comp, nr):
verify = job.getDebugLevel("conn_tool") verify = job.getDebugLevel("conn_tool")
conn = {} conn = {}
if job.conf.confs.get("tools").get("connsrc") == D.DFILE_TYPE_YML: if job.conf.confs.get("tools").get("connsrc") == D.DFILE_TYPE_YML:
conn = utils.config_tool.getConfig("tool", "conn") conn = utils.config_tool.getConfig("tool", B.SUBJECT_CONN)
xtypes = None xtypes = None
if ("types" in conn["env"][comp]): if ("types" in conn["env"][comp]):
xtypes = conn["env"][comp]["types"] xtypes = conn["env"][comp]["types"]
@ -52,7 +52,7 @@ def getConnections(comp):
conn = {} conn = {}
conns = [] conns = []
if job.conf.confs.get("tools").get("connsrc") in [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV]: if job.conf.confs.get("tools").get("connsrc") in [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV]:
conn = utils.config_tool.getConfig("tool", "conn") conn = utils.config_tool.getConfig("tool", B.SUBJECT_CONN)
if not comp in conn["env"]: if not comp in conn["env"]:
job.m.setFatal("Conn-Tool: Comp not configured " + comp) job.m.setFatal("Conn-Tool: Comp not configured " + comp)
elif job.conf.confs.get("tools").get("connsrc") == "flaskdb": elif job.conf.confs.get("tools").get("connsrc") == "flaskdb":

8
utils/data_const.py

@ -12,6 +12,7 @@ TYPE_FLOAT = "float"
TYPE_DOUBLE = "double" TYPE_DOUBLE = "double"
TYPE_DATE = "date" TYPE_DATE = "date"
TYPE_TIME = "time" TYPE_TIME = "time"
# fields in DDL # fields in DDL
DDL_FNULLABLE = "nullable" DDL_FNULLABLE = "nullable"
DDL_FNAME = "field" DDL_FNAME = "field"
@ -43,6 +44,11 @@ CSV_NODETYPE_KEYS = "_keys"
CSV_BLOCK_OPTION = "option" CSV_BLOCK_OPTION = "option"
CSV_BLOCK_STEP = "step" CSV_BLOCK_STEP = "step"
STEP_COMP_I = 1
STEP_EXECNR_I = 2
STEP_REFNR_I = 3
STEP_ARGS_I = 4
STEP_LIST_I = 4
EXCP_MALFORMAT = "malformated line: " EXCP_MALFORMAT = "malformated line: "
ATTR_SRC_TYPE = "tdtyp" ATTR_SRC_TYPE = "tdtyp"
@ -54,5 +60,3 @@ DEFAULT_DB_PARTITION = "n"
DEFAULT_DB_CONN_JAR = "n" DEFAULT_DB_CONN_JAR = "n"
""" attribute for connection-jar-file instead of connection by ip, port """ """ attribute for connection-jar-file instead of connection by ip, port """
ATTR_TABLE_DATE = "date"
ATTR_TABLE_CNT = "count"

19
utils/date_tool.py

@ -10,6 +10,7 @@ import utils.data_const as D
F_DIR = "%Y-%m-%d_%H-%M-%S" F_DIR = "%Y-%m-%d_%H-%M-%S"
F_DB_DATE = "%Y-%m-%d"
F_DE = "%d.%m.%Y" F_DE = "%d.%m.%Y"
F_N8 = "%Y%m%d" F_N8 = "%Y%m%d"
@ -99,20 +100,21 @@ def parseDate(instring):
sec = 0 sec = 0
if instring[0:2] == "{(" and instring[-2:] == ")}": if instring[0:2] == "{(" and instring[-2:] == ")}":
return parseFormula(instring) return parseFormula(instring)
for d in ["_", " "]: if len(instring) > 8:
if d in instring and instring.find(d) > 8: for d in ["_", " "]:
dstring = instring[0:instring.find(d)] if d in instring and instring.find(d) > 8:
tstring = instring[instring.find(d)+1:] dstring = instring[0:instring.find(d)]
dres = parseDate(dstring) tstring = instring[instring.find(d)+1:]
tres = parseDate(tstring) dres = parseDate(dstring)
return (dres[0], dres[1], dres[2], tres[3], tres[4], tres[5]) tres = parseDate(tstring)
return (dres[0], dres[1], dres[2], tres[3], tres[4], tres[5])
if re.match(r"\d{4}[-./]\d{2}[-./]\d{2}", instring): if re.match(r"\d{4}[-./]\d{2}[-./]\d{2}", instring):
res = re.match(r"(\d{4})[-./](\d{2})[-./](\d{2})", instring) res = re.match(r"(\d{4})[-./](\d{2})[-./](\d{2})", instring)
year = int(res.group(1)) year = int(res.group(1))
mon = int(res.group(2)) mon = int(res.group(2))
day = int(res.group(3)) day = int(res.group(3))
return (year, mon, day, hour, min, sec) return (year, mon, day, hour, min, sec)
if re.match(r"\d{1,2}[-./]\d{1,2}[-./]\d{1,2}", instring): if re.match(r"\d{1,2}[-./]\d{1,2}[-./]\d{4}", instring):
res = re.match(r"(\d{1,2})[-./](\d{1,2})[-./](\d{4})", instring) res = re.match(r"(\d{1,2})[-./](\d{1,2})[-./](\d{4})", instring)
year = int(res.group(3)) year = int(res.group(3))
mon = int(res.group(2)) mon = int(res.group(2))
@ -129,3 +131,4 @@ def parseDate(instring):
min = int(res.group(2)) min = int(res.group(2))
sec = int(res.group(3)) sec = int(res.group(3))
return (year, mon, day, hour, min, sec) return (year, mon, day, hour, min, sec)
return (year, mon, day, hour, min, sec)

120
utils/db_abstract.py

@ -38,7 +38,10 @@ SPECIAL CASES:
* If the table is partitioned tables the functions delete/insert/select calls the callback-functions * If the table is partitioned tables the functions delete/insert/select calls the callback-functions
COMP.nextTable() resp. COMP.nextTdata(). COMP.nextTable() resp. COMP.nextTdata().
""" """
import re
import basic.program import basic.program
import basic.catalog
import utils.config_tool import utils.config_tool
import basic.constants as B import basic.constants as B
import utils.data_const as D import utils.data_const as D
@ -86,6 +89,7 @@ def getDbAttributes(comp, table):
print("f " + attr + " " + out[attr]) print("f " + attr + " " + out[attr])
return out return out
def getStringIndex(text, intern): def getStringIndex(text, intern):
if intern in text: if intern in text:
return text.index(intern) return text.index(intern)
@ -142,12 +146,108 @@ def parseSQLwhere(condition, ddl=None):
conjunctor = "" conjunctor = ""
dbwhere = "" dbwhere = ""
for cond in parts: for cond in parts:
print("exp "+cond[1]+" "+str(ddl))
if cond[1] in ddl[B.DATA_NODE_HEADER]: if cond[1] in ddl[B.DATA_NODE_HEADER]:
dbwhere += " "+conjunctor+" "+cond[1]+" "+cond[0]+" "+cond[2] dbwhere += " "+conjunctor+" "+cond[1]+" "+cond[0]+" "+cond[2]
conjunctor = cond[3] conjunctor = cond[3]
print("exp ok")
return "WHERE "+dbwhere.strip() return "WHERE "+dbwhere.strip()
def getSqlTable(comp, table):
"""
the function gets the technical tablename inclusive necessary schema information
:param comp:
:param table:
:return:
"""
attr = getDbAttributes(comp, table)
if attr[B.ATTR_DB_TABNAME] != "":
sqltable = attr[B.ATTR_DB_TABNAME]
else:
sqltable = table
if attr[B.ATTR_DB_SCHEMA] != "":
sqltable = attr[B.ATTR_DB_SCHEMA] + "." + sqltable
return sqltable
def getTechnicalIDFields(ddl):
ids = []
keys = {}
for f in ddl:
if f[0:1] == "_":
continue
if "T" in ddl[f][D.DDL_KEY]:
keys[ddl[f][D.DDL_KEY]] = f
for k in keys:
ids.append(k)
return ids
def formatDbRows(table, comp, rows):
out = []
fields = comp.conf[B.DATA_NODE_DDL][table]
header = comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
for r in rows:
ro = {}
for f in header:
if f in r:
val = formatDbField(comp, r[f], fields[f])
ro[f] = val
else:
val = formatDbField(comp, B.SVAL_NULL, fields[f])
ro[f] = val
out.append(ro)
return out
def formatDbField(comp, val, field):
if val == B.SVAL_NULL:
if field[D.DDL_FNULLABLE] != B.SVAL_YES:
comp.m.logError("must-field is null "+ field[D.DDL_FNAME])
return None
print("formatDbField "+str(comp))
print("formatDbField "+str(field)+" "+str(val))
return formatDbVal(comp.m, val, field[D.DDL_TYPE])
def formatDbVal(msg, val, dtyp):
ctlg = basic.catalog.Catalog.getInstance()
if dtyp == D.TYPE_STRING:
if not isinstance(val, str):
msg.logError("field must be " + dtyp + ", " + str(val))
return str(val)
if dtyp == D.TYPE_DATE:
if not isinstance(val, str):
msg.logError("field must be " + dtyp + ", " + str(val))
return utils.date_tool.getFormatDatetupel(utils.date_tool.parseDate(val), utils.date_tool.F_DB_DATE)
if dtyp == D.TYPE_INT:
if not (isinstance(val, int) or re.match(r"^\d+$", val)):
msg.logError("field must be " + dtyp + ", " + str(val))
return 0
return int(val)
if dtyp == D.TYPE_FLOAT:
if not (isinstance(val, float) or re.match(r"^\d+[.,]\d+$", val)):
msg.logError("field must be " + dtyp + ", " + str(val))
return 0
return float(val)
else:
pass
def isCompTable(comp, data, table):
""" checks if the table in data relates to the component """
print(str(data))
return isCompRow(comp, data[B.DATA_NODE_TABLES][table])
def isCompRow(comp, row):
""" checks if the table in data relates to the component """
print("isCompRow "+comp.name+" "+str(row))
if comp.name in row[B.ATTR_DATA_COMP] \
and row[B.ATTR_DATA_COMP][comp.name] in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]:
return True
return False
# --------------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------
class DbFcts(): class DbFcts():
@ -184,6 +284,7 @@ class DbFcts():
raise Exception("missing data node in table") raise Exception("missing data node in table")
tdata[subdir][t][D.DATA_ATTR_COUNT] = len(tdata[subdir][t][B.DATA_NODE_DATA]) tdata[subdir][t][D.DATA_ATTR_COUNT] = len(tdata[subdir][t][B.DATA_NODE_DATA])
tdata[subdir][t][D.DATA_ATTR_DATE] = utils.date_tool.getActdate(utils.date_tool.F_DE) tdata[subdir][t][D.DATA_ATTR_DATE] = utils.date_tool.getActdate(utils.date_tool.F_DE)
self.comp.m.logMsg("Tabelle {} mit {} Zeilen gelesen".format(t, len(tdata[subdir][t][B.DATA_NODE_DATA])))
return tdata return tdata
def selectRows(self, statement): def selectRows(self, statement):
@ -215,14 +316,19 @@ class DbFcts():
raise Exception(B.EXCEPT_NOT_IMPLEMENT) raise Exception(B.EXCEPT_NOT_IMPLEMENT)
def insertTables(self, tdata): def insertTables(self, tdata):
""" method to insert rows into a database """
statement written in sql """ method to insert rows into the database of the component
"""
# TODO wird der Tabellenname/DB/Schema unter tdata gespeichert?
plainname = basic.componentHandling.getPlainCompname(self.comp.name) plainname = basic.componentHandling.getPlainCompname(self.comp.name)
self.loadDdl() self.loadDdl()
for t in self.comp.conf[B.DATA_NODE_DDL]: for t in tdata[B.DATA_NODE_TABLES]:
print("einzufuegende Tabelle "+t) print("einzufuegende Tabelle "+self.comp.name+" "+t)
if (t in tdata[plainname]): if isCompTable(self.comp, tdata, t):
self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA]) self.insertRows(t, tdata[B.DATA_NODE_TABLES][t][B.DATA_NODE_DATA])
self.comp.m.logMsg("in Tabelle {} {} Zeilen eingefuegt".format(
t, len(tdata[B.DATA_NODE_TABLES][t][B.DATA_NODE_DATA])))
def insertRows(self, rows): def insertRows(self, rows):
""" method to insert rows into a database """ method to insert rows into a database
@ -253,6 +359,8 @@ class DbFcts():
return "" return ""
def getDbValue(self, fo, value): def getDbValue(self, fo, value):
# TODO Untersceidung csv und echte DB
return formatDbField(self.comp, value, fo)
if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES: if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES:
return self.getDbNull() return self.getDbNull()
if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING: if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING:

57
utils/dbcsv_tool.py

@ -8,7 +8,9 @@ import basic.program
import utils.config_tool import utils.config_tool
import utils.db_abstract import utils.db_abstract
import basic.constants as B import basic.constants as B
import utils.path_tool
import utils.file_tool
import utils.tdata_tool
class DbFcts(utils.db_abstract.DbFcts): class DbFcts(utils.db_abstract.DbFcts):
""" """
@ -22,51 +24,84 @@ class DbFcts(utils.db_abstract.DbFcts):
def selectRows(self, table): def selectRows(self, table):
""" method to select rows from a database """ method to select rows from a database
statement written in sql """ statement written in sql """
sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
header = ""
path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
print(path)
tdata = {} tdata = {}
data = utils.tdata_tool.readCsv(self.comp.m, path, self.comp)
tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
if B.DATA_NODE_TABLES in data \
and table in data[B.DATA_NODE_TABLES]\
and B.DATA_NODE_DATA in data[B.DATA_NODE_TABLES][table]:
tdata[B.DATA_NODE_DATA] = data[B.DATA_NODE_TABLES][table][B.DATA_NODE_DATA]
else:
tdata[B.DATA_NODE_DATA] = {}
return tdata return tdata
def deleteRows(self, table): def deleteRows(self, table):
""" method to delete rows from a database """ method to delete rows from a database
statement written in sql """ statement written in sql """
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool") verify = -1+job.getDebugLevel("db_tool")
cmd = "DELETE FROM "+table+";" sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
header = ""
path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
print(h)
header += ";"+h
cmd = header[1:]+"\n"
utils.file_tool.writeFileText(self.comp.m, path, cmd)
self.comp.m.logInfo(cmd) self.comp.m.logInfo(cmd)
def updateRows(self, statement): def updateRows(self, statement):
""" method to delete rows from a database """ method to delete rows from a database
statement written in sql """ statement written in sql """
raise Exception(B.EXCEPT_NOT_IMPLEMENT) raise Exception(B.EXCEPT_NOT_IMPLEMENT)
def insertRows(self, table, rows): def insertRows(self, table, rows):
""" method to insert rows into a database """ method to insert rows into a database
the rows will be interpreted by the ddl of the component the rows will be interpreted by the ddl of the component
""" """
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool") verify = -1+job.getDebugLevel("db_tool")
cmd = "INSERT INTO "+table+";" sqlTable = utils.db_abstract.getSqlTable(self.comp, table)
header = "" header = ""
path = utils.path_tool.composePattern("{env.dompath}/"+sqlTable+".csv", self.comp)
if len(rows) == 0:
return ""
else:
pass
for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
print(h) print(h)
header += ", "+h header += ";"+h
cmd += " (" + header[1:]+" ) " cmd = "table:" + table + header+"\n"
print("HEADER : "+cmd)
rowvalues = "" rowvalues = ""
for r in rows: for r in rows:
if not utils.db_abstract.isCompRow(self.comp, r):
continue
print("r-----------------") print("r-----------------")
print(r) print(r)
rowvalues = "" rowvalues = self.comp.name+":"+table
cmd += "\n ( "
for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
print("head "+h)
if h in B.LIST_DB_ATTR:
continue
print("h "+h) print("h "+h)
if (h in r): if (h in r):
rowvalues += ", "+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], r[h]) rowvalues += ";"+str(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], r[h]))
else: else:
rowvalues += ", "+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_DATA][h], "") rowvalues += ";"+str(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], ""))
print("rv " + rowvalues) print("rv " + rowvalues)
cmd += rowvalues[1:]+" )," cmd += rowvalues+"\n"
cmd = cmd[0:-1]+";" utils.file_tool.writeFileText(self.comp.m, path, cmd)
self.comp.m.logInfo(cmd) self.comp.m.logInfo(cmd)
def getConnector(self): def getConnector(self):
""" add-on-method to get the connector """ add-on-method to get the connector
this method should only called by the class itself """ this method should only called by the class itself """

31
utils/file_abstract.py

@ -0,0 +1,31 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import basic.program
import basic.catalog
import utils.config_tool
import basic.constants as B
import utils.data_const as D
import utils.date_tool
class FileFcts():
"""
this is an abstract class
"""
def __init__(self):
pass
def parseText(self, text):
"""
this function parses the text and translates it to dict
:param text:
:return:
"""
def file2dict(self):
pass

1
utils/file_tool.py

@ -222,3 +222,4 @@ def writeFileDict(msg, path, dict, enc="utf-8"):
doc = json.dumps(file, indent=4) doc = json.dumps(file, indent=4)
file.write(doc) file.write(doc)
file.close() file.close()

3
utils/path_tool.py

@ -43,7 +43,8 @@ def getKeyValue(key, comp=None):
return utils.config_tool.getAttr(comp, key[5:]) return utils.config_tool.getAttr(comp, key[5:])
return "" return ""
elif 'env.' in key: elif 'env.' in key:
#if key[4:] if key[4:] in comp.conf["conn"]:
return comp.conf["conn"][key[4:]]
pass pass
elif (pt.pattern): elif (pt.pattern):
return pt.pattern[key] return pt.pattern[key]

133
utils/tdata_tool.py

@ -26,6 +26,7 @@ import basic.program
import utils.file_tool import utils.file_tool
import basic.constants as B import basic.constants as B
import utils.data_const as D import utils.data_const as D
import utils.date_tool
TOOL_NAME = "tdata_tool" TOOL_NAME = "tdata_tool"
""" name of the tool in order to switch debug-info on """ """ name of the tool in order to switch debug-info on """
@ -71,13 +72,24 @@ def getTestdata():
# read file in testdata # read file in testdata
job.m.logInfo("Test-Data readed from " + tdata[D.ATTR_SRC_TYPE] + " for " + tdata[D.ATTR_SRC_NAME]) job.m.logInfo("Test-Data readed from " + tdata[D.ATTR_SRC_TYPE] + " for " + tdata[D.ATTR_SRC_NAME])
elif tdata[D.ATTR_SRC_TYPE] == D.DATA_SRC_DIR: elif tdata[D.ATTR_SRC_TYPE] == D.DATA_SRC_DIR:
filename = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.D.ATTR_PATH_TDATA), tdata[D.ATTR_SRC_NAME], "testspec.csv") path = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.ATTR_PATH_TDATA), tdata[D.ATTR_SRC_NAME])
filename = os.path.join(path , "testspec.csv")
data = getCsvSpec(job.m, filename, D.CSV_SPECTYPE_DATA) data = getCsvSpec(job.m, filename, D.CSV_SPECTYPE_DATA)
for k in data: for k in data:
tdata[k] = data[k] tdata[k] = data[k]
if (k == D.CSV_BLOCK_OPTION): if (k == D.CSV_BLOCK_OPTION):
for p in data[k]: for p in data[k]:
setattr(job.par, p, data[k][p]) setattr(job.par, p, data[k][p])
files = utils.file_tool.getFiles(job.m, path, "table_", None)
for f in files:
print(f)
filename = os.path.join(path, f)
data = readCsv(job.m, filename, None)
table = f[6:-4]
print(filename+" "+table)
if B.DATA_NODE_TABLES not in tdata:
tdata[B.DATA_NODE_TABLES] = {}
tdata[B.DATA_NODE_TABLES][table] = data[B.DATA_NODE_TABLES][table]
else: else:
job.m.setFatal("test-Data: reftyp " + tdata[D.ATTR_SRC_TYPE] + " is not implemented") job.m.setFatal("test-Data: reftyp " + tdata[D.ATTR_SRC_TYPE] + " is not implemented")
return tdata return tdata
@ -110,7 +122,7 @@ def parseCsvSpec(msg, lines, type):
header = [] header = []
h = [] # from a[] h = [] # from a[]
status = "start" status = "start"
tableDate = utils.date_tool.getActdate(utils.date_tool.F_DE)
tableDict = {} tableDict = {}
for l in lines: for l in lines:
print("lines "+l) print("lines "+l)
@ -129,11 +141,24 @@ def parseCsvSpec(msg, lines, type):
if (not B.DATA_NODE_STEPS in data): if (not B.DATA_NODE_STEPS in data):
data[B.DATA_NODE_STEPS] = [] data[B.DATA_NODE_STEPS] = []
step = {} step = {}
step[B.DATA_NODE_COMP] = fields[1] step[B.DATA_NODE_COMP] = fields[D.STEP_COMP_I]
step[B.ATTR_DATA_REF] = fields[2] step[B.ATTR_EXEC_REF] = fields[D.STEP_EXECNR_I]
step[B.ATTR_DATA_REF] = fields[D.STEP_REFNR_I]
step[B.ATTR_STEP_ARGS] = {} step[B.ATTR_STEP_ARGS] = {}
a = fields[3].split(",") if D.STEP_ARGS_I == D.STEP_LIST_I:
args = ""
for i in range(D.STEP_ARGS_I, len(fields)):
if len(fields[i]) < 1:
continue
if fields[i][0:1] == "#":
continue
args += "," + fields[i]
args = args[1:]
else:
args = fields[D.STEP_ARGS_I]
a = args.split(",")
for arg in a: for arg in a:
print("arg "+arg)
b = arg.split(":") b = arg.split(":")
if len(b) < 2: if len(b) < 2:
raise Exception(D.EXCP_MALFORMAT + "" + l) raise Exception(D.EXCP_MALFORMAT + "" + l)
@ -145,12 +170,17 @@ def parseCsvSpec(msg, lines, type):
raise Exception(D.EXCP_MALFORMAT+""+l) raise Exception(D.EXCP_MALFORMAT+""+l)
data[a[0]][a[1]] = fields[1] data[a[0]][a[1]] = fields[1]
continue continue
elif a[0].lower() == D.DATA_ATTR_DATE:
tableDate = fields[1]
elif (a[0].lower() in D.CSV_HEADER_START): elif (a[0].lower() in D.CSV_HEADER_START):
# create deep structure a_0 ... a_n # create deep structure a_0 ... a_n
print("tdata 136 CSV_HEADER_START "+str(len(a))) print("tdata 136 CSV_HEADER_START "+str(len(a)))
h = a h = a
data[B.DATA_NODE_TABLES] = {} header = []
if B.DATA_NODE_TABLES not in data:
data[B.DATA_NODE_TABLES] = {}
h[0] = B.DATA_NODE_TABLES h[0] = B.DATA_NODE_TABLES
comps = {}
tableDict = getTabContent(msg, data, h) tableDict = getTabContent(msg, data, h)
i = 0 i = 0
for f in fields: for f in fields:
@ -171,6 +201,7 @@ def parseCsvSpec(msg, lines, type):
headerFields = [] headerFields = []
else: else:
tableDict[B.DATA_NODE_DATA] = [] tableDict[B.DATA_NODE_DATA] = []
tableDict[D.DATA_ATTR_DATE] = tableDate
setTabContent(msg, data, tableDict, h) setTabContent(msg, data, tableDict, h)
status = D.CSV_SPECTYPE_DATA status = D.CSV_SPECTYPE_DATA
continue continue
@ -179,14 +210,25 @@ def parseCsvSpec(msg, lines, type):
# fill data # fill data
tableDict = getTabContent(msg, data, h) tableDict = getTabContent(msg, data, h)
row = {} row = {}
print(fields)
i = 1 i = 1
# case-differentiation DATA or TREE # case-differentiation DATA or TREE
for f in header: for f in header:
print(str(i)+" "+str(len(fields))+" "+str(len(header)))
row[f] = fields[i] row[f] = fields[i]
if type == D.CSV_SPECTYPE_TREE: if type == D.CSV_SPECTYPE_TREE:
tableDict[B.DATA_NODE_DATA][f] = fields[i] tableDict[B.DATA_NODE_DATA][f] = fields[i]
i += 1 i += 1
if type == D.CSV_SPECTYPE_DATA: if type == D.CSV_SPECTYPE_DATA:
print("parseSpec "+ str(fields[0]))
row[B.ATTR_DATA_COMP] = {}
for c in fields[0].split(","):
a = c.split(":")
print("parseSpec " + str(a))
comps[a[0]] = a[1]
row[B.ATTR_DATA_COMP][a[0]] = a[1]
#row[B.ATTR_DATA_COMP] = fields[0].split(",")
tableDict[B.ATTR_DATA_COMP] = comps
tableDict[B.DATA_NODE_DATA].append(row) tableDict[B.DATA_NODE_DATA].append(row)
elif type == D.CSV_SPECTYPE_KEYS: elif type == D.CSV_SPECTYPE_KEYS:
tableDict[D.CSV_NODETYPE_KEYS][fields[1]] = row tableDict[D.CSV_NODETYPE_KEYS][fields[1]] = row
@ -205,6 +247,13 @@ def parseCsvSpec(msg, lines, type):
return data return data
def mergeTableComponents(comps, rowComps):
for c in rowComps.split(","):
a = c.split(":")
comps[a[0]] = a[1]
return comps
def setTabContent(msg, data, tabledata, path): def setTabContent(msg, data, tabledata, path):
if len(path) >= 2 and path[1] not in data[path[0]]: if len(path) >= 2 and path[1] not in data[path[0]]:
data[path[0]][path[1]] = {} data[path[0]][path[1]] = {}
@ -238,13 +287,15 @@ def getTabContent(msg, data, path):
def readCsv(msg, filename, comp, aliasNode=""): def readCsv(msg, filename, comp, aliasNode=""):
lines = utils.file_tool.readFileLines(filename, msg) lines = utils.file_tool.readFileLines(filename, msg)
print("readCsv "+filename)
print(lines)
return parseCsv(msg, filename, lines, comp, aliasNode) return parseCsv(msg, filename, lines, comp, aliasNode)
def parseCsv(msg, filename, lines, comp, aliasNode=""): def parseCsv(msg, filename, lines, comp, aliasNode=""):
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -4+job.getDebugLevel(TOOL_NAME) verify = -4+job.getDebugLevel(TOOL_NAME)
job.debug(verify, "# # # # # # # # parseCsv " + filename + " :" + comp.name + ": " + str(lines)) job.debug(verify, "# # # # # # # # parseCsv " + filename + " :" + str(lines))
fields = [] fields = []
nodes = [] nodes = []
columns = [] columns = []
@ -264,14 +315,22 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
job.debug(verify, str(state) + " line " + line + " :" + str(len(fields)) + ": " + str(fields)) job.debug(verify, str(state) + " line " + line + " :" + str(len(fields)) + ": " + str(fields))
if len(testline) < 2 and state < 1: if len(testline) < 2 and state < 1:
state = 0 state = 0
elif a[0].lower() == D.ATTR_TABLE_DATE: elif a[0].lower() == D.DATA_ATTR_DATE:
tableDate = fields[1] tableDate = fields[1]
elif a[0].lower() == D.ATTR_TABLE_CNT: state = 1
elif a[0].lower() == D.DATA_ATTR_COUNT:
tableCnt = fields[1] tableCnt = fields[1]
elif a[0].lower() in D.CSV_HEADER_START: state = 1
elif a[0].lower() in D.CSV_HEADER_START or \
(comp is not None and state == 1
and isCompTableFile(comp, filename)):
state = 2 state = 2
columns = [] columns = []
h = a h = a
if len(h) < 2 and comp is not None:
a = ["table", basename]
h = a
startCols = 0
cnt = len(fields) cnt = len(fields)
job.debug(verify, str(state) + " cnt " + str(cnt)) job.debug(verify, str(state) + " cnt " + str(cnt))
data[B.DATA_NODE_TABLES] = {} data[B.DATA_NODE_TABLES] = {}
@ -287,10 +346,11 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
nodes.append(h[i]) nodes.append(h[i])
job.debug(verify, str(state) + " nodes " + str(nodes)) job.debug(verify, str(state) + " nodes " + str(nodes))
tableDict = getTabContent(msg, data, h) tableDict = getTabContent(msg, data, h)
tableDict[B.ATTR_DATA_COMP] = {}
if len(tableDate) > 6: if len(tableDate) > 6:
tableDict[D.ATTR_TABLE_DATE] = tableDate tableDict[D.DATA_ATTR_DATE] = tableDate
if int(tableCnt) > 0: if int(tableCnt) > 0:
tableDict[D.ATTR_TABLE_CNT] = tableCnt tableDict[D.DATA_ATTR_COUNT] = tableCnt
j = 0 j = 0
for i in range(1, cnt): for i in range(1, cnt):
if fields[i][0:1] == "_": if fields[i][0:1] == "_":
@ -308,7 +368,14 @@ def parseCsv(msg, filename, lines, comp, aliasNode=""):
tableDict = getTabContent(msg, data, h) tableDict = getTabContent(msg, data, h)
state = 3 state = 3
row = {} row = {}
print(line)
if startCols > 0:
row[B.ATTR_DATA_COMP] = {}
row[B.ATTR_DATA_COMP][a[0]] = a[1]
tableDict[B.ATTR_DATA_COMP][a[0]] = a[1]
for i in range(startCols, cnt+startCols): for i in range(startCols, cnt+startCols):
print("for "+str(i)+" "+str(len(row))+" "+str(startCols)+" "+str(len(fields)))
print(str(fields[i]))
if i >= len(columns)+startCols: if i >= len(columns)+startCols:
break break
row[columns[i-startCols]] = fields[i] row[columns[i-startCols]] = fields[i]
@ -358,25 +425,38 @@ def buildCsvData(filename, tdata, comp):
:param comp: if specific else None :param comp: if specific else None
:return: :return:
""" """
compColumn = not isCompTableFile(comp, filename)
job = basic.program.Job.getInstance() job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel(TOOL_NAME) verify = -1+job.getDebugLevel(TOOL_NAME)
job.debug(verify, "writeDataTable " + str(comp)) job.debug(verify, "writeDataTable " + str(comp))
text = "" text = ""
for k in [D.ATTR_TABLE_DATE, D.ATTR_TABLE_CNT]: for k in [D.DATA_ATTR_DATE, D.DATA_ATTR_COUNT]:
if k in tdata: if k in tdata:
text += k+";"+tdata[k]+"\n" text += k+";"+str(tdata[k])+"\n"
text += "table" header = "table"
for f in tdata[B.DATA_NODE_HEADER]: for f in tdata[B.DATA_NODE_HEADER]:
text += ";"+f header += ";"+f
if compColumn:
text += header
else:
#text += "_nr;" + header[6:] + "\n"
text += header[6:] + "\n"
i = 0
for r in tdata[B.DATA_NODE_DATA]: for r in tdata[B.DATA_NODE_DATA]:
text += "\n" row = ""
i += 1
for f in tdata[B.DATA_NODE_HEADER]: for f in tdata[B.DATA_NODE_HEADER]:
if f in r: if f in r:
text += ";"+str(r[f]) row += ";"+str(r[f])
else: else:
text += ";" row += ";"
text += "\n" if compColumn:
return text text += row
else:
text += row[1:]
#text += str(i) + row
text += "\n"
return text
def writeCsvData(filename, tdata, comp): def writeCsvData(filename, tdata, comp):
@ -386,3 +466,14 @@ def writeCsvData(filename, tdata, comp):
text += buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp) text += buildCsvData(filename, tdata[B.DATA_NODE_TABLES][k], comp)
text += "\n" text += "\n"
utils.file_tool.writeFileText(comp.m, filename, text) utils.file_tool.writeFileText(comp.m, filename, text)
def isCompTableFile(comp, filename):
""" check if the filename belongs to the component """
basetable = os.path.basename(filename)[0:-4]
if comp is None:
return False
if B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_ARTS] and basetable in comp.conf[B.DATA_NODE_DDL] \
and comp.name in filename:
return True
return False
Loading…
Cancel
Save