Browse Source

mv db-tools

refactor
Ulrich 2 years ago
parent
commit
3e10a5780f
  1. 0
      test/test_12component.py
  2. 50
      test/test_912component.py
  3. 14
      tools/db_abstract.py
  4. 13
      tools/dbcsv_tool.py
  5. 5
      tools/dbmysql_tool.py
  6. 12
      tools/dbrel_tool.py

0
test/test_11component.py → test/test_12component.py

50
test/test_912component.py

@ -0,0 +1,50 @@
import unittest
import os
import inspect
import shutil
import tools.path_tool
import basic.program
import basic.component
import basic.componentHandling
import test.constants
import basic.constants as B
import test.constants as T
import basic.componentHandling
import tools.file_tool
HOME_PATH = test.constants.HOME_PATH
PYTHON_CMD = "python"
TEST_FUNCTIONS = ["test_00init"]
PROGRAM_NAME = "clean_workspace"
class MyTestCase(unittest.TestCase):
mymsg = "--------------------------------------------------------------"
def test_00init(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
# simple job instantiate - without parameter and only simple messaging
job = basic.program.Job(PROGRAM_NAME)
print(str(job.__dict__))
# self.checkSimpleJob(job)
sjob = basic.program.SimpleJob(PROGRAM_NAME)
# print(str(sjob.__dict__))
self.checkSimpleComponent(sjob)
def checkSimpleComponent(self, job):
self.assertIn("conf", job.__dict__)
self.assertIn("jobid", job.__dict__)
self.assertIn("program", job.__dict__)
self.assertIn("start", job.__dict__)
self.assertIn("configpath", job.conf)
self.assertIn("paths", job.conf)
if __name__ == '__main__':
unittest.main()

14
utils/db_abstract.py → tools/db_abstract.py

@ -43,10 +43,10 @@ import re
import basic.program import basic.program
import basic.catalog import basic.catalog
import utils.config_tool import tools.config_tool
import basic.constants as B import basic.constants as B
import utils.data_const as D import tools.data_const as D
import utils.date_tool import tools.date_tool
import os import os
@ -235,11 +235,11 @@ def formatDbVal(msg, val, dtyp):
if dtyp == D.TYPE_DATE: if dtyp == D.TYPE_DATE:
if not isinstance(val, str): if not isinstance(val, str):
msg.logError("field must be " + dtyp + ", " + str(val)) msg.logError("field must be " + dtyp + ", " + str(val))
return utils.date_tool.getFormatDatetupel(utils.date_tool.parseDate(val), utils.date_tool.F_DB_DATE) return tools.date_tool.getFormatDatetupel(tools.date_tool.parseDate(val), tools.date_tool.F_DB_DATE)
if dtyp == D.TYPE_TIME: if dtyp == D.TYPE_TIME:
if not isinstance(val, str): if not isinstance(val, str):
msg.logError("field must be " + dtyp + ", " + str(val)) msg.logError("field must be " + dtyp + ", " + str(val))
return utils.date_tool.getFormatDatetupel(utils.date_tool.parseDate(val), utils.date_tool.F_DB_TIME) return tools.date_tool.getFormatDatetupel(tools.date_tool.parseDate(val), tools.date_tool.F_DB_TIME)
if dtyp == D.TYPE_INT: if dtyp == D.TYPE_INT:
if not (isinstance(val, int) or re.match(r"^\d+$", val)): if not (isinstance(val, int) or re.match(r"^\d+$", val)):
msg.logError("field must be " + dtyp + ", " + str(val)) msg.logError("field must be " + dtyp + ", " + str(val))
@ -303,7 +303,7 @@ class DbFcts():
if B.DATA_NODE_DATA not in tdata[subdir][t]: if B.DATA_NODE_DATA not in tdata[subdir][t]:
raise Exception("missing data node in table") raise Exception("missing data node in table")
tdata[subdir][t][D.DATA_ATTR_COUNT] = len(tdata[subdir][t][B.DATA_NODE_DATA]) tdata[subdir][t][D.DATA_ATTR_COUNT] = len(tdata[subdir][t][B.DATA_NODE_DATA])
tdata[subdir][t][D.DATA_ATTR_DATE] = utils.date_tool.getActdate(utils.date_tool.F_DE) tdata[subdir][t][D.DATA_ATTR_DATE] = tools.date_tool.getActdate(tools.date_tool.F_DE)
self.comp.m.logMsg("Tabelle {} mit {} Zeilen gelesen".format(t, len(tdata[subdir][t][B.DATA_NODE_DATA]))) self.comp.m.logMsg("Tabelle {} mit {} Zeilen gelesen".format(t, len(tdata[subdir][t][B.DATA_NODE_DATA])))
return tdata return tdata
@ -369,7 +369,7 @@ class DbFcts():
the ddl are mostly stored as csv in the component-folder """ the ddl are mostly stored as csv in the component-folder """
if (B.DATA_NODE_DDL in self.comp.conf): if (B.DATA_NODE_DDL in self.comp.conf):
return return
conf = utils.config_tool.getConfig(job, D.DDL_FILENAME, self.comp.name) conf = tools.config_tool.getConfig(job, D.DDL_FILENAME, self.comp.name)
self.comp.conf[B.DATA_NODE_DDL] = {} self.comp.conf[B.DATA_NODE_DDL] = {}
for k in conf[self.comp.name]: for k in conf[self.comp.name]:
self.comp.conf[B.DATA_NODE_DDL][k] = conf[self.comp.name][k] self.comp.conf[B.DATA_NODE_DDL][k] = conf[self.comp.name][k]

13
utils/dbcsv_tool.py → tools/dbcsv_tool.py

@ -4,15 +4,14 @@
# Author : Ulrich Carmesin # Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de # Source : gitea.ucarmesin.de
# --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------
import basic.program
import utils.config_tool import utils.config_tool
import utils.db_abstract import tools.db_abstract
import basic.constants as B import basic.constants as B
import utils.path_tool import utils.path_tool
import utils.file_tool import utils.file_tool
import utils.tdata_tool import utils.tdata_tool
class DbFcts(utils.db_abstract.DbFcts): class DbFcts(tools.db_abstract.DbFcts):
""" """
This interface defines each necessary connection to any kind of database. This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
@ -24,7 +23,7 @@ class DbFcts(utils.db_abstract.DbFcts):
def selectRows(self, table, job): def selectRows(self, table, job):
""" method to select rows from a database """ method to select rows from a database
statement written in sql """ statement written in sql """
sqlTable = utils.db_abstract.getSqlTable(self.comp, table) sqlTable = tools.db_abstract.getSqlTable(self.comp, table)
header = "" header = ""
path = utils.path_tool.composePattern(job, "{env.dompath}/"+sqlTable+".csv", self.comp) path = utils.path_tool.composePattern(job, "{env.dompath}/"+sqlTable+".csv", self.comp)
print(path) print(path)
@ -44,7 +43,7 @@ class DbFcts(utils.db_abstract.DbFcts):
""" method to delete rows from a database """ method to delete rows from a database
statement written in sql """ statement written in sql """
verify = -1+job.getDebugLevel("db_tool") verify = -1+job.getDebugLevel("db_tool")
sqlTable = utils.db_abstract.getSqlTable(self.comp, table) sqlTable = tools.db_abstract.getSqlTable(self.comp, table)
header = "" header = ""
path = utils.path_tool.composePattern(job, "{env.dompath}/"+sqlTable+".csv", self.comp) path = utils.path_tool.composePattern(job, "{env.dompath}/"+sqlTable+".csv", self.comp)
for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
@ -66,7 +65,7 @@ class DbFcts(utils.db_abstract.DbFcts):
the rows will be interpreted by the ddl of the component the rows will be interpreted by the ddl of the component
""" """
verify = -1+job.getDebugLevel("db_tool") verify = -1+job.getDebugLevel("db_tool")
sqlTable = utils.db_abstract.getSqlTable(self.comp, table) sqlTable = tools.db_abstract.getSqlTable(self.comp, table)
header = "" header = ""
path = utils.path_tool.composePattern(job, "{env.dompath}/"+sqlTable+".csv", self.comp) path = utils.path_tool.composePattern(job, "{env.dompath}/"+sqlTable+".csv", self.comp)
if len(rows) == 0: if len(rows) == 0:
@ -80,7 +79,7 @@ class DbFcts(utils.db_abstract.DbFcts):
print("HEADER : "+cmd) print("HEADER : "+cmd)
rowvalues = "" rowvalues = ""
for r in rows: for r in rows:
if not utils.db_abstract.isCompRow(self.comp, r): if not tools.db_abstract.isCompRow(self.comp, r):
continue continue
print("r-----------------") print("r-----------------")
print(r) print(r)

5
utils/dbmysql_tool.py → tools/dbmysql_tool.py

@ -4,15 +4,14 @@
# Author : Ulrich Carmesin # Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de # Source : gitea.ucarmesin.de
# --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------
import basic.program
import utils.config_tool import utils.config_tool
import utils.dbrel_tool import tools.dbrel_tool
import mysql.connector import mysql.connector
import basic.constants as B import basic.constants as B
import utils.data_const as D import utils.data_const as D
import utils.date_tool import utils.date_tool
class DbFcts(utils.dbrel_tool.DbFcts): class DbFcts(tools.dbrel_tool.DbFcts):
""" """
This interface defines each necessary connection to any kind of database. This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.

12
utils/dbrel_tool.py → tools/dbrel_tool.py

@ -4,13 +4,11 @@
# Author : Ulrich Carmesin # Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de # Source : gitea.ucarmesin.de
# --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------
import basic.program import tools.db_abstract
import utils.config_tool
import utils.db_abstract
import mysql.connector import mysql.connector
import basic.constants as B import basic.constants as B
import utils.data_const as D import utils.data_const as D
class DbFcts(utils.db_abstract.DbFcts): class DbFcts(tools.db_abstract.DbFcts):
""" """
This interface defines each necessary connection to any kind of database. This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
@ -89,7 +87,7 @@ class DbFcts(utils.db_abstract.DbFcts):
) )
return mysql return mysql
def getCreateTable(self, table): def getCreateTable(self, table):
sql = "CREATE TABLE IF NOT EXISTS " + self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_DB_DATABASE] + "."+table+" (" sql = "CREATE TABLE IF NOT EXISTS " + self.job.conf[B.TOPIC_NODE_DB][B.ATTR_DB_DATABASE] + "."+table+" ("
return sql return sql
def getSchemaAttribut(self, attr, atype): def getSchemaAttribut(self, attr, atype):
@ -111,7 +109,7 @@ class DbFcts(utils.db_abstract.DbFcts):
def getSchemaSubtable(self, parent, attributes): def getSchemaSubtable(self, parent, attributes):
sql = "CREATE TABLE IF NOT EXISTS "+\ sql = "CREATE TABLE IF NOT EXISTS "+\
self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_DB_DATABASE]+"."+self.getSubTableName(parent, attributes[0]["attr"]) + " (" self.job.conf[B.TOPIC_NODE_DB][B.ATTR_DB_DATABASE]+"."+self.getSubTableName(parent, attributes[0]["attr"]) + " ("
sql += self.getSchemaAttribut(self.getSubTableId(parent, attributes[0]["attr"]), "id") + "," sql += self.getSchemaAttribut(self.getSubTableId(parent, attributes[0]["attr"]), "id") + ","
sql += self.getSchemaAttribut(parent+"id", D.TYPE_INT) + "," sql += self.getSchemaAttribut(parent+"id", D.TYPE_INT) + ","
for a in attributes: for a in attributes:
@ -125,7 +123,7 @@ class DbFcts(utils.db_abstract.DbFcts):
table = table[4:] table = table[4:]
sql = "CREATE INDEX " sql = "CREATE INDEX "
sql += "idx_"+table+"_"+attr sql += "idx_"+table+"_"+attr
sql += " ON " + self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_DB_DATABASE] + "." + table sql += " ON " + self.job.conf[B.TOPIC_NODE_DB][B.ATTR_DB_DATABASE] + "." + table
sql += " ( " + attr + " );" sql += " ( " + attr + " );"
return sql return sql
Loading…
Cancel
Save