Testskripter
/
datestex
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

db-tool part1

master
Ulrich Carmesin 3 years ago
parent
ae399fc300
commit
8031b6e8ef
5 changed files with 282 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 17
      basic/constants.py
  2. 34
      components/testexec.py
  3. 54
      test/test_db.py
  4. 85
      utils/db_abstract.py
  5. 92
      utils/dbshive_tool.py

17
basic/constants.py
View File

@ -66,6 +66,16 @@ DATA_NODE_COMP = "comp"
DATA_NODE_PAR = "par"
""" This constant defines """
TOPIC_NODE_DB = "db"
ATTR_DB_PARTITION = "partitioned"
""" attribute if table is partitioned - partitions are parametrized """
ATTR_DB_CONN_JAR = "conn_jar_name"
""" attribute for connection-jar-file instead of connection by ip, port """
ATTR_DB_TABNAME = "tabname"
""" attribute in order to use a different technical name for the db-table """
PAR_DB_WHERE = "dbwhere"
""" optional parameter with a where-clause """
PAR_DB_PARTITION = "dbparts"
""" optional parameter for partitions of a partitioned tables """
TOPIC_NODE_CLI = "cli"
TOPIC_NODE_API = "api"
@ -107,6 +117,13 @@ ATTR_INST_SUBCOMP = "components"
SUBJECT_FCT = "function" # | | | | x | main-programs
SUBJECT_ARTS = "artifact" # | | | | x | Component
"""
in this subject-node are each kind of result of any component with the structure:
* topic (db, cli, api, ...)
* * general attributes - to-know: technical attributes are stored in connection-tree
* * partial-component - to-know: the existence of db-tables can be defined in the ddl-tree
* * * specific attributes - it overrides the general attributes
"""
ATTR_ARTS_DB = "db"
ATTR_ARTS_LOG = "log"
ATTR_ARTS_LOB = "lob"

34
components/testexec.py
View File

@ -35,6 +35,9 @@ import utils.db_abstract
import basic.toolHandling
import components.component
import basic.componentHandling
import utils.db_abstract
import basic.constants as B
class Testexecuter():
def prepare_system(self, granularity):
@ -116,6 +119,37 @@ class Testexecuter():
self.m.setMsg("readInstance for " + self.name + " is OK")
self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper())
def composeSqlClauses(self, sql):
job = basic.program.Job.getInstance()
out = {}
print("composeSqlClauses "+sql)
table = sql[sql.upper().index(" FROM ")+6:].strip()
print(table)
sql_new = sql[0:sql.upper().index(" FROM ")+5]
print(sql_new)
attr = utils.db_abstract.getDbAttributes(self, table)
if attr[B.ATTR_DB_TABNAME] != "":
table = self.conf[B.SUBJECT_ARTS][table][B.ATTR_DB_TABNAME]
sql_new += " " + table
print(sql_new)
if (hasattr(job.par, B.PAR_DB_WHERE)):
sql_new += " WHERE "+getattr(job.par, B.PAR_DB_WHERE)
if sql_new[0:6] == "SELECT":
sql_new += " ORDER BY id"
print(sql_new)
sql_new = sql_new.replace('!', "\'")
if (hasattr(job.par, B.PAR_DB_PARTITION)) and (attr[B.ATTR_DB_PARTITION] != "n"):
parts = getattr(job.par, B.PAR_DB_PARTITION)
a = parts.split(",")
for p in a:
sql_part = sql_new
sql_part = sql_part.replace(attr[B.ATTR_DB_PARTITION], p)
out[p] = sql_part
else:
out["ALL"] = sql_new
return out
def test_System(self, granularity):
"""

54
test/test_db.py
View File

@ -0,0 +1,54 @@
import unittest, os
import basic.program
import utils.path_tool
import basic.toolHandling
import test.constants
import components.component
import basic.constants as B
HOME_PATH = test.constants.HOME_PATH
conf = {}
class MyTestCase(unittest.TestCase):
def test_toolhandling(self):
job = basic.program.Job("unit")
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
"modus": "unit"}
job.par.setParameterArgs(args)
#t = basic.toolHandling.ToolManager()
comp = components.component.Component()
comp.name = "testb"
comp.conf = {}
comp.conf[B.SUBJECT_ARTS] = {}
comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB] = {}
comp.conf[B.SUBJECT_CONN] = {}
comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB] = {}
comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][B.ATTR_TYPE] = "shive"
tool = basic.toolHandling.getDbTool(comp)
self.assertRegex(str(type(tool)), 'dbshive_tool.DbFcts')
attr = tool.getDbAttributes("xx")
self.assertRegex(attr[B.ATTR_DB_PARTITION], 'n')
comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "y"
attr = tool.getDbAttributes("xx")
self.assertRegex(attr[B.ATTR_DB_PARTITION], 'y')
comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "z"
attr = tool.getDbAttributes("xx")
self.assertRegex(attr[B.ATTR_DB_PARTITION], 'z')
#
sqls = comp.composeSqlClauses("SELECT * FROM lofts")
print(sqls)
setattr(job.par, B.PAR_DB_WHERE, "name like !%utz%! and regname = !+reg+!")
comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "+reg+"
sqls = comp.composeSqlClauses("SELECT * FROM lofts")
print(sqls)
setattr(job.par, B.PAR_DB_PARTITION, "BA,FV")
sqls = comp.composeSqlClauses("SELECT * FROM lofts")
print(sqls)
tool.deleteRows("deltable")
tool.selectRows("deltable")
if __name__ == '__main__':
unittest.main()

85
utils/db_abstract.py
View File

@ -43,6 +43,50 @@ import utils.config_tool
import basic.constants as B
import os
DFLT_DB_PARTITION = "n"
""" attribute if table is partitioned - partitions are parametrized """
DFLT_DB_CONN_JAR = "n"
""" attribute for connection-jar-file instead of connection by ip, port """
def getDbAttributes(comp, table):
"""
this function collects all relevant db-attributes from any location where it can be set.
The location could be
* comp.artifact.db.[table].attr
* comp.artifact.[db].[table].attr
"""
out = {
B.ATTR_DB_TABNAME: "",
B.ATTR_DB_PARTITION: DFLT_DB_PARTITION,
B.ATTR_DB_CONN_JAR: DFLT_DB_CONN_JAR
}
for attr in out.keys():
print(attr)
if (table in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \
and (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]):
out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr]
print("a " + attr + " " + out[attr])
elif (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]):
out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr]
print("b " + attr + " " + out[attr])
elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \
and (table in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \
and (attr in comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]):
out[attr] = comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr]
print("c " + attr + " " + out[attr])
elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \
and (attr in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]):
out[attr] = comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr]
print("d " + attr + " " + out[attr])
elif (attr in comp.conf[B.SUBJECT_CONN]):
out[attr] = comp.conf[B.SUBJECT_CONN][attr]
print("e " + attr + " " + out[attr])
else:
print("f " + attr + " " + out[attr])
return out
class DbFcts():
"""
This interface defines each necessary connection to any kind of database.
@ -55,6 +99,47 @@ class DbFcts():
def setComp(self, comp):
self.comp = comp
def getDbAttributes(self, table):
"""
this function collects all relevant db-attributes from any location where it can be set.
The location could be
* comp.artifact.db.[table].attr
* comp.artifact.[db].[table].attr
"""
return getDbAttributes(self.comp, table)
def xxgetDbAttributes(self, table):
out = {
B.ATTR_DB_TABNAME: "",
B.ATTR_DB_PARTITION: DFLT_DB_PARTITION,
B.ATTR_DB_CONN_JAR: DFLT_DB_CONN_JAR
}
for attr in out.keys():
print(attr)
if (table in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \
and (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]):
out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr]
print("a "+attr+" "+out[attr])
elif (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]):
out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr]
print("b "+attr+" "+out[attr])
elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \
and (table in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \
and (attr in self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]):
out[attr] = self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr]
print("c " + attr+" "+out[attr])
elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \
and (attr in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]):
out[attr] = self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr]
print("d "+attr+" "+out[attr])
elif (attr in self.comp.conf[B.SUBJECT_CONN]):
out[attr] = self.comp.conf[B.SUBJECT_CONN][attr]
print("e " + attr+" "+out[attr])
else:
print("f " + attr+" "+out[attr])
return out
def selectTables(self):
""" method to delete rows from a database
statement written in sql """

92
utils/dbshive_tool.py
View File

@ -0,0 +1,92 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import basic.program
import utils.config_tool
import utils.db_abstract
import pyspark
import basic.constants as B
class DbFcts(utils.db_abstract.DbFcts):
"""
This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
"""
def __init__(self):
pass
def selectRows(self, table):
""" method to select rows from a database
statement written in sql """
tdata = {}
dry = 0
# attr = self.getDbAttributes(table)
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "SELECT "+"*" #",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER])
cmd += " FROM "+table
sqls = self.comp.composeSqlClauses(cmd)
data = []
for k in sqls.keys():
sql = sqls[k]
if dry == 1:
spark = self.getConnector()
df = spark.sql(cmd)
for r in df:
data.append(r)
else:
print("select "+sql)
#self.comp.m.logInfo(cmd)
#tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
#tdata[B.DATA_NODE_DATA] = data
return tdata
def deleteRows(self, table):
""" method to delete rows from a database
statement written in sql """
job = basic.program.Job.getInstance()
dry = 0
verify = -1+job.getDebugLevel("db_tool")
cmd = "DELETE FROM "+table
print("deleteRows "+cmd)
sqls = self.comp.composeSqlClauses(cmd)
print("deleteRows "+cmd)
print(sqls)
for k in sqls.keys():
sql = sqls[k]
if dry == 1:
#spark = self.getConnector()
#df = spark.sql(cmd)
pass
else:
print("select "+sql)
#self.comp.m.logInfo(cmd)
def insertRows(self, table, rows):
""" method to insert rows into a database
the rows will be interpreted by the ddl of the component
"""
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
spark = self.getConnector()
df = spark.createDataFrame(rows)
self.comp.m.logInfo("cmd")
def getConnector(self):
""" add-on-method to get the connector
this method should only called by the class itself """
job = basic.program.Job.getInstance()
spark = pyspark.SparkSession\
.builder\
.appName("datest")\
.getOrCreate()
return spark
Write Preview
Loading…
Cancel
Save