From 8031b6e8eff3321ad608899fe9de8ed2e231d4a1 Mon Sep 17 00:00:00 2001 From: Ulrich Carmesin Date: Wed, 23 Feb 2022 23:11:27 +0100 Subject: [PATCH] db-tool part1 --- basic/constants.py | 17 ++++++++ components/testexec.py | 34 ++++++++++++++++ test/test_db.py | 54 +++++++++++++++++++++++++ utils/db_abstract.py | 85 ++++++++++++++++++++++++++++++++++++++ utils/dbshive_tool.py | 92 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 282 insertions(+) create mode 100644 test/test_db.py create mode 100644 utils/dbshive_tool.py diff --git a/basic/constants.py b/basic/constants.py index 4266928..3126f5d 100644 --- a/basic/constants.py +++ b/basic/constants.py @@ -66,6 +66,16 @@ DATA_NODE_COMP = "comp" DATA_NODE_PAR = "par" """ This constant defines """ TOPIC_NODE_DB = "db" +ATTR_DB_PARTITION = "partitioned" +""" attribute if table is partitioned - partitions are parametrized """ +ATTR_DB_CONN_JAR = "conn_jar_name" +""" attribute for connection-jar-file instead of connection by ip, port """ +ATTR_DB_TABNAME = "tabname" +""" attribute in order to use a different technical name for the db-table """ +PAR_DB_WHERE = "dbwhere" +""" optional parameter with a where-clause """ +PAR_DB_PARTITION = "dbparts" +""" optional parameter for partitions of a partitioned tables """ TOPIC_NODE_CLI = "cli" TOPIC_NODE_API = "api" @@ -107,6 +117,13 @@ ATTR_INST_SUBCOMP = "components" SUBJECT_FCT = "function" # | | | | x | main-programs SUBJECT_ARTS = "artifact" # | | | | x | Component +""" +in this subject-node are each kind of result of any component with the structure: + * topic (db, cli, api, ...) + * * general attributes - to-know: technical attributes are stored in connection-tree + * * partial-component - to-know: the existence of db-tables can be defined in the ddl-tree + * * * specific attributes - it overrides the general attributes +""" ATTR_ARTS_DB = "db" ATTR_ARTS_LOG = "log" ATTR_ARTS_LOB = "lob" diff --git a/components/testexec.py b/components/testexec.py index 68bf725..031e465 100644 --- a/components/testexec.py +++ b/components/testexec.py @@ -35,6 +35,9 @@ import utils.db_abstract import basic.toolHandling import components.component import basic.componentHandling +import utils.db_abstract +import basic.constants as B + class Testexecuter(): def prepare_system(self, granularity): @@ -116,6 +119,37 @@ class Testexecuter(): self.m.setMsg("readInstance for " + self.name + " is OK") self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def composeSqlClauses(self, sql): + job = basic.program.Job.getInstance() + out = {} + print("composeSqlClauses "+sql) + table = sql[sql.upper().index(" FROM ")+6:].strip() + print(table) + sql_new = sql[0:sql.upper().index(" FROM ")+5] + print(sql_new) + attr = utils.db_abstract.getDbAttributes(self, table) + if attr[B.ATTR_DB_TABNAME] != "": + table = self.conf[B.SUBJECT_ARTS][table][B.ATTR_DB_TABNAME] + sql_new += " " + table + print(sql_new) + if (hasattr(job.par, B.PAR_DB_WHERE)): + sql_new += " WHERE "+getattr(job.par, B.PAR_DB_WHERE) + if sql_new[0:6] == "SELECT": + sql_new += " ORDER BY id" + print(sql_new) + sql_new = sql_new.replace('!', "\'") + if (hasattr(job.par, B.PAR_DB_PARTITION)) and (attr[B.ATTR_DB_PARTITION] != "n"): + parts = getattr(job.par, B.PAR_DB_PARTITION) + a = parts.split(",") + for p in a: + sql_part = sql_new + sql_part = sql_part.replace(attr[B.ATTR_DB_PARTITION], p) + out[p] = sql_part + else: + out["ALL"] = sql_new + return out + def test_System(self, granularity): """ diff --git a/test/test_db.py b/test/test_db.py new file mode 100644 index 0000000..bf9a587 --- /dev/null +++ b/test/test_db.py @@ -0,0 +1,54 @@ +import unittest, os + +import basic.program +import utils.path_tool +import basic.toolHandling +import test.constants +import components.component +import basic.constants as B +HOME_PATH = test.constants.HOME_PATH + +conf = {} + +class MyTestCase(unittest.TestCase): + + def test_toolhandling(self): + job = basic.program.Job("unit") + args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", + "modus": "unit"} + job.par.setParameterArgs(args) + #t = basic.toolHandling.ToolManager() + comp = components.component.Component() + comp.name = "testb" + comp.conf = {} + comp.conf[B.SUBJECT_ARTS] = {} + comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB] = {} + comp.conf[B.SUBJECT_CONN] = {} + comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB] = {} + comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][B.ATTR_TYPE] = "shive" + tool = basic.toolHandling.getDbTool(comp) + self.assertRegex(str(type(tool)), 'dbshive_tool.DbFcts') + attr = tool.getDbAttributes("xx") + self.assertRegex(attr[B.ATTR_DB_PARTITION], 'n') + comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "y" + attr = tool.getDbAttributes("xx") + self.assertRegex(attr[B.ATTR_DB_PARTITION], 'y') + comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "z" + attr = tool.getDbAttributes("xx") + self.assertRegex(attr[B.ATTR_DB_PARTITION], 'z') + # + sqls = comp.composeSqlClauses("SELECT * FROM lofts") + print(sqls) + setattr(job.par, B.PAR_DB_WHERE, "name like !%utz%! and regname = !+reg+!") + comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "+reg+" + sqls = comp.composeSqlClauses("SELECT * FROM lofts") + print(sqls) + setattr(job.par, B.PAR_DB_PARTITION, "BA,FV") + sqls = comp.composeSqlClauses("SELECT * FROM lofts") + print(sqls) + tool.deleteRows("deltable") + tool.selectRows("deltable") + + +if __name__ == '__main__': + unittest.main() diff --git a/utils/db_abstract.py b/utils/db_abstract.py index f82e0d7..631220c 100644 --- a/utils/db_abstract.py +++ b/utils/db_abstract.py @@ -43,6 +43,50 @@ import utils.config_tool import basic.constants as B import os +DFLT_DB_PARTITION = "n" +""" attribute if table is partitioned - partitions are parametrized """ +DFLT_DB_CONN_JAR = "n" +""" attribute for connection-jar-file instead of connection by ip, port """ + + +def getDbAttributes(comp, table): + """ + this function collects all relevant db-attributes from any location where it can be set. + The location could be + * comp.artifact.db.[table].attr + * comp.artifact.[db].[table].attr + """ + out = { + B.ATTR_DB_TABNAME: "", + B.ATTR_DB_PARTITION: DFLT_DB_PARTITION, + B.ATTR_DB_CONN_JAR: DFLT_DB_CONN_JAR + } + for attr in out.keys(): + print(attr) + if (table in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \ + and (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]): + out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr] + print("a " + attr + " " + out[attr]) + elif (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]): + out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr] + print("b " + attr + " " + out[attr]) + elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \ + and (table in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \ + and (attr in comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]): + out[attr] = comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr] + print("c " + attr + " " + out[attr]) + elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \ + and (attr in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]): + out[attr] = comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr] + print("d " + attr + " " + out[attr]) + elif (attr in comp.conf[B.SUBJECT_CONN]): + out[attr] = comp.conf[B.SUBJECT_CONN][attr] + print("e " + attr + " " + out[attr]) + else: + print("f " + attr + " " + out[attr]) + return out + + class DbFcts(): """ This interface defines each necessary connection to any kind of database. @@ -55,6 +99,47 @@ class DbFcts(): def setComp(self, comp): self.comp = comp + def getDbAttributes(self, table): + """ + this function collects all relevant db-attributes from any location where it can be set. + The location could be + * comp.artifact.db.[table].attr + * comp.artifact.[db].[table].attr + """ + return getDbAttributes(self.comp, table) + + def xxgetDbAttributes(self, table): + out = { + B.ATTR_DB_TABNAME: "", + B.ATTR_DB_PARTITION: DFLT_DB_PARTITION, + B.ATTR_DB_CONN_JAR: DFLT_DB_CONN_JAR + } + for attr in out.keys(): + print(attr) + if (table in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \ + and (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]): + out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr] + print("a "+attr+" "+out[attr]) + elif (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]): + out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr] + print("b "+attr+" "+out[attr]) + elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \ + and (table in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \ + and (attr in self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]): + out[attr] = self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr] + print("c " + attr+" "+out[attr]) + elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \ + and (attr in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]): + out[attr] = self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr] + print("d "+attr+" "+out[attr]) + elif (attr in self.comp.conf[B.SUBJECT_CONN]): + out[attr] = self.comp.conf[B.SUBJECT_CONN][attr] + print("e " + attr+" "+out[attr]) + else: + print("f " + attr+" "+out[attr]) + + return out + def selectTables(self): """ method to delete rows from a database statement written in sql """ diff --git a/utils/dbshive_tool.py b/utils/dbshive_tool.py new file mode 100644 index 0000000..4aaff87 --- /dev/null +++ b/utils/dbshive_tool.py @@ -0,0 +1,92 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +import basic.program +import utils.config_tool +import utils.db_abstract +import pyspark +import basic.constants as B + +class DbFcts(utils.db_abstract.DbFcts): + """ + This interface defines each necessary connection to any kind of database. + The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. + """ + def __init__(self): + pass + + + def selectRows(self, table): + """ method to select rows from a database + statement written in sql """ + tdata = {} + dry = 0 + # attr = self.getDbAttributes(table) + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "SELECT "+"*" #",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]) + cmd += " FROM "+table + sqls = self.comp.composeSqlClauses(cmd) + data = [] + for k in sqls.keys(): + sql = sqls[k] + if dry == 1: + spark = self.getConnector() + df = spark.sql(cmd) + for r in df: + data.append(r) + else: + print("select "+sql) + #self.comp.m.logInfo(cmd) + #tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER] + #tdata[B.DATA_NODE_DATA] = data + return tdata + + def deleteRows(self, table): + """ method to delete rows from a database + statement written in sql """ + job = basic.program.Job.getInstance() + dry = 0 + verify = -1+job.getDebugLevel("db_tool") + cmd = "DELETE FROM "+table + print("deleteRows "+cmd) + sqls = self.comp.composeSqlClauses(cmd) + print("deleteRows "+cmd) + print(sqls) + for k in sqls.keys(): + sql = sqls[k] + if dry == 1: + #spark = self.getConnector() + #df = spark.sql(cmd) + pass + else: + print("select "+sql) + #self.comp.m.logInfo(cmd) + + def insertRows(self, table, rows): + """ method to insert rows into a database + the rows will be interpreted by the ddl of the component + """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + spark = self.getConnector() + df = spark.createDataFrame(rows) + + self.comp.m.logInfo("cmd") + + def getConnector(self): + """ add-on-method to get the connector + this method should only called by the class itself """ + job = basic.program.Job.getInstance() + spark = pyspark.SparkSession\ + .builder\ + .appName("datest")\ + .getOrCreate() + return spark + + + +