db-tool part1

basic/constants.py

@@ -66,6 +66,16 @@ DATA_NODE_COMP = "comp"
 DATA_NODE_PAR  = "par"
 """ This constant defines """
 TOPIC_NODE_DB = "db"
+ATTR_DB_PARTITION = "partitioned"
+""" attribute if table is partitioned - partitions are parametrized """
+ATTR_DB_CONN_JAR = "conn_jar_name"
+""" attribute for connection-jar-file instead of connection by ip, port """
+ATTR_DB_TABNAME = "tabname"
+""" attribute in order to use a different technical name for the db-table """
+PAR_DB_WHERE = "dbwhere"
+""" optional parameter with a where-clause """
+PAR_DB_PARTITION = "dbparts"
+""" optional parameter for partitions of a partitioned tables """
 TOPIC_NODE_CLI = "cli"
 TOPIC_NODE_API = "api"
 
@@ -107,6 +117,13 @@ ATTR_INST_SUBCOMP = "components"
 SUBJECT_FCT  = "function"      #     |     |     |    | x  |         main-programs
 
 SUBJECT_ARTS = "artifact"      #     |     |     |    | x  |         Component
+""" 
+in this subject-node are each kind of result of any component with the structure:
+  * topic (db, cli, api, ...)
+  * * general attributes - to-know: technical attributes are stored in connection-tree
+  * * partial-component  - to-know: the existence of db-tables can be defined in the ddl-tree
+  * * * specific attributes - it overrides the general attributes
+"""
 ATTR_ARTS_DB = "db"
 ATTR_ARTS_LOG = "log"
 ATTR_ARTS_LOB = "lob"

components/testexec.py

@@ -35,6 +35,9 @@ import utils.db_abstract
 import basic.toolHandling
 import components.component
 import basic.componentHandling
+import utils.db_abstract
+import basic.constants as B
+
 
 class Testexecuter():
     def prepare_system(self, granularity):
@@ -116,6 +119,37 @@ class Testexecuter():
         self.m.setMsg("readInstance for " + self.name + " is OK")
         self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper())
 
+
+    def composeSqlClauses(self, sql):
+        job = basic.program.Job.getInstance()
+        out = {}
+        print("composeSqlClauses "+sql)
+        table = sql[sql.upper().index(" FROM ")+6:].strip()
+        print(table)
+        sql_new = sql[0:sql.upper().index(" FROM ")+5]
+        print(sql_new)
+        attr = utils.db_abstract.getDbAttributes(self, table)
+        if attr[B.ATTR_DB_TABNAME] != "":
+            table = self.conf[B.SUBJECT_ARTS][table][B.ATTR_DB_TABNAME]
+        sql_new += " " + table
+        print(sql_new)
+        if (hasattr(job.par, B.PAR_DB_WHERE)):
+            sql_new += " WHERE "+getattr(job.par, B.PAR_DB_WHERE)
+        if sql_new[0:6] == "SELECT":
+            sql_new += " ORDER BY id"
+        print(sql_new)
+        sql_new = sql_new.replace('!', "\'")
+        if (hasattr(job.par, B.PAR_DB_PARTITION)) and (attr[B.ATTR_DB_PARTITION] != "n"):
+            parts = getattr(job.par, B.PAR_DB_PARTITION)
+            a = parts.split(",")
+            for p in a:
+                sql_part = sql_new
+                sql_part = sql_part.replace(attr[B.ATTR_DB_PARTITION], p)
+                out[p] = sql_part
+        else:
+            out["ALL"] = sql_new
+        return out
+
     def test_System(self, granularity):
         """
 

test/test_db.py

@@ -0,0 +1,54 @@
+import unittest, os
+
+import basic.program
+import utils.path_tool
+import basic.toolHandling
+import test.constants
+import components.component
+import basic.constants as B
+HOME_PATH = test.constants.HOME_PATH
+
+conf = {}
+
+class MyTestCase(unittest.TestCase):
+
+    def test_toolhandling(self):
+        job = basic.program.Job("unit")
+        args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
+                "modus": "unit"}
+        job.par.setParameterArgs(args)
+        #t = basic.toolHandling.ToolManager()
+        comp = components.component.Component()
+        comp.name = "testb"
+        comp.conf = {}
+        comp.conf[B.SUBJECT_ARTS] = {}
+        comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB] = {}
+        comp.conf[B.SUBJECT_CONN] = {}
+        comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB] = {}
+        comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][B.ATTR_TYPE] = "shive"
+        tool = basic.toolHandling.getDbTool(comp)
+        self.assertRegex(str(type(tool)), 'dbshive_tool.DbFcts')
+        attr = tool.getDbAttributes("xx")
+        self.assertRegex(attr[B.ATTR_DB_PARTITION], 'n')
+        comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "y"
+        attr = tool.getDbAttributes("xx")
+        self.assertRegex(attr[B.ATTR_DB_PARTITION], 'y')
+        comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "z"
+        attr = tool.getDbAttributes("xx")
+        self.assertRegex(attr[B.ATTR_DB_PARTITION], 'z')
+        #
+        sqls = comp.composeSqlClauses("SELECT * FROM lofts")
+        print(sqls)
+        setattr(job.par, B.PAR_DB_WHERE, "name like !%utz%! and regname = !+reg+!")
+        comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][B.ATTR_DB_PARTITION] = "+reg+"
+        sqls = comp.composeSqlClauses("SELECT * FROM lofts")
+        print(sqls)
+        setattr(job.par, B.PAR_DB_PARTITION, "BA,FV")
+        sqls = comp.composeSqlClauses("SELECT * FROM lofts")
+        print(sqls)
+        tool.deleteRows("deltable")
+        tool.selectRows("deltable")
+
+
+if __name__ == '__main__':
+    unittest.main()

utils/db_abstract.py

@@ -43,6 +43,50 @@ import utils.config_tool
 import basic.constants as B
 import os
 
+DFLT_DB_PARTITION = "n"
+""" attribute if table is partitioned - partitions are parametrized """
+DFLT_DB_CONN_JAR = "n"
+""" attribute for connection-jar-file instead of connection by ip, port """
+
+
+def getDbAttributes(comp, table):
+    """
+    this function collects all relevant db-attributes from any location where it can be set.
+    The location could be
+      * comp.artifact.db.[table].attr
+      * comp.artifact.[db].[table].attr
+    """
+    out = {
+        B.ATTR_DB_TABNAME: "",
+        B.ATTR_DB_PARTITION: DFLT_DB_PARTITION,
+        B.ATTR_DB_CONN_JAR: DFLT_DB_CONN_JAR
+    }
+    for attr in out.keys():
+        print(attr)
+        if (table in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \
+                and (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]):
+            out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr]
+            print("a " + attr + " " + out[attr])
+        elif (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]):
+            out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr]
+            print("b " + attr + " " + out[attr])
+        elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \
+                and (table in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \
+                and (attr in comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]):
+            out[attr] = comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr]
+            print("c " + attr + " " + out[attr])
+        elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \
+                and (attr in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]):
+            out[attr] = comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr]
+            print("d " + attr + " " + out[attr])
+        elif (attr in comp.conf[B.SUBJECT_CONN]):
+            out[attr] = comp.conf[B.SUBJECT_CONN][attr]
+            print("e " + attr + " " + out[attr])
+        else:
+            print("f " + attr + " " + out[attr])
+    return out
+
+
 class DbFcts():
     """
     This interface defines each necessary connection to any kind of database.
@@ -55,6 +99,47 @@ class DbFcts():
     def setComp(self, comp):
         self.comp = comp
 
+    def getDbAttributes(self, table):
+        """
+        this function collects all relevant db-attributes from any location where it can be set.
+        The location could be
+          * comp.artifact.db.[table].attr
+          * comp.artifact.[db].[table].attr
+        """
+        return getDbAttributes(self.comp, table)
+
+    def xxgetDbAttributes(self, table):
+        out = {
+            B.ATTR_DB_TABNAME: "",
+            B.ATTR_DB_PARTITION: DFLT_DB_PARTITION,
+            B.ATTR_DB_CONN_JAR: DFLT_DB_CONN_JAR
+        }
+        for attr in out.keys():
+            print(attr)
+            if (table in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \
+            and (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]):
+                out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr]
+                print("a "+attr+" "+out[attr])
+            elif (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]):
+                out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr]
+                print("b "+attr+" "+out[attr])
+            elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \
+            and (table in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \
+            and (attr in self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]):
+                out[attr] = self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr]
+                print("c " + attr+" "+out[attr])
+            elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \
+            and (attr in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]):
+                 out[attr] = self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr]
+                 print("d "+attr+" "+out[attr])
+            elif (attr in self.comp.conf[B.SUBJECT_CONN]):
+                out[attr] = self.comp.conf[B.SUBJECT_CONN][attr]
+                print("e " + attr+" "+out[attr])
+            else:
+                print("f " + attr+" "+out[attr])
+
+        return out
+
     def selectTables(self):
         """ method to delete rows from a database
         statement written in sql """

utils/dbshive_tool.py

@@ -0,0 +1,92 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
+import basic.program
+import utils.config_tool
+import utils.db_abstract
+import pyspark
+import basic.constants as B
+
+class DbFcts(utils.db_abstract.DbFcts):
+    """
+    This interface defines each necessary connection to any kind of database.
+    The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
+    """
+    def __init__(self):
+        pass
+
+
+    def selectRows(self, table):
+        """ method to select rows from a database
+        statement written in sql """
+        tdata = {}
+        dry = 0
+        # attr = self.getDbAttributes(table)
+        job = basic.program.Job.getInstance()
+        verify = -1+job.getDebugLevel("db_tool")
+        cmd = "SELECT "+"*" #",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER])
+        cmd += " FROM "+table
+        sqls = self.comp.composeSqlClauses(cmd)
+        data = []
+        for k in sqls.keys():
+            sql = sqls[k]
+            if dry == 1:
+                spark = self.getConnector()
+                df = spark.sql(cmd)
+                for r in df:
+                    data.append(r)
+            else:
+                print("select "+sql)
+            #self.comp.m.logInfo(cmd)
+        #tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
+        #tdata[B.DATA_NODE_DATA] = data
+        return tdata
+
+    def deleteRows(self, table):
+        """ method to delete rows from a database
+        statement written in sql """
+        job = basic.program.Job.getInstance()
+        dry = 0
+        verify = -1+job.getDebugLevel("db_tool")
+        cmd = "DELETE FROM "+table
+        print("deleteRows "+cmd)
+        sqls = self.comp.composeSqlClauses(cmd)
+        print("deleteRows "+cmd)
+        print(sqls)
+        for k in sqls.keys():
+            sql = sqls[k]
+            if dry == 1:
+                #spark = self.getConnector()
+                #df = spark.sql(cmd)
+                pass
+            else:
+                print("select "+sql)
+            #self.comp.m.logInfo(cmd)
+
+    def insertRows(self, table, rows):
+        """ method to insert rows into a database
+        the rows will be interpreted by the ddl of the component
+        """
+        job = basic.program.Job.getInstance()
+        verify = -1+job.getDebugLevel("db_tool")
+        spark = self.getConnector()
+        df = spark.createDataFrame(rows)
+
+        self.comp.m.logInfo("cmd")
+
+    def getConnector(self):
+        """ add-on-method to get the connector
+        this method should only called by the class itself """
+        job = basic.program.Job.getInstance()
+        spark = pyspark.SparkSession\
+            .builder\
+            .appName("datest")\
+            .getOrCreate()
+        return spark
+
+
+
+