frame for specific tool-classes db and cli

test/test_toolhandling.py

@@ -0,0 +1,42 @@
+import unittest, os
+
+import basic.program
+import utils.path_tool
+import basic.toolHandling
+import test.constants
+import components.component
+HOME_PATH = test.constants.HOME_PATH
+
+conf = {}
+
+class MyTestCase(unittest.TestCase):
+    def test_toolhandling(self):
+        job = basic.program.Job("unit")
+        args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
+                "modus": "unit"}
+        job.par.setParameterArgs(args)
+        #t = basic.toolHandling.ToolManager()
+        comp = components.component.Component()
+        comp.name = "testb"
+        comp.conf = {}
+        comp.conf["conn"] = {}
+        self.assertRaises(LookupError, basic.toolHandling.getDbTool, comp)
+
+        comp = components.component.Component()
+        comp.name = "testb"
+        comp.conf = {}
+        comp.conf["conn"] = {}
+        comp.conf["conn"]["dbtype"] = "mysql"
+        comp.conf["conn"]["clitype"] = "ssh"
+        tool = basic.toolHandling.getDbTool(comp)
+        self.assertRegex(str(type(tool)), 'dbmysql_tool.DbFcts')
+        tool = basic.toolHandling.getCliTool(comp)
+        self.assertRegex(str(type(tool)), 'clissh_tool.CliFcts')
+
+        comp.conf["conn"]["dbtype"] = "dxx"
+        comp.conf["conn"]["clitype"] = "sxx"
+        self.assertRaises(FileNotFoundError, basic.toolHandling.getDbTool, comp)
+        self.assertRaises(FileNotFoundError, basic.toolHandling.getCliTool, comp)
+
+if __name__ == '__main__':
+    unittest.main()

utils/cli_abstract.py

@@ -0,0 +1,19 @@
+import basic.program
+import utils.config_tool
+
+class CliFcts():
+    """
+    This interface defines each necessary connection to any kind of database.
+    The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
+    """
+    def __init__(self):
+        self.comp = None
+        pass
+
+    def setComp(self, comp):
+        self.comp = comp
+
+    def execCommand(self, comp, command):
+        """ method to execute the statement
+        this method should only called by the class itself """
+        raise Exception("method is not implemented")

utils/clibash_tool.py

@@ -0,0 +1,21 @@
+#
+# ----------------------------------------------------------
+"""
+This module implements the technique to interact via bash to the test-object.
+The class has to be constructed by the tool-Handling because of keyword "bash" in the configuration,
+then it will be called with the interface / abstract-class cli_abstract
+"""
+import os
+import utils.cli_abstract
+import basic
+
+class CliFcts(utils.cli_abstract.CliFcts):
+    def execCmd(self, cmds):
+        """
+
+        :param cmds:
+        :return:
+        """
+        for cmd in cmds:
+            rc = os.system(cmd)
+            self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd)

utils/clicmd_tool.py

@@ -0,0 +1,27 @@
+#
+# ----------------------------------------------------------
+"""
+This module implements the technique to interact via win-cmd to the test-object.
+The class has to be constructed by the tool-Handling because of keyword "cmd" in the configuration,
+then it will be called with the interface / abstract-class cli_abstract
+"""
+import os
+import utils.cli_abstract
+import basic
+
+class CliFcts(utils.cli_abstract.CliFcts):
+    def execCmd(self, cmds):
+        """
+        executes an array of commands on a windows-machine - the commands will be intern translated before execution
+        :param cmds: written in linux-bash
+        :return:
+        """
+        for cmd in cmds:
+            cmd = self.translate(cmd)
+            rc = os.system(cmd)
+            self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd)
+        return "ok"
+
+    def translate(self, cmd):
+        """ translates a bash-cmd (default) into a windows-cmd """
+        return cmd

utils/clihadoop_tool.py

@@ -0,0 +1,27 @@
+#
+# ----------------------------------------------------------
+"""
+This module implements the technique to interact via hadoop-cmd to the test-object.
+The class has to be constructed by the tool-Handling because of keyword "hadoop" in the configuration,
+then it will be called with the interface / abstract-class cli_abstract
+"""
+import os
+import utils.cli_abstract
+import basic
+
+class CliFcts(utils.cli_abstract.CliFcts):
+    def execCmd(self, cmds):
+        """
+        executes an array of commands on a hadoop-machine - the commands will be intern translated before execution
+        :param cmds: written in linux-bash
+        :return:
+        """
+        for cmd in cmds:
+            cmd = self.translate(cmd)
+            rc = os.system(cmd)
+            self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd)
+        return "ok"
+
+    def translate(self, cmd):
+        """ translates a bash-cmd (default) into a windows-cmd """
+        return cmd

utils/ssh_tool.py → utils/clissh_tool.py

@@ -1,29 +1,16 @@
 #
 # ----------------------------------------------------------
 """
-
+This module implements the technique to interact via ssh to the test-object.
+The class has to be constructed by the tool-Handling because of keyword "ssh" in the configuration,
+then it will be called with the interface / abstract-class cli_abstract
 """
 import os
-
+import utils.cli_abstract
 import basic
 import paramiko
 
-def getRemoteClass(comp):
-    job = basic.program.Job.getInstance()
-    verify = job.getDebugLevel("config_tool")
-    if job.conf.confs.get("tools").get("remotetyp") == "ssh":
-        return SshCmd(comp)
-
-class RemoteCmd:
-    def __init__(self):
-        self.rc = 0
-        self.sysout = ""
-    pass
-
-class SshCmd(RemoteCmd):
-    def __init__(self, comp):
-        self.conn = comp
-
+class CliFcts(utils.cli_abstract.CliFcts):
     def execCmd(self, cmds):
         """
 

utils/db_abstract.py

@@ -0,0 +1,115 @@
+import basic.program
+import utils.config_tool
+import os
+
+class DbFcts():
+    """
+    This interface defines each necessary connection to any kind of database.
+    The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
+    """
+    def __init__(self):
+        self.comp = None
+        pass
+
+    def setComp(self, comp):
+        self.comp = comp
+
+    def selectTables(self):
+        """ method to delete rows from a database
+        statement written in sql """
+        self.loadDdl()
+        tdata = {}
+        for t in self.comp.conf["ddl"]:
+            tdata[t] = self.selectRows(t)
+        return tdata
+
+    def selectRows(self, statement):
+        """ method to select rows from a database
+        statement written in sql """
+        raise Exception("method is not implemented")
+
+    def deleteTables(self):
+        """ method to delete rows from a database
+        statement written in sql """
+        self.loadDdl()
+        for t in self.comp.conf["ddl"]:
+            print("zu loeschende Tabelle "+t)
+            self.deleteRows(t)
+
+    def deleteRows(self, table):
+        """ method to delete rows from a database
+        statement written in sql """
+        raise Exception("method is not implemented")
+
+    def updateRows(self, statement):
+        """ method to delete rows from a database
+        statement written in sql """
+        raise Exception("method is not implemented")
+
+    def getConnector(self):
+        """ add-on-method to get the connector
+        this method should only called by the class itself """
+        raise Exception("method is not implemented")
+
+    def insertTables(self, tdata):
+        """ method to insert rows into a database
+        statement written in sql """
+        plainname = basic.componentHandling.getPlainCompname(self.comp.name)
+        self.loadDdl()
+        for t in self.comp.conf["ddl"]:
+            print("einzufuegende Tabelle "+t)
+            if (t in tdata[plainname]):
+                self.insertRows(t, tdata[plainname][t]["_data"])
+
+    def insertRows(self, rows):
+        """ method to insert rows into a database
+        the rows will be interpreted by the ddl of the component
+        """
+        raise Exception("method is not implemented")
+
+    def execStatement(self, statement):
+        """ add-on-method to execute the statement
+        this method should only called by the class itself """
+        raise Exception("method is not implemented")
+
+    def loadDdl(self):
+        job = basic.program.Job.getInstance()
+        if ("ddl" in self.comp.conf):
+            return
+        conf = utils.config_tool.getConfig("DATASTRUCTURE", self.comp.name)
+        self.comp.conf["ddl"] = {}
+        for k in conf[self.comp.name]:
+            self.comp.conf["ddl"][k] = conf[self.comp.name][k]
+        return conf[self.comp.name]
+
+    def getWhere(self):
+        return ""
+    def getOrder(self):
+        return ""
+
+    def getDbValue(self, fo, value):
+        if len(value.strip()) == 0 and fo["nullable"] == "y":
+            return self.getDbNull()
+        if fo["type"] == "string":
+            return "'"+value.strip()+"'"
+        elif fo["type"] == "int":
+            return value.strip()
+        elif fo["type"] == "double":
+            return self.getDbDouble(value)
+        elif fo["type"] == "float":
+            return self.getDbFloat(value)
+        elif fo["type"] == "date":
+            return self.getDbDate(value)
+        elif fo["type"] == "time":
+            return self.getDbTime(value)
+
+    def getDbDouble(self, value):
+        return value
+    def getDbFloat(self, value):
+        return value
+    def getDbDate(self, value):
+        return value
+    def getDbTime(self, value):
+        return value
+    def getDbNull(self):
+        return "null"

utils/dbcsv_tool.py

@@ -0,0 +1,83 @@
+import basic.program
+import utils.config_tool
+import utils.db_abstract
+import mysql.connector
+
+
+class DbFcts(utils.db_abstract.DbFcts):
+  """
+  This interface defines each necessary connection to any kind of database.
+  The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
+  """
+
+  def __init__(self):
+    pass
+
+  def selectRows(self, table):
+    """ method to select rows from a database
+    statement written in sql """
+    tdata={}
+    return tdata
+
+  def deleteRows(self, table):
+    """ method to delete rows from a database
+    statement written in sql """
+    job = basic.program.Job.getInstance()
+    verify = -1+job.getDebugLevel("db_tool")
+    cmd = "DELETE FROM "+table+";"
+    self.comp.m.logInfo(cmd)
+
+  def updateRows(self, statement):
+    """ method to delete rows from a database
+    statement written in sql """
+    raise Exception("method is not implemented")
+
+  def insertRows(self, table, rows):
+    """ method to insert rows into a database
+    the rows will be interpreted by the ddl of the component
+    """
+    job = basic.program.Job.getInstance()
+    verify = -1+job.getDebugLevel("db_tool")
+    cmd = "INSERT INTO "+table+";"
+    header = ""
+    for h in self.comp.conf["ddl"][table]["fields"]:
+      print(h)
+      header += ", "+h
+    cmd += " (" + header[1:]+" ) "
+    rowvalues = ""
+    for r in rows:
+      print("r-----------------")
+      print(r)
+      rowvalues = ""
+      cmd += "\n ( "
+      for h in self.comp.conf["ddl"][table]["fields"]:
+        print("h "+h)
+        if (h in r):
+          rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], r[h])
+        else:
+          rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], "")
+        print("rv " + rowvalues)
+      cmd += rowvalues[1:]+" ),"
+    cmd = cmd[0:-1]+";"
+    self.comp.m.logInfo(cmd)
+
+  def getConnector(self):
+    """ add-on-method to get the connector
+    this method should only called by the class itself """
+    job = basic.program.Job.getInstance()
+    mydb = mysql.connector.connect(
+      host = "localhost",
+      user = "datest",
+      password = "Advent!2021",
+      database = "datest"
+    )
+    return mysql
+
+  @staticmethod
+  def execStatement(self, comp, conn, statement):
+    """ add-on-method to execute the statement
+    this method should only called by the class itself """
+    raise Exception("method is not implemented")
+
+
+

utils/dbmysql_tool.py

@@ -0,0 +1,96 @@
+import basic.program
+import utils.config_tool
+import utils.db_abstract
+import mysql.connector
+
+
+class DbFcts(utils.db_abstract.DbFcts):
+  """
+  This interface defines each necessary connection to any kind of database.
+  The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
+  """
+
+  def __init__(self):
+    pass
+
+  def selectRows(self, table):
+    """ method to select rows from a database
+    statement written in sql """
+    tdata={}
+    job = basic.program.Job.getInstance()
+    verify = -1+job.getDebugLevel("db_tool")
+    cmd = "SELECT * FROM "+table+";"
+    #mycursor = self.getConnector()
+    #mycursor.execute(cmd)
+    #myresult = mycursor.fetchall()
+    tdata["_header"] = []
+    for f in self.comp.conf["ddl"][table]["fields"]:
+      tdata["_header"].append(f)
+    myresult = []
+    for x in myresult:
+      print(x)
+    self.comp.m.logInfo(cmd)
+    return tdata
+
+  def deleteRows(self, table):
+    """ method to delete rows from a database
+    statement written in sql """
+    job = basic.program.Job.getInstance()
+    verify = -1+job.getDebugLevel("db_tool")
+    cmd = "DELETE FROM "+table+";"
+    self.comp.m.logInfo(cmd)
+
+  def updateRows(self, statement):
+    """ method to delete rows from a database
+    statement written in sql """
+    raise Exception("method is not implemented")
+
+  def insertRows(self, table, rows):
+    """ method to insert rows into a database
+    the rows will be interpreted by the ddl of the component
+    """
+    job = basic.program.Job.getInstance()
+    verify = -1+job.getDebugLevel("db_tool")
+    cmd = "INSERT INTO "+table+";"
+    header = ""
+    for h in self.comp.conf["ddl"][table]["fields"]:
+      print(h)
+      header += ", "+h
+    cmd += " (" + header[1:]+" ) "
+    rowvalues = ""
+    for r in rows:
+      print("r-----------------")
+      print(r)
+      rowvalues = ""
+      cmd += "\n ( "
+      for h in self.comp.conf["ddl"][table]["fields"]:
+        print("h "+h)
+        if (h in r):
+          rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], r[h])
+        else:
+          rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], "")
+        print("rv " + rowvalues)
+      cmd += rowvalues[1:]+" ),"
+    cmd = cmd[0:-1]+";"
+    self.comp.m.logInfo(cmd)
+
+  def getConnector(self):
+    """ add-on-method to get the connector
+    this method should only called by the class itself """
+    job = basic.program.Job.getInstance()
+    mydb = mysql.connector.connect(
+      host = "localhost",
+      user = "datest",
+      password = "Advent!2021",
+      database = "datest"
+    )
+    return mysql
+
+  @staticmethod
+  def execStatement(self, comp, conn, statement):
+    """ add-on-method to execute the statement
+    this method should only called by the class itself """
+    raise Exception("method is not implemented")
+
+
+

utils/dbspark_tool.py

@@ -0,0 +1,75 @@
+import basic.program
+import utils.config_tool
+import utils.db_abstract
+import pyspark
+import basic.constants
+
+DATA_NODE_HEADER = basic.constants.DATA_NODE_HEADER
+DATA_NODE_DATA = basic.constants.DATA_NODE_DATA
+
+class DbFcts(utils.db_abstract.DbFcts):
+  """
+  This interface defines each necessary connection to any kind of database.
+  The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
+  """
+  def __init__(self):
+    pass
+  def selectRows(self, table):
+    """ method to select rows from a database
+    statement written in sql """
+    tdata={}
+    job = basic.program.Job.getInstance()
+    verify = -1+job.getDebugLevel("db_tool")
+    cmd = "SELECT "+self.comp.conf["ddl"][table]["fields"].join(",")\
+          +" FROM "+table+""+self.getWhere()+""+self.getOrder()";"
+    spark = self.getConnector()
+    df = spark.sql(cmd)
+    data = []
+    for r in df:
+      data.append(r)
+    tdata[DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"]
+    tdata[DATA_NODE_DATA] = data
+    return tdata
+
+  def deleteRows(self, table):
+    """ method to delete rows from a database
+    statement written in sql """
+    job = basic.program.Job.getInstance()
+    verify = -1+job.getDebugLevel("db_tool")
+    cmd = "DELETE FROM "+table+";"
+    self.comp.m.logInfo(cmd)
+
+  def updateRows(self, statement):
+    """ method to delete rows from a database
+    statement written in sql """
+    raise Exception("method is not implemented")
+
+  def insertRows(self, table, rows):
+    """ method to insert rows into a database
+    the rows will be interpreted by the ddl of the component
+    """
+    job = basic.program.Job.getInstance()
+    verify = -1+job.getDebugLevel("db_tool")
+    spark = self.getConnector()
+    df = spark.createDataFrame(rows)
+
+    self.comp.m.logInfo("cmd")
+
+  def getConnector(self):
+    """ add-on-method to get the connector
+    this method should only called by the class itself """
+    job = basic.program.Job.getInstance()
+    spark = pyspark.SparkSession.builder()\
+      .master("local[1]")\
+      .appName("SparkByExamples.com")\
+      .getOrCreate()
+    return spark
+
+  @staticmethod
+  def execStatement(self, comp, conn, statement):
+    """ add-on-method to execute the statement
+    this method should only called by the class itself """
+    raise Exception("method is not implemented")
+
+
+