From 5577b815060a5f0afbd8e2c8cd6ce2843905f175 Mon Sep 17 00:00:00 2001 From: Ulrich Carmesin Date: Tue, 1 Feb 2022 22:38:02 +0100 Subject: [PATCH] frame for specific tool-classes db and cli --- test/test_toolhandling.py | 42 ++++++++++ utils/cli_abstract.py | 19 +++++ utils/clibash_tool.py | 21 +++++ utils/clicmd_tool.py | 27 ++++++ utils/clihadoop_tool.py | 27 ++++++ utils/{ssh_tool.py => clissh_tool.py} | 23 ++---- utils/db_abstract.py | 115 ++++++++++++++++++++++++++ utils/dbcsv_tool.py | 83 +++++++++++++++++++ utils/dbmysql_tool.py | 96 +++++++++++++++++++++ utils/dbspark_tool.py | 75 +++++++++++++++++ 10 files changed, 510 insertions(+), 18 deletions(-) create mode 100644 test/test_toolhandling.py create mode 100644 utils/cli_abstract.py create mode 100644 utils/clibash_tool.py create mode 100644 utils/clicmd_tool.py create mode 100644 utils/clihadoop_tool.py rename utils/{ssh_tool.py => clissh_tool.py} (65%) create mode 100644 utils/db_abstract.py create mode 100644 utils/dbcsv_tool.py create mode 100644 utils/dbmysql_tool.py create mode 100644 utils/dbspark_tool.py diff --git a/test/test_toolhandling.py b/test/test_toolhandling.py new file mode 100644 index 0000000..daae2d1 --- /dev/null +++ b/test/test_toolhandling.py @@ -0,0 +1,42 @@ +import unittest, os + +import basic.program +import utils.path_tool +import basic.toolHandling +import test.constants +import components.component +HOME_PATH = test.constants.HOME_PATH + +conf = {} + +class MyTestCase(unittest.TestCase): + def test_toolhandling(self): + job = basic.program.Job("unit") + args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", + "modus": "unit"} + job.par.setParameterArgs(args) + #t = basic.toolHandling.ToolManager() + comp = components.component.Component() + comp.name = "testb" + comp.conf = {} + comp.conf["conn"] = {} + self.assertRaises(LookupError, basic.toolHandling.getDbTool, comp) + + comp = components.component.Component() + comp.name = "testb" + comp.conf = {} + comp.conf["conn"] = {} + comp.conf["conn"]["dbtype"] = "mysql" + comp.conf["conn"]["clitype"] = "ssh" + tool = basic.toolHandling.getDbTool(comp) + self.assertRegex(str(type(tool)), 'dbmysql_tool.DbFcts') + tool = basic.toolHandling.getCliTool(comp) + self.assertRegex(str(type(tool)), 'clissh_tool.CliFcts') + + comp.conf["conn"]["dbtype"] = "dxx" + comp.conf["conn"]["clitype"] = "sxx" + self.assertRaises(FileNotFoundError, basic.toolHandling.getDbTool, comp) + self.assertRaises(FileNotFoundError, basic.toolHandling.getCliTool, comp) + +if __name__ == '__main__': + unittest.main() diff --git a/utils/cli_abstract.py b/utils/cli_abstract.py new file mode 100644 index 0000000..9e70982 --- /dev/null +++ b/utils/cli_abstract.py @@ -0,0 +1,19 @@ +import basic.program +import utils.config_tool + +class CliFcts(): + """ + This interface defines each necessary connection to any kind of database. + The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. + """ + def __init__(self): + self.comp = None + pass + + def setComp(self, comp): + self.comp = comp + + def execCommand(self, comp, command): + """ method to execute the statement + this method should only called by the class itself """ + raise Exception("method is not implemented") diff --git a/utils/clibash_tool.py b/utils/clibash_tool.py new file mode 100644 index 0000000..3322e2f --- /dev/null +++ b/utils/clibash_tool.py @@ -0,0 +1,21 @@ +# +# ---------------------------------------------------------- +""" +This module implements the technique to interact via bash to the test-object. +The class has to be constructed by the tool-Handling because of keyword "bash" in the configuration, +then it will be called with the interface / abstract-class cli_abstract +""" +import os +import utils.cli_abstract +import basic + +class CliFcts(utils.cli_abstract.CliFcts): + def execCmd(self, cmds): + """ + + :param cmds: + :return: + """ + for cmd in cmds: + rc = os.system(cmd) + self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd) \ No newline at end of file diff --git a/utils/clicmd_tool.py b/utils/clicmd_tool.py new file mode 100644 index 0000000..5bb7294 --- /dev/null +++ b/utils/clicmd_tool.py @@ -0,0 +1,27 @@ +# +# ---------------------------------------------------------- +""" +This module implements the technique to interact via win-cmd to the test-object. +The class has to be constructed by the tool-Handling because of keyword "cmd" in the configuration, +then it will be called with the interface / abstract-class cli_abstract +""" +import os +import utils.cli_abstract +import basic + +class CliFcts(utils.cli_abstract.CliFcts): + def execCmd(self, cmds): + """ + executes an array of commands on a windows-machine - the commands will be intern translated before execution + :param cmds: written in linux-bash + :return: + """ + for cmd in cmds: + cmd = self.translate(cmd) + rc = os.system(cmd) + self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd) + return "ok" + + def translate(self, cmd): + """ translates a bash-cmd (default) into a windows-cmd """ + return cmd \ No newline at end of file diff --git a/utils/clihadoop_tool.py b/utils/clihadoop_tool.py new file mode 100644 index 0000000..fcb5862 --- /dev/null +++ b/utils/clihadoop_tool.py @@ -0,0 +1,27 @@ +# +# ---------------------------------------------------------- +""" +This module implements the technique to interact via hadoop-cmd to the test-object. +The class has to be constructed by the tool-Handling because of keyword "hadoop" in the configuration, +then it will be called with the interface / abstract-class cli_abstract +""" +import os +import utils.cli_abstract +import basic + +class CliFcts(utils.cli_abstract.CliFcts): + def execCmd(self, cmds): + """ + executes an array of commands on a hadoop-machine - the commands will be intern translated before execution + :param cmds: written in linux-bash + :return: + """ + for cmd in cmds: + cmd = self.translate(cmd) + rc = os.system(cmd) + self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd) + return "ok" + + def translate(self, cmd): + """ translates a bash-cmd (default) into a windows-cmd """ + return cmd \ No newline at end of file diff --git a/utils/ssh_tool.py b/utils/clissh_tool.py similarity index 65% rename from utils/ssh_tool.py rename to utils/clissh_tool.py index a52160c..ebbc61e 100644 --- a/utils/ssh_tool.py +++ b/utils/clissh_tool.py @@ -1,29 +1,16 @@ # # ---------------------------------------------------------- """ - +This module implements the technique to interact via ssh to the test-object. +The class has to be constructed by the tool-Handling because of keyword "ssh" in the configuration, +then it will be called with the interface / abstract-class cli_abstract """ import os - +import utils.cli_abstract import basic import paramiko -def getRemoteClass(comp): - job = basic.program.Job.getInstance() - verify = job.getDebugLevel("config_tool") - if job.conf.confs.get("tools").get("remotetyp") == "ssh": - return SshCmd(comp) - -class RemoteCmd: - def __init__(self): - self.rc = 0 - self.sysout = "" - pass - -class SshCmd(RemoteCmd): - def __init__(self, comp): - self.conn = comp - +class CliFcts(utils.cli_abstract.CliFcts): def execCmd(self, cmds): """ diff --git a/utils/db_abstract.py b/utils/db_abstract.py new file mode 100644 index 0000000..4a213e5 --- /dev/null +++ b/utils/db_abstract.py @@ -0,0 +1,115 @@ +import basic.program +import utils.config_tool +import os + +class DbFcts(): + """ + This interface defines each necessary connection to any kind of database. + The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. + """ + def __init__(self): + self.comp = None + pass + + def setComp(self, comp): + self.comp = comp + + def selectTables(self): + """ method to delete rows from a database + statement written in sql """ + self.loadDdl() + tdata = {} + for t in self.comp.conf["ddl"]: + tdata[t] = self.selectRows(t) + return tdata + + def selectRows(self, statement): + """ method to select rows from a database + statement written in sql """ + raise Exception("method is not implemented") + + def deleteTables(self): + """ method to delete rows from a database + statement written in sql """ + self.loadDdl() + for t in self.comp.conf["ddl"]: + print("zu loeschende Tabelle "+t) + self.deleteRows(t) + + def deleteRows(self, table): + """ method to delete rows from a database + statement written in sql """ + raise Exception("method is not implemented") + + def updateRows(self, statement): + """ method to delete rows from a database + statement written in sql """ + raise Exception("method is not implemented") + + def getConnector(self): + """ add-on-method to get the connector + this method should only called by the class itself """ + raise Exception("method is not implemented") + + def insertTables(self, tdata): + """ method to insert rows into a database + statement written in sql """ + plainname = basic.componentHandling.getPlainCompname(self.comp.name) + self.loadDdl() + for t in self.comp.conf["ddl"]: + print("einzufuegende Tabelle "+t) + if (t in tdata[plainname]): + self.insertRows(t, tdata[plainname][t]["_data"]) + + def insertRows(self, rows): + """ method to insert rows into a database + the rows will be interpreted by the ddl of the component + """ + raise Exception("method is not implemented") + + def execStatement(self, statement): + """ add-on-method to execute the statement + this method should only called by the class itself """ + raise Exception("method is not implemented") + + def loadDdl(self): + job = basic.program.Job.getInstance() + if ("ddl" in self.comp.conf): + return + conf = utils.config_tool.getConfig("DATASTRUCTURE", self.comp.name) + self.comp.conf["ddl"] = {} + for k in conf[self.comp.name]: + self.comp.conf["ddl"][k] = conf[self.comp.name][k] + return conf[self.comp.name] + + def getWhere(self): + return "" + def getOrder(self): + return "" + + def getDbValue(self, fo, value): + if len(value.strip()) == 0 and fo["nullable"] == "y": + return self.getDbNull() + if fo["type"] == "string": + return "'"+value.strip()+"'" + elif fo["type"] == "int": + return value.strip() + elif fo["type"] == "double": + return self.getDbDouble(value) + elif fo["type"] == "float": + return self.getDbFloat(value) + elif fo["type"] == "date": + return self.getDbDate(value) + elif fo["type"] == "time": + return self.getDbTime(value) + + def getDbDouble(self, value): + return value + def getDbFloat(self, value): + return value + def getDbDate(self, value): + return value + def getDbTime(self, value): + return value + def getDbNull(self): + return "null" \ No newline at end of file diff --git a/utils/dbcsv_tool.py b/utils/dbcsv_tool.py new file mode 100644 index 0000000..df25117 --- /dev/null +++ b/utils/dbcsv_tool.py @@ -0,0 +1,83 @@ +import basic.program +import utils.config_tool +import utils.db_abstract +import mysql.connector + + +class DbFcts(utils.db_abstract.DbFcts): + """ + This interface defines each necessary connection to any kind of database. + The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. + """ + + def __init__(self): + pass + + def selectRows(self, table): + """ method to select rows from a database + statement written in sql """ + tdata={} + return tdata + + def deleteRows(self, table): + """ method to delete rows from a database + statement written in sql """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "DELETE FROM "+table+";" + self.comp.m.logInfo(cmd) + + def updateRows(self, statement): + """ method to delete rows from a database + statement written in sql """ + raise Exception("method is not implemented") + + def insertRows(self, table, rows): + """ method to insert rows into a database + the rows will be interpreted by the ddl of the component + """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "INSERT INTO "+table+";" + header = "" + for h in self.comp.conf["ddl"][table]["fields"]: + print(h) + header += ", "+h + cmd += " (" + header[1:]+" ) " + rowvalues = "" + for r in rows: + print("r-----------------") + print(r) + rowvalues = "" + cmd += "\n ( " + for h in self.comp.conf["ddl"][table]["fields"]: + print("h "+h) + if (h in r): + rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], r[h]) + else: + rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], "") + print("rv " + rowvalues) + cmd += rowvalues[1:]+" )," + cmd = cmd[0:-1]+";" + self.comp.m.logInfo(cmd) + + def getConnector(self): + """ add-on-method to get the connector + this method should only called by the class itself """ + job = basic.program.Job.getInstance() + mydb = mysql.connector.connect( + host = "localhost", + user = "datest", + password = "Advent!2021", + database = "datest" + ) + return mysql + + @staticmethod + def execStatement(self, comp, conn, statement): + """ add-on-method to execute the statement + this method should only called by the class itself """ + raise Exception("method is not implemented") + + + diff --git a/utils/dbmysql_tool.py b/utils/dbmysql_tool.py new file mode 100644 index 0000000..64d665a --- /dev/null +++ b/utils/dbmysql_tool.py @@ -0,0 +1,96 @@ +import basic.program +import utils.config_tool +import utils.db_abstract +import mysql.connector + + +class DbFcts(utils.db_abstract.DbFcts): + """ + This interface defines each necessary connection to any kind of database. + The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. + """ + + def __init__(self): + pass + + def selectRows(self, table): + """ method to select rows from a database + statement written in sql """ + tdata={} + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "SELECT * FROM "+table+";" + #mycursor = self.getConnector() + #mycursor.execute(cmd) + #myresult = mycursor.fetchall() + tdata["_header"] = [] + for f in self.comp.conf["ddl"][table]["fields"]: + tdata["_header"].append(f) + myresult = [] + for x in myresult: + print(x) + self.comp.m.logInfo(cmd) + return tdata + + def deleteRows(self, table): + """ method to delete rows from a database + statement written in sql """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "DELETE FROM "+table+";" + self.comp.m.logInfo(cmd) + + def updateRows(self, statement): + """ method to delete rows from a database + statement written in sql """ + raise Exception("method is not implemented") + + def insertRows(self, table, rows): + """ method to insert rows into a database + the rows will be interpreted by the ddl of the component + """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "INSERT INTO "+table+";" + header = "" + for h in self.comp.conf["ddl"][table]["fields"]: + print(h) + header += ", "+h + cmd += " (" + header[1:]+" ) " + rowvalues = "" + for r in rows: + print("r-----------------") + print(r) + rowvalues = "" + cmd += "\n ( " + for h in self.comp.conf["ddl"][table]["fields"]: + print("h "+h) + if (h in r): + rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], r[h]) + else: + rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], "") + print("rv " + rowvalues) + cmd += rowvalues[1:]+" )," + cmd = cmd[0:-1]+";" + self.comp.m.logInfo(cmd) + + def getConnector(self): + """ add-on-method to get the connector + this method should only called by the class itself """ + job = basic.program.Job.getInstance() + mydb = mysql.connector.connect( + host = "localhost", + user = "datest", + password = "Advent!2021", + database = "datest" + ) + return mysql + + @staticmethod + def execStatement(self, comp, conn, statement): + """ add-on-method to execute the statement + this method should only called by the class itself """ + raise Exception("method is not implemented") + + + diff --git a/utils/dbspark_tool.py b/utils/dbspark_tool.py new file mode 100644 index 0000000..311258b --- /dev/null +++ b/utils/dbspark_tool.py @@ -0,0 +1,75 @@ +import basic.program +import utils.config_tool +import utils.db_abstract +import pyspark +import basic.constants + +DATA_NODE_HEADER = basic.constants.DATA_NODE_HEADER +DATA_NODE_DATA = basic.constants.DATA_NODE_DATA + +class DbFcts(utils.db_abstract.DbFcts): + """ + This interface defines each necessary connection to any kind of database. + The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. + """ + def __init__(self): + pass + def selectRows(self, table): + """ method to select rows from a database + statement written in sql """ + tdata={} + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "SELECT "+self.comp.conf["ddl"][table]["fields"].join(",")\ + +" FROM "+table+""+self.getWhere()+""+self.getOrder()";" + spark = self.getConnector() + df = spark.sql(cmd) + data = [] + for r in df: + data.append(r) + tdata[DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"] + tdata[DATA_NODE_DATA] = data + return tdata + + def deleteRows(self, table): + """ method to delete rows from a database + statement written in sql """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "DELETE FROM "+table+";" + self.comp.m.logInfo(cmd) + + def updateRows(self, statement): + """ method to delete rows from a database + statement written in sql """ + raise Exception("method is not implemented") + + def insertRows(self, table, rows): + """ method to insert rows into a database + the rows will be interpreted by the ddl of the component + """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + spark = self.getConnector() + df = spark.createDataFrame(rows) + + self.comp.m.logInfo("cmd") + + def getConnector(self): + """ add-on-method to get the connector + this method should only called by the class itself """ + job = basic.program.Job.getInstance() + spark = pyspark.SparkSession.builder()\ + .master("local[1]")\ + .appName("SparkByExamples.com")\ + .getOrCreate() + return spark + + @staticmethod + def execStatement(self, comp, conn, statement): + """ add-on-method to execute the statement + this method should only called by the class itself """ + raise Exception("method is not implemented") + + +