Browse Source

frame for specific tool-classes db and cli

master
Ulrich Carmesin 3 years ago
parent
commit
5577b81506
  1. 42
      test/test_toolhandling.py
  2. 19
      utils/cli_abstract.py
  3. 21
      utils/clibash_tool.py
  4. 27
      utils/clicmd_tool.py
  5. 27
      utils/clihadoop_tool.py
  6. 23
      utils/clissh_tool.py
  7. 115
      utils/db_abstract.py
  8. 83
      utils/dbcsv_tool.py
  9. 96
      utils/dbmysql_tool.py
  10. 75
      utils/dbspark_tool.py

42
test/test_toolhandling.py

@ -0,0 +1,42 @@
import unittest, os
import basic.program
import utils.path_tool
import basic.toolHandling
import test.constants
import components.component
HOME_PATH = test.constants.HOME_PATH
conf = {}
class MyTestCase(unittest.TestCase):
def test_toolhandling(self):
job = basic.program.Job("unit")
args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool",
"modus": "unit"}
job.par.setParameterArgs(args)
#t = basic.toolHandling.ToolManager()
comp = components.component.Component()
comp.name = "testb"
comp.conf = {}
comp.conf["conn"] = {}
self.assertRaises(LookupError, basic.toolHandling.getDbTool, comp)
comp = components.component.Component()
comp.name = "testb"
comp.conf = {}
comp.conf["conn"] = {}
comp.conf["conn"]["dbtype"] = "mysql"
comp.conf["conn"]["clitype"] = "ssh"
tool = basic.toolHandling.getDbTool(comp)
self.assertRegex(str(type(tool)), 'dbmysql_tool.DbFcts')
tool = basic.toolHandling.getCliTool(comp)
self.assertRegex(str(type(tool)), 'clissh_tool.CliFcts')
comp.conf["conn"]["dbtype"] = "dxx"
comp.conf["conn"]["clitype"] = "sxx"
self.assertRaises(FileNotFoundError, basic.toolHandling.getDbTool, comp)
self.assertRaises(FileNotFoundError, basic.toolHandling.getCliTool, comp)
if __name__ == '__main__':
unittest.main()

19
utils/cli_abstract.py

@ -0,0 +1,19 @@
import basic.program
import utils.config_tool
class CliFcts():
"""
This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
"""
def __init__(self):
self.comp = None
pass
def setComp(self, comp):
self.comp = comp
def execCommand(self, comp, command):
""" method to execute the statement
this method should only called by the class itself """
raise Exception("method is not implemented")

21
utils/clibash_tool.py

@ -0,0 +1,21 @@
#
# ----------------------------------------------------------
"""
This module implements the technique to interact via bash to the test-object.
The class has to be constructed by the tool-Handling because of keyword "bash" in the configuration,
then it will be called with the interface / abstract-class cli_abstract
"""
import os
import utils.cli_abstract
import basic
class CliFcts(utils.cli_abstract.CliFcts):
def execCmd(self, cmds):
"""
:param cmds:
:return:
"""
for cmd in cmds:
rc = os.system(cmd)
self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd)

27
utils/clicmd_tool.py

@ -0,0 +1,27 @@
#
# ----------------------------------------------------------
"""
This module implements the technique to interact via win-cmd to the test-object.
The class has to be constructed by the tool-Handling because of keyword "cmd" in the configuration,
then it will be called with the interface / abstract-class cli_abstract
"""
import os
import utils.cli_abstract
import basic
class CliFcts(utils.cli_abstract.CliFcts):
def execCmd(self, cmds):
"""
executes an array of commands on a windows-machine - the commands will be intern translated before execution
:param cmds: written in linux-bash
:return:
"""
for cmd in cmds:
cmd = self.translate(cmd)
rc = os.system(cmd)
self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd)
return "ok"
def translate(self, cmd):
""" translates a bash-cmd (default) into a windows-cmd """
return cmd

27
utils/clihadoop_tool.py

@ -0,0 +1,27 @@
#
# ----------------------------------------------------------
"""
This module implements the technique to interact via hadoop-cmd to the test-object.
The class has to be constructed by the tool-Handling because of keyword "hadoop" in the configuration,
then it will be called with the interface / abstract-class cli_abstract
"""
import os
import utils.cli_abstract
import basic
class CliFcts(utils.cli_abstract.CliFcts):
def execCmd(self, cmds):
"""
executes an array of commands on a hadoop-machine - the commands will be intern translated before execution
:param cmds: written in linux-bash
:return:
"""
for cmd in cmds:
cmd = self.translate(cmd)
rc = os.system(cmd)
self.comp.m.logInfo("RC "+str(rc)+" zu CMD "+cmd)
return "ok"
def translate(self, cmd):
""" translates a bash-cmd (default) into a windows-cmd """
return cmd

23
utils/ssh_tool.py → utils/clissh_tool.py

@ -1,29 +1,16 @@
# #
# ---------------------------------------------------------- # ----------------------------------------------------------
""" """
This module implements the technique to interact via ssh to the test-object.
The class has to be constructed by the tool-Handling because of keyword "ssh" in the configuration,
then it will be called with the interface / abstract-class cli_abstract
""" """
import os import os
import utils.cli_abstract
import basic import basic
import paramiko import paramiko
def getRemoteClass(comp): class CliFcts(utils.cli_abstract.CliFcts):
job = basic.program.Job.getInstance()
verify = job.getDebugLevel("config_tool")
if job.conf.confs.get("tools").get("remotetyp") == "ssh":
return SshCmd(comp)
class RemoteCmd:
def __init__(self):
self.rc = 0
self.sysout = ""
pass
class SshCmd(RemoteCmd):
def __init__(self, comp):
self.conn = comp
def execCmd(self, cmds): def execCmd(self, cmds):
""" """

115
utils/db_abstract.py

@ -0,0 +1,115 @@
import basic.program
import utils.config_tool
import os
class DbFcts():
"""
This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
"""
def __init__(self):
self.comp = None
pass
def setComp(self, comp):
self.comp = comp
def selectTables(self):
""" method to delete rows from a database
statement written in sql """
self.loadDdl()
tdata = {}
for t in self.comp.conf["ddl"]:
tdata[t] = self.selectRows(t)
return tdata
def selectRows(self, statement):
""" method to select rows from a database
statement written in sql """
raise Exception("method is not implemented")
def deleteTables(self):
""" method to delete rows from a database
statement written in sql """
self.loadDdl()
for t in self.comp.conf["ddl"]:
print("zu loeschende Tabelle "+t)
self.deleteRows(t)
def deleteRows(self, table):
""" method to delete rows from a database
statement written in sql """
raise Exception("method is not implemented")
def updateRows(self, statement):
""" method to delete rows from a database
statement written in sql """
raise Exception("method is not implemented")
def getConnector(self):
""" add-on-method to get the connector
this method should only called by the class itself """
raise Exception("method is not implemented")
def insertTables(self, tdata):
""" method to insert rows into a database
statement written in sql """
plainname = basic.componentHandling.getPlainCompname(self.comp.name)
self.loadDdl()
for t in self.comp.conf["ddl"]:
print("einzufuegende Tabelle "+t)
if (t in tdata[plainname]):
self.insertRows(t, tdata[plainname][t]["_data"])
def insertRows(self, rows):
""" method to insert rows into a database
the rows will be interpreted by the ddl of the component
"""
raise Exception("method is not implemented")
def execStatement(self, statement):
""" add-on-method to execute the statement
this method should only called by the class itself """
raise Exception("method is not implemented")
def loadDdl(self):
job = basic.program.Job.getInstance()
if ("ddl" in self.comp.conf):
return
conf = utils.config_tool.getConfig("DATASTRUCTURE", self.comp.name)
self.comp.conf["ddl"] = {}
for k in conf[self.comp.name]:
self.comp.conf["ddl"][k] = conf[self.comp.name][k]
return conf[self.comp.name]
def getWhere(self):
return ""
def getOrder(self):
return ""
def getDbValue(self, fo, value):
if len(value.strip()) == 0 and fo["nullable"] == "y":
return self.getDbNull()
if fo["type"] == "string":
return "'"+value.strip()+"'"
elif fo["type"] == "int":
return value.strip()
elif fo["type"] == "double":
return self.getDbDouble(value)
elif fo["type"] == "float":
return self.getDbFloat(value)
elif fo["type"] == "date":
return self.getDbDate(value)
elif fo["type"] == "time":
return self.getDbTime(value)
def getDbDouble(self, value):
return value
def getDbFloat(self, value):
return value
def getDbDate(self, value):
return value
def getDbTime(self, value):
return value
def getDbNull(self):
return "null"

83
utils/dbcsv_tool.py

@ -0,0 +1,83 @@
import basic.program
import utils.config_tool
import utils.db_abstract
import mysql.connector
class DbFcts(utils.db_abstract.DbFcts):
"""
This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
"""
def __init__(self):
pass
def selectRows(self, table):
""" method to select rows from a database
statement written in sql """
tdata={}
return tdata
def deleteRows(self, table):
""" method to delete rows from a database
statement written in sql """
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "DELETE FROM "+table+";"
self.comp.m.logInfo(cmd)
def updateRows(self, statement):
""" method to delete rows from a database
statement written in sql """
raise Exception("method is not implemented")
def insertRows(self, table, rows):
""" method to insert rows into a database
the rows will be interpreted by the ddl of the component
"""
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "INSERT INTO "+table+";"
header = ""
for h in self.comp.conf["ddl"][table]["fields"]:
print(h)
header += ", "+h
cmd += " (" + header[1:]+" ) "
rowvalues = ""
for r in rows:
print("r-----------------")
print(r)
rowvalues = ""
cmd += "\n ( "
for h in self.comp.conf["ddl"][table]["fields"]:
print("h "+h)
if (h in r):
rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], r[h])
else:
rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], "")
print("rv " + rowvalues)
cmd += rowvalues[1:]+" ),"
cmd = cmd[0:-1]+";"
self.comp.m.logInfo(cmd)
def getConnector(self):
""" add-on-method to get the connector
this method should only called by the class itself """
job = basic.program.Job.getInstance()
mydb = mysql.connector.connect(
host = "localhost",
user = "datest",
password = "Advent!2021",
database = "datest"
)
return mysql
@staticmethod
def execStatement(self, comp, conn, statement):
""" add-on-method to execute the statement
this method should only called by the class itself """
raise Exception("method is not implemented")

96
utils/dbmysql_tool.py

@ -0,0 +1,96 @@
import basic.program
import utils.config_tool
import utils.db_abstract
import mysql.connector
class DbFcts(utils.db_abstract.DbFcts):
"""
This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
"""
def __init__(self):
pass
def selectRows(self, table):
""" method to select rows from a database
statement written in sql """
tdata={}
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "SELECT * FROM "+table+";"
#mycursor = self.getConnector()
#mycursor.execute(cmd)
#myresult = mycursor.fetchall()
tdata["_header"] = []
for f in self.comp.conf["ddl"][table]["fields"]:
tdata["_header"].append(f)
myresult = []
for x in myresult:
print(x)
self.comp.m.logInfo(cmd)
return tdata
def deleteRows(self, table):
""" method to delete rows from a database
statement written in sql """
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "DELETE FROM "+table+";"
self.comp.m.logInfo(cmd)
def updateRows(self, statement):
""" method to delete rows from a database
statement written in sql """
raise Exception("method is not implemented")
def insertRows(self, table, rows):
""" method to insert rows into a database
the rows will be interpreted by the ddl of the component
"""
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "INSERT INTO "+table+";"
header = ""
for h in self.comp.conf["ddl"][table]["fields"]:
print(h)
header += ", "+h
cmd += " (" + header[1:]+" ) "
rowvalues = ""
for r in rows:
print("r-----------------")
print(r)
rowvalues = ""
cmd += "\n ( "
for h in self.comp.conf["ddl"][table]["fields"]:
print("h "+h)
if (h in r):
rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], r[h])
else:
rowvalues += ", "+self.getDbValue(self.comp.conf["ddl"][table]["data"][h], "")
print("rv " + rowvalues)
cmd += rowvalues[1:]+" ),"
cmd = cmd[0:-1]+";"
self.comp.m.logInfo(cmd)
def getConnector(self):
""" add-on-method to get the connector
this method should only called by the class itself """
job = basic.program.Job.getInstance()
mydb = mysql.connector.connect(
host = "localhost",
user = "datest",
password = "Advent!2021",
database = "datest"
)
return mysql
@staticmethod
def execStatement(self, comp, conn, statement):
""" add-on-method to execute the statement
this method should only called by the class itself """
raise Exception("method is not implemented")

75
utils/dbspark_tool.py

@ -0,0 +1,75 @@
import basic.program
import utils.config_tool
import utils.db_abstract
import pyspark
import basic.constants
DATA_NODE_HEADER = basic.constants.DATA_NODE_HEADER
DATA_NODE_DATA = basic.constants.DATA_NODE_DATA
class DbFcts(utils.db_abstract.DbFcts):
"""
This interface defines each necessary connection to any kind of database.
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
"""
def __init__(self):
pass
def selectRows(self, table):
""" method to select rows from a database
statement written in sql """
tdata={}
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "SELECT "+self.comp.conf["ddl"][table]["fields"].join(",")\
+" FROM "+table+""+self.getWhere()+""+self.getOrder()";"
spark = self.getConnector()
df = spark.sql(cmd)
data = []
for r in df:
data.append(r)
tdata[DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"]
tdata[DATA_NODE_DATA] = data
return tdata
def deleteRows(self, table):
""" method to delete rows from a database
statement written in sql """
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "DELETE FROM "+table+";"
self.comp.m.logInfo(cmd)
def updateRows(self, statement):
""" method to delete rows from a database
statement written in sql """
raise Exception("method is not implemented")
def insertRows(self, table, rows):
""" method to insert rows into a database
the rows will be interpreted by the ddl of the component
"""
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
spark = self.getConnector()
df = spark.createDataFrame(rows)
self.comp.m.logInfo("cmd")
def getConnector(self):
""" add-on-method to get the connector
this method should only called by the class itself """
job = basic.program.Job.getInstance()
spark = pyspark.SparkSession.builder()\
.master("local[1]")\
.appName("SparkByExamples.com")\
.getOrCreate()
return spark
@staticmethod
def execStatement(self, comp, conn, statement):
""" add-on-method to execute the statement
this method should only called by the class itself """
raise Exception("method is not implemented")
Loading…
Cancel
Save