From efb414413139f0e11ee1b5258aa14bde0d8b3b4a Mon Sep 17 00:00:00 2001 From: Ulrich Carmesin Date: Tue, 1 Mar 2022 22:11:04 +0100 Subject: [PATCH] central get attributes --- utils/api_abstract.py | 41 ++++++++++++++++++ utils/dbsfile_tool.py | 96 +++++++++++++++++++++++++++++++++++++++++++ utils/tdata_tool.py | 39 ++++++++++++------ 3 files changed, 163 insertions(+), 13 deletions(-) create mode 100644 utils/api_abstract.py create mode 100644 utils/dbsfile_tool.py diff --git a/utils/api_abstract.py b/utils/api_abstract.py new file mode 100644 index 0000000..5337ff6 --- /dev/null +++ b/utils/api_abstract.py @@ -0,0 +1,41 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +""" +This abstract class CliFcts defines the interface for the execution of any command-line-functions. +It uses the following configuration + .a) COMP.conf->artifact->cli->[system] : for structural attributes of the operating-system \n + .c) COMP.conf->conn->[cli] : for connection-attributes and structural attributes, + maybe specialized for the operating-system + +The attribute type resolves which technique is used, implemented in a specific tool-class: + * cmd,bash,powersh ... for specific local operating-system + * ssh,hadoop ... for specific remote operating-system +The main tasks are: \n +.1. executing the command-array - with attributes + * conn.ip, host, port, user, password, ... for synchronous db-connection + * conn.root, ... for path-definitions for file-implementations (csv, ) + +""" +import basic.program +import utils.config_tool + +class CliFcts(): + """ + This interface defines each necessary connection to any kind of database. + The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. + """ + def __init__(self): + self.comp = None + pass + + def setComp(self, comp): + self.comp = comp + + def execCommand(self, comp, command): + """ method to execute the statement + this method should only called by the class itself """ + raise Exception("method is not implemented") diff --git a/utils/dbsfile_tool.py b/utils/dbsfile_tool.py new file mode 100644 index 0000000..9599167 --- /dev/null +++ b/utils/dbsfile_tool.py @@ -0,0 +1,96 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +import basic.program +import utils.config_tool +import utils.db_abstract +import pyspark +import basic.constants as B + +class DbFcts(utils.db_abstract.DbFcts): + """ + This interface defines each necessary connection to any kind of database. + The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. + """ + def __init__(self): + pass + + + def selectRows(self, table): + """ method to select rows from a database + statement written in sql """ + tdata = {} + dry = 0 + # attr = self.getDbAttributes(table) + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + cmd = "SELECT "+"*" #",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]) + cmd += " FROM "+table + sqls = self.comp.composeSqlClauses(cmd) + data = [] + for k in sqls.keys(): + sql = sqls[k] + if dry == 1: + spark = self.getConnector() + df = spark.sql(sql) + for r in df: + data.append(r) + else: + print("select "+sql) + #self.comp.m.logInfo(cmd) + #tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER] + #tdata[B.DATA_NODE_DATA] = data + return tdata + + def deleteRows(self, table): + """ method to delete rows from a database + statement written in sql """ + job = basic.program.Job.getInstance() + dry = 0 + verify = -1+job.getDebugLevel("db_tool") + cmd = "DELETE FROM "+table + print("deleteRows "+cmd) + sqls = self.comp.composeSqlClauses(cmd) + print("deleteRows "+cmd) + print(sqls) + for k in sqls.keys(): + sql = sqls[k] + if dry == 1: + #spark = self.getConnector() + #df = spark.sql(cmd) + pass + else: + print("select "+sql) + #self.comp.m.logInfo(cmd) + + def insertRows(self, table, rows): + """ method to insert rows into a database + the rows will be interpreted by the ddl of the component + """ + job = basic.program.Job.getInstance() + verify = -1+job.getDebugLevel("db_tool") + spark = self.getConnector() + df = spark.createDataFrame(rows) + + self.comp.m.logInfo("cmd") + + def getConnector(self): + """ add-on-method to get the connector + this method should only called by the class itself """ + job = basic.program.Job.getInstance() + attr = self.getDbAttributes("null") + spark = None + if B.ATTR_DB_CONN_JAR in attr: + spark = pyspark.SparkSession\ + .builder\ + .appName("datest")\ + .config("sparkjar", "")\ + .getOrCreate() + return spark + + + + diff --git a/utils/tdata_tool.py b/utils/tdata_tool.py index f1b59fe..f2db61a 100644 --- a/utils/tdata_tool.py +++ b/utils/tdata_tool.py @@ -26,8 +26,8 @@ import basic.program import utils.file_tool import basic.constants as B -DATA_SOURCE_DIR = "dir" -DATA_SOURCE_CSV = "csv" +DATA_SRC_DIR = "dir" +DATA_SRC_CSV = "csv" CSV_HEADER_START = ["node", "table", "tabelle"] CSV_DELIMITER = ";" @@ -35,8 +35,21 @@ CSV_DELIMITER = ";" CSV_SPECTYPE_DATA = "data" CSV_SPECTYPE_TREE = "tree" -def getSourcefile(filename): - pass +ATTR_SRC_TYPE = "tdtyp" +ATTR_SRC_DATA = "tdsrc" +ATTR_SRC_NAME = "tdname" + +def getTdataAttr(): + job = basic.program.Job.getInstance() + out = {} # + out[ATTR_SRC_TYPE] = DATA_SRC_DIR + out[ATTR_SRC_NAME] = getattr(job.par, B.PAR_TESTCASE) + for p in [ATTR_SRC_TYPE, ATTR_SRC_DATA, ATTR_SRC_NAME]: + out[p] = "" + if getattr(job.par, p): + out[p] = getattr(job.par, p) + return out + def getTestdata(): """ @@ -50,22 +63,22 @@ def getTestdata(): reftyp = getattr(job.par, "tdtyp") source = getattr(job.par, "tdsrc") criteria = getattr(job.par, "tdname") - tdata = {"reftyp": reftyp, "source": source, "criteria": criteria} - if reftyp == "flaskdb": + tdata = getTdataAttr() # {"reftyp": reftyp, "source": source, "criteria": criteria} + if tdata[ATTR_SRC_TYPE] == "flaskdb": # read data-structure with sourcename # connect to source # select with all data with datastructure - job.m.setInfo("Test-Data readed from " + reftyp + " for " + criteria) - elif reftyp == "csv": + job.m.setInfo("Test-Data readed from " + tdata[ATTR_SRC_TYPE] + " for " + tdata[ATTR_SRC_NAME]) + elif tdata[ATTR_SRC_TYPE] == DATA_SRC_CSV: # read file in testdata - job.m.logInfo("Test-Data readed from " + reftyp + " for " + criteria) - elif reftyp == "dir": - filename = os.path.join(job.conf.getJobConf("paths:testdata"), source, "testspec.csv") + job.m.logInfo("Test-Data readed from " + tdata[ATTR_SRC_TYPE] + " for " + tdata[ATTR_SRC_NAME]) + elif tdata[ATTR_SRC_TYPE] == DATA_SRC_DIR: + filename = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.ATTR_PATH_TDATA), tdata[ATTR_SRC_NAME], "testspec.csv") data = getCsvSpec(job.m, filename, CSV_SPECTYPE_DATA) for k in data: tdata[k] = data[k] else: - job.m.setFatal("test-Data: reftyp " + reftyp + " is not implemented") + job.m.setFatal("test-Data: reftyp " + tdata[ATTR_SRC_TYPE] + " is not implemented") return tdata def getCsvSpec(msg, filename, type): @@ -248,7 +261,7 @@ def writeDataTable(filename, tdata, comp): text += "\n" for f in tdata[B.DATA_NODE_HEADER]: if f in r: - text += ";"+r[f] + text += ";"+str(r[f]) else: text += ";" text += "\n"