central get attributes

utils/api_abstract.py

@@ -0,0 +1,41 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
+"""
+This abstract class CliFcts defines the interface for the execution of any command-line-functions.
+It uses the following configuration
+  .a) COMP.conf->artifact->cli->[system] : for structural attributes of the operating-system \n
+  .c) COMP.conf->conn->[cli] : for connection-attributes and structural attributes,
+  maybe specialized for the operating-system
+
+The attribute type resolves which technique is used, implemented in a specific tool-class:
+  * cmd,bash,powersh ... for specific local operating-system
+  * ssh,hadoop ... for specific remote operating-system
+The main tasks are: \n
+.1. executing the command-array - with attributes
+   * conn.ip, host, port, user, password, ... for synchronous db-connection
+   * conn.root, ... for path-definitions for file-implementations (csv, )
+
+"""
+import basic.program
+import utils.config_tool
+
+class CliFcts():
+    """
+    This interface defines each necessary connection to any kind of database.
+    The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
+    """
+    def __init__(self):
+        self.comp = None
+        pass
+
+    def setComp(self, comp):
+        self.comp = comp
+
+    def execCommand(self, comp, command):
+        """ method to execute the statement
+        this method should only called by the class itself """
+        raise Exception("method is not implemented")

utils/dbsfile_tool.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
+import basic.program
+import utils.config_tool
+import utils.db_abstract
+import pyspark
+import basic.constants as B
+
+class DbFcts(utils.db_abstract.DbFcts):
+    """
+    This interface defines each necessary connection to any kind of database.
+    The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
+    """
+    def __init__(self):
+        pass
+
+
+    def selectRows(self, table):
+        """ method to select rows from a database
+        statement written in sql """
+        tdata = {}
+        dry = 0
+        # attr = self.getDbAttributes(table)
+        job = basic.program.Job.getInstance()
+        verify = -1+job.getDebugLevel("db_tool")
+        cmd = "SELECT "+"*" #",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER])
+        cmd += " FROM "+table
+        sqls = self.comp.composeSqlClauses(cmd)
+        data = []
+        for k in sqls.keys():
+            sql = sqls[k]
+            if dry == 1:
+                spark = self.getConnector()
+                df = spark.sql(sql)
+                for r in df:
+                    data.append(r)
+            else:
+                print("select "+sql)
+            #self.comp.m.logInfo(cmd)
+        #tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]
+        #tdata[B.DATA_NODE_DATA] = data
+        return tdata
+
+    def deleteRows(self, table):
+        """ method to delete rows from a database
+        statement written in sql """
+        job = basic.program.Job.getInstance()
+        dry = 0
+        verify = -1+job.getDebugLevel("db_tool")
+        cmd = "DELETE FROM "+table
+        print("deleteRows "+cmd)
+        sqls = self.comp.composeSqlClauses(cmd)
+        print("deleteRows "+cmd)
+        print(sqls)
+        for k in sqls.keys():
+            sql = sqls[k]
+            if dry == 1:
+                #spark = self.getConnector()
+                #df = spark.sql(cmd)
+                pass
+            else:
+                print("select "+sql)
+            #self.comp.m.logInfo(cmd)
+
+    def insertRows(self, table, rows):
+        """ method to insert rows into a database
+        the rows will be interpreted by the ddl of the component
+        """
+        job = basic.program.Job.getInstance()
+        verify = -1+job.getDebugLevel("db_tool")
+        spark = self.getConnector()
+        df = spark.createDataFrame(rows)
+
+        self.comp.m.logInfo("cmd")
+
+    def getConnector(self):
+        """ add-on-method to get the connector
+        this method should only called by the class itself """
+        job = basic.program.Job.getInstance()
+        attr = self.getDbAttributes("null")
+        spark = None
+        if B.ATTR_DB_CONN_JAR in attr:
+            spark = pyspark.SparkSession\
+                .builder\
+                .appName("datest")\
+                .config("sparkjar", "")\
+                .getOrCreate()
+        return spark
+
+
+
+

utils/tdata_tool.py

@@ -26,8 +26,8 @@ import basic.program
 import utils.file_tool
 import basic.constants as B
 
-DATA_SOURCE_DIR = "dir"
+DATA_SRC_DIR = "dir"
-DATA_SOURCE_CSV = "csv"
+DATA_SRC_CSV = "csv"
 
 CSV_HEADER_START = ["node", "table", "tabelle"]
 CSV_DELIMITER = ";"
@@ -35,8 +35,21 @@ CSV_DELIMITER = ";"
 CSV_SPECTYPE_DATA = "data"
 CSV_SPECTYPE_TREE = "tree"
 
-def getSourcefile(filename):
+ATTR_SRC_TYPE = "tdtyp"
-    pass
+ATTR_SRC_DATA = "tdsrc"
+ATTR_SRC_NAME = "tdname"
+
+def getTdataAttr():
+    job = basic.program.Job.getInstance()
+    out = {} #
+    out[ATTR_SRC_TYPE] = DATA_SRC_DIR
+    out[ATTR_SRC_NAME] = getattr(job.par, B.PAR_TESTCASE)
+    for p in [ATTR_SRC_TYPE, ATTR_SRC_DATA, ATTR_SRC_NAME]:
+        out[p] = ""
+        if getattr(job.par, p):
+            out[p] = getattr(job.par, p)
+    return out
+
 
 def getTestdata():
     """
@@ -50,22 +63,22 @@ def getTestdata():
     reftyp = getattr(job.par, "tdtyp")
     source = getattr(job.par, "tdsrc")
     criteria = getattr(job.par, "tdname")
-    tdata = {"reftyp": reftyp, "source": source, "criteria": criteria}
+    tdata = getTdataAttr()   # {"reftyp": reftyp, "source": source, "criteria": criteria}
-    if reftyp == "flaskdb":
+    if tdata[ATTR_SRC_TYPE] == "flaskdb":
         # read data-structure with sourcename
         # connect to source
         # select with all data with datastructure
-        job.m.setInfo("Test-Data readed from " + reftyp + " for " + criteria)
+        job.m.setInfo("Test-Data readed from " + tdata[ATTR_SRC_TYPE] + " for " + tdata[ATTR_SRC_NAME])
-    elif reftyp == "csv":
+    elif tdata[ATTR_SRC_TYPE] == DATA_SRC_CSV:
         # read file in testdata
-        job.m.logInfo("Test-Data readed from " + reftyp + " for " + criteria)
+        job.m.logInfo("Test-Data readed from " + tdata[ATTR_SRC_TYPE] + " for " + tdata[ATTR_SRC_NAME])
-    elif reftyp == "dir":
+    elif tdata[ATTR_SRC_TYPE] == DATA_SRC_DIR:
-        filename = os.path.join(job.conf.getJobConf("paths:testdata"), source, "testspec.csv")
+        filename = os.path.join(job.conf.getJobConf(B.SUBJECT_PATH+":"+B.ATTR_PATH_TDATA), tdata[ATTR_SRC_NAME], "testspec.csv")
         data = getCsvSpec(job.m, filename, CSV_SPECTYPE_DATA)
         for k in data:
             tdata[k] = data[k]
     else:
-        job.m.setFatal("test-Data: reftyp " + reftyp + " is not implemented")
+        job.m.setFatal("test-Data: reftyp " + tdata[ATTR_SRC_TYPE] + " is not implemented")
     return tdata
 
 def getCsvSpec(msg, filename, type):
@@ -248,7 +261,7 @@ def writeDataTable(filename, tdata, comp):
         text += "\n"
         for f in tdata[B.DATA_NODE_HEADER]:
             if f in r:
-                text += ";"+r[f]
+                text += ";"+str(r[f])
             else:
                 text += ";"
     text += "\n"