some comments with constants

basic/componentHandling.py

@@ -1,3 +1,9 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 # managing the components
 # -----------------------------------------------------------------------------
 """
@@ -16,6 +22,7 @@ import basic.message
 import components.component
 import importlib
 import copy
+import basic.constants as B
 
 comps = {}
 PARAM_NOSUBNODE = ["artifact", "components", "instance"]
@@ -127,7 +134,7 @@ class ComponentManager:
             c = class_()
             c.name = name
             c.conf = confs["conf"]
-            c.conf["conn"] = conns[0]
+            c.conf[B.SUBJECT_CONN] = conns[0]
             c.m = basic.message.Message(basic.message.LIMIT_DEBUG, "logTime", name)
             c.init()
             print("createComponent 4 a " + componentName)
@@ -148,7 +155,7 @@ class ComponentManager:
                 c = class_()
                 c.name = name
                 c.conf = confs["conf"]
-                c.conf["conn"] = conns[0]
+                c.conf[B.SUBJECT_CONN] = conns[0]
                 c.m = basic.message.Message(basic.message.LIMIT_DEBUG, "logTime", name)
                 c.init()
                 print("createComponent 4 b " + componentName)

basic/constants.py

@@ -1,17 +1,84 @@
-#
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
+"""
+The constants desribes the keywords of the main datastructures, these are
+* the configurations of
+  * basic-configuration for global variables in job - stored in external conf-folder
+  * comp-configuration for component-attributes  - stored in internal component-folder in
+    + ddl-configuration of an entity of the component as attributes - stored in internal component-folder
+    + environment-config for connection-attributes of the component - stored in external environment-folder
+  * tool-configuration
+  * test-specification with testdata - stored in external testdata-folder
+* the internal datastructure
+
+"""
 # the internal datastructure is a tree with this design:
 # root {                  : constant
 # + option {              : constant
-#   +
 # + steps
 # + comp {                : variable component-name
 #   + substructure {      : variable maybe scheme, table of a database-component
 #   + + _header    [      : constant
 #       - fields          : variable field-name
-DATA_NODE_DATA = "_data"
+DATA_NODE_HEADER = "_header"
+""" This constant defines a subnode of a table for the column-names """
+DATA_NODE_DATA   = "_data"
+""" This constant defines a subnode of a table for the data which are set as key-value-pair with the column-names as key """
+DATA_NODE_STEPS  = "_steps"
+""" This constant defines the main node in the testdata for the steps to execute """
+DATA_NODE_OPTION = "_option"
+""" This constant defines main node in the testdata for testcase specific parameters """
+DATA_NODE_DDL  = "ddl"
+""" This constant defines """
 
 # the configuration of a component or tool
 # entity {              : variable name of the group, basic, component-name or tool-name
 # + subject       {     : variable subject-name - it correspondends to a tool
 # + + sub-subject {     : variable subject-name - it correspondends to a tool
 #     + attributes      : constant of the tool in which the attribute ist implemented
+
+# the main subjects            # prog basic envir tool comp testcase main implentation module
+SUBJECT_PATH = "paths"         #     | x   |     | x  |    |         path_tool, config_tool
+""" This constant defines the subject in order to define paths of filesystem of any testuse """
+ATTR_PATH_MODE = "mode"
+""" This constant defines the home-folder in filesystem of test """
+ATTR_PATH_HOME = "home"
+""" This constant defines the home-folder in testing-filesystem  """
+ATTR_PATH_DEBUG = "debugs"
+""" This constant defines the debug-folder in testing-filesystem  """
+ATTR_PATH_ARCHIV = "archiv"
+""" This constant defines the archiv-folder in testing-filesystem  """
+ATTR_PATH_SPECIAL = "results"
+""" This constant defines the debug-folder in testing-filesystem  """
+ATTR_PATH_PROGRAM = "program"
+""" This constant defines the program-folder in testing-filesystem  """
+ATTR_PATH_ENV = "environment"
+""" This constant defines the environment-folder in testing-filesystem, used for configs related to environments """
+ATTR_PATH_RELEASE = "release"
+""" This constant defines the release-folder in testing-filesystem, used for configs related to release """
+ATTR_PATH_TDATA = "testdata"
+""" This constant defines the testdata-folder in testing-filesystem with the testcase-specifications """
+ATTR_PATH_PATTN = "pattern"
+""" This constant defines the debug-folder in testing-filesystem  """
+SUBJECT_APPS = "applications"  #     | x   |     |    |    |         CompHandling
+
+SUBJECT_INST = "instance"      #     |     |     |    | x  |         CompHanlding
+ATTR_COUNT   = "count"         #     |     |     |    | x  |         CompHanlding
+
+SUBJECT_FCT  = "function"      #     |     |     |    | x  |         main-programs
+
+SUBJECT_ARTS = "artifact"      #     |     |     |    | x  |         Component
+ATTR_ARTS_DB = "db"
+ATTR_ARTS_LOG = "log"
+ATTR_ARTS_LOB = "lob"
+SUBJECT_DB   = "databases"     #     |     |     |    | #  |         db*_tools, match_tool
+
+SUBJECT_CONN = "conn"          #     |     | x   |    |    |         conn_tool, db*_tools, cli*_toold
+ATTR_CONN_DBTYPE = "dbtype"    #     | x   | x   |    | x  |         conn_tool, toolHandling, db*_tools
+ATTR_CONN_CLITYPE = "clitype"  #     | x   | x   |    | x  |         conn_tool, toolHandling, cli*_tools
+
+RULE_ACCEPTANCE = "acceptance" #     |     |     |    | x  |         tools_match

basic/message.py

@@ -58,8 +58,8 @@ class Message:
         self.componente = componente  # dezantrales Logsystem
         #self.level = level  # Vorgabe zu level zum Filtern, ob auszugebe
         job = basic.program.Job.getInstance()
-        print(vars(job))
-        print(globals())
+        # print(vars(job))
+        # print(globals())
         # exit(5)
         verify = LIMIT_DEBUG
         self.initErrorTyp()

basic/program.py

@@ -1,11 +1,16 @@
-#!/usr/bin/python3
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 # Template Batchrahmen
 #
 #import sys, getopt
 import argparse
 import copy
 
-import yaml
+import yaml, os
 from datetime import datetime
 
 import basic.message
@@ -198,7 +203,7 @@ class Job:
 	def getTraceLevel(self, elem):
 		return self.getMessageLevel("trace", elem)
 	def debug(self, prio, text):
-		print("job.debug "+str(prio)+" "+text)
+		#print("job.debug "+str(prio)+" "+text)
 		if hasattr(self, "m"):
 			self.m.debug(prio, text)
 		else:
@@ -318,6 +323,10 @@ class Configuration:
 		self.program = program
 		print (f"job initialisiert {self.program}")
 		if program == "unit":
+			if (os.path.exists('../conf/basis.yml')):
+				self.setConfiguration('../conf/basis.yml')
+				return
+		elif (os.path.exists('conf/basis.yml')):
 			self.setConfiguration('../conf/basis.yml')
 			return
 		self.setConfiguration('conf/basis.yml')
@@ -326,8 +335,12 @@ class Configuration:
 		self.confs = {}
 		with open(path, "r") as file:
 			doc = yaml.full_load(file)
-			for i, v in doc.items():
-				self.confs[i] = v
+			if "basic" in doc:
+				for i, v in doc["basic"].items():
+					self.confs[i] = v
+			else:
+				for i, v in doc.items():
+					self.confs[i] = v
 	def setConfig(self, path, val):
 		a = path.split(".")
 		if len(a) == 1:

basic/toolHandling.py

@@ -1,6 +1,12 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 import importlib, os
-
 import basic.program
+import basic.constants as B
 # -------------------------------------------------
 
 def hasAttr(o, name):
@@ -24,9 +30,9 @@ Toolmanager
 def getDbTool(comp):
     job = basic.program.Job.getInstance()
     verify = int(job.getDebugLevel("db_tool"))
-    if not hasAttr(comp.conf["conn"], "dbtype"):
-        if hasAttr(comp.conf["conn"], "types") and hasAttr(comp.conf["conn"]["types"], "dbtype"):
-            dbtype = comp.conf["conn"]["types"]["dbtype"]
+    if not hasAttr(comp.conf[B.SUBJECT_CONN], "dbtype"):
+        if hasAttr(comp.conf[B.SUBJECT_CONN], "types") and hasAttr(comp.conf[B.SUBJECT_CONN]["types"], "dbtype"):
+            dbtype = comp.conf[B.SUBJECT_CONN]["types"]["dbtype"]
         else:
             raise LookupError("dbtype is not set in comp " + comp.name)
     else:

components/component.py

@@ -1,3 +1,9 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 # abstract class for instance components
 # ---------------------------------------------------------------------
 from datetime import datetime
@@ -23,7 +29,7 @@ class CompData:
 
 #class Component(components.sysmonitor.SystemMonitor, components.testexec.Testexecuter, components.report.Report,
 #                components.maintain.Maintainer, components.catalog, threading.Thread):
-class Component():
+class Component(components.testexec.Testexecuter):
     """
     A component represents an application of the system-under-test or a data-artifact which is created from the system-under-test.
     As the representation it has to knowlegde of the url, which other components depends on this component.

utils/config_tool.py

@@ -1,6 +1,12 @@
-#
-# --------------------------------------------------------------
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 import sys
+import basic.constants as B
+
 try:
     import basic.program
 except ImportError:

utils/db_abstract.py

@@ -1,5 +1,12 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 import basic.program
 import utils.config_tool
+import basic.constants as B
 import os
 
 class DbFcts():
@@ -19,7 +26,7 @@ class DbFcts():
         statement written in sql """
         self.loadDdl()
         tdata = {}
-        for t in self.comp.conf["ddl"]:
+        for t in self.comp.conf[B.DATA_NODE_DDL]:
             tdata[t] = self.selectRows(t)
         return tdata
 
@@ -32,7 +39,7 @@ class DbFcts():
         """ method to delete rows from a database
         statement written in sql """
         self.loadDdl()
-        for t in self.comp.conf["ddl"]:
+        for t in self.comp.conf[B.DATA_NODE_DDL]:
             print("zu loeschende Tabelle "+t)
             self.deleteRows(t)
 
@@ -59,7 +66,7 @@ class DbFcts():
         for t in self.comp.conf["ddl"]:
             print("einzufuegende Tabelle "+t)
             if (t in tdata[plainname]):
-                self.insertRows(t, tdata[plainname][t]["_data"])
+                self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA])
 
     def insertRows(self, rows):
         """ method to insert rows into a database
@@ -74,12 +81,12 @@ class DbFcts():
 
     def loadDdl(self):
         job = basic.program.Job.getInstance()
-        if ("ddl" in self.comp.conf):
+        if (B.DATA_NODE_DDL in self.comp.conf):
             return
         conf = utils.config_tool.getConfig("DATASTRUCTURE", self.comp.name)
-        self.comp.conf["ddl"] = {}
+        self.comp.conf[B.DATA_NODE_DDL] = {}
         for k in conf[self.comp.name]:
-            self.comp.conf["ddl"][k] = conf[self.comp.name][k]
+            self.comp.conf[B.DATA_NODE_DDL][k] = conf[self.comp.name][k]
         return conf[self.comp.name]
 
     def getWhere(self):

utils/dbcsv_tool.py

@@ -1,7 +1,12 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 import basic.program
 import utils.config_tool
 import utils.db_abstract
-import mysql.connector
 
 
 class DbFcts(utils.db_abstract.DbFcts):
@@ -16,7 +21,7 @@ class DbFcts(utils.db_abstract.DbFcts):
   def selectRows(self, table):
     """ method to select rows from a database
     statement written in sql """
-    tdata={}
+    tdata = {}
     return tdata
 
   def deleteRows(self, table):
@@ -65,19 +70,7 @@ class DbFcts(utils.db_abstract.DbFcts):
     """ add-on-method to get the connector
     this method should only called by the class itself """
     job = basic.program.Job.getInstance()
-    mydb = mysql.connector.connect(
-      host = "localhost",
-      user = "datest",
-      password = "Advent!2021",
-      database = "datest"
-    )
-    return mysql
-
-  @staticmethod
-  def execStatement(self, comp, conn, statement):
-    """ add-on-method to execute the statement
-    this method should only called by the class itself """
-    raise Exception("method is not implemented")
+    return ""
 
 
 

utils/dbmysql_tool.py

@@ -1,8 +1,14 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 import basic.program
 import utils.config_tool
 import utils.db_abstract
 import mysql.connector
-
+import basic.constants as B
 
 class DbFcts(utils.db_abstract.DbFcts):
   """
@@ -16,16 +22,16 @@ class DbFcts(utils.db_abstract.DbFcts):
   def selectRows(self, table):
     """ method to select rows from a database
     statement written in sql """
-    tdata={}
+    tdata = {}
     job = basic.program.Job.getInstance()
     verify = -1+job.getDebugLevel("db_tool")
     cmd = "SELECT * FROM "+table+";"
     #mycursor = self.getConnector()
     #mycursor.execute(cmd)
     #myresult = mycursor.fetchall()
-    tdata["_header"] = []
-    for f in self.comp.conf["ddl"][table]["fields"]:
-      tdata["_header"].append(f)
+    tdata[B.DATA_NODE_HEADER] = []
+    for f in self.comp.conf["ddl"][table][B.DATA_NODE_HEADER]:
+      tdata[B.DATA_NODE_HEADER].append(f)
     myresult = []
     for x in myresult:
       print(x)

utils/dbspark_tool.py

@@ -1,11 +1,14 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 import basic.program
 import utils.config_tool
 import utils.db_abstract
 import pyspark
-import basic.constants
-
-DATA_NODE_HEADER = basic.constants.DATA_NODE_HEADER
-DATA_NODE_DATA = basic.constants.DATA_NODE_DATA
+import basic.constants as B
 
 class DbFcts(utils.db_abstract.DbFcts):
   """
@@ -17,18 +20,18 @@ class DbFcts(utils.db_abstract.DbFcts):
   def selectRows(self, table):
     """ method to select rows from a database
     statement written in sql """
-    tdata={}
+    tdata = {}
     job = basic.program.Job.getInstance()
     verify = -1+job.getDebugLevel("db_tool")
-    cmd = "SELECT "+self.comp.conf["ddl"][table]["fields"].join(",")\
-          +" FROM "+table+""+self.getWhere()+""+self.getOrder()";"
+    cmd = "SELECT "+",".join(self.comp.conf["ddl"][table]["fields"])
+    cmd += " FROM "+table+""+self.getWhere()+""+self.getOrder()
     spark = self.getConnector()
     df = spark.sql(cmd)
     data = []
     for r in df:
       data.append(r)
-    tdata[DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"]
-    tdata[DATA_NODE_DATA] = data
+    tdata[B.DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"]
+    tdata[B.DATA_NODE_DATA] = data
     return tdata
 
   def deleteRows(self, table):
@@ -39,11 +42,6 @@ class DbFcts(utils.db_abstract.DbFcts):
     cmd = "DELETE FROM "+table+";"
     self.comp.m.logInfo(cmd)
 
-  def updateRows(self, statement):
-    """ method to delete rows from a database
-    statement written in sql """
-    raise Exception("method is not implemented")
-
   def insertRows(self, table, rows):
     """ method to insert rows into a database
     the rows will be interpreted by the ddl of the component
@@ -59,17 +57,12 @@ class DbFcts(utils.db_abstract.DbFcts):
     """ add-on-method to get the connector
     this method should only called by the class itself """
     job = basic.program.Job.getInstance()
-    spark = pyspark.SparkSession.builder()\
-      .master("local[1]")\
-      .appName("SparkByExamples.com")\
+    spark = pyspark.SparkSession\
+      .builder\
+      .appName("datest")\
       .getOrCreate()
     return spark
 
-  @staticmethod
-  def execStatement(self, comp, conn, statement):
-    """ add-on-method to execute the statement
-    this method should only called by the class itself """
-    raise Exception("method is not implemented")
 
 
 

utils/match_tool.py

@@ -1,7 +1,13 @@
-#
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 import json
 import utils.css_tool
 import basic.program
+import basic.constants as B
 # ------------------------------------------------------------
 """
 
@@ -366,7 +372,7 @@ def compareRows(matching, path):
     htmltext = "<p>Tabelle : "+table+"</p>"
     htmltext += "<table>"
     htmltext += "<tr><th></th>"
-    for f in ddl["_header"]:
+    for f in ddl[B.DATA_NODE_HEADER]:
         job.debug(verify, "ddl " + f + " ")
         header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
         htmltext += "<th>"+f+"</th>"

utils/path_tool.py

@@ -1,11 +1,15 @@
-# All functions related to the full path.
-# It implements the team conventions of the directory structure
-# -----------------------------------------------------------------------
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 """ In diesem Modul werden alle Funktionen zusammengefasst zur Generierung und Ermittlung von pathsn """
 import sys
 import basic.program
 import utils.config_tool
 import re
+import basic.constants as B
 
 
 def getKeyValue(key):
@@ -17,7 +21,7 @@ def getKeyValue(key):
         neu = job.getParameter(key[8:])
         return neu
     elif 'job.conf' in key:
-        neu = job.conf.confs["paths"][key[9:]]
+        neu = job.conf.confs[B.SUBJECT_PATH][key[9:]]
         job.debug(verify, neu)
         return neu
         # return job.conf.confs["paths"][key[9:]]

utils/tdata_tool.py

@@ -1,5 +1,9 @@
-#
-# ---------------------------------------------------------------------
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
 """
 the issue of this tool is to transform extern data to the internal structure and the internal structure into extern data - i.e. mostly test-results.
 * * * * * * * *
@@ -111,7 +115,7 @@ def getCsvSpec(msg, filename, type):
             if (a[0] not in data):
                 data[a[0]] = {}
                 if len(a) == 1 and type == CSV_SPECTYPE_DATA:
-                    data[a[0]]["_data"] = []
+                    data[a[0]][B.DATA_NODE_DATA] = []
                 # its a component
             if len(a) > 1 and a[1] not in data[a[0]]:
                 data[a[0]][a[1]] = {}
@@ -120,7 +124,7 @@ def getCsvSpec(msg, filename, type):
             if len(a) > 2 and a[2] not in data[a[0]][a[1]]:
                 data[a[0]][a[1]][a[2]] = {}
                 if len(a) == 3 and type == CSV_SPECTYPE_DATA:
-                    data[a[0]][a[1]][a[1]]["_data"] = []
+                    data[a[0]][a[1]][a[1]][B.DATA_NODE_DATA] = []
             # fill data
             row = {}
             i = 1
@@ -128,13 +132,13 @@ def getCsvSpec(msg, filename, type):
                 row[f] = fields[i]
                 i += 1
             if len(a) == 1 and type == CSV_SPECTYPE_DATA:
-                data[a[0]]["_data"].append(row)
+                data[a[0]][B.DATA_NODE_DATA].append(row)
             elif len(a) == 1 and type == CSV_SPECTYPE_DATA:
                 data[a[0]] = {f: row}
             elif len(a) == 2 and type == CSV_SPECTYPE_DATA:
-                data[a[0]][a[1]]["_data"].append(row)
+                data[a[0]][a[1]][B.DATA_NODE_DATA].append(row)
             elif len(a) == 1 and type == CSV_SPECTYPE_DATA:
-                data[a[0]][a[1]]["_data"] = {f: row}
+                data[a[0]][a[1]][B.DATA_NODE_DATA] = {f: row}
             elif len(a) == 3 and type == CSV_SPECTYPE_DATA:
                 data[a[0]][a[1]][a[2]] = row
             elif len(a) == 1 and type == CSV_SPECTYPE_DATA: