Browse Source

some comments with constants

master
Ulrich Carmesin 3 years ago
parent
commit
5caa3f52c7
  1. 11
      basic/componentHandling.py
  2. 73
      basic/constants.py
  3. 4
      basic/message.py
  4. 23
      basic/program.py
  5. 14
      basic/toolHandling.py
  6. 8
      components/component.py
  7. 0
      test_run.py
  8. 10
      utils/config_tool.py
  9. 19
      utils/db_abstract.py
  10. 23
      utils/dbcsv_tool.py
  11. 16
      utils/dbmysql_tool.py
  12. 37
      utils/dbspark_tool.py
  13. 10
      utils/match_tool.py
  14. 12
      utils/path_tool.py
  15. 18
      utils/tdata_tool.py

11
basic/componentHandling.py

@ -1,3 +1,9 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
# managing the components
# -----------------------------------------------------------------------------
"""
@ -16,6 +22,7 @@ import basic.message
import components.component
import importlib
import copy
import basic.constants as B
comps = {}
PARAM_NOSUBNODE = ["artifact", "components", "instance"]
@ -127,7 +134,7 @@ class ComponentManager:
c = class_()
c.name = name
c.conf = confs["conf"]
c.conf["conn"] = conns[0]
c.conf[B.SUBJECT_CONN] = conns[0]
c.m = basic.message.Message(basic.message.LIMIT_DEBUG, "logTime", name)
c.init()
print("createComponent 4 a " + componentName)
@ -148,7 +155,7 @@ class ComponentManager:
c = class_()
c.name = name
c.conf = confs["conf"]
c.conf["conn"] = conns[0]
c.conf[B.SUBJECT_CONN] = conns[0]
c.m = basic.message.Message(basic.message.LIMIT_DEBUG, "logTime", name)
c.init()
print("createComponent 4 b " + componentName)

73
basic/constants.py

@ -1,17 +1,84 @@
#
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
"""
The constants desribes the keywords of the main datastructures, these are
* the configurations of
* basic-configuration for global variables in job - stored in external conf-folder
* comp-configuration for component-attributes - stored in internal component-folder in
+ ddl-configuration of an entity of the component as attributes - stored in internal component-folder
+ environment-config for connection-attributes of the component - stored in external environment-folder
* tool-configuration
* test-specification with testdata - stored in external testdata-folder
* the internal datastructure
"""
# the internal datastructure is a tree with this design:
# root { : constant
# + option { : constant
# +
# + steps
# + comp { : variable component-name
# + substructure { : variable maybe scheme, table of a database-component
# + + _header [ : constant
# - fields : variable field-name
DATA_NODE_DATA = "_data"
DATA_NODE_HEADER = "_header"
""" This constant defines a subnode of a table for the column-names """
DATA_NODE_DATA = "_data"
""" This constant defines a subnode of a table for the data which are set as key-value-pair with the column-names as key """
DATA_NODE_STEPS = "_steps"
""" This constant defines the main node in the testdata for the steps to execute """
DATA_NODE_OPTION = "_option"
""" This constant defines main node in the testdata for testcase specific parameters """
DATA_NODE_DDL = "ddl"
""" This constant defines """
# the configuration of a component or tool
# entity { : variable name of the group, basic, component-name or tool-name
# + subject { : variable subject-name - it correspondends to a tool
# + + sub-subject { : variable subject-name - it correspondends to a tool
# + attributes : constant of the tool in which the attribute ist implemented
# the main subjects # prog basic envir tool comp testcase main implentation module
SUBJECT_PATH = "paths" # | x | | x | | path_tool, config_tool
""" This constant defines the subject in order to define paths of filesystem of any testuse """
ATTR_PATH_MODE = "mode"
""" This constant defines the home-folder in filesystem of test """
ATTR_PATH_HOME = "home"
""" This constant defines the home-folder in testing-filesystem """
ATTR_PATH_DEBUG = "debugs"
""" This constant defines the debug-folder in testing-filesystem """
ATTR_PATH_ARCHIV = "archiv"
""" This constant defines the archiv-folder in testing-filesystem """
ATTR_PATH_SPECIAL = "results"
""" This constant defines the debug-folder in testing-filesystem """
ATTR_PATH_PROGRAM = "program"
""" This constant defines the program-folder in testing-filesystem """
ATTR_PATH_ENV = "environment"
""" This constant defines the environment-folder in testing-filesystem, used for configs related to environments """
ATTR_PATH_RELEASE = "release"
""" This constant defines the release-folder in testing-filesystem, used for configs related to release """
ATTR_PATH_TDATA = "testdata"
""" This constant defines the testdata-folder in testing-filesystem with the testcase-specifications """
ATTR_PATH_PATTN = "pattern"
""" This constant defines the debug-folder in testing-filesystem """
SUBJECT_APPS = "applications" # | x | | | | CompHandling
SUBJECT_INST = "instance" # | | | | x | CompHanlding
ATTR_COUNT = "count" # | | | | x | CompHanlding
SUBJECT_FCT = "function" # | | | | x | main-programs
SUBJECT_ARTS = "artifact" # | | | | x | Component
ATTR_ARTS_DB = "db"
ATTR_ARTS_LOG = "log"
ATTR_ARTS_LOB = "lob"
SUBJECT_DB = "databases" # | | | | # | db*_tools, match_tool
SUBJECT_CONN = "conn" # | | x | | | conn_tool, db*_tools, cli*_toold
ATTR_CONN_DBTYPE = "dbtype" # | x | x | | x | conn_tool, toolHandling, db*_tools
ATTR_CONN_CLITYPE = "clitype" # | x | x | | x | conn_tool, toolHandling, cli*_tools
RULE_ACCEPTANCE = "acceptance" # | | | | x | tools_match

4
basic/message.py

@ -58,8 +58,8 @@ class Message:
self.componente = componente # dezantrales Logsystem
#self.level = level # Vorgabe zu level zum Filtern, ob auszugebe
job = basic.program.Job.getInstance()
print(vars(job))
print(globals())
# print(vars(job))
# print(globals())
# exit(5)
verify = LIMIT_DEBUG
self.initErrorTyp()

23
basic/program.py

@ -1,11 +1,16 @@
#!/usr/bin/python3
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
# Template Batchrahmen
#
#import sys, getopt
import argparse
import copy
import yaml
import yaml, os
from datetime import datetime
import basic.message
@ -198,7 +203,7 @@ class Job:
def getTraceLevel(self, elem):
return self.getMessageLevel("trace", elem)
def debug(self, prio, text):
print("job.debug "+str(prio)+" "+text)
#print("job.debug "+str(prio)+" "+text)
if hasattr(self, "m"):
self.m.debug(prio, text)
else:
@ -318,6 +323,10 @@ class Configuration:
self.program = program
print (f"job initialisiert {self.program}")
if program == "unit":
if (os.path.exists('../conf/basis.yml')):
self.setConfiguration('../conf/basis.yml')
return
elif (os.path.exists('conf/basis.yml')):
self.setConfiguration('../conf/basis.yml')
return
self.setConfiguration('conf/basis.yml')
@ -326,8 +335,12 @@ class Configuration:
self.confs = {}
with open(path, "r") as file:
doc = yaml.full_load(file)
for i, v in doc.items():
self.confs[i] = v
if "basic" in doc:
for i, v in doc["basic"].items():
self.confs[i] = v
else:
for i, v in doc.items():
self.confs[i] = v
def setConfig(self, path, val):
a = path.split(".")
if len(a) == 1:

14
basic/toolHandling.py

@ -1,6 +1,12 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import importlib, os
import basic.program
import basic.constants as B
# -------------------------------------------------
def hasAttr(o, name):
@ -24,9 +30,9 @@ Toolmanager
def getDbTool(comp):
job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("db_tool"))
if not hasAttr(comp.conf["conn"], "dbtype"):
if hasAttr(comp.conf["conn"], "types") and hasAttr(comp.conf["conn"]["types"], "dbtype"):
dbtype = comp.conf["conn"]["types"]["dbtype"]
if not hasAttr(comp.conf[B.SUBJECT_CONN], "dbtype"):
if hasAttr(comp.conf[B.SUBJECT_CONN], "types") and hasAttr(comp.conf[B.SUBJECT_CONN]["types"], "dbtype"):
dbtype = comp.conf[B.SUBJECT_CONN]["types"]["dbtype"]
else:
raise LookupError("dbtype is not set in comp " + comp.name)
else:

8
components/component.py

@ -1,3 +1,9 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
# abstract class for instance components
# ---------------------------------------------------------------------
from datetime import datetime
@ -23,7 +29,7 @@ class CompData:
#class Component(components.sysmonitor.SystemMonitor, components.testexec.Testexecuter, components.report.Report,
# components.maintain.Maintainer, components.catalog, threading.Thread):
class Component():
class Component(components.testexec.Testexecuter):
"""
A component represents an application of the system-under-test or a data-artifact which is created from the system-under-test.
As the representation it has to knowlegde of the url, which other components depends on this component.

0
test_run.py

10
utils/config_tool.py

@ -1,6 +1,12 @@
#
# --------------------------------------------------------------
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import sys
import basic.constants as B
try:
import basic.program
except ImportError:

19
utils/db_abstract.py

@ -1,5 +1,12 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import basic.program
import utils.config_tool
import basic.constants as B
import os
class DbFcts():
@ -19,7 +26,7 @@ class DbFcts():
statement written in sql """
self.loadDdl()
tdata = {}
for t in self.comp.conf["ddl"]:
for t in self.comp.conf[B.DATA_NODE_DDL]:
tdata[t] = self.selectRows(t)
return tdata
@ -32,7 +39,7 @@ class DbFcts():
""" method to delete rows from a database
statement written in sql """
self.loadDdl()
for t in self.comp.conf["ddl"]:
for t in self.comp.conf[B.DATA_NODE_DDL]:
print("zu loeschende Tabelle "+t)
self.deleteRows(t)
@ -59,7 +66,7 @@ class DbFcts():
for t in self.comp.conf["ddl"]:
print("einzufuegende Tabelle "+t)
if (t in tdata[plainname]):
self.insertRows(t, tdata[plainname][t]["_data"])
self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA])
def insertRows(self, rows):
""" method to insert rows into a database
@ -74,12 +81,12 @@ class DbFcts():
def loadDdl(self):
job = basic.program.Job.getInstance()
if ("ddl" in self.comp.conf):
if (B.DATA_NODE_DDL in self.comp.conf):
return
conf = utils.config_tool.getConfig("DATASTRUCTURE", self.comp.name)
self.comp.conf["ddl"] = {}
self.comp.conf[B.DATA_NODE_DDL] = {}
for k in conf[self.comp.name]:
self.comp.conf["ddl"][k] = conf[self.comp.name][k]
self.comp.conf[B.DATA_NODE_DDL][k] = conf[self.comp.name][k]
return conf[self.comp.name]
def getWhere(self):

23
utils/dbcsv_tool.py

@ -1,7 +1,12 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import basic.program
import utils.config_tool
import utils.db_abstract
import mysql.connector
class DbFcts(utils.db_abstract.DbFcts):
@ -16,7 +21,7 @@ class DbFcts(utils.db_abstract.DbFcts):
def selectRows(self, table):
""" method to select rows from a database
statement written in sql """
tdata={}
tdata = {}
return tdata
def deleteRows(self, table):
@ -65,19 +70,7 @@ class DbFcts(utils.db_abstract.DbFcts):
""" add-on-method to get the connector
this method should only called by the class itself """
job = basic.program.Job.getInstance()
mydb = mysql.connector.connect(
host = "localhost",
user = "datest",
password = "Advent!2021",
database = "datest"
)
return mysql
@staticmethod
def execStatement(self, comp, conn, statement):
""" add-on-method to execute the statement
this method should only called by the class itself """
raise Exception("method is not implemented")
return ""

16
utils/dbmysql_tool.py

@ -1,8 +1,14 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import basic.program
import utils.config_tool
import utils.db_abstract
import mysql.connector
import basic.constants as B
class DbFcts(utils.db_abstract.DbFcts):
"""
@ -16,16 +22,16 @@ class DbFcts(utils.db_abstract.DbFcts):
def selectRows(self, table):
""" method to select rows from a database
statement written in sql """
tdata={}
tdata = {}
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "SELECT * FROM "+table+";"
#mycursor = self.getConnector()
#mycursor.execute(cmd)
#myresult = mycursor.fetchall()
tdata["_header"] = []
for f in self.comp.conf["ddl"][table]["fields"]:
tdata["_header"].append(f)
tdata[B.DATA_NODE_HEADER] = []
for f in self.comp.conf["ddl"][table][B.DATA_NODE_HEADER]:
tdata[B.DATA_NODE_HEADER].append(f)
myresult = []
for x in myresult:
print(x)

37
utils/dbspark_tool.py

@ -1,11 +1,14 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import basic.program
import utils.config_tool
import utils.db_abstract
import pyspark
import basic.constants
DATA_NODE_HEADER = basic.constants.DATA_NODE_HEADER
DATA_NODE_DATA = basic.constants.DATA_NODE_DATA
import basic.constants as B
class DbFcts(utils.db_abstract.DbFcts):
"""
@ -17,18 +20,18 @@ class DbFcts(utils.db_abstract.DbFcts):
def selectRows(self, table):
""" method to select rows from a database
statement written in sql """
tdata={}
tdata = {}
job = basic.program.Job.getInstance()
verify = -1+job.getDebugLevel("db_tool")
cmd = "SELECT "+self.comp.conf["ddl"][table]["fields"].join(",")\
+" FROM "+table+""+self.getWhere()+""+self.getOrder()";"
cmd = "SELECT "+",".join(self.comp.conf["ddl"][table]["fields"])
cmd += " FROM "+table+""+self.getWhere()+""+self.getOrder()
spark = self.getConnector()
df = spark.sql(cmd)
data = []
for r in df:
data.append(r)
tdata[DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"]
tdata[DATA_NODE_DATA] = data
tdata[B.DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"]
tdata[B.DATA_NODE_DATA] = data
return tdata
def deleteRows(self, table):
@ -39,11 +42,6 @@ class DbFcts(utils.db_abstract.DbFcts):
cmd = "DELETE FROM "+table+";"
self.comp.m.logInfo(cmd)
def updateRows(self, statement):
""" method to delete rows from a database
statement written in sql """
raise Exception("method is not implemented")
def insertRows(self, table, rows):
""" method to insert rows into a database
the rows will be interpreted by the ddl of the component
@ -59,17 +57,12 @@ class DbFcts(utils.db_abstract.DbFcts):
""" add-on-method to get the connector
this method should only called by the class itself """
job = basic.program.Job.getInstance()
spark = pyspark.SparkSession.builder()\
.master("local[1]")\
.appName("SparkByExamples.com")\
spark = pyspark.SparkSession\
.builder\
.appName("datest")\
.getOrCreate()
return spark
@staticmethod
def execStatement(self, comp, conn, statement):
""" add-on-method to execute the statement
this method should only called by the class itself """
raise Exception("method is not implemented")

10
utils/match_tool.py

@ -1,7 +1,13 @@
#
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import json
import utils.css_tool
import basic.program
import basic.constants as B
# ------------------------------------------------------------
"""
@ -366,7 +372,7 @@ def compareRows(matching, path):
htmltext = "<p>Tabelle : "+table+"</p>"
htmltext += "<table>"
htmltext += "<tr><th></th>"
for f in ddl["_header"]:
for f in ddl[B.DATA_NODE_HEADER]:
job.debug(verify, "ddl " + f + " ")
header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]})
htmltext += "<th>"+f+"</th>"

12
utils/path_tool.py

@ -1,11 +1,15 @@
# All functions related to the full path.
# It implements the team conventions of the directory structure
# -----------------------------------------------------------------------
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
""" In diesem Modul werden alle Funktionen zusammengefasst zur Generierung und Ermittlung von pathsn """
import sys
import basic.program
import utils.config_tool
import re
import basic.constants as B
def getKeyValue(key):
@ -17,7 +21,7 @@ def getKeyValue(key):
neu = job.getParameter(key[8:])
return neu
elif 'job.conf' in key:
neu = job.conf.confs["paths"][key[9:]]
neu = job.conf.confs[B.SUBJECT_PATH][key[9:]]
job.debug(verify, neu)
return neu
# return job.conf.confs["paths"][key[9:]]

18
utils/tdata_tool.py

@ -1,5 +1,9 @@
#
# ---------------------------------------------------------------------
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
"""
the issue of this tool is to transform extern data to the internal structure and the internal structure into extern data - i.e. mostly test-results.
* * * * * * * *
@ -111,7 +115,7 @@ def getCsvSpec(msg, filename, type):
if (a[0] not in data):
data[a[0]] = {}
if len(a) == 1 and type == CSV_SPECTYPE_DATA:
data[a[0]]["_data"] = []
data[a[0]][B.DATA_NODE_DATA] = []
# its a component
if len(a) > 1 and a[1] not in data[a[0]]:
data[a[0]][a[1]] = {}
@ -120,7 +124,7 @@ def getCsvSpec(msg, filename, type):
if len(a) > 2 and a[2] not in data[a[0]][a[1]]:
data[a[0]][a[1]][a[2]] = {}
if len(a) == 3 and type == CSV_SPECTYPE_DATA:
data[a[0]][a[1]][a[1]]["_data"] = []
data[a[0]][a[1]][a[1]][B.DATA_NODE_DATA] = []
# fill data
row = {}
i = 1
@ -128,13 +132,13 @@ def getCsvSpec(msg, filename, type):
row[f] = fields[i]
i += 1
if len(a) == 1 and type == CSV_SPECTYPE_DATA:
data[a[0]]["_data"].append(row)
data[a[0]][B.DATA_NODE_DATA].append(row)
elif len(a) == 1 and type == CSV_SPECTYPE_DATA:
data[a[0]] = {f: row}
elif len(a) == 2 and type == CSV_SPECTYPE_DATA:
data[a[0]][a[1]]["_data"].append(row)
data[a[0]][a[1]][B.DATA_NODE_DATA].append(row)
elif len(a) == 1 and type == CSV_SPECTYPE_DATA:
data[a[0]][a[1]]["_data"] = {f: row}
data[a[0]][a[1]][B.DATA_NODE_DATA] = {f: row}
elif len(a) == 3 and type == CSV_SPECTYPE_DATA:
data[a[0]][a[1]][a[2]] = row
elif len(a) == 1 and type == CSV_SPECTYPE_DATA:

Loading…
Cancel
Save