diff --git a/basic/componentHandling.py b/basic/componentHandling.py index 4a349cb..cb08cc1 100644 --- a/basic/componentHandling.py +++ b/basic/componentHandling.py @@ -1,3 +1,9 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- # managing the components # ----------------------------------------------------------------------------- """ @@ -16,6 +22,7 @@ import basic.message import components.component import importlib import copy +import basic.constants as B comps = {} PARAM_NOSUBNODE = ["artifact", "components", "instance"] @@ -127,7 +134,7 @@ class ComponentManager: c = class_() c.name = name c.conf = confs["conf"] - c.conf["conn"] = conns[0] + c.conf[B.SUBJECT_CONN] = conns[0] c.m = basic.message.Message(basic.message.LIMIT_DEBUG, "logTime", name) c.init() print("createComponent 4 a " + componentName) @@ -148,7 +155,7 @@ class ComponentManager: c = class_() c.name = name c.conf = confs["conf"] - c.conf["conn"] = conns[0] + c.conf[B.SUBJECT_CONN] = conns[0] c.m = basic.message.Message(basic.message.LIMIT_DEBUG, "logTime", name) c.init() print("createComponent 4 b " + componentName) diff --git a/basic/constants.py b/basic/constants.py index 8aa5683..e63c951 100644 --- a/basic/constants.py +++ b/basic/constants.py @@ -1,17 +1,84 @@ -# +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +""" +The constants desribes the keywords of the main datastructures, these are +* the configurations of + * basic-configuration for global variables in job - stored in external conf-folder + * comp-configuration for component-attributes - stored in internal component-folder in + + ddl-configuration of an entity of the component as attributes - stored in internal component-folder + + environment-config for connection-attributes of the component - stored in external environment-folder + * tool-configuration + * test-specification with testdata - stored in external testdata-folder +* the internal datastructure + +""" # the internal datastructure is a tree with this design: # root { : constant # + option { : constant -# + # + steps # + comp { : variable component-name # + substructure { : variable maybe scheme, table of a database-component # + + _header [ : constant # - fields : variable field-name -DATA_NODE_DATA = "_data" +DATA_NODE_HEADER = "_header" +""" This constant defines a subnode of a table for the column-names """ +DATA_NODE_DATA = "_data" +""" This constant defines a subnode of a table for the data which are set as key-value-pair with the column-names as key """ +DATA_NODE_STEPS = "_steps" +""" This constant defines the main node in the testdata for the steps to execute """ +DATA_NODE_OPTION = "_option" +""" This constant defines main node in the testdata for testcase specific parameters """ +DATA_NODE_DDL = "ddl" +""" This constant defines """ # the configuration of a component or tool # entity { : variable name of the group, basic, component-name or tool-name # + subject { : variable subject-name - it correspondends to a tool # + + sub-subject { : variable subject-name - it correspondends to a tool # + attributes : constant of the tool in which the attribute ist implemented + +# the main subjects # prog basic envir tool comp testcase main implentation module +SUBJECT_PATH = "paths" # | x | | x | | path_tool, config_tool +""" This constant defines the subject in order to define paths of filesystem of any testuse """ +ATTR_PATH_MODE = "mode" +""" This constant defines the home-folder in filesystem of test """ +ATTR_PATH_HOME = "home" +""" This constant defines the home-folder in testing-filesystem """ +ATTR_PATH_DEBUG = "debugs" +""" This constant defines the debug-folder in testing-filesystem """ +ATTR_PATH_ARCHIV = "archiv" +""" This constant defines the archiv-folder in testing-filesystem """ +ATTR_PATH_SPECIAL = "results" +""" This constant defines the debug-folder in testing-filesystem """ +ATTR_PATH_PROGRAM = "program" +""" This constant defines the program-folder in testing-filesystem """ +ATTR_PATH_ENV = "environment" +""" This constant defines the environment-folder in testing-filesystem, used for configs related to environments """ +ATTR_PATH_RELEASE = "release" +""" This constant defines the release-folder in testing-filesystem, used for configs related to release """ +ATTR_PATH_TDATA = "testdata" +""" This constant defines the testdata-folder in testing-filesystem with the testcase-specifications """ +ATTR_PATH_PATTN = "pattern" +""" This constant defines the debug-folder in testing-filesystem """ +SUBJECT_APPS = "applications" # | x | | | | CompHandling + +SUBJECT_INST = "instance" # | | | | x | CompHanlding +ATTR_COUNT = "count" # | | | | x | CompHanlding + +SUBJECT_FCT = "function" # | | | | x | main-programs + +SUBJECT_ARTS = "artifact" # | | | | x | Component +ATTR_ARTS_DB = "db" +ATTR_ARTS_LOG = "log" +ATTR_ARTS_LOB = "lob" +SUBJECT_DB = "databases" # | | | | # | db*_tools, match_tool + +SUBJECT_CONN = "conn" # | | x | | | conn_tool, db*_tools, cli*_toold +ATTR_CONN_DBTYPE = "dbtype" # | x | x | | x | conn_tool, toolHandling, db*_tools +ATTR_CONN_CLITYPE = "clitype" # | x | x | | x | conn_tool, toolHandling, cli*_tools + +RULE_ACCEPTANCE = "acceptance" # | | | | x | tools_match \ No newline at end of file diff --git a/basic/message.py b/basic/message.py index 721be74..6f5508b 100644 --- a/basic/message.py +++ b/basic/message.py @@ -58,8 +58,8 @@ class Message: self.componente = componente # dezantrales Logsystem #self.level = level # Vorgabe zu level zum Filtern, ob auszugebe job = basic.program.Job.getInstance() - print(vars(job)) - print(globals()) + # print(vars(job)) + # print(globals()) # exit(5) verify = LIMIT_DEBUG self.initErrorTyp() diff --git a/basic/program.py b/basic/program.py index 2529519..f5740bf 100644 --- a/basic/program.py +++ b/basic/program.py @@ -1,11 +1,16 @@ -#!/usr/bin/python3 +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- # Template Batchrahmen # #import sys, getopt import argparse import copy -import yaml +import yaml, os from datetime import datetime import basic.message @@ -198,7 +203,7 @@ class Job: def getTraceLevel(self, elem): return self.getMessageLevel("trace", elem) def debug(self, prio, text): - print("job.debug "+str(prio)+" "+text) + #print("job.debug "+str(prio)+" "+text) if hasattr(self, "m"): self.m.debug(prio, text) else: @@ -318,6 +323,10 @@ class Configuration: self.program = program print (f"job initialisiert {self.program}") if program == "unit": + if (os.path.exists('../conf/basis.yml')): + self.setConfiguration('../conf/basis.yml') + return + elif (os.path.exists('conf/basis.yml')): self.setConfiguration('../conf/basis.yml') return self.setConfiguration('conf/basis.yml') @@ -326,8 +335,12 @@ class Configuration: self.confs = {} with open(path, "r") as file: doc = yaml.full_load(file) - for i, v in doc.items(): - self.confs[i] = v + if "basic" in doc: + for i, v in doc["basic"].items(): + self.confs[i] = v + else: + for i, v in doc.items(): + self.confs[i] = v def setConfig(self, path, val): a = path.split(".") if len(a) == 1: diff --git a/basic/toolHandling.py b/basic/toolHandling.py index 05c7d97..d27925c 100644 --- a/basic/toolHandling.py +++ b/basic/toolHandling.py @@ -1,6 +1,12 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- import importlib, os - import basic.program +import basic.constants as B # ------------------------------------------------- def hasAttr(o, name): @@ -24,9 +30,9 @@ Toolmanager def getDbTool(comp): job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("db_tool")) - if not hasAttr(comp.conf["conn"], "dbtype"): - if hasAttr(comp.conf["conn"], "types") and hasAttr(comp.conf["conn"]["types"], "dbtype"): - dbtype = comp.conf["conn"]["types"]["dbtype"] + if not hasAttr(comp.conf[B.SUBJECT_CONN], "dbtype"): + if hasAttr(comp.conf[B.SUBJECT_CONN], "types") and hasAttr(comp.conf[B.SUBJECT_CONN]["types"], "dbtype"): + dbtype = comp.conf[B.SUBJECT_CONN]["types"]["dbtype"] else: raise LookupError("dbtype is not set in comp " + comp.name) else: diff --git a/components/component.py b/components/component.py index 7801004..f0c3c18 100644 --- a/components/component.py +++ b/components/component.py @@ -1,3 +1,9 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- # abstract class for instance components # --------------------------------------------------------------------- from datetime import datetime @@ -23,7 +29,7 @@ class CompData: #class Component(components.sysmonitor.SystemMonitor, components.testexec.Testexecuter, components.report.Report, # components.maintain.Maintainer, components.catalog, threading.Thread): -class Component(): +class Component(components.testexec.Testexecuter): """ A component represents an application of the system-under-test or a data-artifact which is created from the system-under-test. As the representation it has to knowlegde of the url, which other components depends on this component. diff --git a/test_run.py b/test_run.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils/config_tool.py b/utils/config_tool.py index 38c193d..71435cc 100644 --- a/utils/config_tool.py +++ b/utils/config_tool.py @@ -1,6 +1,12 @@ -# -# -------------------------------------------------------------- +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- import sys +import basic.constants as B + try: import basic.program except ImportError: diff --git a/utils/db_abstract.py b/utils/db_abstract.py index 4a213e5..a49f332 100644 --- a/utils/db_abstract.py +++ b/utils/db_abstract.py @@ -1,5 +1,12 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- import basic.program import utils.config_tool +import basic.constants as B import os class DbFcts(): @@ -19,7 +26,7 @@ class DbFcts(): statement written in sql """ self.loadDdl() tdata = {} - for t in self.comp.conf["ddl"]: + for t in self.comp.conf[B.DATA_NODE_DDL]: tdata[t] = self.selectRows(t) return tdata @@ -32,7 +39,7 @@ class DbFcts(): """ method to delete rows from a database statement written in sql """ self.loadDdl() - for t in self.comp.conf["ddl"]: + for t in self.comp.conf[B.DATA_NODE_DDL]: print("zu loeschende Tabelle "+t) self.deleteRows(t) @@ -59,7 +66,7 @@ class DbFcts(): for t in self.comp.conf["ddl"]: print("einzufuegende Tabelle "+t) if (t in tdata[plainname]): - self.insertRows(t, tdata[plainname][t]["_data"]) + self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA]) def insertRows(self, rows): """ method to insert rows into a database @@ -74,12 +81,12 @@ class DbFcts(): def loadDdl(self): job = basic.program.Job.getInstance() - if ("ddl" in self.comp.conf): + if (B.DATA_NODE_DDL in self.comp.conf): return conf = utils.config_tool.getConfig("DATASTRUCTURE", self.comp.name) - self.comp.conf["ddl"] = {} + self.comp.conf[B.DATA_NODE_DDL] = {} for k in conf[self.comp.name]: - self.comp.conf["ddl"][k] = conf[self.comp.name][k] + self.comp.conf[B.DATA_NODE_DDL][k] = conf[self.comp.name][k] return conf[self.comp.name] def getWhere(self): diff --git a/utils/dbcsv_tool.py b/utils/dbcsv_tool.py index df25117..4886252 100644 --- a/utils/dbcsv_tool.py +++ b/utils/dbcsv_tool.py @@ -1,7 +1,12 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- import basic.program import utils.config_tool import utils.db_abstract -import mysql.connector class DbFcts(utils.db_abstract.DbFcts): @@ -16,7 +21,7 @@ class DbFcts(utils.db_abstract.DbFcts): def selectRows(self, table): """ method to select rows from a database statement written in sql """ - tdata={} + tdata = {} return tdata def deleteRows(self, table): @@ -65,19 +70,7 @@ class DbFcts(utils.db_abstract.DbFcts): """ add-on-method to get the connector this method should only called by the class itself """ job = basic.program.Job.getInstance() - mydb = mysql.connector.connect( - host = "localhost", - user = "datest", - password = "Advent!2021", - database = "datest" - ) - return mysql - - @staticmethod - def execStatement(self, comp, conn, statement): - """ add-on-method to execute the statement - this method should only called by the class itself """ - raise Exception("method is not implemented") + return "" diff --git a/utils/dbmysql_tool.py b/utils/dbmysql_tool.py index 64d665a..75fa0da 100644 --- a/utils/dbmysql_tool.py +++ b/utils/dbmysql_tool.py @@ -1,8 +1,14 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- import basic.program import utils.config_tool import utils.db_abstract import mysql.connector - +import basic.constants as B class DbFcts(utils.db_abstract.DbFcts): """ @@ -16,16 +22,16 @@ class DbFcts(utils.db_abstract.DbFcts): def selectRows(self, table): """ method to select rows from a database statement written in sql """ - tdata={} + tdata = {} job = basic.program.Job.getInstance() verify = -1+job.getDebugLevel("db_tool") cmd = "SELECT * FROM "+table+";" #mycursor = self.getConnector() #mycursor.execute(cmd) #myresult = mycursor.fetchall() - tdata["_header"] = [] - for f in self.comp.conf["ddl"][table]["fields"]: - tdata["_header"].append(f) + tdata[B.DATA_NODE_HEADER] = [] + for f in self.comp.conf["ddl"][table][B.DATA_NODE_HEADER]: + tdata[B.DATA_NODE_HEADER].append(f) myresult = [] for x in myresult: print(x) diff --git a/utils/dbspark_tool.py b/utils/dbspark_tool.py index 311258b..106bbd3 100644 --- a/utils/dbspark_tool.py +++ b/utils/dbspark_tool.py @@ -1,11 +1,14 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- import basic.program import utils.config_tool import utils.db_abstract import pyspark -import basic.constants - -DATA_NODE_HEADER = basic.constants.DATA_NODE_HEADER -DATA_NODE_DATA = basic.constants.DATA_NODE_DATA +import basic.constants as B class DbFcts(utils.db_abstract.DbFcts): """ @@ -17,18 +20,18 @@ class DbFcts(utils.db_abstract.DbFcts): def selectRows(self, table): """ method to select rows from a database statement written in sql """ - tdata={} + tdata = {} job = basic.program.Job.getInstance() verify = -1+job.getDebugLevel("db_tool") - cmd = "SELECT "+self.comp.conf["ddl"][table]["fields"].join(",")\ - +" FROM "+table+""+self.getWhere()+""+self.getOrder()";" + cmd = "SELECT "+",".join(self.comp.conf["ddl"][table]["fields"]) + cmd += " FROM "+table+""+self.getWhere()+""+self.getOrder() spark = self.getConnector() df = spark.sql(cmd) data = [] for r in df: data.append(r) - tdata[DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"] - tdata[DATA_NODE_DATA] = data + tdata[B.DATA_NODE_HEADER] = self.comp.conf["ddl"][table]["fields"] + tdata[B.DATA_NODE_DATA] = data return tdata def deleteRows(self, table): @@ -39,11 +42,6 @@ class DbFcts(utils.db_abstract.DbFcts): cmd = "DELETE FROM "+table+";" self.comp.m.logInfo(cmd) - def updateRows(self, statement): - """ method to delete rows from a database - statement written in sql """ - raise Exception("method is not implemented") - def insertRows(self, table, rows): """ method to insert rows into a database the rows will be interpreted by the ddl of the component @@ -59,17 +57,12 @@ class DbFcts(utils.db_abstract.DbFcts): """ add-on-method to get the connector this method should only called by the class itself """ job = basic.program.Job.getInstance() - spark = pyspark.SparkSession.builder()\ - .master("local[1]")\ - .appName("SparkByExamples.com")\ + spark = pyspark.SparkSession\ + .builder\ + .appName("datest")\ .getOrCreate() return spark - @staticmethod - def execStatement(self, comp, conn, statement): - """ add-on-method to execute the statement - this method should only called by the class itself """ - raise Exception("method is not implemented") diff --git a/utils/match_tool.py b/utils/match_tool.py index a9a6762..5df38b2 100644 --- a/utils/match_tool.py +++ b/utils/match_tool.py @@ -1,7 +1,13 @@ -# +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- import json import utils.css_tool import basic.program +import basic.constants as B # ------------------------------------------------------------ """ @@ -366,7 +372,7 @@ def compareRows(matching, path): htmltext = "

Tabelle : "+table+"

" htmltext += "" htmltext += "" - for f in ddl["_header"]: + for f in ddl[B.DATA_NODE_HEADER]: job.debug(verify, "ddl " + f + " ") header.append({ "field": f, "type": ddl[f]["type"], "acceptance": ddl[f]["acceptance"]}) htmltext += "" diff --git a/utils/path_tool.py b/utils/path_tool.py index f97b6e7..94a2d84 100644 --- a/utils/path_tool.py +++ b/utils/path_tool.py @@ -1,11 +1,15 @@ -# All functions related to the full path. -# It implements the team conventions of the directory structure -# ----------------------------------------------------------------------- +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- """ In diesem Modul werden alle Funktionen zusammengefasst zur Generierung und Ermittlung von pathsn """ import sys import basic.program import utils.config_tool import re +import basic.constants as B def getKeyValue(key): @@ -17,7 +21,7 @@ def getKeyValue(key): neu = job.getParameter(key[8:]) return neu elif 'job.conf' in key: - neu = job.conf.confs["paths"][key[9:]] + neu = job.conf.confs[B.SUBJECT_PATH][key[9:]] job.debug(verify, neu) return neu # return job.conf.confs["paths"][key[9:]] diff --git a/utils/tdata_tool.py b/utils/tdata_tool.py index d598807..929cdc2 100644 --- a/utils/tdata_tool.py +++ b/utils/tdata_tool.py @@ -1,5 +1,9 @@ -# -# --------------------------------------------------------------------- +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- """ the issue of this tool is to transform extern data to the internal structure and the internal structure into extern data - i.e. mostly test-results. * * * * * * * * @@ -111,7 +115,7 @@ def getCsvSpec(msg, filename, type): if (a[0] not in data): data[a[0]] = {} if len(a) == 1 and type == CSV_SPECTYPE_DATA: - data[a[0]]["_data"] = [] + data[a[0]][B.DATA_NODE_DATA] = [] # its a component if len(a) > 1 and a[1] not in data[a[0]]: data[a[0]][a[1]] = {} @@ -120,7 +124,7 @@ def getCsvSpec(msg, filename, type): if len(a) > 2 and a[2] not in data[a[0]][a[1]]: data[a[0]][a[1]][a[2]] = {} if len(a) == 3 and type == CSV_SPECTYPE_DATA: - data[a[0]][a[1]][a[1]]["_data"] = [] + data[a[0]][a[1]][a[1]][B.DATA_NODE_DATA] = [] # fill data row = {} i = 1 @@ -128,13 +132,13 @@ def getCsvSpec(msg, filename, type): row[f] = fields[i] i += 1 if len(a) == 1 and type == CSV_SPECTYPE_DATA: - data[a[0]]["_data"].append(row) + data[a[0]][B.DATA_NODE_DATA].append(row) elif len(a) == 1 and type == CSV_SPECTYPE_DATA: data[a[0]] = {f: row} elif len(a) == 2 and type == CSV_SPECTYPE_DATA: - data[a[0]][a[1]]["_data"].append(row) + data[a[0]][a[1]][B.DATA_NODE_DATA].append(row) elif len(a) == 1 and type == CSV_SPECTYPE_DATA: - data[a[0]][a[1]]["_data"] = {f: row} + data[a[0]][a[1]][B.DATA_NODE_DATA] = {f: row} elif len(a) == 3 and type == CSV_SPECTYPE_DATA: data[a[0]][a[1]][a[2]] = row elif len(a) == 1 and type == CSV_SPECTYPE_DATA:
"+f+"