Browse Source

POC datamodel

refactor
Ulrich 2 years ago
parent
commit
eff0669cbc
  1. 28
      basic/DATASTRUCTURE.yml
  2. 12
      basic/Testserver.py
  3. 26
      basic/application.py
  4. 14
      basic/compexec.py
  5. 10
      basic/component.py
  6. 281
      basic/componentHandling.py
  7. 3
      basic/connection.py
  8. 13
      basic/entity.py
  9. 3
      basic/testcase.py
  10. 2
      basic/testexecution.py
  11. 2
      basic/testplan.py
  12. 2
      basic/testsuite.py
  13. 2
      basic/user.py
  14. 320
      utils/path_tool.py

28
basic/DATASTRUCTURE.yml

@ -17,8 +17,8 @@ application:
type: pk type: pk
name: name:
field: name field: name
index: I
type: str type: str
index: I
description: description:
field: description field: description
type: string type: string
@ -48,8 +48,8 @@ application:
type: time type: time
actual: actual:
field: actual field: actual
index: I
type: int type: int
index: I
ap_component: ap_component:
_header: _header:
- apcomid - apcomid
@ -60,12 +60,12 @@ ap_component:
type: pk type: pk
apid: apid:
field: apid field: apid
index: I
type: int type: int
index: I
component: component:
field: component field: component
index: I
type: str type: str
index: I
ap_application: ap_application:
_header: _header:
- apappid - apappid
@ -76,12 +76,12 @@ ap_application:
type: pk type: pk
apid: apid:
field: apid field: apid
index: I
type: int type: int
index: I
application: application:
field: component field: component
index: I
type: str type: str
index: I
ap_project: ap_project:
_header: _header:
- approid - approid
@ -94,12 +94,12 @@ ap_project:
type: pk type: pk
apid: apid:
field: apid field: apid
index: I
type: int type: int
index: I
project: project:
field: project field: project
index: I
type: str type: str
index: I
description: description:
field: description field: description
type: string type: string
@ -125,8 +125,8 @@ environment:
type: pk type: pk
name: name:
field: name field: name
index: I
type: str type: str
index: I
description: description:
field: description field: description
type: string type: string
@ -156,8 +156,8 @@ environment:
type: time type: time
actual: actual:
field: actual field: actual
index: I
type: int type: int
index: I
en_project: en_project:
_header: _header:
- enproid - enproid
@ -168,12 +168,12 @@ en_project:
type: pk type: pk
enid: enid:
field: enid field: enid
index: I
type: int type: int
index: I
project: project:
field: project field: project
index: I
type: str type: str
index: I
en_component: en_component:
_header: _header:
- encomid - encomid
@ -245,8 +245,8 @@ component:
type: pk type: pk
name: name:
field: name field: name
index: I
type: str type: str
index: I
description: description:
field: description field: description
type: string type: string
@ -276,6 +276,6 @@ component:
type: time type: time
actual: actual:
field: actual field: actual
index: I
type: int type: int
index: I

12
basic/Testserver.py

@ -13,12 +13,12 @@ class Testserver(basic.component.Component):
print('init '+COMP_NAME) print('init '+COMP_NAME)
self.m = job.m self.m = job.m
self.conf = {} self.conf = {}
if B.TOPIC_NODE_DB in job.conf.confs: if B.TOPIC_NODE_DB in job.conf:
self.conf[B.SUBJECT_CONN] = {} self.conf[B.SUBJECT_CONN] = {}
self.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB] = {} self.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB] = {}
for attr in B.LIST_DB_ATTR: for attr in B.LIST_DB_ATTR:
if attr in job.conf.confs[B.TOPIC_NODE_DB]: if attr in job.conf[B.TOPIC_NODE_DB]:
self.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr] = job.conf.confs[B.TOPIC_NODE_DB][attr] self.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr] = job.conf[B.TOPIC_NODE_DB][attr]
if not B.DATA_NODE_DDL in self.conf: if not B.DATA_NODE_DDL in self.conf:
self.conf[B.DATA_NODE_DDL] = {} self.conf[B.DATA_NODE_DDL] = {}
for table in COMP_TABLES: for table in COMP_TABLES:
@ -35,8 +35,8 @@ class Testserver(basic.component.Component):
self.conf[B.DATA_NODE_DDL][table] = ddl self.conf[B.DATA_NODE_DDL][table] = ddl
def createDBTables(self, job): def createDBTables(self, job):
if B.TOPIC_NODE_DB in job.conf.confs: if B.TOPIC_NODE_DB in job.conf:
dbi = basic.toolHandling.getDbTool(job, self, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) dbi = basic.toolHandling.getDbTool(job, self, job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE])
else: else:
return "No DB in job-config" return "No DB in job-config"
for t in COMP_TABLES: for t in COMP_TABLES:
@ -45,7 +45,7 @@ class Testserver(basic.component.Component):
if len(s) < 3: if len(s) < 3:
continue continue
try: try:
dbi.execStatement(s+";", job.conf.confs[B.TOPIC_NODE_DB]) dbi.execStatement(s+";", job.conf[B.TOPIC_NODE_DB])
print("SQL executed: "+s) print("SQL executed: "+s)
except Exception as e: except Exception as e:
raise Exception("Fehler bei createSchema "+s) raise Exception("Fehler bei createSchema "+s)

26
basic/application.py

@ -3,9 +3,7 @@
# Source : gitea.ucarmesin.de # Source : gitea.ucarmesin.de
# --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------
import os import os
import utils.db_abstract
import basic.toolHandling import basic.toolHandling
import utils.data_const as D
import basic.constants as B import basic.constants as B
import basic.entity import basic.entity
import utils.path_const as P import utils.path_const as P
@ -34,8 +32,8 @@ def searchProjects(job, appl):
:return: :return:
""" """
projects = {} projects = {}
if B.SUBJECT_PROJECTS in job.conf.confs: if B.SUBJECT_PROJECTS in job.conf:
for k in job.conf.confs[B.SUBJECT_PROJECTS]: for k in job.conf[B.SUBJECT_PROJECTS]:
if k in B.LIST_SUBJECTS: if k in B.LIST_SUBJECTS:
continue continue
if hasattr(job.par, B.PAR_PROJ) and k != getattr(job.par, B.PAR_PROJ): if hasattr(job.par, B.PAR_PROJ) and k != getattr(job.par, B.PAR_PROJ):
@ -46,7 +44,7 @@ def searchProjects(job, appl):
projects[k] = appl[B.SUBJECT_PROJECTS][k] projects[k] = appl[B.SUBJECT_PROJECTS][k]
projects[k][B.SUBJECT_ENV] = [] projects[k][B.SUBJECT_ENV] = []
else: else:
job.conf.confs[B.SUBJECT_PROJECTS] = appl[B.SUBJECT_PROJECTS] job.conf[B.SUBJECT_PROJECTS] = appl[B.SUBJECT_PROJECTS]
return projects return projects
def getEnvironments(job, projectList): def getEnvironments(job, projectList):
@ -57,7 +55,7 @@ def getEnvironments(job, projectList):
:return: :return:
""" """
projects = {} projects = {}
path = job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ENV] path = job.conf[B.SUBJECT_PATH][B.ATTR_PATH_ENV]
if os.path.exists(path): if os.path.exists(path):
raise Exception("Umgebungsverzeichnis existiert nicht "+path) raise Exception("Umgebungsverzeichnis existiert nicht "+path)
for envdir in os.listdir(path): for envdir in os.listdir(path):
@ -113,8 +111,8 @@ def syncEnitities(job):
:return: :return:
""" """
syncMethod = DEFAULT_SYNC syncMethod = DEFAULT_SYNC
if B.SUBJECT_ENTITY in job.conf.confs: if B.SUBJECT_ENTITY in job.conf:
syncMethod = job.conf.confs["entity"][TABLE_NAMES[0]]["storage"] syncMethod = job.conf["entity"][TABLE_NAMES[0]]["storage"]
if syncMethod.count("-") < 2: if syncMethod.count("-") < 2:
return return
fileTime = basic.entity.VAL_ZERO_TIME fileTime = basic.entity.VAL_ZERO_TIME
@ -122,13 +120,13 @@ def syncEnitities(job):
# get git-commit # get git-commit
if "git" in syncMethod: if "git" in syncMethod:
apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "") apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "")
repopath = apppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:] repopath = apppath[len(job.conf[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:]
gitresult = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1) gitresult = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1)
fileTime = gitresult[0]["date"] fileTime = gitresult[0]["date"]
print(str(gitresult)) print(str(gitresult))
if "db" in syncMethod: if "db" in syncMethod:
if B.TOPIC_NODE_DB in job.conf.confs: if B.TOPIC_NODE_DB in job.conf:
dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE])
else: else:
return "No DB in job-config" return "No DB in job-config"
data = dbi.selectRows(TABLE_NAMES[0], job) data = dbi.selectRows(TABLE_NAMES[0], job)
@ -221,7 +219,7 @@ class Application(basic.entity.Entity):
self.getEntity(job, name) self.getEntity(job, name)
def getEntity(self, job, name): def getEntity(self, job, name):
if B.TOPIC_NODE_DB in job.conf.confs: if B.TOPIC_NODE_DB in job.conf:
self.selectEntity(job, name) self.selectEntity(job, name)
#self.readEntity(job, name) #self.readEntity(job, name)
else: else:
@ -229,7 +227,7 @@ class Application(basic.entity.Entity):
def readEntity(self, job, app): def readEntity(self, job, app):
apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "") apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "")
repopath = apppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:] repopath = apppath[len(job.conf[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:]
gitresult = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1) gitresult = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1)
applData = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS) applData = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS)
# main object # main object
@ -298,7 +296,7 @@ class Application(basic.entity.Entity):
return rows return rows
def selectEntity(self, job, app): def selectEntity(self, job, app):
dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE])
data = dbi.selectRows(TABLE_NAMES[0], job, "WHERE name = \'"+app+"\' AND actual = "+basic.entity.ENTITY_ACTUAL) data = dbi.selectRows(TABLE_NAMES[0], job, "WHERE name = \'"+app+"\' AND actual = "+basic.entity.ENTITY_ACTUAL)
# main object # main object
self.setAppRow(data[B.DATA_NODE_DATA][0], app) self.setAppRow(data[B.DATA_NODE_DATA][0], app)

14
basic/compexec.py

@ -38,8 +38,7 @@ import basic.message
import basic.program import basic.program
import inspect import inspect
import os import os
import re import tools.db_abstract
import utils.db_abstract
import basic.toolHandling import basic.toolHandling
import basic.component import basic.component
import basic.componentHandling import basic.componentHandling
@ -50,7 +49,6 @@ import utils.match_const as M
import utils.tdata_tool import utils.tdata_tool
import basic.constants as B import basic.constants as B
import basic.text_const as T import basic.text_const as T
import utils.data_const as D
import utils.path_const as P import utils.path_const as P
class Testexecuter(): class Testexecuter():
@ -112,7 +110,7 @@ class Testexecuter():
if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS] and B.DATA_NODE_TABLES in tdata: if B.TOPIC_NODE_DB in self.conf[B.SUBJECT_ARTS] and B.DATA_NODE_TABLES in tdata:
for t in tdata[B.DATA_NODE_TABLES]: for t in tdata[B.DATA_NODE_TABLES]:
print (t) print (t)
if utils.db_abstract.isCompTable(self, job, tdata, t): if tools.db_abstract.isCompTable(self, job, tdata, t):
self.m.logInfo("insert content "+ self.name) self.m.logInfo("insert content "+ self.name)
dbi = basic.toolHandling.getDbTool(job, self) dbi = basic.toolHandling.getDbTool(job, self)
dbi.insertTables(tdata, job) dbi.insertTables(tdata, job)
@ -158,7 +156,7 @@ class Testexecuter():
def composeFileClauses(self, job, pattern): def composeFileClauses(self, job, pattern):
#job = basic.program.Job.getInstance() #job = basic.program.Job.getInstance()
out = {} out = {}
attr = utils.db_abstract.getDbAttributes(self, job, "null") attr = tools.db_abstract.getDbAttributes(self, job, "null")
while "{" in pattern: while "{" in pattern:
pre = pattern[0:pattern.index("{")] pre = pattern[0:pattern.index("{")]
pat = pattern[pattern.index("{"):pattern.index("}")] pat = pattern[pattern.index("{"):pattern.index("}")]
@ -200,7 +198,7 @@ class Testexecuter():
print("table "+table) print("table "+table)
sql_new = sql[0:sql.upper().index(" FROM ")+5] sql_new = sql[0:sql.upper().index(" FROM ")+5]
print("sql_new "+sql_new) print("sql_new "+sql_new)
attr = utils.db_abstract.getDbAttributes(self, table) attr = tools.db_abstract.getDbAttributes(self, table)
if attr[B.ATTR_DB_TABNAME] != "": if attr[B.ATTR_DB_TABNAME] != "":
table = attr[B.ATTR_DB_TABNAME] table = attr[B.ATTR_DB_TABNAME]
if attr[B.ATTR_DB_SCHEMA] != "": if attr[B.ATTR_DB_SCHEMA] != "":
@ -209,12 +207,12 @@ class Testexecuter():
print("sql_new "+sql_new) print("sql_new "+sql_new)
if (hasattr(job.par, B.PAR_DB_WHERE)): if (hasattr(job.par, B.PAR_DB_WHERE)):
# actual it parses only conjunct or disjunct normalform without parentesis # actual it parses only conjunct or disjunct normalform without parentesis
parts = utils.db_abstract.parseSQLwhere(getattr(job.par, B.PAR_DB_WHERE), self.conf[B.DATA_NODE_DDL][table]) parts = tools.db_abstract.parseSQLwhere(getattr(job.par, B.PAR_DB_WHERE), self.conf[B.DATA_NODE_DDL][table])
# Felder und Operationen # Felder und Operationen
# print(dbwhere) # print(dbwhere)
sql_new += " WHERE "+parts sql_new += " WHERE "+parts
if sql_new[0:6] == "SELECT": if sql_new[0:6] == "SELECT":
ids = utils.db_abstract.getTechnicalIDFields(self.conf["ddl"][table]) ids = tools.db_abstract.getTechnicalIDFields(self.conf["ddl"][table])
sql_new += " ORDER BY "+",".join(ids) sql_new += " ORDER BY "+",".join(ids)
print("sql_new "+sql_new) print("sql_new "+sql_new)
sql_new = sql_new.replace('!', "\'") sql_new = sql_new.replace('!', "\'")

10
basic/component.py

@ -35,8 +35,8 @@ def syncEnitity(job, elem):
:return: :return:
""" """
syncMethod = DEFAULT_SYNC syncMethod = DEFAULT_SYNC
if B.SUBJECT_ENTITY in job.conf.confs: if B.SUBJECT_ENTITY in job.conf:
syncMethod = job.conf.confs["entity"][TABLE_NAMES[0]]["storage"] syncMethod = job.conf["entity"][TABLE_NAMES[0]]["storage"]
if syncMethod.count("-") < 2: if syncMethod.count("-") < 2:
return return
fileTime = basic.entity.VAL_ZERO_TIME fileTime = basic.entity.VAL_ZERO_TIME
@ -44,13 +44,13 @@ def syncEnitity(job, elem):
# get git-commit # get git-commit
if "git" in syncMethod: if "git" in syncMethod:
comppath = tools.config_tool.getConfigPath(job, P.KEY_COMP, elem) comppath = tools.config_tool.getConfigPath(job, P.KEY_COMP, elem)
repopath = comppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:] repopath = comppath[len(job.conf[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:]
gitresult = tools.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1) gitresult = tools.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1)
fileTime = gitresult[0]["date"] fileTime = gitresult[0]["date"]
print(str(gitresult)) print(str(gitresult))
if "db" in syncMethod: if "db" in syncMethod:
if B.TOPIC_NODE_DB in job.conf.confs: if B.TOPIC_NODE_DB in job.conf:
dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE])
else: else:
return "No DB in job-config" return "No DB in job-config"
data = dbi.selectRows(TABLE_NAMES[0], job) data = dbi.selectRows(TABLE_NAMES[0], job)

281
basic/componentHandling.py

@ -1,281 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
# managing the components
# -----------------------------------------------------------------------------
"""
component has to be created in relation of the application (basis.yml).
Each componente could be created mostly once, but not everytime:
* the same instance of a component is used in different contexts
* there could be exist more instances
* there could be alternatives of an instance
Each kind of instance has its component-class and for each use should be an object be created.
Each crated component-onject are documented in the parameter-file.
"""
import tools.config_tool
import tools.conn_tool
import basic.program
import basic.message
import basic.component
import importlib
import copy
import basic.constants as B
import tools.data_const as D
comps = {}
PARAM_NOSUBNODE = [B.SUBJECT_ARTS, "components", "instance"]
DEFAULT_INST_CNT = 1
DEFAULT_INST_SGL = "y"
def getInstanceAttributes(conf):
"""
the attributes for instancing the component are set from configuration or from default
:param conf:
:return: a complete set of these attributes
"""
out = {
B.ATTR_INST_CNT: DEFAULT_INST_CNT,
B.ATTR_INST_SGL: DEFAULT_INST_SGL
}
if B.SUBJECT_INST in conf:
for attr in [B.ATTR_INST_CNT, B.ATTR_INST_SGL]:
if attr in conf[B.SUBJECT_INST]:
out[attr] = conf[B.SUBJECT_INST][attr]
return out
def getComponents(job, mainfct):
#job = basic.program.Job.getInstance()
verify = -2 + job.getDebugLevel("job_tool")
job.debug(verify, "getComponents " + mainfct)
out = []
for c in comps:
job.debug(verify, "getComponents " + c + ": " + str(comps[c].conf))
print("getComponents " + c + ": " + str(comps[c].conf))
if mainfct in comps[c].conf["function"]:
out.append(c)
return out
class ComponentManager:
__instance = None
__instances = {}
"""
initializes the Manager with all necessary components
"""
def __init__(self, job, option=""):
#job = basic.program.Job.getInstance()
job.m.logDebug("applicationscomponente -- " + str(type(job.par)))
self.components = {}
self.comps = {}
self.job = job
ComponentManager.__instances[job.jobid] = self
ComponentManager.__instance = self
print ("init ComponentHandling "+str(self))
def initComponents(self):
# sets components the first time
# afterwards set components from parameter-file
job = self.job # basic.program.Job.getInstance()
anw = job.par.application
job.m.logDebug("applicationscomponente -- " + str(type(job.par)))
if not job.conf.confs[B.SUBJECT_APPS].get(anw):
job.m.setFatal("application " + job.par.application + " is not configured")
return
for k in job.conf.confs[B.SUBJECT_APPS].get(anw):
if k == B.ATTR_APPS_PROJECT:
continue
job.m.logDebug("applicationscomponente -- " + k + ":")
print("applicationscomponente -- " + k + ":")
self.createComponent(k, 0, "")
def getComponent(self, compobjname):
job = self.job #basic.program.Job.getInstance()
verify = -2 + job.getDebugLevel("job_tool")
job.debug(verify, "getComponents " + compobjname)
if compobjname in self.comps:
return self.comps[compobjname]
return None
def getComponents(self, mainfct):
job = self.job #basic.program.Job.getInstance()
verify = -2 + job.getDebugLevel("job_tool")
job.debug(verify, "getComponents " + mainfct)
out = []
for c in self.comps:
job.debug(verify, "getComponents " + c + ": " + str(self.comps[c].conf))
#print("getComponents " + c + ": " + str(self.comps[c].conf))
if mainfct in self.comps[c].conf["function"]:
out.append(c)
return out
@staticmethod
def getInstance(job, init="N"):
if (job.jobid in ComponentManager.__instances):
return ComponentManager.__instances[job.jobid]
else:
return ComponentManager(job)
def createComponent(self, componentName, nr, suffix):
"""
in order to create a component it must be loaded
* knogwedge of the application - which components should be created
* technical-knowledge of the instanciated component, especially the connection, user, password
* business-knowledge of the component, especially of their interfaces resp. artifacts
:param componentName: Name of the component
:param nr: for numbered instance if component is multiple
:param suffix: suffix for specific context of the component
:return:
"""
job = self.job #basic.program.Job.getInstance()
verify = job.getDebugLevel("job_tool")
componentName = componentName.lower()
job.debug(verify, "createComponent " + componentName)
confs = tools.config_tool.getConfig(job, "comp", componentName)
conns = tools.conn_tool.getConnections(job, componentName)
instAttr = getInstanceAttributes(confs)
job.debug(verify, "createComponent -91- " + componentName + " : " + str(confs))
if nr > 0 and int(instAttr[B.ATTR_INST_CNT]) > 1:
job.m.setError("for multiple callers are multiple calls not implemented ")
if nr > 0 and len(conns) == 0:
job.m.setError("for multiple calls has only one call configured")
#print(confs)
parContent = job.loadParameter()
if len(conns) == 1:
c = self.createInstance(componentName, parContent, confs, conns, 0)
#print("createComponent 3 a " + componentName)
self.createSubComponents(c, nr, suffix)
else:
i = 1
#print("createComponent 3 b " + componentName)
for cn in conns:
c = self.createInstance(componentName, parContent, confs, conns, i)
self.createSubComponents(c, i, suffix)
i = i + 1
#print("createComponent 9 " + componentName)
#print(self.comps)
def createInstance(self, compName, parContent, confs, conns, nr):
"""
instance a component
:param compName: name without suffix or number
:param parContent: content of the parameter-file which is dumped from a pre-step
:param confs: configuration of the component
:param conns: connection-attributes for the specific environment
:param nr: number if component is multiple
:return: instance of the component with all necessary attributes
"""
job = self.job #basic.program.Job.getInstance()
cmodul = importlib.import_module(getComponentPath(compName))
class_ = getattr(cmodul, getComponentClass(compName))
c = class_()
if nr > 0:
name = compName + "_0" + str(nr)
i = nr - 1
else:
name = compName
i = 0
c.name = name
c.classname = compName
c.m = basic.message.Message(job, basic.message.LIMIT_DEBUG, "logTime", name)
c.conf = tools.config_tool.mergeConn(c.m, confs["conf"], conns[i])
c.conf[B.SUBJECT_CONN] = conns[i]
c.init(job)
if parContent is not None:
print("createComponent 5 a " + compName + " : " + str(parContent))
if B.SUBJECT_COMPS in parContent and compName in parContent[B.SUBJECT_COMPS]:
for k in parContent[B.SUBJECT_COMPS][compName].keys():
c.conf[k] = parContent[B.SUBJECT_COMPS][compName][k]
if B.SUBJECT_ARTS in c.conf and B.TOPIC_NODE_DB in c.conf[B.SUBJECT_ARTS]:
if not B.DATA_NODE_DDL in c.conf:
c.conf[B.DATA_NODE_DDL] = {}
for table in c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]:
if table in B.LIST_DB_ATTR:
continue
conf = tools.config_tool.getConfig(job, D.DDL_FILENAME, compName, table)
if B.DATA_NODE_TABLES in conf and table in conf[B.DATA_NODE_TABLES]:
c.conf[B.DATA_NODE_DDL][table] = conf[B.DATA_NODE_TABLES][table]
elif table in conf:
c.conf[B.DATA_NODE_DDL][table] = conf[table]
else:
c.conf[B.DATA_NODE_DDL][table] = conf
comps[name] = c
self.comps[name] = c
return c
def createSubComponents(self, comp, nr, suffix):
job = self.job #basic.program.Job.getInstance()
verify = -2 + job.getDebugLevel("job_tool")
job.debug(verify, "getComponents " + str(comp.conf[B.ATTR_INST_SUBCOMP]))
for c in comp.conf[B.ATTR_INST_SUBCOMP].keys():
if c == "none":
continue
self.createComponent(c, nr, suffix)
def getComponentDict(self):
job = self.job #basic.program.Job.getInstance()
verify = -2 + job.getDebugLevel("job_tool")
job.debug(verify, "getComponents ")
out = {}
for c in self.comps:
out[self.comps[c].name] = {}
for k in self.comps[c].conf.keys():
if isParameterSubnode(k): # B.SUBJECT_ARTS in k or "components" in k or "instance" in k:
out[self.comps[c].name][k] = copy.deepcopy(self.comps[c].conf[k])
return out
def getComponentFolder(comp):
return comp.lower()
def getComponentModul(comp):
return comp[0:1].upper() + comp[1:].lower()
def getComponentClass(comp):
return comp[0:1].upper() + comp[1:].lower()
def getComponentPath(comp):
return "components." + getComponentFolder(comp) + "." + getComponentModul(comp)
def getComponentDict(job = None):
#job = basic.program.Job.getInstance()
#verify = -2 + job.getDebugLevel("job_tool")
#job.debug(verify, "getComponents ")
out = {}
for c in comps:
out[comps[c].name] = {}
for k in comps[c].conf.keys():
if isParameterSubnode(k): # B.SUBJECT_ARTS in k or "components" in k or "instance" in k:
out[comps[c].name][k] = copy.deepcopy(comps[c].conf[k])
return out
def isParameterSubnode(key):
for k in PARAM_NOSUBNODE:
if key in k:
return False
return True
def getPlainCompname(name):
if "_0" in name:
return name[0:-3]
return name

3
basic/connection.py

@ -4,7 +4,6 @@
# Author : Ulrich Carmesin # Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de # Source : gitea.ucarmesin.de
# --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------
import utils.db_abstract
import basic.toolHandling import basic.toolHandling
import utils.data_const as D import utils.data_const as D
import basic.constants as B import basic.constants as B
@ -28,7 +27,7 @@ class Connection(basic.entity.Entity):
def getSchema(self): def getSchema(self):
dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE] dbtype = self.job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE]
dbi = basic.toolHandling.getDbTool(self.job, None, dbtype) dbi = basic.toolHandling.getDbTool(self.job, None, dbtype)
print(str(dbi)) print(str(dbi))
sql = dbi.getCreateTable("connection") sql = dbi.getCreateTable("connection")

13
basic/entity.py

@ -1,5 +1,4 @@
import getpass import getpass
import utils.db_abstract
import basic.toolHandling import basic.toolHandling
import utils.data_const as D import utils.data_const as D
import basic.constants as B import basic.constants as B
@ -53,7 +52,7 @@ class Entity:
def getDbAttr(self, job): def getDbAttr(self, job):
out = {} out = {}
for attr in [B.ATTR_DB_HOST, B.ATTR_DB_USER, B.ATTR_DB_DATABASE, B.ATTR_DB_PASSWD]: for attr in [B.ATTR_DB_HOST, B.ATTR_DB_USER, B.ATTR_DB_DATABASE, B.ATTR_DB_PASSWD]:
out[attr] = job.conf.confs[B.TOPIC_NODE_DB][attr] out[attr] = job.conf[B.TOPIC_NODE_DB][attr]
return out return out
def getDdl(self, job, ddl): def getDdl(self, job, ddl):
@ -70,8 +69,8 @@ class Entity:
return out return out
def createSchema(self, testserver): def createSchema(self, testserver):
if B.TOPIC_NODE_DB in self.job.conf.confs: if B.TOPIC_NODE_DB in self.job.conf:
dbi = basic.toolHandling.getDbTool(self.job, testserver, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) dbi = basic.toolHandling.getDbTool(self.job, testserver, self.job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE])
else: else:
return "No DB in job-config" return "No DB in job-config"
sql = self.getSchema() sql = self.getSchema()
@ -79,7 +78,7 @@ class Entity:
for s in sql.split(";\n"): for s in sql.split(";\n"):
if len(s) < 3: continue if len(s) < 3: continue
try: try:
dbi.execStatement(s+";", self.job.conf.confs[B.TOPIC_NODE_DB]) dbi.execStatement(s+";", self.job.conf[B.TOPIC_NODE_DB])
print("SQL executed: "+s) print("SQL executed: "+s)
except Exception as e: except Exception as e:
raise Exception("Fehler bei createSchema "+s) raise Exception("Fehler bei createSchema "+s)
@ -88,7 +87,7 @@ class Entity:
raise Exception(B.EXCEPT_NOT_IMPLEMENT) raise Exception(B.EXCEPT_NOT_IMPLEMENT)
def getHistoryFields(self): def getHistoryFields(self):
dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE] dbtype = self.job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE]
dbi = basic.toolHandling.getDbTool(self.job, None, dbtype) dbi = basic.toolHandling.getDbTool(self.job, None, dbtype)
sql = dbi.getSchemaAttribut("inscommit", D.TYPE_STR)+"," sql = dbi.getSchemaAttribut("inscommit", D.TYPE_STR)+","
sql += dbi.getSchemaAttribut("insauthor", D.TYPE_STR)+"," sql += dbi.getSchemaAttribut("insauthor", D.TYPE_STR)+","
@ -100,7 +99,7 @@ class Entity:
return sql return sql
def selectHistoryFields(self): def selectHistoryFields(self):
if B.TOPIC_NODE_DB in self.job.conf.confs: if B.TOPIC_NODE_DB in self.job.conf:
dbi = basic.toolHandling.getDbTool(self.job, self.testserver, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) dbi = basic.toolHandling.getDbTool(self.job, self.testserver, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE])
else: else:
return "No DB in job-config" return "No DB in job-config"

3
basic/testcase.py

@ -4,7 +4,6 @@
# Author : Ulrich Carmesin # Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de # Source : gitea.ucarmesin.de
# --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------
import utils.db_abstract
import basic.toolHandling import basic.toolHandling
import utils.data_const as D import utils.data_const as D
import basic.constants as B import basic.constants as B
@ -26,7 +25,7 @@ class Testcase(basic.entity.Entity):
self.job = job self.job = job
def getSchema(self): def getSchema(self):
dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE] dbtype = self.job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE]
dbi = basic.toolHandling.getDbTool(self.job, None, dbtype) dbi = basic.toolHandling.getDbTool(self.job, None, dbtype)
sql = dbi.getCreateTable("testcase") sql = dbi.getCreateTable("testcase")
sql += dbi.getSchemaAttribut("tcid", "id")+"," sql += dbi.getSchemaAttribut("tcid", "id")+","

2
basic/testexecution.py

@ -25,7 +25,7 @@ class Testexecution(basic.entity.Entity):
self.job = job self.job = job
def getSchema(self): def getSchema(self):
dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE] dbtype = self.job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE]
dbi = basic.toolHandling.getDbTool(self.job, None, dbtype) dbi = basic.toolHandling.getDbTool(self.job, None, dbtype)
sql = dbi.getCreateTable("testexecution") sql = dbi.getCreateTable("testexecution")
sql += dbi.getSchemaAttribut("teid", "id")+"," sql += dbi.getSchemaAttribut("teid", "id")+","

2
basic/testplan.py

@ -24,7 +24,7 @@ class Testplan(basic.entity.Entity):
self.job = job self.job = job
def getSchema(self): def getSchema(self):
dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE] dbtype = self.job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE]
dbi = basic.toolHandling.getDbTool(self.job, None, dbtype) dbi = basic.toolHandling.getDbTool(self.job, None, dbtype)
sql = dbi.getCreateTable("testplan") sql = dbi.getCreateTable("testplan")
sql += dbi.getSchemaAttribut("tpid", "id")+"," sql += dbi.getSchemaAttribut("tpid", "id")+","

2
basic/testsuite.py

@ -26,7 +26,7 @@ class Testsuite(basic.entity.Entity):
self.job = job self.job = job
def getSchema(self): def getSchema(self):
dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE] dbtype = self.job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE]
dbi = basic.toolHandling.getDbTool(self.job, None, dbtype) dbi = basic.toolHandling.getDbTool(self.job, None, dbtype)
sql = dbi.getCreateTable("testsuite") sql = dbi.getCreateTable("testsuite")
sql += dbi.getSchemaAttribut("tsid", "id")+"," sql += dbi.getSchemaAttribut("tsid", "id")+","

2
basic/user.py

@ -39,7 +39,7 @@ class User(basic.entity.Entity):
self.m = job.m self.m = job.m
def getSchema(self): def getSchema(self):
dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE] dbtype = self.job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE]
dbi = basic.toolHandling.getDbTool(self.job, None, dbtype) dbi = basic.toolHandling.getDbTool(self.job, None, dbtype)
sql = dbi.getCreateTable("user") sql = dbi.getCreateTable("user")
sql += dbi.getSchemaAttribut("id", "id")+"," sql += dbi.getSchemaAttribut("id", "id")+","

320
utils/path_tool.py

@ -1,320 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
""" In diesem Modul werden alle Funktionen zusammengefasst zur Generierung und Ermittlung von pathsn """
import os.path
import sys
import basic.program
import utils.config_tool
import re
import basic.constants as B
import utils.path_const as P
import utils.date_tool
import getpass
TOOL_NAME = "path_tool"
def getHome():
home = os.getcwd()
if home[-4:] == "test" and home[-6:] != "datest":
home = home[0:-5]
if home[-10:] == "components":
home = home[0:-11]
if home[-6:] == "datest":
prgdir = home[-6:]
home = home[0:-7]
elif home[-7:] == "program":
prgdir = home[-7:]
home = home[0:-8]
return home
def getBasisConfigPath():
home = os.getcwd()
a = home.split(os.path.sep)
for i in range(1, len(a)):
path = os.path.sep.join(a[0:-i])
path = os.path.join(path, P.VAL_CONFIG, B.BASIS_FILE)
for format in utils.config_tool.CONFIG_FORMAT:
filepath = path+"."+format
if os.path.isfile(filepath):
return filepath
if os.path.exists(filepath):
return filepath
raise Exception("no basis-configuration found")
def getActualJsonPath(job):
username = getpass.getuser()
path = os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_DEBUG], username+"Job.json")
print("------ path "+path)
#if os.path.exists(path):
# return path
return path
def getKeyValue(job, key, comp=None):
"""
this function gets the value for the key which relates to an attribute in the job or in the component
:param key:
:param comp:
:return:
"""
#job = basic.program.Job.getInstance()
try:
verify = job.getDebugLevel(TOOL_NAME)-4
except:
verify = False
pt = PathConf.getInstance(job)
if verify: job.debug(verify, "getKeyValue " + key)
if 'job.par' in key:
val = job.getParameter(key[8:])
return val
elif 'job.conf' in key:
val = job.conf.confs[B.SUBJECT_PATH][key[9:]]
if verify: job.debug(verify, val)
return val
elif 'job.' in key:
a = key[4:].split(":")
val = getattr(job, a[0])
# only date with hours
if a[0] in ["start"]:
print("++++++++++++++"+str(val))
val = utils.date_tool.formatParsedDate(str(val), utils.date_tool.F_LOG)
print("++++++++++++++"+val)
if len(a) > 1 and a[1] == "H":
val = val[0:-4]+"00"
if verify: job.debug(verify, val)
return val
# return job.conf.confs["paths"][key[9:]]
elif 'comp.' in key:
if comp is None:
raise Exception(P.EXP_COMP_MISSING.format(key))
if utils.config_tool.hasAttr(comp.conf, key[5:]):
return utils.config_tool.getAttr(comp.conf, key[5:])
if utils.config_tool.hasAttr(comp, key[5:]):
return utils.config_tool.getAttr(comp, key[5:])
return ""
elif 'env.' in key:
if key[4:] in comp.conf["conn"]:
return comp.conf["conn"][key[4:]]
pass
elif key in pt.pattern:
return pt.pattern[key]
elif "time" in key and hasattr(job, "start"):
return getattr(job, "start")
else:
return "xx-"+key+"-xx"
def composePath(job, pathname, comp):
"""
this function composes a concrete path by the structured pathname
- the key of pathname is declared in path_const and the structure is configurated in config/path.yml.
:param pathname - plain keyword
:param comp:
:return:
"""
#job = basic.program.Job.getInstance()
verify = job.getDebugLevel(TOOL_NAME)
pt = PathConf.getInstance(job)
job.debug(verify, "composePath " + pathname + " zu " + str(pt) + "mit ")
job.debug(verify, str(pt.pattern))
if pt.pattern[pathname]:
return composePattern(job, pt.pattern[pathname], comp)
else:
job.debug(verify, "in Pattern nicht vorhanden: " + pathname)
def composePattern(job, pattern, comp):
"""
the function composes the pattern to the standardarized path with the attributes
which are stored in the job and the component
- the key of pathname is declared in path_const and the structure is configurated in config/path.yml.
:param pattern: - keyword surroundet with {}
:param comp:
:return: path
"""
#job = basic.program.Job.getInstance()
try:
verify = job.getDebugLevel(TOOL_NAME)
except:
verify = False
verbose = not False
#job.debug(verify, "composePattern " + pattern)
max=5
l = re.findall('\{.*?\}', pattern)
#job.debug(verify, l)
print(l)
for pat in l:
if verbose: print(str(max) + ": " + pattern + ": " + pat)
pit = getKeyValue(job, pat[1:-1], comp)
if verbose: print(str(pit) + ": " + pattern + ": " + pat)
#job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
pattern = pattern.replace(pat, pit)
#job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
while ("{" in pattern):
max = max-1
#job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
pattern = composePattern(job, pattern, comp)
#job.debug(verify, str(max) + ": " + pattern + ": " + pat + ": " + pit)
if (max < 3) :
break
return pattern
def rejoinPath(a, b="", c="", d="", e="", f=""):
"""
this function concatenates the arguments to a path in the correct format for the operating-system
:param a:
:param b: optional
:param c: optional
:param d: optional
:param e: optional
:param f: optional
:return: path
"""
work = a+"/"+b+"/"+c+"/"+d+"/"+e+"/"+f
if a.find("://") > 1:
protocol = True
else:
protocol = False
work = re.sub(r'\\', '/', work)
work = re.sub(r'\/', '/', work)
work = re.sub(r'//', '/', work)
while work[-1:] == "/":
work = work[0:-1]
l = work.split("/")
out = ""
for x in l:
if len(x) < 1:
continue
if protocol:
if len(out) < 1:
out = x
else:
out = out+"/"+x
else:
out = os.path.join(out, x)
if out[1:2] == ":" and out[2:3] != "\\":
out = out[0:2]+"\\"+out[2:]
elif protocol:
if "://" not in out or out.index("://") > 8 or out.index("://") < 1:
i = out.index(":/")
out = out[0:i+1] + "/" + out[i+1:]
pass
if not protocol and out.count("\\") < 1 and out[0:1] != "/" and out[0:2] != "..":
out = "/"+out
return out
def extractPattern(job, pathtyp, comp=None):
"""
this function extracts recoursively all parts of the pathstrucure as key and gets the values from the
job-parameter and job-configuration
:param pathtyp: the name of the path-structure
:param comp:
:return: dictionary of all part (key) with their valuess
"""
#job = basic.program.Job.getInstance()
verify = job.getDebugLevel(TOOL_NAME)
out = []
pt = PathConf.getInstance(job)
pattern = pt.pattern[pathtyp]
work = pattern
while "{" in work:
i = work.index("{")
j = work.index("}")
pre = work[0:i]
pat = work[i+1:j]
job.debug(verify, work + " von " + str(i) + "-" + str(j) + " pre " + pre + "pat " + pat)
pit = getKeyValue(job, pat, comp)
tup = (pre, pat, pit)
out.append(tup)
work = work[j+1:]
return out
def extractPath(job, pathtyp, path):
"""
this function extracts parts of a concrete structered path and stores the parts
as attributes into the actual job. So these attributes can read from the concrete
path instead of the related parameter-arguments.
It stores the values into the job-parameter
:param pathtyp: the structure of the concrete path
:param path: the concrete path - it should be the directory in the parameter of the job
:return:
"""
#job = basic.program.Job.getInstance()
patterlist = extractPattern(job, pathtyp)
verbose = False
work = path
i = 0
if verbose: print("-- extractPatternList -- " + pathtyp + ":" + str(patterlist))
for p in patterlist:
if len(p) < 1 : continue
delim = p[0]
key = p[1]
val = p[2]
nextdelim = ""
if i >= len(patterlist) - 1:
nextdelim = ""
else:
nextdelim = patterlist[i+1][0]
if verbose: print("xPath delim " + delim + " " + str(len(delim)) + ", " + nextdelim + " work " + work)
work = work[len(delim):]
if verbose: print("xPath key " + key + " i " + str(i) + " work " + work)
if val is not None:
if verbose: print("val not none " + val)
if val in work:
if verbose: print("val ok")
work = work.replace(val, "")
elif "time" in key and "job.par" in key:
prop = ""
if i < len(patterlist) - 1:
prop = work[0:work.index(nextdelim)]
else:
prop = work
key = key[8:]
if verbose: print("setprop " + key + " = " + prop)
if hasattr(job.par, key): delattr(job.par, key)
setattr(job.par, key, val)
else:
if verbose: print("val not not ok " + val + " zu " + key)
elif "job.par" in key:
prop = ""
if i < len(patterlist) - 1:
if verbose: print("job.par nextdelim " + nextdelim)
prop = work[0:work.index(nextdelim)]
else:
prop = work
key = key[8:]
if verbose: print("setprop " + key + " = " + prop)
if hasattr(job.par, key): delattr(job.par, key)
setattr(job.par, key, prop)
work = work.replace(prop, "")
else:
if verbose: print("val is none " + key)
i = i +1
class PathConf:
"""
this class contains the structure-informations of the testrelevant directories
"""
__instance = None
def __init__(self, job=None):
#print('init pathConf')
confs = utils.config_tool.getConfig(job, "tool", "path")
self.pattern = confs["pattern"]
#print(self.pattern)
PathConf.__instance = self
@staticmethod
def getInstance(job = None):
#print("PathConf getInstance " + str(PathConf.__instance))
if (PathConf.__instance is None):
PathConf(job)
#print("PathConf getInstance " + str(PathConf.__instance))
return PathConf.__instance
Loading…
Cancel
Save