Browse Source

refactorings for new workspace

refactor
Ulrich 2 years ago
parent
commit
13bbfc6875
  1. 12
      basic/componentHandling.py
  2. 0
      basic/connection.py
  3. 30
      basic/program.py
  4. 0
      basic/testcase.py
  5. 0
      basic/testsuite.py
  6. 0
      make_workspace.py
  7. 4
      requirements.txt
  8. 18
      test/test_04config.py
  9. 17
      utils/config_tool.py
  10. 15
      utils/conn_tool.py
  11. 1
      utils/data_const.py
  12. 2
      utils/file_tool.py
  13. 6
      utils/path_const.py
  14. 17
      utils/path_tool.py

12
basic/componentHandling.py

@ -112,7 +112,7 @@ class ComponentManager:
out = [] out = []
for c in self.comps: for c in self.comps:
job.debug(verify, "getComponents " + c + ": " + str(self.comps[c].conf)) job.debug(verify, "getComponents " + c + ": " + str(self.comps[c].conf))
print("getComponents " + c + ": " + str(self.comps[c].conf)) #print("getComponents " + c + ": " + str(self.comps[c].conf))
if mainfct in self.comps[c].conf["function"]: if mainfct in self.comps[c].conf["function"]:
out.append(c) out.append(c)
return out return out
@ -149,21 +149,21 @@ class ComponentManager:
job.m.setError("for multiple callers are multiple calls not implemented ") job.m.setError("for multiple callers are multiple calls not implemented ")
if nr > 0 and len(conns) == 0: if nr > 0 and len(conns) == 0:
job.m.setError("for multiple calls has only one call configured") job.m.setError("for multiple calls has only one call configured")
print(confs) #print(confs)
parContent = job.loadParameter() parContent = job.loadParameter()
if len(conns) == 1: if len(conns) == 1:
c = self.createInstance(componentName, parContent, confs, conns, 0) c = self.createInstance(componentName, parContent, confs, conns, 0)
print("createComponent 3 a " + componentName) #print("createComponent 3 a " + componentName)
self.createSubComponents(c, nr, suffix) self.createSubComponents(c, nr, suffix)
else: else:
i = 1 i = 1
print("createComponent 3 b " + componentName) #print("createComponent 3 b " + componentName)
for cn in conns: for cn in conns:
c = self.createInstance(componentName, parContent, confs, conns, i) c = self.createInstance(componentName, parContent, confs, conns, i)
self.createSubComponents(c, i, suffix) self.createSubComponents(c, i, suffix)
i = i + 1 i = i + 1
print("createComponent 9 " + componentName) #print("createComponent 9 " + componentName)
print(self.comps) #print(self.comps)
def createInstance(self, compName, parContent, confs, conns, nr): def createInstance(self, compName, parContent, confs, conns, nr):

0
basic/connection.py

30
basic/program.py

@ -21,7 +21,7 @@ import utils.path_tool
import utils.file_tool import utils.file_tool
import utils.config_tool import utils.config_tool
import test.constants as T import test.constants as T
import utils.path_const as P
jobdef = { jobdef = {
"webflask": { "webflask": {
@ -201,7 +201,13 @@ class Job:
print("prog-42 " + str(self.par.basedir)) print("prog-42 " + str(self.par.basedir))
conf = Configuration(self, program) conf = Configuration(self, program)
self.conf = conf self.conf = conf
appl = utils.config_tool.getConfig(self, "basic", B.SUBJECT_APPS) try:
path = utils.config_tool.getConfigPath(self, P.KEY_BASIC, B.BASIS_FILE)
print("comps.basispath "+path)
self.conf.setConfiguration(self, path)
except:
pass # the special path is not necessary
appl = utils.config_tool.getConfig(self, P.KEY_BASIC, B.SUBJECT_APPS)
print(appl) print(appl)
if appl is not None: if appl is not None:
self.conf.confs[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS] self.conf.confs[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS]
@ -503,16 +509,24 @@ class Configuration:
self.setConfiguration(job, path) self.setConfiguration(job, path)
return return
def debug(self, verify, text):
if hasattr(self, "m"):
self.m.debug(verify, text)
def setConfiguration(self, job, path): def setConfiguration(self, job, path):
self.confs = {} if not hasattr(self, "confs"):
self.confs = {}
self.confs["configpath"] = []
doc = utils.file_tool.readFileDict(job, path, None) doc = utils.file_tool.readFileDict(job, path, None)
self.confs["configpath"] = path self.confs["configpath"].append(path)
if "basic" in doc: if "basic" in doc:
for i, v in doc["basic"].items(): for k, v in doc["basic"].items():
self.confs[i] = v if k not in self.confs:
self.confs[k] = v
else: else:
for i, v in doc.items(): for k, v in doc.items():
self.confs[i] = v if k not in self.confs:
self.confs[k] = v
def setConfig(self, path, val): def setConfig(self, path, val):
a = path.split(".") a = path.split(".")

0
basic/testcase.py

0
basic/testsuite.py

0
make_workspace.py

4
requirements.txt

@ -1,6 +1,6 @@
pyyaml~=6.0 pyyaml~=6.0
paramiko~=2.9.2 paramiko~=2.9.2
mysql-connector-python
cryptography~=36.0.1 cryptography~=36.0.1
pip~=21.3.1 pip~=21.3.1
MarkupSafe~=2.1.1 MarkupSafe~=2.1.1
@ -12,4 +12,4 @@ setuptools~=60.5.0
zipp~=3.8.1 zipp~=3.8.1
xmltodict~=0.12.0 xmltodict~=0.12.0
pyspark~=3.2.1 pyspark~=3.2.1
Flask~=2.2.2 Flask~=2.2.2

18
test/test_04config.py

@ -18,7 +18,7 @@ import utils.path_const as P
import basic.constants as B import basic.constants as B
TEST_FUNCTIONS = ["test_01getConfig", "test_02mergeAttributes", "test_03getAttributes"] TEST_FUNCTIONS = ["test_01getConfig", "test_02mergeAttributes", "test_03getAttributes"]
TEST_FUNCTIONS = ["test_03getAttributes"] #TEST_FUNCTIONS = ["test_03getAttributes"]
verbose = False verbose = False
class MyTestCase(unittest.TestCase): class MyTestCase(unittest.TestCase):
@ -32,18 +32,18 @@ class MyTestCase(unittest.TestCase):
return return
job = test.testtools.getJob() job = test.testtools.getJob()
x = B.SUBJECT_APPS x = B.SUBJECT_APPS
r = utils.config_tool.getConfigPath(x, P.KEY_BASIC) r = utils.config_tool.getConfigPath(job, P.KEY_BASIC, x)
self.assertIn(os.path.join(T.COMP_PATH, B.SUBJECT_APPS), r) self.assertIn(os.path.join(T.COMP_PATH, P.VAL_CONFIG, B.SUBJECT_APPS), r)
cnttest += 1 cnttest += 1
x = "path" x = "path"
r = utils.config_tool.getConfigPath(x, P.KEY_TOOL) r = utils.config_tool.getConfigPath(job, P.KEY_TOOL, x)
self.assertIn(os.path.join(T.PROG_PATH, P.VAL_UTIL, P.VAL_CONFIG), r) self.assertIn(os.path.join(T.PROG_PATH, P.VAL_UTIL, P.VAL_CONFIG), r)
cnttest += 1 cnttest += 1
x = "conn" x = "conn"
r = utils.config_tool.getConfigPath(x, P.KEY_TOOL) r = utils.config_tool.getConfigPath(job, P.KEY_TOOL, x)
self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_ENV)), r) self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_ENV)), r)
cnttest += 1 cnttest += 1
self.assertRaises(Exception, utils.config_tool.getConfigPath, (P.KEY_COMP, "TestX2")) self.assertRaises(Exception, utils.config_tool.getConfigPath, (job, P.KEY_COMP, "TestX2"))
# self.assertEqual(r, None) # self.assertEqual(r, None)
cnttest += 1 cnttest += 1
r = utils.config_tool.getConfigPath(job, P.KEY_COMP, "testcrm") r = utils.config_tool.getConfigPath(job, P.KEY_COMP, "testcrm")
@ -98,9 +98,9 @@ class MyTestCase(unittest.TestCase):
comp = test.testtools.getComp(job, "testrest") comp = test.testtools.getComp(job, "testrest")
path = "file.xmlrest" path = "file.xmlrest"
attrList = utils.config_tool.getAttributeList(comp, path, job) attrList = utils.config_tool.getAttributeList(comp, path, job)
print(str(comp.conf["conn"])) #print(str(comp.conf["conn"]))
print(str(comp.conf[B.SUBJECT_ARTS])) #print(str(comp.conf[B.SUBJECT_ARTS]))
print(str(attrList)) #print(str(attrList))
MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)

17
utils/config_tool.py

@ -49,7 +49,7 @@ def getConfigPath(job, modul, name, subname=""):
if modul == P.KEY_TOOL: if modul == P.KEY_TOOL:
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
P.KEY_TOOL+"_"+name+"."+format) P.VAL_CONFIG, P.KEY_TOOL+"_"+name+"."+format)
job.debug(verify, "1 " + pathname) job.debug(verify, "1 " + pathname)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
@ -105,7 +105,7 @@ def getConfigPath(job, modul, name, subname=""):
elif modul == P.KEY_BASIC: elif modul == P.KEY_BASIC:
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
name + "."+format) P.VAL_CONFIG , name + "."+format)
job.debug(verify, "4 " + pathname) job.debug(verify, "4 " + pathname)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
@ -172,7 +172,7 @@ def getConfigPath(job, modul, name, subname=""):
job.debug(verify, "9 " + pathname) job.debug(verify, "9 " + pathname)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name) raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name)
def getConfValue(attribute, comp): def getConfValue(attribute, comp):
@ -189,41 +189,30 @@ def getConfValue(attribute, comp):
def getAttr(o, name): def getAttr(o, name):
#print("hasAttr " + str(type(o))+" "+name)
if (isinstance(o, dict)): if (isinstance(o, dict)):
if (name in o.keys()): if (name in o.keys()):
#print("hasAttr dict ok " + str(type(o)))
return o[name] return o[name]
#print("hasAttr dict "+str(type(o)))
elif (isinstance(o, list)): elif (isinstance(o, list)):
pass pass
#print("hasAttr list "+str(type(o)))
elif hasattr(o, name): elif hasattr(o, name):
#print("hasAttr class ok "+str(type(o)))
return getattr(o, name) return getattr(o, name)
return False return False
def hasAttr(o, name): def hasAttr(o, name):
#print("hasAttr " + str(type(o))+" "+name)
if (isinstance(o, dict)): if (isinstance(o, dict)):
if (name in o.keys()): if (name in o.keys()):
#print("hasAttr dict ok " + str(type(o)))
return True return True
#print("hasAttr dict "+str(type(o)))
elif (isinstance(o, list)): elif (isinstance(o, list)):
pass pass
#print("hasAttr list "+str(type(o)))
elif hasattr(o, name): elif hasattr(o, name):
#print("hasAttr class ok "+str(type(o)))
return True return True
return False return False
def getConfig(job, modul, name, subname=""): def getConfig(job, modul, name, subname=""):
#job = basic.program.Job.getInstance()
if job is None: if job is None:
verify = 24 verify = 24
else: else:

15
utils/conn_tool.py

@ -51,13 +51,16 @@ def getConnections(job, comp):
msg.debug(verify, "getConnections " + comp) msg.debug(verify, "getConnections " + comp)
conn = {} conn = {}
conns = [] conns = []
if job.conf.confs.get("tools").get("connsrc") in [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV]: # if a datest-database exists read the connections
conn = utils.config_tool.getConfig(job, "tool", B.SUBJECT_CONN) conndb = {}
if not comp in conn["env"]: if job.conf.confs.get("db"):
job.m.setFatal("Conn-Tool: Comp not configured " + comp) # select
elif job.conf.confs.get("tools").get("connsrc") == "flaskdb":
pass pass
conn = utils.config_tool.getConfig(job, "tool", B.SUBJECT_CONN)
if not comp in conn["env"]:
job.m.setFatal("Conn-Tool: Comp not configured " + comp)
attr = {} attr = {}
if "general" in conn["env"]: if "general" in conn["env"]:
for a in conn["env"]["general"]: for a in conn["env"]["general"]:
@ -69,7 +72,7 @@ def getConnections(job, comp):
#if ("types" in conn["env"][comp]): #if ("types" in conn["env"][comp]):
# xtypes = conn["env"][comp]["types"] # xtypes = conn["env"][comp]["types"]
for i in range(conn["env"][comp][B.SUBJECT_INST][B.ATTR_INST_CNT]): for i in range(conn["env"][comp][B.SUBJECT_INST][B.ATTR_INST_CNT]):
print("range " + str(i + 1)) #print("range " + str(i + 1))
instnr = "inst" + str(i + 1) instnr = "inst" + str(i + 1)
#if (xtypes is not None): #if (xtypes is not None):
# conn["env"][comp][instnr]["types"] = xtypes # conn["env"][comp][instnr]["types"] = xtypes

1
utils/data_const.py

@ -9,6 +9,7 @@ DDL_FILENAME = "DATASTRUCTURE"
DATA_NODE_TYPE = "type" DATA_NODE_TYPE = "type"
TYPE_STRING = "string" TYPE_STRING = "string"
TYPE_STR = "str" TYPE_STR = "str"
TYPE_TEXT = "text"
TYPE_INT = "int" TYPE_INT = "int"
TYPE_FLOAT = "float" TYPE_FLOAT = "float"
TYPE_DOUBLE = "double" TYPE_DOUBLE = "double"

2
utils/file_tool.py

@ -127,7 +127,7 @@ def getTree(msg, job, pfad):
def mkPaths(job, path, msg): def mkPaths(job, path, msg):
# job = basic.program.Job.getInstance() # job = basic.program.Job.getInstance()
verify = int(job.getDebugLevel("file_tool")) verify = int(job.getDebugLevel("file_tool"))
modus = job.conf.confs["paths"]["mode"] #modus = job.conf.confs["paths"]["mode"]
dirname = os.path.dirname(path) dirname = os.path.dirname(path)
if os.path.exists(dirname): if os.path.exists(dirname):
return return

6
utils/path_const.py

@ -28,6 +28,10 @@ VAL_COMPS = "components"
""" subdir for the plugin components """ """ subdir for the plugin components """
VAL_BASIC = "basic" VAL_BASIC = "basic"
""" subdir for the basic job-framework """ """ subdir for the basic job-framework """
VAL_BASE_DATA = "data"
""" subdir for the basis data-folder """
VAL_TDATA = "testdata"
""" subdir for the basis data-folder """
# ------------------------------------------------------------- # -------------------------------------------------------------
# parameter with arguments # parameter with arguments
PAR_APP = "job.par." + B.PAR_APP PAR_APP = "job.par." + B.PAR_APP
@ -73,6 +77,8 @@ ATTR_PATH_RELEASE = "release"
""" This constant defines the folder in testing-filesystem, used for configs related to release """ """ This constant defines the folder in testing-filesystem, used for configs related to release """
ATTR_PATH_TDATA = "testdata" ATTR_PATH_TDATA = "testdata"
""" This constant defines the folder in testing-filesystem with the testcase-specifications """ """ This constant defines the folder in testing-filesystem with the testcase-specifications """
ATTR_PATH_TEMP = "temp"
""" This constant defines the debug-folder in testing-filesystem """
ATTR_PATH_PATTN = "pattern" ATTR_PATH_PATTN = "pattern"
""" This constant defines the debug-folder in testing-filesystem """ """ This constant defines the debug-folder in testing-filesystem """

17
utils/path_tool.py

@ -35,11 +35,14 @@ def getBasisConfigPath():
a = home.split(os.path.sep) a = home.split(os.path.sep)
for i in range(0, len(a)): for i in range(0, len(a)):
path = os.path.sep.join(a[0:-i]) path = os.path.sep.join(a[0:-i])
path = os.path.join(path, "config", B.BASIS_FILE) path = os.path.join(path, P.VAL_CONFIG, B.BASIS_FILE)
for format in utils.config_tool.CONFIG_FORMAT: for format in utils.config_tool.CONFIG_FORMAT:
path += "."+format filepath = path+"."+format
if os.path.exists(path): if os.path.isfile(filepath):
return path return filepath
if os.path.exists(filepath):
return filepath
raise Exception("no basis-configuration found")
def getKeyValue(job, key, comp=None): def getKeyValue(job, key, comp=None):
@ -110,7 +113,7 @@ def composePattern(job, pattern, comp):
""" """
#job = basic.program.Job.getInstance() #job = basic.program.Job.getInstance()
verify = job.getDebugLevel(TOOL_NAME) verify = job.getDebugLevel(TOOL_NAME)
verbose = True verbose = False
job.debug(verify, "composePattern " + pattern) job.debug(verify, "composePattern " + pattern)
max=5 max=5
l = re.findall('\{.*?\}', pattern) l = re.findall('\{.*?\}', pattern)
@ -271,10 +274,10 @@ class PathConf:
""" """
__instance = None __instance = None
def __init__(self, job=None): def __init__(self, job=None):
print('init pathConf') #print('init pathConf')
confs = utils.config_tool.getConfig(job, "tool", "path") confs = utils.config_tool.getConfig(job, "tool", "path")
self.pattern = confs["pattern"] self.pattern = confs["pattern"]
print(self.pattern) #print(self.pattern)
PathConf.__instance = self PathConf.__instance = self
@staticmethod @staticmethod

Loading…
Cancel
Save