refactorings for new workspace

basic/componentHandling.py

@@ -112,7 +112,7 @@ class ComponentManager:
         out = []
         for c in self.comps:
             job.debug(verify, "getComponents " + c + ": " + str(self.comps[c].conf))
-            print("getComponents " + c + ": " + str(self.comps[c].conf))
+            #print("getComponents " + c + ": " + str(self.comps[c].conf))
             if mainfct in self.comps[c].conf["function"]:
                 out.append(c)
         return out
@@ -149,21 +149,21 @@ class ComponentManager:
             job.m.setError("for multiple callers are multiple calls not implemented ")
         if nr > 0 and len(conns) == 0:
             job.m.setError("for multiple calls has only one call configured")
-        print(confs)
+        #print(confs)
         parContent = job.loadParameter()
         if len(conns) == 1:
             c = self.createInstance(componentName, parContent, confs, conns, 0)
-            print("createComponent 3 a " + componentName)
+            #print("createComponent 3 a " + componentName)
             self.createSubComponents(c, nr, suffix)
         else:
             i = 1
-            print("createComponent 3 b " + componentName)
+            #print("createComponent 3 b " + componentName)
             for cn in conns:
                 c = self.createInstance(componentName, parContent, confs, conns, i)
                 self.createSubComponents(c, i, suffix)
                 i = i + 1
-        print("createComponent 9 " + componentName)
-        print(self.comps)
+        #print("createComponent 9 " + componentName)
+        #print(self.comps)
 
 
     def createInstance(self, compName, parContent, confs, conns, nr):

basic/program.py

@@ -21,7 +21,7 @@ import utils.path_tool
 import utils.file_tool
 import utils.config_tool
 import test.constants as T
-
+import utils.path_const as P
 
 jobdef = {
 	"webflask": {
@@ -201,7 +201,13 @@ class Job:
 		print("prog-42 " + str(self.par.basedir))
 		conf = Configuration(self, program)
 		self.conf = conf
-		appl = utils.config_tool.getConfig(self, "basic", B.SUBJECT_APPS)
+		try:
+			path = utils.config_tool.getConfigPath(self, P.KEY_BASIC, B.BASIS_FILE)
+			print("comps.basispath "+path)
+			self.conf.setConfiguration(self, path)
+		except:
+			pass # the special path is not necessary
+		appl = utils.config_tool.getConfig(self, P.KEY_BASIC, B.SUBJECT_APPS)
 		print(appl)
 		if appl is not None:
 			self.conf.confs[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS]
@@ -503,16 +509,24 @@ class Configuration:
 		self.setConfiguration(job, path)
 		return
 
+	def debug(self, verify, text):
+		if hasattr(self, "m"):
+			self.m.debug(verify, text)
+
 	def setConfiguration(self, job, path):
-		self.confs = {}
+		if not hasattr(self, "confs"):
+			self.confs = {}
+			self.confs["configpath"] = []
 		doc = utils.file_tool.readFileDict(job, path, None)
-		self.confs["configpath"] = path
+		self.confs["configpath"].append(path)
 		if "basic" in doc:
-			for i, v in doc["basic"].items():
-				self.confs[i] = v
+			for k, v in doc["basic"].items():
+				if k not in self.confs:
+					self.confs[k] = v
 		else:
-			for i, v in doc.items():
-				self.confs[i] = v
+			for k, v in doc.items():
+				if k not in self.confs:
+					self.confs[k] = v
 
 	def setConfig(self, path, val):
 		a = path.split(".")

requirements.txt

@@ -1,6 +1,6 @@
 pyyaml~=6.0
 paramiko~=2.9.2
-
+mysql-connector-python
 cryptography~=36.0.1
 pip~=21.3.1
 MarkupSafe~=2.1.1
@@ -12,4 +12,4 @@ setuptools~=60.5.0
 zipp~=3.8.1
 xmltodict~=0.12.0
 pyspark~=3.2.1
-Flask~=2.2.2
+Flask~=2.2.2

test/test_04config.py

@@ -18,7 +18,7 @@ import utils.path_const as P
 import basic.constants as B
 
 TEST_FUNCTIONS = ["test_01getConfig", "test_02mergeAttributes", "test_03getAttributes"]
-TEST_FUNCTIONS = ["test_03getAttributes"]
+#TEST_FUNCTIONS = ["test_03getAttributes"]
 verbose = False
 
 class MyTestCase(unittest.TestCase):
@@ -32,18 +32,18 @@ class MyTestCase(unittest.TestCase):
             return
         job = test.testtools.getJob()
         x = B.SUBJECT_APPS
-        r = utils.config_tool.getConfigPath(x, P.KEY_BASIC)
-        self.assertIn(os.path.join(T.COMP_PATH, B.SUBJECT_APPS), r)
+        r = utils.config_tool.getConfigPath(job, P.KEY_BASIC, x)
+        self.assertIn(os.path.join(T.COMP_PATH, P.VAL_CONFIG, B.SUBJECT_APPS), r)
         cnttest += 1
         x = "path"
-        r = utils.config_tool.getConfigPath(x, P.KEY_TOOL)
+        r = utils.config_tool.getConfigPath(job, P.KEY_TOOL, x)
         self.assertIn(os.path.join(T.PROG_PATH, P.VAL_UTIL, P.VAL_CONFIG), r)
         cnttest += 1
         x = "conn"
-        r = utils.config_tool.getConfigPath(x, P.KEY_TOOL)
+        r = utils.config_tool.getConfigPath(job, P.KEY_TOOL, x)
         self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_ENV)), r)
         cnttest += 1
-        self.assertRaises(Exception, utils.config_tool.getConfigPath, (P.KEY_COMP, "TestX2"))
+        self.assertRaises(Exception, utils.config_tool.getConfigPath, (job, P.KEY_COMP, "TestX2"))
         # self.assertEqual(r, None)
         cnttest += 1
         r = utils.config_tool.getConfigPath(job, P.KEY_COMP, "testcrm")
@@ -98,9 +98,9 @@ class MyTestCase(unittest.TestCase):
         comp = test.testtools.getComp(job, "testrest")
         path = "file.xmlrest"
         attrList = utils.config_tool.getAttributeList(comp, path, job)
-        print(str(comp.conf["conn"]))
-        print(str(comp.conf[B.SUBJECT_ARTS]))
-        print(str(attrList))
+        #print(str(comp.conf["conn"]))
+        #print(str(comp.conf[B.SUBJECT_ARTS]))
+        #print(str(attrList))
         MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
 
 

utils/config_tool.py

@@ -49,7 +49,7 @@ def getConfigPath(job, modul, name, subname=""):
     if modul == P.KEY_TOOL:
         for format in CONFIG_FORMAT:
             pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
-                                    P.KEY_TOOL+"_"+name+"."+format)
+                                    P.VAL_CONFIG, P.KEY_TOOL+"_"+name+"."+format)
             job.debug(verify, "1 " + pathname)
             if os.path.exists(pathname):
                 return pathname
@@ -105,7 +105,7 @@ def getConfigPath(job, modul, name, subname=""):
     elif modul == P.KEY_BASIC:
         for format in CONFIG_FORMAT:
             pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
-                                    name + "."+format)
+                        P.VAL_CONFIG ,  name + "."+format)
             job.debug(verify, "4 " + pathname)
             if os.path.exists(pathname):
                 return pathname
@@ -172,7 +172,7 @@ def getConfigPath(job, modul, name, subname=""):
             job.debug(verify, "9 " + pathname)
             if os.path.exists(pathname):
                 return pathname
-        raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name)
+    raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name)
 
 
 def getConfValue(attribute, comp):
@@ -189,41 +189,30 @@ def getConfValue(attribute, comp):
 
 
 def getAttr(o, name):
-    #print("hasAttr " + str(type(o))+" "+name)
     if (isinstance(o, dict)):
         if (name in o.keys()):
-            #print("hasAttr dict ok " + str(type(o)))
             return o[name]
-        #print("hasAttr dict "+str(type(o)))
     elif (isinstance(o, list)):
         pass
-        #print("hasAttr list "+str(type(o)))
     elif hasattr(o, name):
-        #print("hasAttr class ok "+str(type(o)))
         return getattr(o, name)
     return False
 
 
 def hasAttr(o, name):
-    #print("hasAttr " + str(type(o))+" "+name)
     if (isinstance(o, dict)):
         if (name in o.keys()):
-            #print("hasAttr dict ok " + str(type(o)))
             return True
-        #print("hasAttr dict "+str(type(o)))
     elif (isinstance(o, list)):
         pass
-        #print("hasAttr list "+str(type(o)))
 
 
     elif hasattr(o, name):
-        #print("hasAttr class ok "+str(type(o)))
         return True
     return False
 
 
 def getConfig(job, modul, name, subname=""):
-    #job = basic.program.Job.getInstance()
     if job is None:
         verify = 24
     else:

utils/conn_tool.py

@@ -51,13 +51,16 @@ def getConnections(job, comp):
     msg.debug(verify, "getConnections " + comp)
     conn = {}
     conns = []
-    if job.conf.confs.get("tools").get("connsrc") in [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV]:
-        conn = utils.config_tool.getConfig(job, "tool", B.SUBJECT_CONN)
-        if not comp in conn["env"]:
-            job.m.setFatal("Conn-Tool: Comp not configured " + comp)
-    elif job.conf.confs.get("tools").get("connsrc") == "flaskdb":
+    # if a datest-database exists read the connections
+    conndb = {}
+    if job.conf.confs.get("db"):
+        # select
         pass
 
+    conn = utils.config_tool.getConfig(job, "tool", B.SUBJECT_CONN)
+    if not comp in conn["env"]:
+        job.m.setFatal("Conn-Tool: Comp not configured " + comp)
+
     attr = {}
     if "general" in conn["env"]:
         for a in conn["env"]["general"]:
@@ -69,7 +72,7 @@ def getConnections(job, comp):
     #if ("types" in conn["env"][comp]):
     #    xtypes = conn["env"][comp]["types"]
     for i in range(conn["env"][comp][B.SUBJECT_INST][B.ATTR_INST_CNT]):
-        print("range " + str(i + 1))
+        #print("range " + str(i + 1))
         instnr = "inst" + str(i + 1)
         #if (xtypes is not None):
         #    conn["env"][comp][instnr]["types"] = xtypes

utils/data_const.py

@@ -9,6 +9,7 @@ DDL_FILENAME = "DATASTRUCTURE"
 DATA_NODE_TYPE = "type"
 TYPE_STRING = "string"
 TYPE_STR = "str"
+TYPE_TEXT = "text"
 TYPE_INT = "int"
 TYPE_FLOAT = "float"
 TYPE_DOUBLE = "double"

utils/file_tool.py

@@ -127,7 +127,7 @@ def getTree(msg, job, pfad):
 def mkPaths(job, path, msg):
     # job = basic.program.Job.getInstance()
     verify = int(job.getDebugLevel("file_tool"))
-    modus = job.conf.confs["paths"]["mode"]
+    #modus = job.conf.confs["paths"]["mode"]
     dirname = os.path.dirname(path)
     if os.path.exists(dirname):
         return

utils/path_const.py

@@ -28,6 +28,10 @@ VAL_COMPS = "components"
 """ subdir for the plugin components """
 VAL_BASIC = "basic"
 """ subdir for the basic job-framework """
+VAL_BASE_DATA = "data"
+""" subdir for the basis data-folder """
+VAL_TDATA = "testdata"
+""" subdir for the basis data-folder """
 # -------------------------------------------------------------
 # parameter with arguments
 PAR_APP = "job.par." + B.PAR_APP
@@ -73,6 +77,8 @@ ATTR_PATH_RELEASE = "release"
 """ This constant defines the folder in testing-filesystem, used for configs related to release """
 ATTR_PATH_TDATA = "testdata"
 """ This constant defines the folder in testing-filesystem with the testcase-specifications """
+ATTR_PATH_TEMP = "temp"
+""" This constant defines the debug-folder in testing-filesystem  """
 ATTR_PATH_PATTN = "pattern"
 """ This constant defines the debug-folder in testing-filesystem  """
 

utils/path_tool.py

@@ -35,11 +35,14 @@ def getBasisConfigPath():
     a = home.split(os.path.sep)
     for i in range(0, len(a)):
         path = os.path.sep.join(a[0:-i])
-        path = os.path.join(path, "config", B.BASIS_FILE)
+        path = os.path.join(path, P.VAL_CONFIG, B.BASIS_FILE)
         for format in utils.config_tool.CONFIG_FORMAT:
-            path += "."+format
-            if os.path.exists(path):
-                return path
+            filepath = path+"."+format
+            if os.path.isfile(filepath):
+                return filepath
+            if os.path.exists(filepath):
+                return filepath
+    raise Exception("no basis-configuration found")
 
 
 def getKeyValue(job, key, comp=None):
@@ -110,7 +113,7 @@ def composePattern(job, pattern, comp):
     """
     #job = basic.program.Job.getInstance()
     verify = job.getDebugLevel(TOOL_NAME)
-    verbose = True
+    verbose = False
     job.debug(verify, "composePattern " + pattern)
     max=5
     l = re.findall('\{.*?\}', pattern)
@@ -271,10 +274,10 @@ class PathConf:
     """
     __instance = None
     def __init__(self, job=None):
-        print('init pathConf')
+        #print('init pathConf')
         confs = utils.config_tool.getConfig(job, "tool", "path")
         self.pattern = confs["pattern"]
-        print(self.pattern)
+        #print(self.pattern)
         PathConf.__instance = self
 
     @staticmethod