merge conn into config

basic/constants.py

@@ -88,6 +88,8 @@ DATA_NODE_OPTION = "_option"
 """ This constant defines main node in the testdata for testcase specific parameters """
 DATA_NODE_TABLES = "_tables"
 """ This constant defines the main node in the testdata for the steps to execute """
+DATA_NODE_KEYS = "_keys"
+""" This constant defines the main node in the testdata for the steps to execute """
 DATA_NODE_DDL  = "ddl"
 """ This constant defines the node for data scheme (DataDefinitionLanguage) 
 The fields are defined in data_const (D) """
@@ -98,25 +100,40 @@ DATA_NODE_PAR  = "par"
 
 """ This constant defines """
 TOPIC_NODE_DB = "db"
-
 # testexec, db_abstr
+ATTR_DB_TYPE = "type"
 ATTR_DB_PARTITION = "partitioned"
 """ optional attribute if table is partitioned
  - this keyword delimited by "+" will be replaced by partition-names which are parametrized """
-ATTR_DB_CONN_JAR = "conn_jar_name"
-""" optional attribute for connection-jar-file instead of connection by ip, port """
-ATTR_CONN_HOST = "hostname"
-""" optional attribute for connection-jar-file instead of connection by ip, port """
-ATTR_CONN_TENANT = "tenant"
-""" optional attribute for connection-jar-file instead of connection by ip, port """
 ATTR_DB_DATABASE = "database"
 """ attribute for technical name of the database """
 ATTR_DB_SCHEMA = "schema"
 """ optional attribute for technical name of the schema  """
 ATTR_DB_TABNAME = "tabname"
 """ optional attribute in order to use a different technical name for the db-table """
+LIST_DB_ATTR = [ATTR_DB_TYPE, ATTR_DB_PARTITION, ATTR_DB_DATABASE, ATTR_DB_SCHEMA, ATTR_DB_TABNAME]
+ATTR_DB_CONN_JAR = "conn_jar_name"
+""" optional attribute for connection-jar-file instead of connection by ip, port """
+ATTR_CONN_HOST = "hostname"
+""" optional attribute for connection-jar-file instead of connection by ip, port """
+ATTR_CONN_TENANT = "tenant"
+""" optional attribute for connection-jar-file instead of connection by ip, port """
+ATTR_CONN_IP = "ip"
+ATTR_CONN_PORT = "port"
+ATTR_CONN_DOMPATH = "dompath"
+""" directory where the component is stored in the filesystem """
+ATTR_CONN_USER = "user"
+ATTR_CONN_PASSWD = "password"
+LIST_CONN_ATTR = [ATTR_DB_CONN_JAR, ATTR_CONN_HOST, ATTR_CONN_IP, ATTR_CONN_PORT, ATTR_CONN_DOMPATH, ATTR_CONN_USER, ATTR_CONN_PASSWD]
 TOPIC_NODE_CLI = "cli"
+ATTR_CLI_TYPE = "type"
+LIST_CLI_ATTR = [ATTR_CLI_TYPE]
 TOPIC_NODE_API = "api"
+ATTR_API_TYPE = "type"
+LIST_API_ATTR = [ATTR_API_TYPE]
+TOPIC_NODE_FILE = "file"
+ATTR_FILE_TYPE = "type"
+LIST_FILE_ATTR = [ATTR_FILE_TYPE]
 
 # the configuration of a component or tool
 # entity {              : variable name of the group, basic, component-name or tool-name
@@ -153,6 +170,7 @@ ATTR_PATH_PATTN = "pattern"
 SUBJECT_INST = "instance"      #     |     |     |    | x  |         CompHanlding
 ATTR_INST_CNT = "count"        #     |     |     |    | x  |         CompHanlding
 ATTR_INST_SGL = "single"
+LIST_INST_ATTR = [ATTR_INST_CNT, ATTR_INST_SGL]
 ATTR_INST_SUBCOMP = "components"
 #SUBJECT_FCT  = "function"      #     |     |     |    | x  |         main-programs
 

basic/report.py

@@ -0,0 +1,10 @@
+#
+# -------------------------------------------------
+"""
+reporting-applications
+"""
+
+
+class Report():
+
+    pass

components/testexec.py → basic/testexec.py

@@ -34,7 +34,7 @@ import inspect
 import os
 import utils.db_abstract
 import basic.toolHandling
-import components.component
+import basic.component
 import basic.componentHandling
 import utils.path_tool
 import utils.file_tool

components/catalog.py

@@ -1,30 +0,0 @@
-#
-# ------------------------------------
-"""
-a lot of test-results are created dynamically - so these dont belog to the testdata.
-In order to get an overview over the testcases these results should be imported into a database.
-"""
-from datetime import datetime
-import basic.message
-import basic.program
-import inspect
-import components.component
-
-
-class Catalog():
-
-    def init_Catalog(self):
-        """
-        initialize special tables in the catalog-database
-        :return:
-        """
-        pass
-
-    def check_Instance(self):
-        """
-        checks system-instances and writes it into the parameter-file
-        """
-        job = basic.program.Job.getInstance()
-        verify = -1 + job.getDebugLevel(self.name)
-        self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime(
-            "%Y%m%d_%H%M%S") + " for " + str(self.name).upper())

components/maintain.py

@@ -1,18 +0,0 @@
-#
-# --------------------------------
-"""
-outside the testrun there are some tasks in order maintain the tests, for example:
-# testrunner to control or execute the testrun
-# set the target-results
-# generate testcases
-# partial cleaning of the storage
-"""
-from datetime import datetime
-import basic.message
-import basic.program
-import inspect
-import components.component
-
-
-class Maintainer():
-    pass

components/report.py

@@ -1,15 +0,0 @@
-#
-# -------------------------------------------------
-"""
-reporting-applications
-"""
-from datetime import datetime
-import basic.message
-import basic.program
-import inspect
-import components.component
-
-
-class Report():
-
-    pass

test/test_config.py

@@ -1,14 +1,15 @@
 import unittest
 import os
 import inspect
-import utils.config_tool as t
+import utils.config_tool
-from basic.program import Job
+import utils.conn_tool
 import test.constants
 import test.testtools
 import utils.path_const as P
+import basic.constants as B
 
 HOME_PATH = test.constants.HOME_PATH
-TEST_FUNCTIONS = ["test_getConfig"]
+TEST_FUNCTIONS = ["test_getConfig", "test_mergeAttributes"]
 VERIFY = False
 
 class MyTestCase(unittest.TestCase):
@@ -22,24 +23,52 @@ class MyTestCase(unittest.TestCase):
             return
         job = test.testtools.getJob()
         x = "path"
-        r = t.getConfigPath(P.KEY_TOOL, x)
+        r = utils.config_tool.getConfigPath(P.KEY_TOOL, x)
         self.assertIn(os.path.join(HOME_PATH, P.VAL_UTIL, P.VAL_CONFIG), r)
         cnttest += 1
         x = "conn"
-        r = t.getConfigPath(P.KEY_TOOL, x)
+        r = utils.config_tool.getConfigPath(P.KEY_TOOL, x)
         self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_ENV)), r)
         cnttest += 1
-        self.assertRaises(Exception, t.getConfigPath, (P.KEY_COMP, "TestX2"))
+        self.assertRaises(Exception, utils.config_tool.getConfigPath, (P.KEY_COMP, "TestX2"))
         # self.assertEqual(r, None)
         cnttest += 1
-        r = t.getConfigPath(P.KEY_COMP, "testA2")
+        r = utils.config_tool.getConfigPath(P.KEY_COMP, "testA2")
         self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_COMPS, "testa2", "CONFIG"), r)
-        r = t.getConfig(P.KEY_TOOL, "path")
+        r = utils.config_tool.getConfig(P.KEY_TOOL, "path")
         if VERIFY: print("pattern " + r["pattern"]["log"])
         if VERIFY: print("pattern " + r["pattern"]["precond"])
         MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
 
 
+    def test_mergeAttributes(self):
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
+        job = test.testtools.getJob()
+        componentName = "testa"
+        confs = utils.config_tool.getConfig("comp", componentName)
+        conns = utils.conn_tool.getConnections(componentName)
+        self.assertEqual(confs["conf"][B.SUBJECT_INST][B.ATTR_INST_CNT], 1)
+        self.assertEqual(conns[0][B.SUBJECT_INST][B.ATTR_INST_CNT], 2)
+        self.assertNotIn(B.ATTR_INST_SGL, conns[0][B.SUBJECT_INST])
+        confs["conf"] = utils.config_tool.mergeConn(job.m, confs["conf"], conns[0])
+        self.assertEqual(confs["conf"][B.SUBJECT_INST][B.ATTR_INST_CNT], 2)
+        cnttest += 1 # it overwrites
+        self.assertEqual(confs["conf"][B.SUBJECT_INST][B.ATTR_INST_SGL], "n")
+        cnttest += 1 # it keep
+        componentName = "testa1"
+        confs = utils.config_tool.getConfig("comp", componentName)
+        conns = utils.conn_tool.getConnections(componentName)
+        self.assertNotIn(B.ATTR_DB_TYPE, confs["conf"][B.SUBJECT_ARTS][B.TOPIC_NODE_DB])
+        confs["conf"] = utils.config_tool.mergeConn(job.m, confs["conf"], conns[0])
+        self.assertIn(B.ATTR_DB_TYPE, confs["conf"][B.SUBJECT_ARTS][B.TOPIC_NODE_DB])
+        cnttest += 1 # new attribute
+        MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
+
+
     def test_getAttributes(self):
         global mymsg
         actfunction = str(inspect.currentframe().f_code.co_name)
@@ -49,6 +78,11 @@ class MyTestCase(unittest.TestCase):
         job = test.testtools.getJob()
         MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
 
+
+    def test_zzz(self):
+        print(MyTestCase.mymsg)
+
+
 if __name__ == '__main__':
     VERIFY = True
     unittest.main()

utils/config_tool.py

@@ -186,4 +186,42 @@ def getConfig(modul, name, subname=""):
     doc = utils.file_tool.readFileDict(pathname, msg)
     for i, v in doc.items():
         confs[i] = v
-    return confs
+    return confs
+
+
+def mergeConn(msg, conf, conn):
+    """
+    merges the config-attributes from the connection-attributes
+    because the connection-attributes has to overwrite the config-attributes if the subject is configured
+    :param conf:
+    :param conn:
+    :return:
+    """
+    for a in conn[B.SUBJECT_INST]:
+        conf[B.SUBJECT_INST][a] = conn[B.SUBJECT_INST][a]
+    for topic in [B.TOPIC_NODE_DB, B.TOPIC_NODE_CLI, B.TOPIC_NODE_API, B.TOPIC_NODE_FILE]:
+        if topic not in conf[B.SUBJECT_ARTS]:
+            continue
+        if topic == B.TOPIC_NODE_DB:
+            list = B.LIST_DB_ATTR
+        if topic == B.TOPIC_NODE_CLI:
+            list = B.LIST_CLI_ATTR
+        if topic == B.TOPIC_NODE_API:
+            list = B.LIST_API_ATTR
+        if topic == B.TOPIC_NODE_FILE:
+            list = B.LIST_FILE_ATTR
+        for a in conf[B.SUBJECT_ARTS][topic]:
+            if a in list:
+                if a in conn[topic]:
+                    conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a]
+            else:
+                for b in conf[B.SUBJECT_ARTS][topic][a]:
+                    if b not in list:
+                        msg.logError("not-topic-attribute in topic-connection: "+topic+", "+b)
+                        continue
+                    if b in conn[topic][a]:
+                        conf[B.SUBJECT_ARTS][topic][a][b] = conn[topic][a][b]
+        for a in list:
+            if a in conn[topic]:
+                conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a]
+    return conf

utils/conn_tool.py

@@ -33,6 +33,12 @@ def getConnection(comp, nr):
 
 
 def getConnections(comp):
+    """
+    it reads the connection-attributes for each instances of this component
+    general attributes are added to the connection-attributes
+    :param comp:
+    :return:
+    """
     job = basic.program.Job.getInstance()
     verify = job.getDebugLevel("conn_tool")
     msg = None
@@ -45,24 +51,32 @@ def getConnections(comp):
     msg.debug(verify, "getConnections " + comp)
     conn = {}
     conns = []
-    if job.conf.confs.get("tools").get("connsrc") == D.DFILE_TYPE_YML:
+    if job.conf.confs.get("tools").get("connsrc") in [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV]:
         conn = utils.config_tool.getConfig("tool", "conn")
         if not comp in conn["env"]:
             job.m.setFatal("Conn-Tool: Comp not configured " + comp)
     elif job.conf.confs.get("tools").get("connsrc") == "flaskdb":
         pass
-    elif job.conf.confs.get("tools").get("connsrc") == D.DFILE_TYPE_CSV:
+
-        pass
+    attr = {}
-    #print(comp)
+    if "general" in conn["env"]:
-    #print(conn["env"].keys())
+        for a in conn["env"]["general"]:
-    #print(conn["env"][comp][B.SUBJECT_INST])
+            attr[a] = conn["env"]["general"]
-    xtypes = None
+    for a in conn["env"][comp]:
-    if ("types" in conn["env"][comp]):
+        if "inst" in a and a != B.SUBJECT_INST:
-        xtypes = conn["env"][comp]["types"]
+            continue
-    for i in range(conn["env"][comp][B.SUBJECT_INST]):
+        attr[a] = conn["env"][comp][a]
+    #if ("types" in conn["env"][comp]):
+    #    xtypes = conn["env"][comp]["types"]
+    for i in range(conn["env"][comp][B.SUBJECT_INST][B.ATTR_INST_CNT]):
         print("range " + str(i + 1))
         instnr = "inst" + str(i + 1)
-        if (xtypes is not None):
+        #if (xtypes is not None):
-            conn["env"][comp][instnr]["types"] = xtypes
+        #    conn["env"][comp][instnr]["types"] = xtypes
+        for a in attr:
+            if a in conn["env"][comp][instnr]:
+                continue # dont overwrite an instance-specific value
+            conn["env"][comp][instnr][a] = attr[a]
         conns.append(conn["env"][comp][instnr])
+
     return conns