From a70fdd1ad0a925fa3b311ec5765b4275874f9559 Mon Sep 17 00:00:00 2001 From: Ulrich Carmesin Date: Thu, 16 Jun 2022 18:38:56 +0200 Subject: [PATCH] merge conn into config --- {components => basic}/component.py | 0 basic/constants.py | 32 ++++++++++++++---- basic/report.py | 10 ++++++ {components => basic}/sysmonitor.py | 0 {components => basic}/testexec.py | 2 +- components/catalog.py | 30 ----------------- components/maintain.py | 18 ----------- components/report.py | 15 --------- test/test_component.py | 0 test/test_config.py | 50 ++++++++++++++++++++++++----- utils/config_tool.py | 40 ++++++++++++++++++++++- utils/conn_tool.py | 38 +++++++++++++++------- 12 files changed, 143 insertions(+), 92 deletions(-) rename {components => basic}/component.py (100%) create mode 100644 basic/report.py rename {components => basic}/sysmonitor.py (100%) rename {components => basic}/testexec.py (99%) delete mode 100644 components/catalog.py delete mode 100644 components/maintain.py delete mode 100644 components/report.py create mode 100644 test/test_component.py diff --git a/components/component.py b/basic/component.py similarity index 100% rename from components/component.py rename to basic/component.py diff --git a/basic/constants.py b/basic/constants.py index 1f98cf4..60801e7 100644 --- a/basic/constants.py +++ b/basic/constants.py @@ -88,6 +88,8 @@ DATA_NODE_OPTION = "_option" """ This constant defines main node in the testdata for testcase specific parameters """ DATA_NODE_TABLES = "_tables" """ This constant defines the main node in the testdata for the steps to execute """ +DATA_NODE_KEYS = "_keys" +""" This constant defines the main node in the testdata for the steps to execute """ DATA_NODE_DDL = "ddl" """ This constant defines the node for data scheme (DataDefinitionLanguage) The fields are defined in data_const (D) """ @@ -98,25 +100,40 @@ DATA_NODE_PAR = "par" """ This constant defines """ TOPIC_NODE_DB = "db" - # testexec, db_abstr +ATTR_DB_TYPE = "type" ATTR_DB_PARTITION = "partitioned" """ optional attribute if table is partitioned - this keyword delimited by "+" will be replaced by partition-names which are parametrized """ -ATTR_DB_CONN_JAR = "conn_jar_name" -""" optional attribute for connection-jar-file instead of connection by ip, port """ -ATTR_CONN_HOST = "hostname" -""" optional attribute for connection-jar-file instead of connection by ip, port """ -ATTR_CONN_TENANT = "tenant" -""" optional attribute for connection-jar-file instead of connection by ip, port """ ATTR_DB_DATABASE = "database" """ attribute for technical name of the database """ ATTR_DB_SCHEMA = "schema" """ optional attribute for technical name of the schema """ ATTR_DB_TABNAME = "tabname" """ optional attribute in order to use a different technical name for the db-table """ +LIST_DB_ATTR = [ATTR_DB_TYPE, ATTR_DB_PARTITION, ATTR_DB_DATABASE, ATTR_DB_SCHEMA, ATTR_DB_TABNAME] +ATTR_DB_CONN_JAR = "conn_jar_name" +""" optional attribute for connection-jar-file instead of connection by ip, port """ +ATTR_CONN_HOST = "hostname" +""" optional attribute for connection-jar-file instead of connection by ip, port """ +ATTR_CONN_TENANT = "tenant" +""" optional attribute for connection-jar-file instead of connection by ip, port """ +ATTR_CONN_IP = "ip" +ATTR_CONN_PORT = "port" +ATTR_CONN_DOMPATH = "dompath" +""" directory where the component is stored in the filesystem """ +ATTR_CONN_USER = "user" +ATTR_CONN_PASSWD = "password" +LIST_CONN_ATTR = [ATTR_DB_CONN_JAR, ATTR_CONN_HOST, ATTR_CONN_IP, ATTR_CONN_PORT, ATTR_CONN_DOMPATH, ATTR_CONN_USER, ATTR_CONN_PASSWD] TOPIC_NODE_CLI = "cli" +ATTR_CLI_TYPE = "type" +LIST_CLI_ATTR = [ATTR_CLI_TYPE] TOPIC_NODE_API = "api" +ATTR_API_TYPE = "type" +LIST_API_ATTR = [ATTR_API_TYPE] +TOPIC_NODE_FILE = "file" +ATTR_FILE_TYPE = "type" +LIST_FILE_ATTR = [ATTR_FILE_TYPE] # the configuration of a component or tool # entity { : variable name of the group, basic, component-name or tool-name @@ -153,6 +170,7 @@ ATTR_PATH_PATTN = "pattern" SUBJECT_INST = "instance" # | | | | x | CompHanlding ATTR_INST_CNT = "count" # | | | | x | CompHanlding ATTR_INST_SGL = "single" +LIST_INST_ATTR = [ATTR_INST_CNT, ATTR_INST_SGL] ATTR_INST_SUBCOMP = "components" #SUBJECT_FCT = "function" # | | | | x | main-programs diff --git a/basic/report.py b/basic/report.py new file mode 100644 index 0000000..4176c45 --- /dev/null +++ b/basic/report.py @@ -0,0 +1,10 @@ +# +# ------------------------------------------------- +""" +reporting-applications +""" + + +class Report(): + + pass diff --git a/components/sysmonitor.py b/basic/sysmonitor.py similarity index 100% rename from components/sysmonitor.py rename to basic/sysmonitor.py diff --git a/components/testexec.py b/basic/testexec.py similarity index 99% rename from components/testexec.py rename to basic/testexec.py index aa06f12..f514fa8 100644 --- a/components/testexec.py +++ b/basic/testexec.py @@ -34,7 +34,7 @@ import inspect import os import utils.db_abstract import basic.toolHandling -import components.component +import basic.component import basic.componentHandling import utils.path_tool import utils.file_tool diff --git a/components/catalog.py b/components/catalog.py deleted file mode 100644 index 7bdde26..0000000 --- a/components/catalog.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# ------------------------------------ -""" -a lot of test-results are created dynamically - so these dont belog to the testdata. -In order to get an overview over the testcases these results should be imported into a database. -""" -from datetime import datetime -import basic.message -import basic.program -import inspect -import components.component - - -class Catalog(): - - def init_Catalog(self): - """ - initialize special tables in the catalog-database - :return: - """ - pass - - def check_Instance(self): - """ - checks system-instances and writes it into the parameter-file - """ - job = basic.program.Job.getInstance() - verify = -1 + job.getDebugLevel(self.name) - self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime( - "%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) diff --git a/components/maintain.py b/components/maintain.py deleted file mode 100644 index b815091..0000000 --- a/components/maintain.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# -------------------------------- -""" -outside the testrun there are some tasks in order maintain the tests, for example: -# testrunner to control or execute the testrun -# set the target-results -# generate testcases -# partial cleaning of the storage -""" -from datetime import datetime -import basic.message -import basic.program -import inspect -import components.component - - -class Maintainer(): - pass \ No newline at end of file diff --git a/components/report.py b/components/report.py deleted file mode 100644 index 66c1576..0000000 --- a/components/report.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# ------------------------------------------------- -""" -reporting-applications -""" -from datetime import datetime -import basic.message -import basic.program -import inspect -import components.component - - -class Report(): - - pass diff --git a/test/test_component.py b/test/test_component.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_config.py b/test/test_config.py index f3d6032..8c54cae 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -1,14 +1,15 @@ import unittest import os import inspect -import utils.config_tool as t -from basic.program import Job +import utils.config_tool +import utils.conn_tool import test.constants import test.testtools import utils.path_const as P +import basic.constants as B HOME_PATH = test.constants.HOME_PATH -TEST_FUNCTIONS = ["test_getConfig"] +TEST_FUNCTIONS = ["test_getConfig", "test_mergeAttributes"] VERIFY = False class MyTestCase(unittest.TestCase): @@ -22,24 +23,52 @@ class MyTestCase(unittest.TestCase): return job = test.testtools.getJob() x = "path" - r = t.getConfigPath(P.KEY_TOOL, x) + r = utils.config_tool.getConfigPath(P.KEY_TOOL, x) self.assertIn(os.path.join(HOME_PATH, P.VAL_UTIL, P.VAL_CONFIG), r) cnttest += 1 x = "conn" - r = t.getConfigPath(P.KEY_TOOL, x) + r = utils.config_tool.getConfigPath(P.KEY_TOOL, x) self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_ENV)), r) cnttest += 1 - self.assertRaises(Exception, t.getConfigPath, (P.KEY_COMP, "TestX2")) + self.assertRaises(Exception, utils.config_tool.getConfigPath, (P.KEY_COMP, "TestX2")) # self.assertEqual(r, None) cnttest += 1 - r = t.getConfigPath(P.KEY_COMP, "testA2") + r = utils.config_tool.getConfigPath(P.KEY_COMP, "testA2") self.assertIn(os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_COMPS, "testa2", "CONFIG"), r) - r = t.getConfig(P.KEY_TOOL, "path") + r = utils.config_tool.getConfig(P.KEY_TOOL, "path") if VERIFY: print("pattern " + r["pattern"]["log"]) if VERIFY: print("pattern " + r["pattern"]["precond"]) MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) + def test_mergeAttributes(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return + job = test.testtools.getJob() + componentName = "testa" + confs = utils.config_tool.getConfig("comp", componentName) + conns = utils.conn_tool.getConnections(componentName) + self.assertEqual(confs["conf"][B.SUBJECT_INST][B.ATTR_INST_CNT], 1) + self.assertEqual(conns[0][B.SUBJECT_INST][B.ATTR_INST_CNT], 2) + self.assertNotIn(B.ATTR_INST_SGL, conns[0][B.SUBJECT_INST]) + confs["conf"] = utils.config_tool.mergeConn(job.m, confs["conf"], conns[0]) + self.assertEqual(confs["conf"][B.SUBJECT_INST][B.ATTR_INST_CNT], 2) + cnttest += 1 # it overwrites + self.assertEqual(confs["conf"][B.SUBJECT_INST][B.ATTR_INST_SGL], "n") + cnttest += 1 # it keep + componentName = "testa1" + confs = utils.config_tool.getConfig("comp", componentName) + conns = utils.conn_tool.getConnections(componentName) + self.assertNotIn(B.ATTR_DB_TYPE, confs["conf"][B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) + confs["conf"] = utils.config_tool.mergeConn(job.m, confs["conf"], conns[0]) + self.assertIn(B.ATTR_DB_TYPE, confs["conf"][B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) + cnttest += 1 # new attribute + MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) + + def test_getAttributes(self): global mymsg actfunction = str(inspect.currentframe().f_code.co_name) @@ -49,6 +78,11 @@ class MyTestCase(unittest.TestCase): job = test.testtools.getJob() MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) + + def test_zzz(self): + print(MyTestCase.mymsg) + + if __name__ == '__main__': VERIFY = True unittest.main() diff --git a/utils/config_tool.py b/utils/config_tool.py index 4f312b4..9b4d2c4 100644 --- a/utils/config_tool.py +++ b/utils/config_tool.py @@ -186,4 +186,42 @@ def getConfig(modul, name, subname=""): doc = utils.file_tool.readFileDict(pathname, msg) for i, v in doc.items(): confs[i] = v - return confs \ No newline at end of file + return confs + + +def mergeConn(msg, conf, conn): + """ + merges the config-attributes from the connection-attributes + because the connection-attributes has to overwrite the config-attributes if the subject is configured + :param conf: + :param conn: + :return: + """ + for a in conn[B.SUBJECT_INST]: + conf[B.SUBJECT_INST][a] = conn[B.SUBJECT_INST][a] + for topic in [B.TOPIC_NODE_DB, B.TOPIC_NODE_CLI, B.TOPIC_NODE_API, B.TOPIC_NODE_FILE]: + if topic not in conf[B.SUBJECT_ARTS]: + continue + if topic == B.TOPIC_NODE_DB: + list = B.LIST_DB_ATTR + if topic == B.TOPIC_NODE_CLI: + list = B.LIST_CLI_ATTR + if topic == B.TOPIC_NODE_API: + list = B.LIST_API_ATTR + if topic == B.TOPIC_NODE_FILE: + list = B.LIST_FILE_ATTR + for a in conf[B.SUBJECT_ARTS][topic]: + if a in list: + if a in conn[topic]: + conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] + else: + for b in conf[B.SUBJECT_ARTS][topic][a]: + if b not in list: + msg.logError("not-topic-attribute in topic-connection: "+topic+", "+b) + continue + if b in conn[topic][a]: + conf[B.SUBJECT_ARTS][topic][a][b] = conn[topic][a][b] + for a in list: + if a in conn[topic]: + conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] + return conf \ No newline at end of file diff --git a/utils/conn_tool.py b/utils/conn_tool.py index 45344ce..6380634 100644 --- a/utils/conn_tool.py +++ b/utils/conn_tool.py @@ -33,6 +33,12 @@ def getConnection(comp, nr): def getConnections(comp): + """ + it reads the connection-attributes for each instances of this component + general attributes are added to the connection-attributes + :param comp: + :return: + """ job = basic.program.Job.getInstance() verify = job.getDebugLevel("conn_tool") msg = None @@ -45,24 +51,32 @@ def getConnections(comp): msg.debug(verify, "getConnections " + comp) conn = {} conns = [] - if job.conf.confs.get("tools").get("connsrc") == D.DFILE_TYPE_YML: + if job.conf.confs.get("tools").get("connsrc") in [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV]: conn = utils.config_tool.getConfig("tool", "conn") if not comp in conn["env"]: job.m.setFatal("Conn-Tool: Comp not configured " + comp) elif job.conf.confs.get("tools").get("connsrc") == "flaskdb": pass - elif job.conf.confs.get("tools").get("connsrc") == D.DFILE_TYPE_CSV: - pass - #print(comp) - #print(conn["env"].keys()) - #print(conn["env"][comp][B.SUBJECT_INST]) - xtypes = None - if ("types" in conn["env"][comp]): - xtypes = conn["env"][comp]["types"] - for i in range(conn["env"][comp][B.SUBJECT_INST]): + + attr = {} + if "general" in conn["env"]: + for a in conn["env"]["general"]: + attr[a] = conn["env"]["general"] + for a in conn["env"][comp]: + if "inst" in a and a != B.SUBJECT_INST: + continue + attr[a] = conn["env"][comp][a] + #if ("types" in conn["env"][comp]): + # xtypes = conn["env"][comp]["types"] + for i in range(conn["env"][comp][B.SUBJECT_INST][B.ATTR_INST_CNT]): print("range " + str(i + 1)) instnr = "inst" + str(i + 1) - if (xtypes is not None): - conn["env"][comp][instnr]["types"] = xtypes + #if (xtypes is not None): + # conn["env"][comp][instnr]["types"] = xtypes + for a in attr: + if a in conn["env"][comp][instnr]: + continue # dont overwrite an instance-specific value + conn["env"][comp][instnr][a] = attr[a] conns.append(conn["env"][comp][instnr]) + return conns