commit 2726a1f51866c447562799070b9daa6edd5e5c5b Author: Ulrich Carmesin Date: Mon Sep 20 23:57:31 2021 +0200 first init diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..974233b --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +../test/log/*.txt +../test/environment/ENV01/log/*.txt diff --git a/.idea/externalDependencies.xml b/.idea/externalDependencies.xml new file mode 100644 index 0000000..7e24ed1 --- /dev/null +++ b/.idea/externalDependencies.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..13fecf3 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..e15ec35 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/pythonProject.iml b/.idea/pythonProject.iml new file mode 100644 index 0000000..786dd58 --- /dev/null +++ b/.idea/pythonProject.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Readme.txt b/Readme.txt new file mode 100644 index 0000000..af45194 --- /dev/null +++ b/Readme.txt @@ -0,0 +1,4 @@ +This program is a test-framework which controls especially the system-under-test and which makes comparsions for diversity-tests. +It should be installed near by the application-system which has to be tested and with access-rights to the application-system. + +The business-test-frameworks on diff --git a/check_environment.py b/check_environment.py new file mode 100644 index 0000000..d2b4905 --- /dev/null +++ b/check_environment.py @@ -0,0 +1,33 @@ +# This is a sample Python script. +import sys# +# import jsonpickle # pip install jsonpickle +import yaml # pip install pyyaml +import ulrich.program +import ulrich.componentHandling +import ulrich.message +import utils.tdata_tool + +PROGRAM_NAME = "check_environment" + +if __name__ == '__main__': + x = ulrich.program.Job(PROGRAM_NAME) + print ("x "+str(x)) + x.startJob() + x.m.logDebug(str(vars(x.par)) + "\n" + str(vars(x.conf))) + if x.m.isRc("fatal"): + x.stopJob() + exit(x.m.rc * (-1) + 3) + # now in theory the program is runnable + x.m.setMsg("# job initialized") + cm = ulrich.componentHandling.ComponentManager() + cm.initComponents() + comps = cm.getComponents(PROGRAM_NAME) + x.m.setMsg("# Components initialized with these relevant components " + str(comps)) + for c in comps: + comp = cm.getComponents(c) + comp.check_Instance() + x.m.merge(comp.m) + comp.confs["function"][PROGRAM_NAME] = comp.m.topmessage + + x.stopJob() +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/components/__init__.py b/components/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/components/component.py b/components/component.py new file mode 100644 index 0000000..9f31565 --- /dev/null +++ b/components/component.py @@ -0,0 +1,380 @@ +# abstract class for instance components +# --------------------------------------------------------------------- +from datetime import datetime + +import ulrich.message +import inspect +""" +A component represents an application of the system-under-test +As the representation it has to knowlegde of the url, which other components depends on this component. +During a test-run the component must be checked, prepared, appfiles has to be collected, etc. For this doing there are some standard-methods implemented. +""" +class Component: + def init(self): + """ + The range of the test-system is defined by the dependancy graph which starts at the application.and indate each component. + By this way it is possible to use the same component in different test-systems and so the + application-knowlegde of this component must be defined only once. + A component can be a software-instance or a data. + Initialisation of concrete component + + it is controlled by their configuration + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + + + def prepare_system(self): + """ + In order to preparate the test system test data should be cleaned and loaded with actual data. + This can happen on different granularities: for each testcase or for each test sequences or for a test set or + for a whole test. + The loaded test data should be selected for a later comparison. + :return: + """ + pass + def init_Testset(self): + """ + main methode for system-preparation + :param self: + :return: + """ + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def reset_TsData(self): + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + if "log" in self.conf["artifact"]: + self.m.logInfo("log rotate in "+ self.name) + if "db" in self.conf["artifact"]: + self.m.logInfo("delete db-content "+ self.name) + if "lob" in self.conf["artifact"]: + self.m.logInfo("lob is deleted with db "+ self.name) + if "file" in self.conf["artifact"]: + self.m.logInfo("rm files in "+ self.name) + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + def load_TsData(self, testdata): + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + if "testdata" in self.conf: + if self.conf["testdata"] == "db": + self.m.logInfo("insert db-content " + self.name) + self.m.setMsg("data loaded for " + self.name + " is OK") + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def read_TsData(self): + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + if "db" in self.conf["artifact"]: + self.m.logInfo("select db-content "+ self.name) + if "lob" in self.conf["artifact"]: + self.m.logInfo("check lob if is deleted with db "+ self.name) + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def init_Testcase(self): + """ + main methode for system-preparation + :param self: + :return: + """ + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + self.m.logInfo("something in "+ self.name) + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def reset_TcData(self): + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + if "log" in self.conf["artifact"]: + self.m.logInfo("log rotate in "+ self.name) + if "db" in self.conf["artifact"]: + self.m.logInfo("delete db-content "+ self.name) + if "lob" in self.conf["artifact"]: + self.m.logInfo("lob is deleted with db "+ self.name) + if "file" in self.conf["artifact"]: + self.m.logInfo("rm files in "+ self.name) + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def load_TcData(self): + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + if "testdata" in self.conf: + if self.conf["testdata"] == "db": + self.m.logInfo("insert db-content " + self.name) + self.m.setMsg("data loaded for " + self.name + " is OK") + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def read_TcData(self): + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + if "db" in self.conf["artifact"]: + self.m.logInfo("select db-content "+ self.name) + if "lob" in self.conf["artifact"]: + self.m.logInfo("check lob if is deleted with db "+ self.name) + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logDebug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def test_system(self): + """ + + :return: + """ + pass + + def spiele_ein(self): + pass + def erzeuge_Anfrage(self): + pass + def sende_Anfrage(self, testdata): + pass + def lese_Antwort(self): + pass + + def finish_Testcase(self): + """ + initialization-routine for finish-step + :return: + """ + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + self.m.logInfo("something in "+ self.name) + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def collect_system(self): + """ + After the test the test data of each component should be selected again - to get the post-condition. + In each component is configured how these data can be collected. + In this phase the collected data have to be transformed to comparable files. + :return: + """ + pass + + def collect_TcArtifact(self): + """ + collects the artifacts from the test-system. + the result is written as original in subfolder {tcorigin} + post: a further contact zo the test-system is not necessary + :return: + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + if "log" in self.conf["artifact"]: + self.m.logInfo("get files in for "+ self.name + " in " + self.conf["artifact"]["log"]["path"]) + if "db" in self.conf["artifact"]: + self.m.logInfo("select db-content "+ self.name) + if "lob" in self.conf["artifact"]: + pass # after selection get file from db + if "file" in self.conf["artifact"]: + self.m.logInfo("get files in for "+ self.name + " in " + self.conf["artifact"]["file"]["path"]) + + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + + def split_TcResult(self): + """ + transforms the result from subfolder {tcorigin}. + the result is written as utf-8-readable parts in subfolder {tcparts} + :return: + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + if "log" in self.conf["artifact"]: + pass # + if "db" in self.conf["artifact"]: + pass # stored in table + if "lob" in self.conf["artifact"]: + self.m.logInfo("move file from db "+ self.name) + self.m.logInfo("tidy files in for " + self.name + " in " + self.conf["artifact"]["lob"]["format"]) + if "file" in self.conf["artifact"]: + self.m.logInfo("tidy files in for "+ self.name + " in " + self.conf["artifact"]["file"]["format"]) + + def fix_TcResult(self): + """ + fixes the result which is collected and transformed from the test-system. + the result is written in comparable form in folder {tcresult} + with the identifiable name - like in target + :return: + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + + def compare_results(self): + """ + to compare the actual result with the target result has three steps: + 1 matching each element so you get a comparable pair + 2 comparing this pair so maybe you get differences + 3 rating the difference if this can be accepted + :return: + """ + pass + + def collect_TcResult(self): + """ + collects the result from the folder {tcresult}. + the content is stored intern for comparison + :return: + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + self.m.logInfo("get files in for " + self.name + " in tcresult ") + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + + def collect_Target(self): + """ + pre: only for components which be collected at the end of the test-set + collects the result from the folder {rsresult}. + post: a further contact zo the test-system is not necessary + :return: + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + + def compare_TcResults(self): + """ + compares the result with the target + (1) the elements of both sides are assigned + [2) looks for differences in assigned pairs + (3) try to accept the difference with inherent rules + (4) writes the result of comparison as html in folder {diff*} + (5) the summary result is returned + :return: + """ + pass + + def getHitscore(self, typ, rs, tg): + """ + calculates the difference between the given elements. + the score is a number [00000-99999] with prefix h_ - 00000 is the best, 99999 the worst + :param typ: + :param rs: + :param tg: + :return: + """ + def report_TcResults(self): + """ + reports the result-code + :return: + """ + pass + + def finish_Testset(self): + pass + + def collect_TsArtifact(self): + """ + collects the artifacts from the test-system. + the result is written as original in subfolder {tsorigin} + :return: + """ + pass + def split_TsResult(self): + """ + transforms the result which is collected from the test-system. + the result is written as utf-8-readable parts in the specific subfolder {tcparts} + the relevant testcases will be called + :return: + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + # for tc in testcases: + # self.fix_TcResult(self) + + def compare_TsResults(self): + """ + controles the comparison the result with the target + :return: + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + # for tc in testcases: + # self.collect_TcResult + # self.collect_Target + # self.compare_TcResults + + def report_TsResults(self): + """ + (1.2) extraction of diff-files (only not accepted differences) as html-file in test-set - result-report - + (2.3) visualization of final result-code of each component and each testcase in test-set - result-report - + (2.4) visualization of statistical result-codes of each component and each test-set in test-context - result-report - + :return: + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel(self.name) + self.m.debug(verify, "--- " + str(inspect.currentframe().f_code.co_name) + "() " + str(self.name)) + reportheader = '' + reportbody = '' + testreport = "" + # if job.par.context == "tset": + # for tc in testcases: + # header = utils.report_tool.getTcHeader() + # body = utils.report_tool.getTcExtraction() + # if job.par.context == "tcontext": + # for ts in testsets: + reportheader = reportheader +'<\head>' + reportbody = reportbody + '<\body>' + testreport = reportheader + reportbody + return testreport + + def report_result(self): + """ + When you finish your test run you have to report the result for your customer. + 1 Your testers have recherche the results which are not acceptable. + They need detailed information of all test results in order to proof if the difference has their cause + in test-technicals or in system-technicals. They should be able to declare the cause. + 2 Your testmanager wants to know what is done - the percentage of the necessary test metrik and the status of each test - + and which principle faults are found and which cause the faults have. + 3 Your projectmanager wants to know if the system is working correct - the percentage of the necessary test metrik + and founded system errors. + :return: + """ + pass + + def maintain_tests(self): + """ + + :return: + """ + pass + + def declare_Target(self): + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel(self.name) + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + self.m.logInfo("something in "+ self.name) + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime("%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + def catalog_tests(self): + """ + Its not only a nice-to-have to know which tests are exactly running and which special cases they tests. + Each test running in a special context a special case. Each of such cases are defined by some attributes. + But often nobody knows in which testcases these attributes are tested. If you want to proof your test metrik + for some very special cases a database of these attributes can be very helpful! + Otherwise you must specify always new test cases for a correction of such special case, + the existent test cases must be found afterwards and must be correct in their expectation. + :return: + """ + pass diff --git a/components/mariadb/CONFIG.yml b/components/mariadb/CONFIG.yml new file mode 100644 index 0000000..c793383 --- /dev/null +++ b/components/mariadb/CONFIG.yml @@ -0,0 +1,17 @@ +# Example af a component with +# # one instance -> connection +# # mo subcomponents +# # artifact-tyoe db -> precondition, load testdata, postcondition +conf: + instance: + count: 1 + components: none + function: + check_environment: "todo" + init_testcase: "todo" # -> precondition, load testdata + finish_testcase: "todo" # -> postcondition, check data + artifact: + db: + reset: testcase + checks: + type: systemctl diff --git a/components/systemInstance.py b/components/systemInstance.py new file mode 100644 index 0000000..3b7971d --- /dev/null +++ b/components/systemInstance.py @@ -0,0 +1,34 @@ +# +# -------------------------------------------------------- +""" +When the test system is installed - especially in a continous pipeline - you dont know which versions are +installed and if each instance is running. In order to know these fundamental points it should be checked. +The result could be inserted into a database so you can get an overview over all test environments. +""" + +class SystemMonitor: + + def check_Instance(self): + """ + checks system-instances and writes it into the parameter-file + """ + job = ulrich.program.Job.getInstance() + verify = -1 + job.getDebugLevel(self.name) + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() started at " + datetime.now().strftime( + "%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + for c in self.confs["components"].keys(): + if self.confs["components"][c] == "none": + pass + elif self.confs["components"][c]["relationtyp"] == "call": + self.m.logInfo("check connection from " + self.name + " to " + c) + elif self.confs["components"][c]["relationtyp"] == "called": + self.m.logInfo("check connection from " + c + " to " + self.name) + elif self.confs["components"][c]["relationtyp"] == "queue": + self.m.logInfo("check queue from " + self.name + " to " + c) + self.m.setMsg("checkInstance for " + self.name + " is OK") + self.m.logInfo("--- " + str(inspect.currentframe().f_code.co_name) + "() finished at " + datetime.now().strftime( + "%Y%m%d_%H%M%S") + " for " + str(self.name).upper()) + + + def restart_Instance(self): + pass diff --git a/components/testa/CONFIG.yml b/components/testa/CONFIG.yml new file mode 100644 index 0000000..f3cc29f --- /dev/null +++ b/components/testa/CONFIG.yml @@ -0,0 +1,32 @@ +# Example af a component with +# # more instance -> connection +# # some subcomponents +# # artifact-tyoe log +conf: + instance: + count: 2 # count of instances + single: n + components: + testA1: + relationtyp: call # call|called|queue + conffile: "{dompath}/config/dbconn.xml" + filetyp: xml + ippattern: ".*?(.*?)" + hostpattern: ".*?(.*?)" + testA2: + relationtyp: call # call : is called by this comp + conffile: "{dompath}/config/settings.xml" + filetyp: xml + urlpattern: ".*?(.*?)" + function: + check_environment: "todo" + init_testcase: "todo" + finish_testcase: "todo" + artifact: + log: # log|db|file|lob + path: "{dompath}/log/debug.log" + rotate: jmx + reset: testcase + oldfile: "{dompath}/log/debug_{timepattern}.log" + + diff --git a/components/testa/Testa.py b/components/testa/Testa.py new file mode 100644 index 0000000..2541903 --- /dev/null +++ b/components/testa/Testa.py @@ -0,0 +1,7 @@ +import components.component + +class Testa(components.component.Component): + def __init__(self): + print('init Testa') + + diff --git a/components/testa/__init__.py b/components/testa/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/components/testa1/CONFIG.yml b/components/testa1/CONFIG.yml new file mode 100644 index 0000000..f1ab498 --- /dev/null +++ b/components/testa1/CONFIG.yml @@ -0,0 +1,24 @@ +# Example af a component with +# # one instance -> connection +# # mo subcomponents +# # artifact-tyoe db -> precondition, load testdata, postcondition +conf: + instance: + count: 1 + single: y + components: + testa11: + relationtyp: a # these attributes can be checked in statical way if connection is correctly set + conffile: "{dompath}/config/dbconn.xml" + filetyp: xml + ippattern: ".*?(.*?)" + hostpattern: ".*?(.*?)" + function: + check_environment: "todo" + init_testcase: "todo" # -> precondition, load testdata + finish_testcase: "todo" # -> postcondition, check data + artifact: + db: + reset: testset + testdata: db # + diff --git a/components/testa1/DATASTRUCTURE.yml b/components/testa1/DATASTRUCTURE.yml new file mode 100644 index 0000000..551bbff --- /dev/null +++ b/components/testa1/DATASTRUCTURE.yml @@ -0,0 +1,193 @@ +# Example of a data-structure with +# # different count of tables, id-fields +testa1: # database + immo: # schema + lofts: # table + character: inventory + fields: + - street + - city + - zip + - state + - beds + - baths + - sqft + - type + - price + - latitude + - longitude + data: + city: + id: b_2 + acceptance: norm + type: string + nullable: n + zip: + id: b_1 + acceptance: norm + type: string + nullable: n + state: + id: n + acceptance: norm + type: string + nullable: n + beds: + id: n + acceptance: norm + type: int + nullable: y + baths: + id: n + acceptance: norm + type: int + nullable: y + sqft: + id: n + acceptance: norm + type: double + nullable: y + type: + id: n + acceptance: norm + type: string + nullable: y + price: + id: n + acceptance: norm + type: double + nullable: y + latitude: + id: n + acceptance: norm + type: float + nullable: y + longitude: + id: n + acceptance: norm + type: float + nullable: y + user: + name: + character: inventory + fields: + - Username + - Identifier + - First_name + - Last_name + data: + Username: + id: b_1 + acceptance: norm + type: string + nullable: y + Identifier: + id: t_1 + acceptance: exist + type: int + nullable: y + First_name: + id: n + acceptance: norm + type: string + nullable: y + Last_name: + id: n + acceptance: norm + type: string + nullable: y + mail: + character: inventory + fields: + - Email + - Identifier + - First_name + - Last_name + data: + Email: + id: b_1 + acceptance: norm + type: string + nullable: y + Identifier: + id: t_1 + acceptance: exist + type: int + nullable: n + First_name: + id: n + acceptance: norm + type: string + nullable: y + Last_name: + id: n + acceptance: norm + type: string + nullable: y + action: + character: transaction + fields: + - id + - Identifier + - login_ok + - login_time + - session_state + - end_time + - cnt_transactions + data: + id: + id: t_1 + acceptance: exist + type: int + nullable: n + Identifier: + id: b_1 + acceptance: norm + type: int + nullable: n + login_ok: + login_time: + session_state: + end_time: + cnt_transactions: + biblio: + books: + character: inventory + fields: + - id + - title + - autor + - library + - subject + - description + data: + id: + id: t_1 + acceptance: norm + type: int + nullable: n + title: + id: b_1 + acceptance: norm + type: string + nullable: n + autor: + id: b_2 + acceptance: norm + type: string + nullable: n + library: + id: n + acceptance: norm + type: string + nullable: y + subject: + id: n + acceptance: norm + type: string + nullable: y + description: + id: n + acceptance: exist + type: clob + nullable: y diff --git a/components/testa1/Testa1.py b/components/testa1/Testa1.py new file mode 100644 index 0000000..005fc56 --- /dev/null +++ b/components/testa1/Testa1.py @@ -0,0 +1,7 @@ +import components.component + +class Testa1(components.component.Component): + def __init__(self): + print('init Testa1') + + diff --git a/components/testa1/__init__.py b/components/testa1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/components/testa11/CONFIG.yml b/components/testa11/CONFIG.yml new file mode 100644 index 0000000..171a5f7 --- /dev/null +++ b/components/testa11/CONFIG.yml @@ -0,0 +1,16 @@ +# Example af a component with +# # one instance -> connection +# # mo subcomponents +# # artifact-tyoe db -> precondition, load testdata, postcondition +conf: + instance: + count: 1 + single: y + components: + none: none + function: + finish_testcase: "todo" # -> postcondition, check data + artifact: + lob: + reset: none + path: "testa1:biblio:books.description" diff --git a/components/testa11/DATASTRUCTURE.yml b/components/testa11/DATASTRUCTURE.yml new file mode 100644 index 0000000..9097a03 --- /dev/null +++ b/components/testa11/DATASTRUCTURE.yml @@ -0,0 +1,187 @@ +# Example of a data-structure with +# # different count of tables, id-fields +testa1: # database + immo: # schema + lofts: # table + character: inventory + fields: + - street + - city + - zip + - state + - beds + - baths + - sqft + - type + - price + - latitude + - longitude + data: + city: + id: b_2 + acceptance: norm + type: string + nullable: n + zip: + id: b_1 + acceptance: norm + type: string + nullable: n + state: + id: n + acceptance: norm + type: string + nullable: n + beds: + id: n + acceptance: norm + type: int + nullable: y + baths: + id: n + acceptance: norm + type: int + nullable: y + sqft: + id: n + acceptance: norm + type: double + nullable: y + type: + id: n + acceptance: norm + type: string + nullable: y + price: + id: n + acceptance: norm + type: double + nullable: y + latitude: + id: n + acceptance: norm + type: float + nullable: y + longitude: + id: n + acceptance: norm + type: float + nullable: y + user: + name: + character: inventory + fields: + - Username + - Identifier + - First_name + - Last_name + data: + Username: + id: b_1 + acceptance: norm + type: string + nullable: y + Identifier: + id: t_1 + acceptance: exist + type: int + nullable: y + First_name: + id: n + acceptance: norm + type: string + nullable: y + Last_name: + id: n + acceptance: norm + type: string + nullable: y + mail: + character: inventory + fields: + - Email + - Identifier + - First_name + - Last_name + data: + Email: + id: b_1 + acceptance: norm + type: string + nullable: y + Identifier: + id: t_1 + acceptance: exist + type: int + nullable: n + First_name: + id: n + acceptance: norm + type: string + nullable: y + Last_name: + id: n + acceptance: norm + type: string + nullable: y + action: + character: transaction + fields: + - id + - Identifier + - login_ok + - login_time + - session_state + - end_time + - cnt_transactions + data: + id: + id: t_1 + acceptance: exist + type: int + nullable: n + Identifier: + id: b_1 + acceptance: norm + type: int + nullable: n + login_ok: + login_time: + session_state: + end_time: + cnt_transactions: + biblio: + books: + character: inventory + fields: + - id + - title + - autor + - library + - subject + data: + id: + id: t_1 + acceptance: norm + type: int + nullable: n + title: + id: b_1 + acceptance: norm + type: string + nullable: n + autor: + id: b_2 + acceptance: norm + type: string + nullable: n + library: + id: n + acceptance: norm + type: string + nullable: y + subject: + id: n + acceptance: norm + type: string + nullable: y diff --git a/components/testa11/Testa11.py b/components/testa11/Testa11.py new file mode 100644 index 0000000..f07f1d5 --- /dev/null +++ b/components/testa11/Testa11.py @@ -0,0 +1,7 @@ +import components.component + +class Testa11(components.component.Component): + def __init__(self): + print('init Testa1') + + diff --git a/components/testa11/__init__.py b/components/testa11/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/components/testa2/CONFIG.yml b/components/testa2/CONFIG.yml new file mode 100644 index 0000000..f32e04c --- /dev/null +++ b/components/testa2/CONFIG.yml @@ -0,0 +1,24 @@ +# Example af a component like a sub-app with +# # one interface - +# # one subcomponents +# # artifact-tyoe log +conf: + instance: + count: 1 + single: n + components: + testa21: + relationtyp: call + conffile: "{dompath}/config/dbconn.xml" + filetyp: xml + ippattern: ".*?(.*?)" + hostpattern: ".*?(.*?)" + function: + check_environment: "todo" + init_testcase: "todo" + finish_testcase: "todo" + artifact: + lob: + path: "testa" + rotate: jmx + reset: testcase diff --git a/components/testa2/Testa2.py b/components/testa2/Testa2.py new file mode 100644 index 0000000..45ca5ec --- /dev/null +++ b/components/testa2/Testa2.py @@ -0,0 +1,7 @@ +import components.component + +class Testa2(components.component.Component): + def __init__(self): + print('init Testa1') + + diff --git a/components/testa2/__init__.py b/components/testa2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/components/testa21/CONFIG.yml b/components/testa21/CONFIG.yml new file mode 100644 index 0000000..64119cd --- /dev/null +++ b/components/testa21/CONFIG.yml @@ -0,0 +1,19 @@ +# Example af a component like a sub-app with +# # no subcomponents +# # artifact-tyoe log +conf: + instance: + count: 1 + single: n + components: + none: none + function: + check_environment: "todo" + init_testcase: "todo" + finish_testcase: "todo" + artifact: + file: + path: "{dompath}/log/debug.log" + reset: testcase + format: xml + match: lines # lines|tree diff --git a/components/testa21/Testa21.py b/components/testa21/Testa21.py new file mode 100644 index 0000000..fc533af --- /dev/null +++ b/components/testa21/Testa21.py @@ -0,0 +1,5 @@ +import components.component + +class Testa21(components.component.Component): + def __init__(self): + print('init Testa1') diff --git a/components/testa21/__init__.py b/components/testa21/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/components/testb/CONFIG.yml b/components/testb/CONFIG.yml new file mode 100644 index 0000000..2a45192 --- /dev/null +++ b/components/testb/CONFIG.yml @@ -0,0 +1,24 @@ +# Example af a component like a main app with +# # one instance -> connection +# # one subcomponents +# # artifact-tyoe log +conf: + instance: + count: 1 + single: y + components: + testB1: + relationtyp: db + conffile: "{dompath}/config/dbconn.xml" + filetyp: xml + ippattern: ".*?(.*?)" + hostpattern: ".*?(.*?)" + function: + check_environment: "todo" + init_testcase: "todo" + finish_testcase: "todo" + artifact: + log: + path: "{dompath}/log/debug.log" + rotate: jmx + reset: testcase diff --git a/components/testb/Testb.py b/components/testb/Testb.py new file mode 100644 index 0000000..b48e799 --- /dev/null +++ b/components/testb/Testb.py @@ -0,0 +1,5 @@ +import components.component + +class Testb(components.component.Component): + def __init__(self): + print('init Testb') diff --git a/components/testb/__init__.py b/components/testb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/components/testb1/CONFIG.yml b/components/testb1/CONFIG.yml new file mode 100644 index 0000000..5b3c842 --- /dev/null +++ b/components/testb1/CONFIG.yml @@ -0,0 +1,17 @@ +# Example af a component with +# # one instance -> connection +# # mo subcomponents +# # artifact-tyoe db -> precondition, load testdata, postcondition +conf: + instance: + count: 1 + single: y + components: + none: none + function: + check_environment: "todo" + init_testcase: "todo" # -> precondition, load testdata + finish_testcase: "todo" # -> postcondition, check data + artifact: + db: + reset: testset diff --git a/components/testb1/Testb1.py b/components/testb1/Testb1.py new file mode 100644 index 0000000..89ade9b --- /dev/null +++ b/components/testb1/Testb1.py @@ -0,0 +1,5 @@ +import components.component + +class Testb1(components.component.Component): + def __init__(self): + print('init Testa1') diff --git a/components/testb1/__init__.py b/components/testb1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/declare_result.py b/declare_result.py new file mode 100644 index 0000000..656a696 --- /dev/null +++ b/declare_result.py @@ -0,0 +1,28 @@ +# This is a sample Python script. +import sys# +# import jsonpickle # pip install jsonpickle +import yaml # pip install pyyaml +import ulrich.program as program +from ulrich.componentHandling import ComponentManager +import ulrich.message as message +# Press Umschalt+F10 to execute it or replace it with your code. +# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. + + + + +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + x = program.Job("check_environment") + #m = message.Message(3) + #m.closeMessage() + x.startJob() + x.m.logInfo("hier eine LogInfo") + x.m.logDebug("hier eine DbugMeldung") + x.m.logDebug(str(vars(x.par)) + "\n" + str(vars(x.conf))) + cm = ComponentManager() + if x.m.isRc("fatal"): + x.stopJob() + exit(x.m.rc * (-1) + 3) + x.stopJob() +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/finish_testcase.py b/finish_testcase.py new file mode 100644 index 0000000..ec0675e --- /dev/null +++ b/finish_testcase.py @@ -0,0 +1,29 @@ +# This is a sample Python script. +import sys# +# import jsonpickle # pip install jsonpickle +import yaml # pip install pyyaml +import ulrich.program +from ulrich.componentHandling import ComponentManager +import ulrich.message +import utils.tdata_tool + +PROGRAM_NAME = "finish_testcase" + +if __name__ == '__main__': + x = ulrich.program.Job(PROGRAM_NAME) + x.startJob() + x.m.logInfo("hier eine LogInfo") + x.m.logDebug("hier eine DbugMeldung") + x.m.logDebug(str(vars(x.par)) + "\n" + str(vars(x.conf))) + if x.m.isRc("fatal"): + print("fatal Error at begin") + x.stopJob() + exit(x.m.rc * (-1) + 3) + # now in theory the program is runnable + cm = ComponentManager() + cm.initComponents() + comps = cm.getComponents(PROGRAM_NAME) + print(" relevant components for this job: " + str(comps)) + tdata = utils.tdata_tool.getTestdata() + x.stopJob() +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/finish_testset.py b/finish_testset.py new file mode 100644 index 0000000..656a696 --- /dev/null +++ b/finish_testset.py @@ -0,0 +1,28 @@ +# This is a sample Python script. +import sys# +# import jsonpickle # pip install jsonpickle +import yaml # pip install pyyaml +import ulrich.program as program +from ulrich.componentHandling import ComponentManager +import ulrich.message as message +# Press Umschalt+F10 to execute it or replace it with your code. +# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. + + + + +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + x = program.Job("check_environment") + #m = message.Message(3) + #m.closeMessage() + x.startJob() + x.m.logInfo("hier eine LogInfo") + x.m.logDebug("hier eine DbugMeldung") + x.m.logDebug(str(vars(x.par)) + "\n" + str(vars(x.conf))) + cm = ComponentManager() + if x.m.isRc("fatal"): + x.stopJob() + exit(x.m.rc * (-1) + 3) + x.stopJob() +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/init_testcase.py b/init_testcase.py new file mode 100644 index 0000000..656a696 --- /dev/null +++ b/init_testcase.py @@ -0,0 +1,28 @@ +# This is a sample Python script. +import sys# +# import jsonpickle # pip install jsonpickle +import yaml # pip install pyyaml +import ulrich.program as program +from ulrich.componentHandling import ComponentManager +import ulrich.message as message +# Press Umschalt+F10 to execute it or replace it with your code. +# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. + + + + +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + x = program.Job("check_environment") + #m = message.Message(3) + #m.closeMessage() + x.startJob() + x.m.logInfo("hier eine LogInfo") + x.m.logDebug("hier eine DbugMeldung") + x.m.logDebug(str(vars(x.par)) + "\n" + str(vars(x.conf))) + cm = ComponentManager() + if x.m.isRc("fatal"): + x.stopJob() + exit(x.m.rc * (-1) + 3) + x.stopJob() +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/init_testset.py b/init_testset.py new file mode 100644 index 0000000..719d4fe --- /dev/null +++ b/init_testset.py @@ -0,0 +1,29 @@ +# This is a sample Python script. +import sys# +# import jsonpickle # pip install jsonpickle +import yaml # pip install pyyaml +import ulrich.program +from ulrich.componentHandling import ComponentManager +import ulrich.message +import utils.tdata_tool + +PROGRAM_NAME = "init_testset" + +if __name__ == '__main__': + x = ulrich.program.Job(PROGRAM_NAME) + x.startJob() + x.m.logInfo("hier eine LogInfo") + x.m.logDebug("hier eine DbugMeldung") + x.m.logDebug(str(vars(x.par)) + "\n" + str(vars(x.conf))) + if x.m.isRc("fatal"): + print("fatal Error at begin") + x.stopJob() + exit(x.m.rc * (-1) + 3) + # now in theory the program is runnable + cm = ComponentManager() + cm.initComponents() + comps = cm.getComponents(PROGRAM_NAME) + print(" relevant components for this job: " + str(comps)) + tdata = utils.tdata_tool.getTestdata() + x.stopJob() +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4818cc5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pyyaml \ No newline at end of file diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/basis.yml b/test/conf/basis.yml new file mode 100644 index 0000000..d9aad9e --- /dev/null +++ b/test/conf/basis.yml @@ -0,0 +1,34 @@ +tools: + connsrc: yml + remotetyp: ssh +toolcls: + pathTool: path_tool +paths: + mode: "0o775" + home: /home/ulrich/6_Projekte/Programme/pythonProject + debugs: /home/ulrich/6_Projekte/Programme/pythonProject/test/log # for temporary debug-logs + archiv: /home/ulrich/6_Projekte/Programme/pythonProject/test/lauf # the results of + results: /home/ulrich/6_Projekte/Programme/pythonProject/test/target # the target results which a former result files + program: /home/ulrich/6_Projekte/Programme/pythonProject # the programs with default configs + environment: /home/ulrich/6_Projekte/Programme/pythonProject/test/environment # for config etc about the system under test + release: /home/ulrich/6_Projekte/Programme/pythonProject/test/release # for configs about special releases + testdata: /home/ulrich/6_Projekte/Programme/pythonProject/test/tdata # for configs about special releases +pattern: + tctime: '\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}' + tltime: '\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}' + testcase: '.*?/' + usecase: '.*?/' +typen: + einzel: "Einzelfaelle separat" + menge: "alle zum Lauf einmal laden und auswerten" + unit: + text: "Units testen" + todo: 1 +applicationen: + TEST: + - testA + - testB +application: + TEST: + - testA + - testB \ No newline at end of file diff --git a/test/environment/ENV01/PARAMETER_TEST_ENV01.yml b/test/environment/ENV01/PARAMETER_TEST_ENV01.yml new file mode 100644 index 0000000..6d65752 --- /dev/null +++ b/test/environment/ENV01/PARAMETER_TEST_ENV01.yml @@ -0,0 +1,7 @@ +comps: {} +par: + application: TEST + basedir: envbase + environment: ENV01 + parstring: python check_environment --application TEST --environment ENV01 + program: check_environment diff --git a/test/environment/ENV01/configs/tool_conn.yml b/test/environment/ENV01/configs/tool_conn.yml new file mode 100644 index 0000000..8f0b1f1 --- /dev/null +++ b/test/environment/ENV01/configs/tool_conn.yml @@ -0,0 +1,87 @@ +env: + testa: + instance: 2 + inst1: + ip: "192.178.168.12" + hostname: my-app-store-1 + port: 54321 + dompath: /opt/app/testa + domscript: bin/manage.sh + user: testa + password: test_secure_a + home: /home/testa + inst2: + ip: 192.178.168.14 + hostname: my-app-store-2 + port: 54321 + dompath: /opt/app/testa + domscript: bin/manage.sh + user: testa + password: test_secure_a + home: /home/testa + testa1: + instance: 1 + inst1: + ip: 192.178.168.12 + hostname: my-app-store-1 + port: 54410 + dompath: /opt/app/testa01 + domscript: bin/manage.sh + user: testa + password: test_secure_a + home: /home/testa + testa11: + instance: 1 + inst1: + ip: 192.178.168.12 + hostname: my-app-store-1 + port: 54410 + dompath: /opt/app/testa01 + domscript: bin/manage.sh + user: testa + password: test_secure_a + home: /home/testa + testa2: + instance: 1 + inst1: + ip: 192.178.168.12 + hostname: my-app-store-1 + port: 54420 + dompath: /opt/app/testa02 + domscript: bin/manage.sh + user: testa + password: test_secure_a + home: /home/testa + testa21: + instance: 1 + inst1: + ip: 192.178.168.12 + hostname: my-app-store-1 + port: 54420 + dompath: /opt/app/testa02 + domscript: bin/manage.sh + user: testa + password: test_secure_a + home: /home/testa + testb: + instance: 1 + inst1: + ip: 192.178.168.14 + hostname: my-app-store-2 + port: 54500 + dompath: /opt/app/testb + domscript: bin/manage.sh + user: testb + password: test_secure_b + home: /home/testb + testb1: + instance: 1 + inst1: + ip: 192.178.168.14 + hostname: my-app-store-2 + port: 54510 + dompath: /opt/app/testb01 + domscript: bin/manage.sh + user: testb + password: test_secure_b + home: /home/testb diff --git a/test/lauf/V0.1/implement_2021-08-28_23-50-51/PARAMETER_TEST_UMGEB1.yml b/test/lauf/V0.1/implement_2021-08-28_23-50-51/PARAMETER_TEST_UMGEB1.yml new file mode 100644 index 0000000..c97a5c5 --- /dev/null +++ b/test/lauf/V0.1/implement_2021-08-28_23-50-51/PARAMETER_TEST_UMGEB1.yml @@ -0,0 +1,43 @@ +comps: + testa_02: + conn: + dompath: /opt/app/testa + domscript: bin/manage.sh + home: /home/testa + hostname: my-app-store-1 + ip: 192.178.168.12 + password: test_secure_a + port: 54321 + user: testa + function: + check_environment: 'INFO: checkInstance for testa_02 is OK' + finish_testcase: todo + init_testcase: todo + testb: + conn: + dompath: /opt/app/testb + domscript: bin/manage.sh + home: /home/testb + hostname: my-app-store-2 + ip: 192.178.168.14 + password: test_secure_b + port: 54500 + user: testb + function: + check_environment: 'INFO: checkInstance for testb is OK' + finish_testcase: todo + init_testcase: todo +par: + application: TEST + basedir: tsbase + environment: ENV01 + parstring: python init_testset --application TEST --environment ENV01 --tsdir /home/ulrich/6_Projekte/Programme/pythonProject/test/lauf/V0.1/implement_2021-08-28_23-50-51 + --tdtyp csv --tdsrc implement --tdname firstunit + program: init_testset + release: V0.1 + tdname: firstunit + tdsrc: implement + tdtyp: csv + tltime: 2021-08-28_23-50-51 + tsdir: /home/ulrich/6_Projekte/Programme/pythonProject/test/lauf/V0.1/implement_2021-08-28_23-50-51 + usecase: implement diff --git a/test/tdata/implement/firstunit.csv b/test/tdata/implement/firstunit.csv new file mode 100644 index 0000000..bb9bc0f --- /dev/null +++ b/test/tdata/implement/firstunit.csv @@ -0,0 +1,24 @@ +head;referencedate;;;;;;;;;;; +;25.08.2021;;;;;;;;;;; +;;;;;;;;;;;; +node;action;street;city;zip;state;beds;baths;sqft;type;price;latitude;longitude +testa1:immo:lofts;;3526 HIGH ST;SACRAMENTO;95838;CA;2;1;836;Residential;59222;38.631913;-121.434879 +testa1:immo:lofts;;51 OMAHA CT;SACRAMENTO;95823;CA;3;1;1167;Residential;68212;38.478902;-121.431028 +;;;;;;;;;;;; +;;;;;;;;;;;; +node;action;Username;Identifier;First_name;Last_name;;;;;;; +testa1:user:name;;booker12;9012;Rachel;Booker;;;;;;; +testa1:user:name;;grey07;2070;Laura;Grey;;;;;;; +;;;;;;;;;;;; +;;;;;;;;;;;; +node;action;Email;Identifier;First_name;Last_name;;;;;;; +testa1:user:mail;;rachel@yourcompany.com;9012;Rachel;Booker;;;;;;; +testa1:user:mail;;laura@yourcompany.com ;2070;Laura;Grey;;;;;;; +;;;;;;;;;;;; +;;;;;;;;;;;; +node;action;id;title;autor;library;subject;description;;;;; +testa1:biblio:books;;1;Eldon Base for stackable storage shelf, platinum;Muhammed MacIntyre;Nunavut;Storage & Organization;;;;;; +testa1:biblio:books;;2;"1.7 Cubic Foot Compact ""Cube"" Office Refrigerators";Barry French;Nunavut;Appliances;;;;;; +testa1:biblio:books;;3;Cardinal Slant-D® Ring Binder, Heavy Gauge Vinyl;Barry French;Nunavut;Binders and Binder Accessories;;;;;; +testa1:biblio:books;;4;R380;Clay Rozendal;Nunavut;Telephones and Communication;;;;;; +testa1:biblio:books;;5;Holmes HEPA Air Purifier;Carlos Soltero;Nunavut;Appliances;;;;;; diff --git a/test/test_config.py b/test/test_config.py new file mode 100644 index 0000000..8e007ab --- /dev/null +++ b/test/test_config.py @@ -0,0 +1,28 @@ +import unittest +import utils.config_tool as t +from ulrich.program import Job + + +class MyTestCase(unittest.TestCase): + def test_something(self): + job = Job("unit") + args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "config_tool", + "modus": "unit"} + job.par.setParameterArgs(args) + r = t.getConfigPath("tool", "path") + print (r) + self.assertEqual(r, "/home/ulrich/6_Projekte/Programme/pythonProject/utils/configs/path.yml") + r = t.getConfigPath("comp", "TestA2") + print (r) + self.assertEqual(r, None) + r = t.getConfigPath("comp", "testA2") + print (r) + r = t.getConfig("tool", "path") + print (r) + #r = t.getConfig("comp", "testA2") + print("pattern " + r["pattern"]["log"]) + print("pattern " + r["pattern"]["precond"]) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_conn.py b/test/test_conn.py new file mode 100644 index 0000000..6d1ba03 --- /dev/null +++ b/test/test_conn.py @@ -0,0 +1,20 @@ +import unittest +import ulrich.program +import utils.conn_tool + +class MyTestCase(unittest.TestCase): + def test_something(self): + print("# # # # tetsComponents # # # # #") + job = ulrich.program.Job("unit") + args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool"} + job.par.setParameterArgs(args) + self.assertEqual(True, True) + conn = utils.conn_tool.getConnection("testb", 1) + print("test-conn " + str(conn)) + conns = utils.conn_tool.getConnections("testb") + print("test-conn " + str(conns)) + conns = utils.conn_tool.getConnections("testa") + print("test-conn " + str(conns)) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_file.py b/test/test_file.py new file mode 100644 index 0000000..ba6e301 --- /dev/null +++ b/test/test_file.py @@ -0,0 +1,32 @@ +import unittest +import utils.file_tool as t +import utils.path_tool +import ulrich.program + + +class MyTestCase(unittest.TestCase): + def test_getFiles(self): + job = ulrich.program.Job("unit") + args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "job_tool", + "modus": "unit"} + job.par.setParameterArgs(args) + r = t.getFiles(job.m, job.conf.confs.get("paths").get("program") + "/utils", "file_.*.py", None) + self.assertEqual((len(r) == 1), True) + r = t.getFiles(job.m, job.conf.confs.get("paths").get("program") + "/utils", "file__.*.py", None) + self.assertEqual((len(r) == 0), True) + r = t.getFiles(job.m, job.conf.confs.get("paths").get("program") + "/utils", ".*_tool.py", None) + self.assertEqual((len(r) > 2), True) + r = t.getFilesRec(job.m, job.conf.confs.get("paths").get("program"), ".*?file.*.py") + print (r) + + def test_pathTool(self): + job = ulrich.program.Job("unit") + args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "job_tool", + "modus": "unit"} + job.par.setParameterArgs(args) + self.assertEqual(utils.path_tool.generatePath("program", "komp", "testA", "CONFIG.yml"), + "/home/ulrich/6_Projekte/PythonProject/komponents/testA/COFIG.yml") + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_job.py b/test/test_job.py new file mode 100644 index 0000000..068be86 --- /dev/null +++ b/test/test_job.py @@ -0,0 +1,35 @@ +import unittest +import os +from ulrich.program import Job +from ulrich.componentHandling import ComponentManager + +class MyTestCase(unittest.TestCase): + def test_parameter(self): + job = Job("unit") + args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool"} + job.par.setParameterArgs(args) + self.assertEqual(job.hascomponente("TestA"), True) + self.assertEqual(job.hasTool("TestA"), False) + self.assertEqual(job.hasTool("job_tool"), True) + self.assertEqual(job.getDebugLevel("file_tool"), 23) + self.assertEqual(job.getDebugLevel("job_tool"), 23) + args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", + "tool" : "job_tool", "tsdir": "/home/ulrich/6_Projekte/Programme/pythonProject/test/lauf/V0.1/startjob/2021-08-21_18-ß2-01"} + job.par.setParameterArgs(args) + def test_components(self): + print("# # # # tetsComponents # # # # #") + job = Job.resetInstance("unit") + args = { "application" : "TEST" , "environment" : "ENV01", "modus" : "unit", "loglevel" : "debug", "tool" : "job_tool"} + job.par.setParameterArgs(args) + cm = ComponentManager() + cm.createComponents("testb", 0, "") + cm.createComponents("testa", 1, "") + + def test_run(self): + os.system("python /home/ulrich/6_Projekte/Programme/pythonProject/check_environment.py -a TEST -e ENV01") + # os.system("python /home/ulrich/6_Projekte/Programme/pythonProject/init_testset.py -a TEST -e ENV01 " + # "-ts /home/ulrich/6_Projekte/Programme/pythonProject/test/lauf/V0.1/implement_2021-08-28_23-50-51 -dt csv -ds implement -dn firstunit") + # os.system("python /home/ulrich/6_Projekte/Programme/pythonProject/init_testset.py") + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_main.py b/test/test_main.py new file mode 100644 index 0000000..52b68d9 --- /dev/null +++ b/test/test_main.py @@ -0,0 +1,12 @@ +import unittest +import utils.path_tool as path_tool +import ulrich.program as program + +class MyTestCase(unittest.TestCase): + def test_pathTool(self): + x = program.Job("test:application=TEST:application=ENV01") + self.assertEqual(path_tool.generatePath("program", "komp", "testA", "CONFIG.yml"), "/home/ulrich/6_Projekte/PythonProject/komponents/testA/COFIG.yml") + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_path.py b/test/test_path.py new file mode 100644 index 0000000..ca6e765 --- /dev/null +++ b/test/test_path.py @@ -0,0 +1,39 @@ +import unittest +import utils.path_tool +from ulrich.program import Job + + +class MyTestCase(unittest.TestCase): + def test_path(self): + job = Job("unit") + t = utils.path_tool.PathTool() + args = {"application": "TEST", "environment": "ENV01", "modus": "unit", "loglevel": "debug", "tool": "job_tool", + "modus": "unit", "testcase": "TESTFALL", "release": "V0.1", "tctime": "2021-08-21_12-02-01" } + job.par.setParameterArgs(args) + pt = t.PathConf() + r = t.getKeyValue("job.par.application") + print(r) + r = t.getKeyValue("job.conf.results") + print(r) + self.assertEqual(r, "/home/ulrich/6_Projekte/Programme/pythonProject/test/target") + r = t.composePath("tcbase", None) + #r = t.replaceNoPython("{job.conf.archiv}/{job.par.release}/{job.par.testcase}/{job.par.tctime}", "{job.conf.archiv}", "/home/ulrich/6_Projekte/Programme/pythonProject/test/lauf") + print(r) + args = { "application" : "TEST" , "application" : "ENV01", "modus" : "unit", "loglevel" : "debug", + "tool" : "job_tool", "tsdir": "/home/ulrich/6_Projekte/Programme/pythonProject/test/lauf/V0.1/startjob/2021-08-21_18-ß2-01"} + job = Job.resetInstance("unit") + job.par.setParameterArgs(args) + # r = t.extractPath("tsbase" , "/home/ulrich/6_Projekte/Programme/pythonProject/test/lauf/V0.1/startjob/2021-08-21_18-ß2-01") + r = t.extractPattern("tsbase" ) + print(r) + self.assertEqual(r[0][1], "job.conf.archiv") + self.assertEqual(r[3][0], "_") + r = t.extractPath("tsbase" , "/home/ulrich/6_Projekte/Programme/pythonProject/test/lauf/V0.1/startjob_2021-08-21_10-02-01") + print("r " + str(r)) + print(vars(job.par)) + self.assertEqual(job.par.release, "V0.1") + self.assertEqual(job.par.usecase, "startjob") + self.assertEqual(job.par.tltime, "2021-08-21_10-02-01") + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_tdata.py b/test/test_tdata.py new file mode 100644 index 0000000..2821e0c --- /dev/null +++ b/test/test_tdata.py @@ -0,0 +1,18 @@ +import unittest +import utils.tdata_tool as t +import ulrich.program + +class MyTestCase(unittest.TestCase): + def test_pathTool(self): + job = ulrich.program.Job("unit") + args = {"application": "TEST", "application": "ENV01", "modus": "unit", "loglevel": "debug", + "tool": "job_tool", "tdtyp": "csv", "tdsrc": "implement", "tdname": "firstunit", + "modus": "unit"} + job.par.setParameterArgs(args) + filename = str(job.conf.confs["paths"]["testdata"]) + "/" + getattr(job.par, "tdsrc") + "/" + getattr(job.par, "tdname") + ".csv" + tdata = t.readCsv(filename) + self.assertEqual(1, 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_run.py b/test_run.py new file mode 100644 index 0000000..e69de29 diff --git a/ulrich/__init__.py b/ulrich/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ulrich/componentHandling.py b/ulrich/componentHandling.py new file mode 100644 index 0000000..1d0e464 --- /dev/null +++ b/ulrich/componentHandling.py @@ -0,0 +1,185 @@ +# managing the components +# ----------------------------------------------------------------------------- +""" +component has to be created in relation of the application (basis.yml). +Each componente could be created mostly once, but not everytime: +* the same instance of a component is used in different contexts +* there could be exist more instances +* there could be alternatives of an instance +Each kind of instance has its component-class and for each use should be an object be created. +Each crated component-onject are documented in the parameter-file. +""" +import utils.config_tool +import utils.conn_tool +import ulrich.program +import ulrich.message +import components.component +import importlib +import copy +components = {} +PARAM_NOSUBNODE = [ "artifact", "components", "instance" ] + +class ComponentManager: + __instance = None + """ + initializes the Manager with all necessary components + """ + def __init__(self): + job = ulrich.program.Job.getInstance() + job.m.logDebug("applicationscomponente -- " + str(type(job.par)) ) + self.components = {} + + def initComponents(self): + # sets components the first time + # afterwards set components from parameter-file + job = ulrich.program.Job.getInstance() + anw = job.par.application + job.m.logDebug("applicationscomponente -- " + str(type(job.par)) ) + if not job.conf.confs["applicationen"].get(anw): + job.m.setFatal("application " + job.par.application + " is not configured") + return + for k in job.conf.confs["applicationen"].get(anw): + job.m.logDebug("applicationscomponente -- " + k + ":" ) + self.createComponents(k, 0, "") + + + def setComponents(self): + # set components from parameter-file + job = ulrich.program.Job.getInstance() + job.m.logDebug("applicationscomponente -- " + str(type(job.par)) ) + + def getComponent(self, compobjname): + job = ulrich.program.Job.getInstance() + verify=-2+job.getDebugLevel("job_tool") + job.debug(verify, "getComponents " + compobjname) + return components[compobjname] + + def getComponents(self, mainfct): + job = ulrich.program.Job.getInstance() + verify=-2+job.getDebugLevel("job_tool") + job.debug(verify, "getComponents " + mainfct) + out = [] + for c in components: + job.debug(verify, "getComponents " + c + ": " + str(components[c].conf)) + print("getComponents " + c + ": " + str(components[c].conf)) + if mainfct in components[c].conf["function"]: + out.append(c) + return out + + @staticmethod + def getInstance(): + if (ComponentManager.__instance is not None): + return ComponentManager.__instance + else: + raise Exception("Klasse noch nicht initialisiert") + + def createComponents(self, componentName, nr, suffix): + """ + in order to create a component it must be loaded + * knogwedge of the application - which components should be created + * technical-knowledge of the instanciated component, especially the connection, user, password + * business-knowledge of the component, especially of their interfaces resp. artifacts + :param componentName: Name of the component + :param nr: + :param add: + :return: + """ + job = ulrich.program.Job.getInstance() + verify=job.getDebugLevel("job_tool") + componentName = componentName.lower() + print("createComponent " + componentName) + confs = utils.config_tool.getConfig("comp", componentName) + conns = utils.conn_tool.getConnections(componentName) + print("createComponent -91- " + componentName + " : " + str(confs)) + if nr > 0 and int(confs["conf"]["instance"]["count"]) > 1: + job.m.setError("for multiple callers are multiple calls not implemented ") + if nr > 0 and len(conns) == 0: + job.m.setError("for multiple calls has only one call configured") + print(confs) + print("createComponent 1 " + componentName) + print(getComponentPath(componentName)) + print("createComponent 2 " + componentName) + cmodul = importlib.import_module(getComponentPath(componentName)) + class_ = getattr(cmodul, getComponentClass(componentName)) + readedPar = job.loadParameter() + if len(conns) == 1: + print("createComponent 3 a " + componentName) + if nr > 0 and confs["conf"]["instance"]["single"] == "n": + name = componentName + "_0" + str(nr) + else: + name = componentName + c = class_() + c.name = name + c.conf = confs["conf"] + c.conf["conn"] = conns[0] + c.m = ulrich.message.Message(ulrich.message.LIMIT_DEBUG, "logTime", name) + c.init() + print("createComponent 4 a " + componentName) + print(vars(c)) + print(vars(c.m)) + if readedPar is not None: + print("createComponent 5 a " + name + " : "+ str(readedPar)) + for k in readedPar["components"][name].keys(): + c.conf[k] = readedPar["components"][name][k] + components[name] = c + self.createComponent(c, nr, suffix) + else: + i = 1 + print("createComponent 3 b " + componentName) + for cn in conns: + name = componentName+ "_0"+str(i) + c = class_() + c.name = name + c.conf = confs["conf"] + c.conf["conn"] = conns[0] + c.m = ulrich.message.Message(ulrich.message.LIMIT_DEBUG, "logTime", name) + c.init() + print("createComponent 4 b " + componentName) + print(vars(c)) + if readedPar is not None: + for k in readedPar["components"][name].keys(): + c.conf[k] = readedPar["components"][name][k] + components[name] = c + self.createComponent(c, i, suffix) + i = i + 1 + print("createComponent 9 " + componentName) + print(components) + + def createComponent(self, comp, nr, suffix): + job = ulrich.program.Job.getInstance() + verify = -2 + job.getDebugLevel("job_tool") + job.debug(verify, "getComponents " + str(comp.conf["components"])) + for c in comp.conf["components"].keys(): + if c == "none": + continue + self.createComponents(c, nr, suffix) + +def getComponentFolder(comp): + return comp.lower() +def getComponentModul(comp): + return comp[0:1].upper() + comp[1:].lower() +def getComponentClass(comp): + return comp[0:1].upper() + comp[1:].lower() +def getComponentPath(comp): + return "components." + getComponentFolder(comp) + "." + getComponentModul(comp) + +def getComponentDict(): + job = ulrich.program.Job.getInstance() + verify=-2+job.getDebugLevel("job_tool") + job.debug(verify, "getComponents ") + out = {} + for c in components: + out[components[c].name] = {} + print("getCompDict " + components[c].name) + for k in components[c].confs.keys(): + print("getCompDict " + k) + if isParameterSubnode(k): # "artifact" in k or "components" in k or "instance" in k: + print("getCompDict -b- " + k) + out[components[c].name][k] = copy.deepcopy(components[c].confs[k]) + return out + +def isParameterSubnode(key): + for k in PARAM_NOSUBNODE: + if key in k: + return False + return True \ No newline at end of file diff --git a/ulrich/message.py b/ulrich/message.py new file mode 100644 index 0000000..9e6d29b --- /dev/null +++ b/ulrich/message.py @@ -0,0 +1,233 @@ +# function for collecting and writing messages i.e. logging and calculate the return-code +# ------------------------------------------------------------------------ +""" +there are two types of logging: +* log everything relevant to document the test-run in relation of the test-system and the test-effort + should be logged in order to archive these logfiles +* debug everything to know how this automatism is running should log into a debug-file. This file can be + removed after a few days +the logging is parametrized by the log-level (fatal ... trace). Especially the debug- and trace-message can be +switched on/of en detail in the functions and over special settings (parameter tool, component: these debugs should be switched on) +there are three types of functions: +* setFatal|Error|...() it logs, sets the top-message and return-code +* logFatal|Error|...() it logs without any effect for the program-result +* log()|debug(int) it logs depending on the special level-setting +at the end of a program there are some results: +* for each component there is a return-code, a top-message for documentation in the parameter-file, and a list of collected messages ++ for the main-program the result (return-code, top-message) will be summarized (the most relevant message will used) +""" +import ulrich.program +import os +import math +from datetime import datetime +import utils.path_tool + +LIMIT_FATAL = 0 +LIMIT_ERROR = 4 +LIMIT_WARN = 8 +LIMIT_MSG = 12 +LIMIT_INFO = 16 +LIMIT_DEBUG = 20 +LIMIT_TRACE = 24 + +RC_OFFSET = 4 +RC_FATAL = 8 +RC_ERROR = 6 +RC_WARN = 5 +RC_MSG = 4 +RC_INFO = 3 +RC_DEBUG = 2 +RC_TRACE = 1 + +class Message: + """ + Ausgaben erfolgen prioritaeten-gesteuert anhand + * Typ (fatal..trace) + * Einstellung (a) ueber Parameter ODER (b) in Funktion + Im Funktionskopf wird Einstellung gesetzt, z.B. verify=job.getDebugLevel (ggf keine Debug-Ausgabe) bzw. verify=job.getDebugLevel-1 (eingeschaltete Debug-Ausgabe) + "fatal": "4", # Abbruchfehlker, wird immer in debug und log ausgegeben, setzt RC + "error": "8", # Fehler, wird immer in debug und log ausgegeben, setzt RC + "warn": "12", # Warnung, wird immer in debug und log ausgegeben, setzt RC + "msg": "16", # Ergebnis, wird immer in debug und log ausgegeben, setzt RC + "info": "20", # Info, wird ggf. in debug und log ausgegeben, setzt RC + "debug": "24", # wird nur in debug ausgegeben, wenn log-level hoechstens auf eingestelltem job-level steht + "trace": "28", # wird nur in debug ausgegeben, wenn log-level hoechstens auf eingestelltem job-level steht + """ + def __init__(self, level, logTime, componente): + # (self, componente, out, level): + self.componente = componente # dezantrales Logsystem + #self.level = level # Vorgabe zu level zum Filtern, ob auszugebe + job = ulrich.program.Job.getInstance() + print(vars(job)) + print(globals()) + # exit(5) + verify = LIMIT_DEBUG + self.initErrorTyp() + self.rc = RC_INFO + if (level == 0): + self.level = LIMIT_DEBUG + else: + self.level = level + # init debugfile - except for each component + if componente is not None: # use general debugfile + self.debugfile = job.m.debugfile + self.debug(verify, "> > > debugfile uebernommen zu " + str(componente)) + else: + debugpath = job.conf.confs["paths"]["debugs"] + "/debug_" + logTime[0:-4] + "00.txt" + self.debugfile = open(debugpath, "a") + self.debug(verify, "> > > debugfile geoeffnet zu " + job.program + " mit " + debugpath) + # init logfile - except for components or unittest + self.logDebug("logfile " + str(componente) + ", " + str(job.par.basedir)) + if componente is not None: # + self.logfile = self.debugfile + elif job.program == "unit": + self.logfile = self.debugfile + else: + basedir = job.par.basedir + basedir = basedir.replace("base", "log") + # basedir = utils.path_tool.composePath(basedir, None) + basedir = utils.path_tool.composePath(basedir, None) + os.makedirs(basedir, exist_ok=True) + logpath = basedir + "/protokoll_" + logTime + ".txt" + self.logDebug("logfile " + logpath) + self.logfile = open(logpath, "w") + self.topmessage = "" + self.messages = [] + + print("message initialisiert mit level " + str(self.level)) + + def initErrorTyp(self): + self.CONST_ERRTYP = { + "fatal": "4", # wird immer in debug und log ausgegeben, setzt RC + "error": "8", # wird immer in debug und log ausgegeben, setzt RC + "warn": "12", # wird immer in debug und log ausgegeben, setzt RC + "msg": "16", # wird immer in debug und log ausgegeben, setzt RC + "info": "20", # wird immer in debug und log ausgegeben, setzt RC + "debug": "24", # wird nur in debug ausgegeben, wenn log-level hoechstens auf eingestelltem job-level steht + "trace": "28", # wird nur in debug ausgegeben, wenn log-level hoechstens auf eingestelltem job-level steht + "rc1": "fatal", + "rc2": "error", + "rc3": "warn", + "rc4": "info", + "rc5": "debug", + "rc6": "trace", + "1": "fatal", + "2": "error", + "3": "warn", + "4": "info", + "5": "debug", + "6": "trace" + } + + def getErrortyp(self, prio): + if prio <= LIMIT_FATAL: + return "FATAL" + elif prio <= LIMIT_ERROR: + return "ERROR" + elif prio <= LIMIT_WARN: + return "WARN" + elif prio <= LIMIT_MSG: + return "MESSAGE" + elif prio <= LIMIT_INFO: + return "INFO" + elif prio <= LIMIT_DEBUG: + return "DEBUG" + elif prio <= LIMIT_TRACE: + return "TRACE" + else: + return "NDEF" + + def closeMessage(self) : + self.debug(LIMIT_INFO, "closeMessage ------------------------------------------- \n") + self.logfile.close() + self.debugfile.close() + + def setRc(self, rc, text): + job = ulrich.program.Job.getInstance() + verify = -0+LIMIT_DEBUG + self.debug(verify, "setRc " + str(rc) + " " + str(self.rc)+ "\n") + if (int(rc) > self.rc): + self.rc = rc + self.topmessage = self.CONST_ERRTYP["rc"+str(rc)].upper() + ": " + text + elif (int(rc) == self.rc): + self.topmessage = self.CONST_ERRTYP["rc"+str(rc)].upper() + ": " + text + + def isRc(self, rc): + rcId = int(int(self.CONST_ERRTYP[rc]) / 4 - RC_OFFSET) + print("< < < isRc " + str(self.rc) + " <=? " + str(rcId)) + if self.rc <= int(rcId): + print("return True") + return True + else: + print("return False") + return False + + def getFinalRc(self): + if (self.rc <= RC_OFFSET): + return 0 + else: + return int(int(self.rc) - RC_OFFSET) + + def setFatal(self, text): + """ Routine zum Setzen des RC und gleichzeitigem Schreiben des Logs """ + self.setRc(RC_FATAL, text) + self.logFatal(text) + + def setError(self, text): + """ Routine zum Setzen des RC und gleichzeitigem Schreiben des Logs """ + self.setRc(RC_ERROR, text) + self.logError(text) + + def setWarn(self, text): + """ Routine zum Setzen des RC und gleichzeitigem Schreiben des Logs """ + self.setRc(RC_WARN, text) + self.logWarn(text) + + def setMsg(self, text): + """ Routine zum Setzen des RC und gleichzeitigem Schreiben des Logs """ + self.setRc(RC_MSG, text) + self.logInfo(text) + + def logFatal(self, text): + self.log(LIMIT_FATAL, "FATAL: " + text) + self.debug(LIMIT_FATAL, "FATAL: " + text) + + def logError(self, text): + self.log(LIMIT_ERROR, "ERROR: " + text) + self.debug(LIMIT_ERROR, "ERROR: " + text) + + def logWarn(self, text): + self.log(LIMIT_WARN, "WARN: " + text) + + def logMsg(self, text): + self.log(LIMIT_MSG, text) + self.log(LIMIT_MSG, "MSG: " + text) + + def logInfo(self, text): + self.log(LIMIT_INFO, text) + + def logDebug(self, text): + self.debug(LIMIT_DEBUG, text) + + def logTrace(self, text): + self.debug(LIMIT_TRACE, text) + + def log(self, prio, text): + """ eigentliche Schreibroutine: hierin wird debug-Level beruecksichtgigt""" + if (int(prio) <= int(self.level)) and (self.componente is None): + self.logfile.write(text + "\n") + elif (int(prio) <= int(self.level)): + self.messages.append(text) + else: + self.debug(prio, self.getErrortyp(prio) + ": " + text) + + def debug(self, prio, text): + """ eigentliche Schreibroutine: hierin wird debug-Level beruecksichtgigt""" + if (int(prio) < int(self.level)+1): + print ("debugfile " + str(self.debugfile)) + self.debugfile.write(text + "\n") + + def merge(self, submsg): + self.setRc(submsg.getFinalRc(), submsg.topmessage) + text = "\n".join(submsg.messages) + self.logInfo("\n"+text+"\n") \ No newline at end of file diff --git a/ulrich/program.py b/ulrich/program.py new file mode 100644 index 0000000..3afb23c --- /dev/null +++ b/ulrich/program.py @@ -0,0 +1,295 @@ +#!/usr/bin/python3 +# Template Batchrahmen +# +#import sys, getopt +import argparse +import copy + +import yaml +from datetime import datetime + +import ulrich.message +import ulrich.message +import ulrich.componentHandling +import utils.date_tool +import utils.path_tool + +jobdef = { + "unit": { + "pardef": "", + "pfilesource" : "", + "pfiletarget" : "envparfile", + "basedir": "envbase", + "logdir": "{job.par.envdir}/{log}/log_{time}.txt" }, + "check_environment": { + "pardef": "", + "pfilesource" : "", + "pfiletarget" : "envparfile", + "basedir": "envbase", + "logdir": "{job.par.envdir}/{log}/log_{time}.txt" }, + "init_testset": { + "pardef": "tsdir,tdtyp,tdsrc,tdname", + "pfilesource" : "envparfile", + "pfiletarget" : "tsparfile", + "basedir": "tsbase", + "logdir": "{job.par.tsdir}/{log}/log_{tstime}.txt" }, + "init_testcase": { + "pardef": "tcdir,tdtyp,tdsrc,tdname", + "pfilesource" : "tsparfile", + "pfiletarget" : "tcparfile", + "basedir": "tcbase", + "logdir": "{job.par.tcdir}/{log}/log_{tctime}.txt" } +} + + +def setGlobal(): + pass + +class Job: + __instance = None + + def __init__ (self, program): + print ("################# init Job ## " + program + " #################") + self.program = program + Job.__instance = self + par = Parameter(program) + self.par = par + print("prog-42 " + str(self.par.basedir)) + conf = Configuration(program) + self.conf = conf + print("prog-45 " + str(self.par.basedir)) + dirpath = self.par.getDirParameter() + setGlobal() + if dirpath is not None: + utils.path_tool.extractPath(dirpath[0], dirpath[1]) + if program == "unit": # no job will be started + self.start = datetime.now() + logTime = self.start.strftime("%Y%m%d_%H%M%S") + self.m = ulrich.message.Message(ulrich.message.LIMIT_DEBUG, logTime, None) + print("prog-50 " + str(self.par.basedir)) + + + def resetInstance(program): + job = Job.getInstance() + if job is not None: + job.stopJob(1) + Job.__instance = None + Job(program) + return Job.__instance + + def getInstance(): + if (Job.__instance is not None): + return Job.__instance + else: + return None + + def startJob(self): + self.start = datetime.now() + print("prog-68 " + str(self.par.basedir)) + logTime = self.start.strftime("%Y%m%d_%H%M%S") + self.m = ulrich.message.Message(ulrich.message.LIMIT_DEBUG, logTime, None) + print("prog-68 " + str(self.m.rc)) + self.par.setParameterLoaded() + self.m.logInfo("# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") + self.m.debug(ulrich.message.LIMIT_INFO, "# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") + self.par.checkParameter() + + def stopJob(self, reboot=0): + self.ende = datetime.now() + self.dumpParameter() + print("stopJob " + str(self.m.messages) + ", " + str(self.m.debugfile)) + self.m.logInfo("# # " + self.m.topmessage + " # # # ") + self.m.logInfo("# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") + self.m.debug(ulrich.message.LIMIT_INFO, "# # " + self.m.topmessage + " # # # ") + self.m.debug(ulrich.message.LIMIT_INFO, "# # # Stop Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # " + self.ende.strftime("%d.%m.%Y %H:%M:%S") + " # # # RC: " + str(self.m.getFinalRc())) + self.m.closeMessage() + rc = self.m.getFinalRc() + print ("rc " + str(rc)) + if reboot == 0: + exit(rc) + + def dumpParameter(self): + parpath = utils.path_tool.composePath(jobdef[self.program]["pfiletarget"], None) + output = {} + output["comps"] = ulrich.componentHandling.getComponentDict() + output["par"] = self.par.__dict__ + print(str(output)) + with open(parpath, "w") as file: + doc = yaml.dump(output, file) + file.close() + + def loadParameter(self): + output = {} + if len(str(jobdef[self.program]["pfilesource"])) < 2: + return None + parpath = utils.path_tool.composePath(jobdef[self.program]["pfilesource"], None) + with open(parpath, "r") as file: + doc = yaml.full_load(file) + for k in doc.keys(): + output[k] = copy.deepcopy(doc[k]) + return output + + def getParameter(self, parameter): + if hasattr(self.par, parameter): + return getattr(self.par, parameter) + elif "xxxtime" in parameter: + neu = utils.date_tool.getActdate(utils.date_tool.F_DIR) + # setattr(self.par, parameter, neu) + return neu + def hasElement(self, parameter, elem): + if hasattr(self.par, parameter): + print (parameter + " in Parameter") + if getattr(self.par, parameter).find(elem) >= 0: + return True + return False + return True + def hascomponente(self, komp): + return self.hasElement("componente", komp) + def hasFunction(self, fct): + return self.hasElement("function", fct) + def hasTool(self, tool): + return self.hasElement("tool", tool) + def getMessageLevel(self, errtyp, elem): + if (not hasattr(self, "m")) or (self.m is None): + return ulrich.message.LIMIT_DEBUG + elif elem.find("tool") > 1: + if not hasattr(self.par, "tool") or getattr(self.par, "tool").find(elem) <= 0: + return int(self.m.CONST_ERRTYP[errtyp]) -1 + else: + return int(self.m.CONST_ERRTYP[errtyp]) + else: + return int(self.m.CONST_ERRTYP[errtyp]) + def getInfoLevel(self, elem): + return self.getMessageLevel("info", elem) + def getDebugLevel(self, elem): + return self.getMessageLevel("debug", elem) + def getTraceLevel(self, elem): + return self.getMessageLevel("trace", elem) + def debug(self, prio, text): + print("job.debug "+str(prio)+" "+text) + if hasattr(self, "m"): + self.m.debug(prio, text) + else: + print(text) + +class Parameter: + print ("class Parameter") + def __init__ (self, program): + print ("# init Parameter for " + program) + self.program = program + if program == "unit": + self.basedir = "debugs" + return + elif jobdef[program]: + self.basedir = jobdef[program]["basedir"] + elif "estcase" in program: + self.basedir = "tcbase" + elif "estset" in program: + self.basedir = "tsbase" + elif "nviron" in program: + self.basedir = "envbase" + else: + self.basedir = "debugs" + print (f"# Parameter initialisiert {self.program} mit basedir {self.basedir}") + self.setParameter() + + def checkParameter(self): + job = Job.getInstance() + print (f"Parameter initialisiert {self.program}") + pardef = jobdef[job.program]["pardef"] + for p in pardef.split(","): + print(p) + if len(p) > 1 and not hasattr(self, p): + job.m.setFatal("Parameter " + p + " is not set!") + print (f"job initialisiert {self.program}") + + def setParameter(self): + """ + 1. Programm -- implementiert in Main-Klasse + 2. anwndung -- steuert zu pruefende System [ in basis_Config ] + 3. amgebung -- steuert zu pruefende Maschine [ in dir/applicationen ] + 4. release -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ] + 5. ~Verz -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch + 6. zyklus -- optional unterscheidet echte und entwicklungsLaeufe + 7. Programmspezifische Parameter + 8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2) + 10. modus -- steuert die Verarbeitung; default echt + - echt-auto Lauf aus Automatisierung (1-7) + - test Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen + - echt-spez Wiederholung einer spezifischen Funktion (1-13) + - unit Ausfuehrung der Unittests + 11. componente -- schraenkt Verarbeitung auf parametriserte componenten ein + 12. funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein + 13. tool -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein + """ + # args = str(sys.argv) + # print ("Job-Programm %s : " % args) + parser = argparse.ArgumentParser() + parser.add_argument('-a', '--application', required=True, action='store') + parser.add_argument('-e', '--environment', required=True, action='store') + parser.add_argument('-r', '--release', action='store') + parser.add_argument('-ts', '--tsdir', action='store') + parser.add_argument('-tc', '--tcdir', action='store') + parser.add_argument('-rs', '--rsdir', action='store') + parser.add_argument('-dt', '--tdtyp', action='store') + parser.add_argument('-ds', '--tdsrc', action='store') + parser.add_argument('-dn', '--tdname', action='store') + parser.add_argument('-l', '--loglevel', action='store') + parser.add_argument('-m', '--modus', action='store') + parser.add_argument('-c', '--componente', action='store') + parser.add_argument('-f', '--funktion', action='store') + parser.add_argument('-t', '--tool', action='store') + # parser.add_argument('-t', '--typ', default='einzel', action='store') + # parser.add_argument('-d', '--dir', action='store') + args = parser.parse_args() + self.setParameterArgs(args) + + def getDirParameter(self): + if hasattr(self, "tcdir"): + return ("tcbase", str(self.tcdir)) + if hasattr(self, "tsdir"): + return ("tsbase", str(self.tsdir)) + return None + + def setParameterArgs(self, args): + job = Job.getInstance() + print("setParArgs " + str(type(args))) + self.parstring = "python " + self.program + if "dict" in str(type(args)): + for k in args: + self.setJobAttr(k, args[k]) + else: + for k in vars(args): + if getattr(args, k) is not None: + self.setJobAttr(k , getattr(args, k)) + + def setParameterLoaded(self): + job = Job.getInstance() + print("setParLoaded " ) + readedPar = job.loadParameter() + if readedPar is not None: + for k in readedPar["par"].keys(): + if not hasattr(self, k): + self.setJobAttr(k, readedPar["par"][k]) + + def setJobAttr(self, key, val): + setattr(self, key, val) + self.parstring = self.parstring + " --" + key + " " + val + +class Configuration: + def __init__ (self, program): + self.program = program + print (f"job initialisiert {self.program}") + if program == "unit": + self.setConfiguration('../conf/basis.yml') + return + self.setConfiguration('conf/basis.yml') + + def setConfiguration(self, path): + self.confs = {} + with open(path, "r") as file: + doc = yaml.full_load(file) + for i, v in doc.items(): + self.confs[i] = v + print ("set conf") + print ("set configuration") diff --git a/ulrich/toolHandling.py b/ulrich/toolHandling.py new file mode 100644 index 0000000..31000d3 --- /dev/null +++ b/ulrich/toolHandling.py @@ -0,0 +1,9 @@ +# +# ------------------------------------------------- +""" +Toolmanager +""" +class ToolManager: + # for c job.conf.toolscls + # c = importlib. [c].capital(c()) + pass \ No newline at end of file diff --git a/unit_run.py b/unit_run.py new file mode 100644 index 0000000..edae501 --- /dev/null +++ b/unit_run.py @@ -0,0 +1,17 @@ +# +# ---------------------------------------------------------- +""" +This program is created for the business-test on the level of unit testing. On this level there is an intensive test-set up to a carthesian product possible. +The test-specification is written in a yaml-config in the data-directory. +workflow: +1. generate test-cases with all inputs into a csv-file of the spec-yaml +2. run the csv-file and collect all results into a new result-csv-file with all inputs and outputs +3. check the result-csv-file and declare it as a target-results +OR instead 2 and 3 on test-first-strategy: +3. create manually the target-result with your inputs and the target system-outputs +4. run and compare the actual-result with the target-result +5. at the end you get an xls-sheet like your result-csv-file but additionally with comparsion-result as detailed result and the only counts of passed and failed tests as minimum-result which can be inserted into management-tools +""" +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + print ("start") \ No newline at end of file diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/config_tool.py b/utils/config_tool.py new file mode 100644 index 0000000..6948c62 --- /dev/null +++ b/utils/config_tool.py @@ -0,0 +1,90 @@ +# +# -------------------------------------------------------------- +import sys +try: + import ulrich.program +except ImportError: + print("ImportError: " + str(ImportError.with_traceback())) + pass +import yaml +import ulrich.componentHandling +import utils.path_tool +import os.path + +def getConfigPath(modul, name): + """ + gets the most specified configuration of different sources + Parameter: + * typ -- (basic, comp, tool) + * name -- the specific class + sources: + * programm << + * install << + * environ << basis-conf + * release << basis-conf + * testset << parameter/environ + * testcase << parameter + """ + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel("config_tool") + job.debug(verify, "getConfig " + modul + ", " + name) + if modul == "tool": + pathname = job.conf.confs.get("paths").get("home") + "/configs/tool_" + name + ".yml" + job.debug(verify, "1 " + pathname) + if os.path.exists(pathname): + return pathname + pathname = job.conf.confs.get("paths").get("program") + "/utils/configs/" + name + ".yml" + job.debug(verify, "2 " + pathname) + if os.path.exists(pathname): + return pathname + pathname = job.conf.confs.get("paths").get("environment") + "/" + job.par.environment + "/configs/tool_" + name + ".yml" + job.debug(verify, "3 " + pathname) + if os.path.exists(pathname): + return pathname + job.debug(verify, "3x " + pathname) + elif modul == "comp": + pathname = job.conf.confs.get("paths").get("home") + "/configs/comp_" + name + ".yml" + job.debug(verify, "4 " + pathname) + if os.path.exists(pathname): + return pathname + pathname = job.conf.confs.get("paths").get("program") + "/components/" + ulrich.componentHandling.getComponentFolder(name) + "/CONFIG.yml" + job.debug(verify, "5 " + pathname) + if os.path.exists(pathname): + return pathname + job.debug(verify, "6 " + pathname) + else: + pathname = utils.path_tool.composePath("tcparfile") + job.debug(verify, "7 " + pathname) + if os.path.exists(pathname): + return pathname + pathname = utils.path_tool.composePath("tsparfile") + job.debug(verify, "8 " + pathname) + if os.path.exists(pathname): + return pathname + pathname = job.conf.confs.get("paths").get("release") + "/configs/basis.yml" + job.debug(verify, "9 " + pathname) + if os.path.exists(pathname): + return pathname + pathname = job.conf.confs.get("paths").get("environment") + "/configs/basis.yml" + job.debug(verify, "10 " + pathname) + if os.path.exists(pathname): + return pathname + pathname = job.conf.confs.get("paths").get("home") + "/config/basis.yml" + job.debug(verify, "11 " + pathname) + if os.path.exists(pathname): + return pathname + job.debug(verify, "12 " + pathname) + +def getConfig(modul, name): + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel("config_tool") + pathname = getConfigPath(modul, name) + confs = {} + job.debug(verify, "getConfig " + pathname) + with open(pathname, "r") as file: + job.debug(verify, "openfile") + doc = yaml.full_load(file) + for i, v in doc.items(): + job.debug(verify, "item " + str(i) + ": " + str(v)) + confs[i] = v + return confs \ No newline at end of file diff --git a/utils/configs/path.yml b/utils/configs/path.yml new file mode 100644 index 0000000..4faac60 --- /dev/null +++ b/utils/configs/path.yml @@ -0,0 +1,44 @@ +# +pattern: + # Keywords + log: log + parfile: PARAMETER_{job.par.application}_{job.par.environment}.yml + precond: vorher + postcond: nachher + diff: diff_fach + prediff: diff_init + rundiff: diff_ablauf + result: Ergebnisse/{comp.name} + origin: original + parts: teilergebnisse + sumfile: xxx + backup: backup + reffile: Herkunft.txt + tc: testfall + ts: testlauf + debugname: debug + logname: log + debugs: "{job.conf.home}/test/log" + # environment + envbase: "{job.conf.environment}/{job.par.environment}" + envlog: "{envbase}/{log}" + envparfile: "{envbase}/{parfile}" + # testcase + tcbase: "{job.conf.archiv}/{job.par.release}/{job.par.testcase}/{job.par.tctime}" + tclog: "{tcbase}/{log}" + tcresult: "{tcbase}/{result}" + tcparfile: "{tcbase}/{parfile}" + tcdiff: "{tcresult}/{diff}" + tcprediff: "{tcresult}/{prediff}" + tcrundiff: "{tcresult}/{rundiff}" + tcprecond: "{tcresult}/vorher" + tcpostcond: "{tcresult}/nachher" + # testset + tsbase: "{job.conf.archiv}/{job.par.release}/{job.par.usecase}_{job.par.tltime}" + tslog: "{tsbase}/{log}" + tsparfile: "{tsbase}/{parfile}" + tssum: "{tsbase}/Ergebnis" + # target-result rs + rsbase: "{job.conf.results}/{job.par.branch}" + rsresult: "{rsbase}/{result}" + rsbackup: "{rsbase}/{result}" diff --git a/utils/configs/txt b/utils/configs/txt new file mode 100644 index 0000000..463ff17 --- /dev/null +++ b/utils/configs/txt @@ -0,0 +1,35 @@ +# +pattern: + # Keywords + log: Protokolle + parfile: PARAMETER_{job.par.application}_{job.par.environ}.yml + precond: vorher + postcond: nachher + diff: diff_fach + prediff: diff_init + rundiff: diff_ablauf + result: Ergebnisse/{comp.name} + origin: original + parts: teilergebnisse + sumfile: + backup: backup + reffile: Herkunft.txt + tc: testfall + ts: testlauf + # testcase + tcbase: {job.conf.archiv}/{job.par.release}/{job.par.testcase}/{job.par.tctime} + tcresult: {tcbase}/{result} + tcparfile: {tcbase}/{parfile} + tcdiff: {tcresult}/{diff} + tcprediff: {tcresult}/{prediff} + tcrundiff: {tcresult}/{rundiff} + precond: {tcresult}/vorher + postcond: {tcresult}/nachher + # testset + tsbase: {job.conf.archiv}/{job.par.release}/{job.par.usecase}_{job.par.tltime} + tsparfile: {tsbase}/{parfile} + tssum: {tsbase}/Ergebnis + # target-result rs + rsbase: {job.conf.results}/{job.par.branch} + rsresult: {rsbase}/{result} + rsbackup: {rsbase}/{result} diff --git a/utils/conn_tool.py b/utils/conn_tool.py new file mode 100644 index 0000000..177cd7c --- /dev/null +++ b/utils/conn_tool.py @@ -0,0 +1,46 @@ +# functions about connections to other instances +# ------------------------------------------------------------------- +""" + +""" +import ulrich.program +import utils.config_tool +def getConnection(comp, nr): + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel("conn_tool") + conn = {} + if job.conf.confs.get("tools").get("connsrc") == "yml": + conn = utils.config_tool.getConfig("tool", "conn") + instnr = "inst" + str(nr) + if conn["env"][comp][instnr]: + return conn["env"][comp][instnr] + else: + job.m.setFatal("Conn-Tool: Comp not configured " + comp + " " + str(nr)) + elif job.conf.confs.get("tools").get("connsrc") == "db": + pass + elif job.conf.confs.get("tools").get("connsrc") == "csv": + pass + return None + +def getConnections(comp): + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel("conn_tool") + print("getConnections " + comp) + conn = {} + conns = [] + if job.conf.confs.get("tools").get("connsrc") == "yml": + conn = utils.config_tool.getConfig("tool", "conn") + if not comp in conn["env"]: + job.m.setFatal("Conn-Tool: Comp not configured " + comp) + elif job.conf.confs.get("tools").get("connsrc") == "db": + pass + elif job.conf.confs.get("tools").get("connsrc") == "csv": + pass + print(comp) + print(conn["env"].keys()) + print(conn["env"][comp]["instance"]) + for i in range(conn["env"][comp]["instance"]): + print("range " + str(i+1)) + instnr = "inst" + str(i+1) + conns.append(conn["env"][comp][instnr]) + return conns diff --git a/utils/date_tool.py b/utils/date_tool.py new file mode 100644 index 0000000..b0278c9 --- /dev/null +++ b/utils/date_tool.py @@ -0,0 +1,12 @@ +# functions related to Date-fields +# ----------------------------------------------------- +""" +additionally functions for calculating date with formulas like [DATE+2M] and for comparison of date related on two reference-dates +""" +import datetime + +F_DIR = "%Y-%m-%d_%H-%M-%S" +def getActdate(format): + return getFormatdate(datetime.datetime.now(), format) +def getFormatdate(date, format): + return date.strftime(format) diff --git a/utils/file_tool.py b/utils/file_tool.py new file mode 100644 index 0000000..38e187a --- /dev/null +++ b/utils/file_tool.py @@ -0,0 +1,90 @@ +# Funktionen zum Dateizugriff mit Suchen, Lesen, Schreiben +# ------------------------------------------------------------ +""" + +""" +import os +import os.path +import re +from ulrich.message import Message +from ulrich.program import Job +from pprint import pp +def getDump(obj): + result="" + print (str(type(obj))) + result = vars(obj) + return str(result) +# if type(obj) == "__dict__" + +def getFiles(msg: Message, path, pattern, conn): + """ + search filenames in the directory - if conn is set search remote + :param msg: -- msg-Objekt + :param path: -- Pfad - String + :param pattern: -- Dateiname als Pattern + :param conn: + :return: Array mit gefundenen Dateien, nur Dateiname + """ + if conn is not None: + return getRemoteFiles(msg, path, pattern, conn) + job = Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + out = [] + msg.debug(verify, "getFiles " + path + " , " + pattern) + for f in os.listdir(path): + msg.debug(verify, "getFiles " + f) + if re.search(pattern, f): + msg.debug(verify, "match " + f) + out.append(f) + return out + +def getRemoteFiles(msg: Message, path, pattern, conn): + """ + search filenames in the directory - if conn is set search remote + :param msg: -- msg-Objekt + :param path: -- Pfad - String + :param pattern: -- Dateiname als Pattern + :param conn: + :return: Array mit gefundenen Dateien, nur Dateiname + """ + + +def getFilesRec(msg: Message, path, pattern): + """ + Sucht Dateien im Verzeichnis rekursiv + :param msg: -- msg-Objekt + :param path: -- Pfad - String + :param pattern: -- Dateiname als Pattern + :return: Array mit gefundenen Dateien, absoluter Pfad + """ + job = Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + out = [] + msg.debug(verify, "getFilesRec " + path + " , " + pattern) + for (r, dirs, files) in os.walk(path): + for f in files: + msg.debug(verify, "getFilesRec " + f) + if re.search(pattern, f): + msg.debug(verify, "match " + f) + out.append(os.path.join(r, f)) + return out + +def getTree(msg: Message, pfad): + job = Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + msg.debug(verify, "getTree " + pfad ) + tree = {} + files = [] + for f in os.listdir(pfad): + if os.path.isDir(os.path.join(pfad, f)): + tree[f] = getTree(msg, os.path.join(pfad, f)) + elif os.path.isFile(os.path.join(pfad, f)): + files.append(f) + tree["_files_"] = files + return tree + +def mkPaths(msg, pfad): + job = Job.getInstance() + verify = int(job.getDebugLevel("file_tool")) + modus = job.conf.confs["paths"]["mode"] + os.makedirs(pfad, exist_ok=True) \ No newline at end of file diff --git a/utils/job_tool.py b/utils/job_tool.py new file mode 100644 index 0000000..67b1562 --- /dev/null +++ b/utils/job_tool.py @@ -0,0 +1,31 @@ +# GrundFunktionen zur Ablaufsteuerung +# +# -------------------------------------------------------- +""" +1. Programm -- implementiert in Main-Klasse +2. Anwndung -- steuert zu pruefende System [ in basis_Config ] +3. application -- steuert zu pruefende Maschine [ in dir/applicationen ] +4. release -- steuert zu prufendes Release [ aus dir/release kann spez. release_Config geladen werden, dir/lauf/release ] +5. ~Verz -- Dokumentationsverzeichnis zu Testlauf/Testfall/Soll-Branch +6. zyklus -- optional unterscheidet echte und entwicklungsLaeufe +7. Programmspezifische Parameter +8. loglevel -- steuert Protokollierung; default debug (fatal/error/warn/msg/info/debug1/debug2/trace1/trace2) +10. Laufart -- steuert die Verarbeitung; default echt +- echt-auto Lauf aus Automatisierung (1-7) +- test Lauf ohne Ausfuehrungen am Testsystem, wohl aber in Testverzeichnissen +- echt-spez Wiederholung einer spezifischen Funktion (1-13) +- unit Ausfuehrung der Unittests +11. Modul -- schraenkt Verarbeitung auf parametriserte componenten ein +12. Funktion -- schraenkt Verarbeitung auf parametriserte Funktionen ein +13. Tool -- schraenkt Protokollierung/Verarbeitung auf parametriserte Tools ein +""" +from ulrich.program import Job +def hasModul(komp): + job = Job.getInstance() + return False +def hasFunction(fct): + job = Job.getInstance() + return False +def hasTool(tool): + job = Job.getInstance() + return False diff --git a/utils/match_tool.py b/utils/match_tool.py new file mode 100644 index 0000000..39a4e2a --- /dev/null +++ b/utils/match_tool.py @@ -0,0 +1,98 @@ +# +# ------------------------------------------------------------ +""" + +""" +class Matching(): + def __init__(self, comp): + self.comp = comp + self.elements = {} + self.resultFiles = comp.resultFiles + self.targetFiles = comp.targetFiles + self.assignedFiles = {} + pass + +def matchFiles(matching): + """ + + post: + :param matching: + :return: + """ + pass + +def matchBestfit(matching): + """ + in this strategy the difference-score of all elements of both sides will be calculated. + the best fit will assigned together until nothing is + * the elements can be each kind of object + * the count of elements should not be high + :param matching: + :return: + """ + hits = {} + output = [] + results = {} + targets = {} + for tg in matching.elements["tgelement"]: + targets[tg] = True + for rs in matching.elements["rselement"]: + results[rs] = True + for tg in matching.elements["tgelement"]: + acthit = matching.comp.getHitscore(matching.elementtyp, matching.elements["rselement"][rs], matching.elements["tgelement"][tg]) + while acthit in hits.keys(): + acthit = getNextHitscore(acthit) + hits[acthit] = {} + hits[acthit]["result"] = rs + hits[acthit]["target"] = tg + for h in sorted(hits.keys()): + if results[h]["result"] and targets[h]["target"]: + results[h]["result"] = False + targets[h]["target"] = False + output.append((hits[acthit]["result"], hits[acthit]["target"])) + for rs in results.keys(): + if results[rs]: + output.append((matching.elements["rselement"][rs], None)) + for tg in targets.keys(): + if results[rs]: + output.append((matching.elements["tgelement"][tg], None)) + return output + +# -------------------------------------------------------------------------- + +def matchTree(matching): + """ + + :param matching: + :return: + """ + pass + + def matchElement(matching, rs, tg): + if "array" in type(rs) and "array" in type(tg): + return matchArray(matching, rs, tg) + elif "dict" in type(rs) and "dict" in type(tg): + return matchDict(matching, rs, tg) + else: + pass + + def matchDict(matching, rs, tg): + pass + def matchArray(matching, rs, tg): + pass # matchBest + +# -------------------------------------------------------------------------- +def matchLines(matching): + pass + +# -------------------------------------------------------------------------- + +def getScoreint(score): + if score.is_integer(): + return score + elif "h_" in score: + return int(score[2:]) +def getHitscore(score): + return "h_" + ':04d'.format(getScoreint(score)) +def getNextHitscore(score): + return "h_" + ':04d'.format(getScoreint(score)+1) diff --git a/utils/path_tool.py b/utils/path_tool.py new file mode 100644 index 0000000..8222802 --- /dev/null +++ b/utils/path_tool.py @@ -0,0 +1,148 @@ +# All functions related to the full path. +# It implements the team conventions of the directory structure +# ----------------------------------------------------------------------- +""" In diesem Modul werden alle Funktionen zusammengefasst zur Generierung und Ermittlung von pathsn """ +import sys +import ulrich.program +import utils.config_tool +import re + + +def getKeyValue(key): + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel("path_tool") + pt = PathConf.getInstance() + print("getKeyValue " + key) + if 'job.par' in key: + neu = job.getParameter(key[8:]) + return neu + elif 'job.conf' in key: + neu = job.conf.confs["paths"][key[9:]] + print(neu) + return neu + # return job.conf.confs["paths"][key[9:]] + elif (pt.pattern): + return pt.pattern[key] + print("pt exists") + else: + return "xx-"+key+"-xx" + +def composePath(pathname, comp): + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel("path_tool") + pt = PathConf.getInstance() + print("composePath " + pathname + " zu " + str(pt) + "mit ") + print(pt.pattern) + if pt.pattern[pathname]: + return composePatttern(pt.pattern[pathname], comp) + else: + print("in Pattern nicht vorhanden: " + pathname) + +def composePatttern(pattern, comp): + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel("path_tool") + print("composePattern " + pattern) + max=5 + l = re.findall('\{.*?\}', pattern) + print(l) + for pat in l: + pit = getKeyValue(pat[1:-1]) + print(str(max) + ": " + pattern + ": " + pat + ": " + pit) + pattern = pattern.replace(pat, pit) + print(str(max) + ": " + pattern + ": " + pat + ": " + pit) + while ("{" in pattern): + max = max-1 + print(str(max) + ": " + pattern + ": " + pat + ": " + pit) + pattern = composePatttern(pattern, comp) + print(str(max) + ": " + pattern + ": " + pat + ": " + pit) + if (max < 3) : + break + return pattern + +def extractPattern(pathtyp): + job = ulrich.program.Job.getInstance() + out = [] + pt = PathConf.getInstance() + pattern = pt.pattern[pathtyp] + work = pattern + while "{" in work: + i = work.index("{") + j = work.index("}") + pre = work[0:i] + pat = work[i+1:j] + print (work + " von " + str(i) + "-" + str(j) + " pre " + pre + "pat " + pat) + pit = getKeyValue(pat) + tup = (pre, pat, pit) + out.append(tup) + work = work[j+1:] + return out + +def extractPath(pathtyp, pathname): + job = ulrich.program.Job.getInstance() + patterlist = extractPattern(pathtyp) + work = pathname + i = 0 + print("-- extractPatternList -- " + pathtyp + ":" + str(patterlist)) + for p in patterlist: + delim = p[0] + key = p[1] + val = p[2] + nextdelim = "" + if i >= len(patterlist) - 1: + nextdelim = "" + else: + nextdelim = patterlist[i+1][0] + print("xPath delim " + delim + " " + str(len(delim)) + ", " + nextdelim + " work " + work) + work = work[len(delim):] + print("xPath key " + key + " i " + str(i) + " work " + work) + if val is not None: + print("val not none " + val) + if val in work: + print("val ok") + work = work.replace(val, "") + elif "time" in key and "job.par" in key: + prop = "" + if i < len(patterlist) - 1: + prop = work[0:work.index(nextdelim)] + else: + prop = work + key = key[8:] + print("setprop " + key + " = " + prop) + if hasattr(job.par, key): delattr(job.par, key) + setattr(job.par, key, val) + else: + print("val not not ok " + val + " zu " + key) + elif "job.par" in key: + prop = "" + if i < len(patterlist) - 1: + print("job.par nextdelim " + nextdelim) + prop = work[0:work.index(nextdelim)] + else: + prop = work + key = key[8:] + print("setprop " + key + " = " + prop) + if hasattr(job.par, key): delattr(job.par, key) + setattr(job.par, key, prop) + work = work.replace(prop, "") + else: + print("val is none " + key) + i = i +1 + + +class PathConf: + __instance = None + def __init__(self): + print('init pathConf') + confs = utils.config_tool.getConfig("tool", "path") + self.pattern = confs["pattern"] + print(self.pattern) + PathConf.__instance = self + + @staticmethod + def getInstance(): + #print("PathConf getInstance " + str(PathConf.__instance)) + if (PathConf.__instance is None): + PathConf() + #print("PathConf getInstance " + str(PathConf.__instance)) + return PathConf.__instance + diff --git a/utils/report_tool.py b/utils/report_tool.py new file mode 100644 index 0000000..defc403 --- /dev/null +++ b/utils/report_tool.py @@ -0,0 +1,58 @@ +# functions in order to report the summaries +# ------------------------------------------------------- +""" +the reporting-system in bottom-up +(0) raw-data +(0.1) artificats created from test-system - raw-files up to work-files - +(0.2) in logs all contacts to test-system - raw-info - +(1) comparison-result +(1.1) comparison-result as difference for each file-comparison in html-files in testcases - diff-files - +(1.2) extraction of diff-files (only not accepted differences) as html-file in test-set - result-report - +(2) result-code +(2.1) result-code of each test-step for each component in testcases - parameter-file - +(2.2) transfer of the summary result-code of each testcase to an extern reporting-system - protocol - +(2.3) visualization of final result-code of each component and each testcase in test-set - result-report - +(2.4) visualization of statistical result-codes of each component and each test-set in test-context - result-report - +""" +import ulrich.program + +def getTcExtraction(tcpath, comp): + """ + extracts the pure differences of diff-files + :param comp: + :return: html-code with links to diff-files + """ + job = ulrich.program.Job.getInstance() + verify = -0 + job.getDebugLevel("report_tool") + job.debug(verify, "writeDataTable " + str(comp)) + body = '

' + # for f in diff-files: + # body = body + extractDiffLines + body = body + '

' + return body + +def getTcBody(tcpath): + """ + + :param tcpath: + :return: + """ + job = ulrich.program.Job.getInstance() + verify = -0+job.getDebugLevel("report_tool") + job.debug(verify, "writeDataTable " + str(tcpath)) + body = '

' + # for c in comps: + # + body = body + '

' + return body + +def getTcHeader(tcpath): + """ + creates header + :param tcpath: + :return: html-code with result-codes of each component + """ + job = ulrich.program.Job.getInstance() + verify = -0+job.getDebugLevel("report_tool") + job.debug(verify, "writeDataTable " + str(tcpath)) + diff --git a/utils/ssh_tool.py b/utils/ssh_tool.py new file mode 100644 index 0000000..5bd2b85 --- /dev/null +++ b/utils/ssh_tool.py @@ -0,0 +1,46 @@ +# +# ---------------------------------------------------------- +""" + +""" +import os + +import ulrich +import paramiko + +def getRemoteClass(comp): + job = ulrich.program.Job.getInstance() + verify = job.getDebugLevel("config_tool") + if job.conf.confs.get("tools").get("remotetyp") == "ssh": + return SshCmd(comp) + +class RemoteCmd: + def __init__(self): + self.rc = 0 + self.sysout = "" + pass + +class SshCmd(RemoteCmd): + def __init__(self, comp): + self.conn = comp + + def execCmd(self, cmds): + """ + + :param cmds: + :return: + """ + ssh = paramiko.SSHClient() + ssh.load_system_host_keys(os.path.expanduser('~/.ssh/known_hosts')) + if self.conn["password"]: + ssh.connect(self.conn["host"], username=self.conn["user"], password=self.conn["password"]) + else: + ssh.connect(self.conn["host"], username=self.conn["user"]) + shell = ssh.invoke_shell() + for cmd in cmds: + stdin, stdout, stderr = ssh.exec_command(cmd + "\n") + self.sysout = stdout.read() + stdin.close() + stderr.close() + stdout.close() + ssh.close() \ No newline at end of file diff --git a/utils/tdata_tool.py b/utils/tdata_tool.py new file mode 100644 index 0000000..1e95110 --- /dev/null +++ b/utils/tdata_tool.py @@ -0,0 +1,135 @@ +# +# --------------------------------------------------------------------- +""" +the issue of this tool is to transform extern data to the internal structure and the internal structure into extern data - i.e. mostly test-results. +* * * * * * * * +the testdata have several elements +* parameter (-td --tdata) : to identify which testdata should be loaded +* source (db: dbname / dir: filename) : always structured in a table (easy to specify) with columns + * node : where the rows are + * action : what should be done - default insert + + fields : dates in relation of a reference 0: + columns.append(fields[i]) + j = j + 1 + cnt = j + job.debug(verify, str(state) + " " + str(cnt) + " cols " + str(columns)) + elif state >= 2 and len(testline) > 2: + if state == 2: + nodes = fields[0].split(":") + job.debug(verify, str(state) + " nodes " + str(nodes)) + state = 3 + row = {} + for i in range(2, cnt-1): + row[columns[i-2]] = fields[i] + job.debug(verify, str(state) + " row " + str(row)) + data.append(row) + elif state == 3: + job.debug(verify, "structure " + str(state) + ": " + str(nodes)) + output = setSubnode(0, nodes, data, output) + data = [] + state = 0 + print(str(output)) + print(str(output)) + output = setSubnode(0, nodes, data, output) + print(str(output)) + file.close() + return output + +def setSubnode(i, nodes, data, tree): + print("setSubnode " + str(i) + ": " + ": " + str(tree)) + if i >= len(nodes): + print("setSubnode a " + str(i)) + tree["data"] = data + elif tree is not None and nodes[i] in tree.keys(): + print("setSubnode b " + str(i)) + tree[nodes[i]] = setSubnode(i+1, nodes, data, tree[nodes[i]]) + else: + print("setSubnode c " + str(i)) + tree[nodes[i]] = setSubnode((i + 1), nodes, data, {}) + return tree + +def getDataStructure(comp): + # gets data-structure from the vml in the component-folder + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel("tdata_tool") + job.debug(verify, "getDataStructure " + comp) + +def normalizeDataRow(dstruct, xpathtupel, row, referencedate): + # normalize data of the row if necessary + # raw-value is saved as new field with _raw as suffix + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel("tdata_tool") + job.debug(verify, "calcDataRow " + row) + +def writeDataTable(teststatus, tdata, comp): + """ + writes the testdata into a csv-file for documentation of the test-run + :param teststatus: + :param tdata: + :param comp: if specific else None + :return: + """ + job = ulrich.program.Job.getInstance() + verify = -1+job.getDebugLevel("tdata_tool") + job.debug(verify, "writeDataTable " + str(comp)) +