From 3f260bf6e5f118e014ae757b861a737ea4ed5b83 Mon Sep 17 00:00:00 2001 From: Ulrich Date: Thu, 5 Jan 2023 23:02:29 +0100 Subject: [PATCH] xample job unit-tester --- basic/componentHandling.py | 281 ++++++++++++++++++ basic/program.py | 83 +----- test/{test_08i18n.py => test_18i18n.py} | 6 +- ...t_10testserver.py => test_90testserver.py} | 6 +- tools/config_tool.py | 3 +- tools/conn_tool.py | 14 +- tools/file_tool.py | 6 - tools/filecsv_fcts.py | 11 +- tools/path_tool.py | 5 +- tools/value_tool.py | 8 +- unit_run.py => unit_tester.py | 31 +- utils/dbsfile_tool.py | 100 ------- utils/dbshive_tool.py | 106 ------- utils/dbspark_tool.py | 73 ----- 14 files changed, 344 insertions(+), 389 deletions(-) create mode 100644 basic/componentHandling.py rename test/{test_08i18n.py => test_18i18n.py} (96%) rename test/{test_10testserver.py => test_90testserver.py} (95%) rename unit_run.py => unit_tester.py (72%) delete mode 100644 utils/dbsfile_tool.py delete mode 100644 utils/dbshive_tool.py delete mode 100644 utils/dbspark_tool.py diff --git a/basic/componentHandling.py b/basic/componentHandling.py new file mode 100644 index 0000000..4ecaa4b --- /dev/null +++ b/basic/componentHandling.py @@ -0,0 +1,281 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +# managing the components +# ----------------------------------------------------------------------------- +""" +component has to be created in relation of the application (basis.yml). +Each componente could be created mostly once, but not everytime: +* the same instance of a component is used in different contexts +* there could be exist more instances +* there could be alternatives of an instance +Each kind of instance has its component-class and for each use should be an object be created. +Each crated component-onject are documented in the parameter-file. +""" +import tools.config_tool +import tools.conn_tool +import basic.program +import basic.message +import basic.component +import importlib +import copy +import basic.constants as B +import tools.data_const as D + +comps = {} +PARAM_NOSUBNODE = [B.SUBJECT_ARTS, "components", "instance"] +DEFAULT_INST_CNT = 1 +DEFAULT_INST_SGL = "y" + + +def getInstanceAttributes(conf): + """ + the attributes for instancing the component are set from configuration or from default + :param conf: + :return: a complete set of these attributes + """ + out = { + B.ATTR_INST_CNT: DEFAULT_INST_CNT, + B.ATTR_INST_SGL: DEFAULT_INST_SGL + } + if B.SUBJECT_INST in conf: + for attr in [B.ATTR_INST_CNT, B.ATTR_INST_SGL]: + if attr in conf[B.SUBJECT_INST]: + out[attr] = conf[B.SUBJECT_INST][attr] + return out + + +def getComponents(job, mainfct): + #job = basic.program.Job.getInstance() + verify = -2 + job.getDebugLevel("job_tool") + job.debug(verify, "getComponents " + mainfct) + out = [] + for c in comps: + job.debug(verify, "getComponents " + c + ": " + str(comps[c].conf)) + print("getComponents " + c + ": " + str(comps[c].conf)) + if mainfct in comps[c].conf["function"]: + out.append(c) + return out + + +class ComponentManager: + __instance = None + __instances = {} + """ + initializes the Manager with all necessary components + """ + + def __init__(self, job, option=""): + #job = basic.program.Job.getInstance() + job.m.logDebug("applicationscomponente -- " + str(type(job.par))) + self.components = {} + self.comps = {} + self.job = job + ComponentManager.__instances[job.jobid] = self + ComponentManager.__instance = self + print ("init ComponentHandling "+str(self)) + + + def initComponents(self): + # sets components the first time + # afterwards set components from parameter-file + job = self.job # basic.program.Job.getInstance() + anw = job.par.application + job.m.logDebug("applicationscomponente -- " + str(type(job.par))) + if not job.conf[B.SUBJECT_APPS].get(anw): + job.m.setFatal("application " + job.par.application + " is not configured") + return + for k in job.conf[B.SUBJECT_APPS].get(anw): + if k == B.ATTR_APPS_PROJECT: + continue + job.m.logDebug("applicationscomponente -- " + k + ":") + print("applicationscomponente -- " + k + ":") + self.createComponent(k, 0, "") + + + def getComponent(self, compobjname): + job = self.job #basic.program.Job.getInstance() + verify = -2 + job.getDebugLevel("job_tool") + job.debug(verify, "getComponents " + compobjname) + if compobjname in self.comps: + return self.comps[compobjname] + return None + + + def getComponents(self, mainfct): + job = self.job #basic.program.Job.getInstance() + verify = -2 + job.getDebugLevel("job_tool") + job.debug(verify, "getComponents " + mainfct) + out = [] + for c in self.comps: + job.debug(verify, "getComponents " + c + ": " + str(self.comps[c].conf)) + #print("getComponents " + c + ": " + str(self.comps[c].conf)) + if mainfct in self.comps[c].conf["function"]: + out.append(c) + return out + + + @staticmethod + def getInstance(job, init="N"): + if (job.jobid in ComponentManager.__instances): + return ComponentManager.__instances[job.jobid] + else: + return ComponentManager(job) + + + def createComponent(self, componentName, nr, suffix): + """ + in order to create a component it must be loaded + * knogwedge of the application - which components should be created + * technical-knowledge of the instanciated component, especially the connection, user, password + * business-knowledge of the component, especially of their interfaces resp. artifacts + :param componentName: Name of the component + :param nr: for numbered instance if component is multiple + :param suffix: suffix for specific context of the component + :return: + """ + job = self.job #basic.program.Job.getInstance() + verify = job.getDebugLevel("job_tool") + componentName = componentName.lower() + job.debug(verify, "createComponent " + componentName) + confs = tools.config_tool.getConfig(job, "comp", componentName) + conns = tools.conn_tool.getConnections(job, componentName) + instAttr = getInstanceAttributes(confs) + job.debug(verify, "createComponent -91- " + componentName + " : " + str(confs)) + if nr > 0 and int(instAttr[B.ATTR_INST_CNT]) > 1: + job.m.setError("for multiple callers are multiple calls not implemented ") + if nr > 0 and len(conns) == 0: + job.m.setError("for multiple calls has only one call configured") + #print(confs) + parContent = job.loadParameter() + if len(conns) == 1: + c = self.createInstance(componentName, parContent, confs, conns, 0) + #print("createComponent 3 a " + componentName) + self.createSubComponents(c, nr, suffix) + else: + i = 1 + #print("createComponent 3 b " + componentName) + for cn in conns: + c = self.createInstance(componentName, parContent, confs, conns, i) + self.createSubComponents(c, i, suffix) + i = i + 1 + #print("createComponent 9 " + componentName) + #print(self.comps) + + + def createInstance(self, compName, parContent, confs, conns, nr): + """ + instance a component + :param compName: name without suffix or number + :param parContent: content of the parameter-file which is dumped from a pre-step + :param confs: configuration of the component + :param conns: connection-attributes for the specific environment + :param nr: number if component is multiple + :return: instance of the component with all necessary attributes + """ + job = self.job #basic.program.Job.getInstance() + cmodul = importlib.import_module(getComponentPath(compName)) + class_ = getattr(cmodul, getComponentClass(compName)) + c = class_() + if nr > 0: + name = compName + "_0" + str(nr) + i = nr - 1 + else: + name = compName + i = 0 + c.name = name + c.classname = compName + + c.m = basic.message.Message(job, basic.message.LIMIT_DEBUG, job.start, c.name) + c.conf = tools.config_tool.mergeConn(c.m, confs["conf"], conns[i]) + c.conf[B.SUBJECT_CONN] = conns[i] + c.init(job) + if parContent is not None: + print("createComponent 5 a " + compName + " : " + str(parContent)) + if B.SUBJECT_COMPS in parContent and compName in parContent[B.SUBJECT_COMPS]: + for k in parContent[B.SUBJECT_COMPS][compName].keys(): + c.conf[k] = parContent[B.SUBJECT_COMPS][compName][k] + if B.SUBJECT_ARTS in c.conf and B.TOPIC_NODE_DB in c.conf[B.SUBJECT_ARTS]: + if not B.DATA_NODE_DDL in c.conf: + c.conf[B.DATA_NODE_DDL] = {} + for table in c.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]: + if table in B.LIST_DB_ATTR: + continue + conf = tools.config_tool.getConfig(job, D.DDL_FILENAME, compName, table) + if B.DATA_NODE_TABLES in conf and table in conf[B.DATA_NODE_TABLES]: + c.conf[B.DATA_NODE_DDL][table] = conf[B.DATA_NODE_TABLES][table] + elif table in conf: + c.conf[B.DATA_NODE_DDL][table] = conf[table] + else: + c.conf[B.DATA_NODE_DDL][table] = conf + comps[name] = c + self.comps[name] = c + return c + + + def createSubComponents(self, comp, nr, suffix): + job = self.job #basic.program.Job.getInstance() + verify = -2 + job.getDebugLevel("job_tool") + job.debug(verify, "getComponents " + str(comp.conf[B.ATTR_INST_SUBCOMP])) + for c in comp.conf[B.ATTR_INST_SUBCOMP].keys(): + if c == "none": + continue + self.createComponent(c, nr, suffix) + + + def getComponentDict(self): + job = self.job #basic.program.Job.getInstance() + verify = -2 + job.getDebugLevel("job_tool") + job.debug(verify, "getComponents ") + out = {} + for c in self.comps: + out[self.comps[c].name] = {} + for k in self.comps[c].conf.keys(): + if isParameterSubnode(k): # B.SUBJECT_ARTS in k or "components" in k or "instance" in k: + out[self.comps[c].name][k] = copy.deepcopy(self.comps[c].conf[k]) + return out + + +def getComponentFolder(comp): + return comp.lower() + + +def getComponentModul(comp): + return comp[0:1].upper() + comp[1:].lower() + + +def getComponentClass(comp): + return comp[0:1].upper() + comp[1:].lower() + + +def getComponentPath(comp): + return "components." + getComponentFolder(comp) + "." + getComponentModul(comp) + + +def getComponentDict(job = None): + #job = basic.program.Job.getInstance() + #verify = -2 + job.getDebugLevel("job_tool") + #job.debug(verify, "getComponents ") + out = {} + for c in comps: + out[comps[c].name] = {} + for k in comps[c].conf.keys(): + if isParameterSubnode(k): # B.SUBJECT_ARTS in k or "components" in k or "instance" in k: + out[comps[c].name][k] = copy.deepcopy(comps[c].conf[k]) + return out + + +def isParameterSubnode(key): + for k in PARAM_NOSUBNODE: + if key in k: + return False + return True + + +def getPlainCompname(name): + if "_0" in name: + return name[0:-3] + return name diff --git a/basic/program.py b/basic/program.py index 6524c13..f704d37 100644 --- a/basic/program.py +++ b/basic/program.py @@ -40,75 +40,8 @@ CTLG_BASEDIR = "basedir" CTLG_LOGPATH = "logpath" CTLG_LOGLEVEL = "loglevel" - -def setGlobal(): - pass - EXCP_CANT_POP = "cant pop this job from the instances" -DEFAULT_ARCHIV_DIR = T.DATA_PATH + "/lauf" -DEFAULT_GRAN = "ws" -DEFAULT_PRG = "webflask" -DEFAULT_GRAN = "ws" -DEFAULT_APP = "WEBFLASK" -DEFAULT_ENV = "Workspace" -DEFAULT_MODE = "test" -DEFAULT_TIME = "2022-08-29_17-29-59" - - - -def createJob(pprg="", pgran="", papp="", penv="", ptstamp="", pmode=""): - """ - this creates a Job-Object with the main arguments ## same as in testtools? - :param pprg: program-name - :param pgran: tc|ts| - :param papp: application - :param penv: environment - :param ptstamp: timestamp - part of specific testfolder - :param pmode: if it is a productive or development execution - :return: - """ - if len(pprg) < 1: - prgname = DEFAULT_PRG - else: - prgname = pprg - if len(pgran) < 1: - gran = DEFAULT_GRAN - else: - gran = pgran - if len(pgran) < 1: - gran = DEFAULT_GRAN - else: - gran = pgran - if len(papp) < 1: - app = DEFAULT_APP - else: - app = papp - if len(penv) < 1: - env = DEFAULT_ENV - else: - env = penv - if len(ptstamp) < 1: - tstamp = DEFAULT_TIME - else: - tstamp = ptstamp - if len(pmode) < 1: - mode = DEFAULT_MODE - else: - mode = pmode - if gran == "tc": - path = DEFAULT_ARCHIV_DIR + "/TC0001/" + tstamp - elif gran == "ts": - path = DEFAULT_ARCHIV_DIR + "/testlauf/TST001_" + tstamp - else: - path = T.DATA_PATH + "/workspace/" - - job = basic.program.Job(prgname) - args = {"application": app, "environment": env, "modus": mode, gran + "time": tstamp, - gran + "dir": path, - "step": 1} - job.par.setParameterArgs(job, args) - return job class SimpleJob: """ @@ -275,9 +208,12 @@ class Job: logTime = self.start.strftime("%Y%m%d_%H%M%S") self.m = basic.message.Message(self, basic.message.LIMIT_DEBUG, logTime, None) tools.job_tool.startJobProcesses(self) - self.par.setParameterLoaded(self) - self.m.logInfo("# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") - self.m.debug(basic.message.LIMIT_INFO, "# # # Start Job " + self.start.strftime("%d.%m.%Y %H:%M:%S") + " # # # ") + if self.programDef[CTLG_PARSOURCE] != "": + self.par.setParameterLoaded(self) + header1 = "# # # Start Job " + tools.date_tool.formatParsedDate(str(self.start), tools.date_tool.F_DE_TSTAMP) + " # # # " + self.m.logInfo(header1) + self.m.debug(basic.message.LIMIT_INFO, header1) + print(header1) self.par.checkParameter(self) self.m.logInfo(self.par.parstring) @@ -285,7 +221,8 @@ class Job: def stopJob(self, reboot=0): tools.job_tool.stopJobProcesses(self) self.ende = datetime.now() - self.dumpParameter() + if self.programDef[CTLG_PARTARGET] != "": + self.dumpParameter() footer1 = "# # " + self.m.topmessage + " # # # " footer2 = "# # # Stop Job " + tools.date_tool.formatParsedDate(str(self.start), tools.date_tool.F_DE_TSTAMP) footer2 += " # " + tools.date_tool.formatParsedDate(str(self.ende), tools.date_tool.F_DE_TSTAMP) + " # # # " @@ -463,8 +400,8 @@ class Parameter: def checkParameter(self, job): print (f"Parameter initialisiert {self.program}") - pardef = self.programDef[CTLG_PARDEF] - for p in pardef.split(","): + pardef = job.programDef[CTLG_PARDEF] + for p in pardef["par"]: if len(p) > 1 and not hasattr(self, p): job.m.setFatal("Parameter " + p + " is not set!") diff --git a/test/test_08i18n.py b/test/test_18i18n.py similarity index 96% rename from test/test_08i18n.py rename to test/test_18i18n.py index 40ebee8..1d1ab28 100644 --- a/test/test_08i18n.py +++ b/test/test_18i18n.py @@ -51,7 +51,7 @@ class MyTestCase(unittest.TestCase): if actfunction not in TEST_FUNCTIONS: return job = test.testtools.getJob() - job.conf.confs["language"] = "de" + job.conf["language"] = "de" i18n = utils.i18n_tool.I18n.getInstance(job) res = i18n.getText(f"{EXP_KEY_MISSING=}", job) if verbose: print("RESULT "+res) @@ -65,7 +65,7 @@ class MyTestCase(unittest.TestCase): if verbose: print("RESULT "+res) self.assertEqual(res, "project") cnttest += 1 - job.conf.confs["language"] = "fr" + job.conf["language"] = "fr" i18n = utils.i18n_tool.I18n.getInstance(job) self.assertRaises(Exception, i18n.getText, (f"{EXP_KEY_MISSING=}", job)) cnttest += 1 @@ -79,7 +79,7 @@ class MyTestCase(unittest.TestCase): if actfunction not in TEST_FUNCTIONS: return job = test.testtools.getJob() - job.conf.confs["language"] = "de" + job.conf["language"] = "de" i18n = utils.i18n_tool.I18n.getInstance(job) # i18n.getText("EXP_KEY_MISSING", EXP_KEY_MISSING, job) res = i18n.getAliasList(f"{EXP_KEY_MISSING=}", job) diff --git a/test/test_10testserver.py b/test/test_90testserver.py similarity index 95% rename from test/test_10testserver.py rename to test/test_90testserver.py index 9035bfb..5d5e674 100644 --- a/test/test_10testserver.py +++ b/test/test_90testserver.py @@ -33,7 +33,7 @@ class MyTestCase(unittest.TestCase): testserver = basic.Testserver.Testserver(job) self.assertIsNotNone(testserver) cnttest += 1 - if B.TOPIC_NODE_DB in job.conf.confs: + if B.TOPIC_NODE_DB in job.conf: self.assertIn(B.TOPIC_NODE_DB, testserver.conf[B.SUBJECT_CONN]) self.assertIn(B.ATTR_DB_DATABASE, testserver.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) self.assertIn(B.DATA_NODE_DDL, testserver.conf) @@ -48,8 +48,8 @@ class MyTestCase(unittest.TestCase): return job = test.testtools.getJob() testserver = basic.Testserver.Testserver(job) - if B.TOPIC_NODE_DB in job.conf.confs: - dbi = basic.toolHandling.getDbTool(job, self, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) + if B.TOPIC_NODE_DB in job.conf: + dbi = basic.toolHandling.getDbTool(job, self, job.conf[B.TOPIC_NODE_DB][B.ATTR_TYPE]) else: return "No DB in job-config" sql = testserver.getDBSchema(job, dbi, "application") diff --git a/tools/config_tool.py b/tools/config_tool.py index d7d39dc..210d4b1 100644 --- a/tools/config_tool.py +++ b/tools/config_tool.py @@ -309,6 +309,7 @@ def mergeConn(msg, conf, conn): :param conn: :return: """ + if B.SUBJECT_INST not in conf: conf[B.SUBJECT_INST] = {} for a in conn[B.SUBJECT_INST]: @@ -324,7 +325,6 @@ def mergeConn(msg, conf, conn): list = B.LIST_API_ATTR if topic == B.TOPIC_NODE_FILE: list = B.LIST_FILE_ATTR - print(" --- merge-conn " + topic + " " + str(list)) for a in conf[B.SUBJECT_ARTS][topic]: if topic not in conn: continue @@ -333,7 +333,6 @@ def mergeConn(msg, conf, conn): conf[B.SUBJECT_ARTS][topic][a] = conn[topic][a] else: for b in conf[B.SUBJECT_ARTS][topic][a]: - print(" --- merge-conn b " + topic + " " + a+" "+b) if b not in list: msg.logError("not-topic-attribute in topic-connection: "+topic+", "+b) continue diff --git a/tools/conn_tool.py b/tools/conn_tool.py index 6453de7..4ea571d 100644 --- a/tools/conn_tool.py +++ b/tools/conn_tool.py @@ -49,9 +49,9 @@ def getConnections(job, comp): conns = [] # if a datest-database exists read the connections conndb = {} - if job.conf.confs.get("db"): - # select - pass + #if job.conf["db"]: + # # select + # pass conn = tools.config_tool.getConfig(job, "tool", B.SUBJECT_CONN) if not comp in conn[B.SUBJECT_ENV]: @@ -64,7 +64,7 @@ def getConnections(job, comp): for a in conn[B.SUBJECT_ENV][comp]: if "inst" in a and a != B.SUBJECT_INST: continue - attr[a] = conn["env"][comp][a] + attr[a] = conn[B.SUBJECT_ENV][comp][a] #if ("types" in conn["env"][comp]): # xtypes = conn["env"][comp]["types"] for i in range(conn[B.SUBJECT_ENV][comp][B.SUBJECT_INST][B.ATTR_INST_CNT]): @@ -73,9 +73,9 @@ def getConnections(job, comp): #if (xtypes is not None): # conn["env"][comp][instnr]["types"] = xtypes for a in attr: - if a in conn["env"][comp][instnr]: + if a in conn[B.SUBJECT_ENV][comp][instnr]: continue # dont overwrite an instance-specific value - conn["env"][comp][instnr][a] = attr[a] - conns.append(conn["env"][comp][instnr]) + conn[B.SUBJECT_ENV][comp][instnr][a] = attr[a] + conns.append(conn[B.SUBJECT_ENV][comp][instnr]) return conns diff --git a/tools/file_tool.py b/tools/file_tool.py index 09c5c70..b49e4e2 100644 --- a/tools/file_tool.py +++ b/tools/file_tool.py @@ -22,8 +22,6 @@ import tools.date_tool def getDump(obj): - result = "" - print(str(type(obj))) result = vars(obj) return str(result) @@ -141,10 +139,8 @@ def mkPaths(job, path, msg): def getFileEncoding(msg, job, path): - print("--- getFileEncoding " + path) encodings = ['utf-8', 'iso-8859-1'] # add more for e in encodings: - print(e) try: fh = codecs.open(path, 'r', encoding=e) fh.readlines() @@ -160,9 +156,7 @@ def getFileEncoding(msg, job, path): def detectFileEncode(job, path, msg): # return "" - # job = basic.program.Job.getInstance() verify = int(job.getDebugLevel("file_tool")) - print(path) cntIso = 0 cntUtf = 0 j = 0 diff --git a/tools/filecsv_fcts.py b/tools/filecsv_fcts.py index 58ac16f..a5b8250 100644 --- a/tools/filecsv_fcts.py +++ b/tools/filecsv_fcts.py @@ -4,6 +4,7 @@ # Author : Ulrich Carmesin # Source : gitea.ucarmesin.de # --------------------------------------------------------------------------------------------------------- +import json import re import basic.program import tools.file_abstract @@ -135,6 +136,14 @@ def splitFields(line, delimiter, job): break if re.match(r"^\"(.*)\"$", fields[i]): fields[i] = fields[i][1:-1] + if fields[i].find("{\"") == 0: + if fields[i].find("{\"\"") == 0: + fields[i] = fields[i].replace("\"\"", "\"") + try: + val = json.loads(fields[i]) + fields[i] = val + except Exception as e: + pass out.append(fields[i]) return out @@ -225,7 +234,7 @@ def setTableData(tableDict, fields, ttype, job): fields = [tableDict[D.DATA_ATTR_ALIAS]] + fields i = 1 for f in tableDict[B.DATA_NODE_HEADER]: - row[f] = fields[i].strip() + row[f] = fields[i] i += 1 if ttype == D.CSV_SPECTYPE_DATA: if B.ATTR_DATA_COMP in tableDict: diff --git a/tools/path_tool.py b/tools/path_tool.py index 03c46c9..7b9bd98 100644 --- a/tools/path_tool.py +++ b/tools/path_tool.py @@ -65,10 +65,7 @@ def compose_path(job, pathname, comp): """ verify = job.getDebugLevel(TOOL_NAME) job.debug(verify, "composePath " + pathname) - if "{" in pathname: - return tools.value_tool.compose_pattern(job, pathname, comp) - else: - job.debug(verify, "in Pattern nicht vorhanden: " + pathname) + return tools.value_tool.compose_pattern(job, pathname, comp) def rejoinPath(a, b="", c="", d="", e="", f=""): diff --git a/tools/value_tool.py b/tools/value_tool.py index 1c0fa05..12751db 100644 --- a/tools/value_tool.py +++ b/tools/value_tool.py @@ -67,7 +67,11 @@ def get_key_value(job, key, comp=None): if not hasattr(job, a[1]): job.m.setError("key, job has not attribute "+a[1]) return "" - return getattr(job, a[1]) + val = getattr(job, a[1]) + if a[1] == "start": + val = tools.date_tool.formatParsedDate(str(val), tools.date_tool.F_LOG) + return val + elif DOM_COMP in key: if comp is None: raise Exception(P.EXP_COMP_MISSING.format(key)) @@ -102,7 +106,7 @@ def get_key_value(job, key, comp=None): def compose_pattern(job, pattern, comp): if "{" in pattern and "}" in pattern: - return compose_string(job, pattern,comp) + return compose_string(job, pattern, comp) vc = ValueConf.getInstance(job) if pattern in vc.pattern: return compose_string(job, "{" + pattern + "}", comp) diff --git a/unit_run.py b/unit_tester.py similarity index 72% rename from unit_run.py rename to unit_tester.py index 53fbabe..754dd29 100644 --- a/unit_run.py +++ b/unit_tester.py @@ -24,6 +24,9 @@ import os, glob import io import sys import test.constants as T +import basic.program + +PROGRAM_NAME = "unit_tester" VERIFY = False @@ -31,15 +34,15 @@ class MyTestLoader(unittest.TestLoader): def start_testcases(self): pass -def create_test_suite(): +def create_test_suite(job): suite = unittest.TestSuite() for testdir in [T.PROG_PATH, T.COMP_PATH]: - print("testdir "+testdir) testFileStrings = sorted(glob.glob(os.path.join(testdir, "test", 'test_*.py'))) - print("testdir "+str(testFileStrings)) for t in testFileStrings: - print("t "+t) - if not t[len(testdir)+11:len(testdir)+13].isnumeric(): + testnum = t[len(testdir)+11:len(testdir)+13] + if not testnum.isnumeric(): + continue + if testnum not in ["01", "03", "08", "10", "11"]: continue if testdir == T.COMP_PATH: v = "components.test." + t[len(testdir) + 6:-3] @@ -53,9 +56,19 @@ def create_test_suite(): suite.addTest(class_(f)) return suite -if __name__ == '__main__': - print ("start") - loader = unittest.TestLoader() - suite = create_test_suite() +def startPyJob(job): + suite = create_test_suite(job) runner = unittest.TextTestRunner(sys.stdout, True, 3, True, True) runner.run(suite) + +if __name__ == '__main__': + job = basic.program.Job(PROGRAM_NAME) + # TODO to move to job_tool + args = {} + if isinstance(job.programDef[basic.program.CTLG_PARDEF], dict) and "par" in job.programDef[basic.program.CTLG_PARDEF]: + for p in job.programDef[basic.program.CTLG_PARDEF]["par"]: + args[p] = job.programDef[basic.program.CTLG_PARDEF]["par"][p] + job.setParameter(args) + job.startJob() + startPyJob(job) + job.stopJob(0) diff --git a/utils/dbsfile_tool.py b/utils/dbsfile_tool.py deleted file mode 100644 index a7dde18..0000000 --- a/utils/dbsfile_tool.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------------------------------------- -# Author : Ulrich Carmesin -# Source : gitea.ucarmesin.de -# --------------------------------------------------------------------------------------------------------- -""" -This class is a technical implementation for Hive-connection with spark - typically used in a -Machine Learning environment for example in hadoop -""" -import json - -import basic.program -import utils.config_tool -import utils.db_abstract -import pyspark -import basic.constants as B - -class DbFcts(utils.db_abstract.DbFcts): - """ - This interface defines each necessary connection to any kind of database. - The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. - """ - def __init__(self): - pass - - - def selectRows(self, table, job): - """ method to select rows from a database - statement written in sql """ - tdata = {} - dry = 0 - # attr = self.getDbAttributes(table) - verify = -1+job.getDebugLevel("db_tool") - pattern = "s3a://{hostname}/data/{tenant}/mt/sandboxes/{job.par.usecae}/{job.par.workspace}/{outfile}/VR_+reg+/" - files = self.comp.composeFileClauses(job, pattern) - data = [] - for k in files.keys(): - sql = files[k] - if dry == 1: - spark = self.getConnector() - df = spark.read.parquet(sql) - dfj = df.toJSON() - for r in dfj.collect(): - data.append(json.loads(r)) - else: - print("select "+sql) - #self.comp.m.logInfo(cmd) - #tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER] - #tdata[B.DATA_NODE_DATA] = data - return tdata - - def deleteRows(self, table, job): - """ method to delete rows from a database - statement written in sql """ - dry = 0 - verify = -1+job.getDebugLevel("db_tool") - cmd = "DELETE FROM "+table - print("deleteRows "+cmd) - sqls = self.comp.composeSqlClauses(job, cmd) - print("deleteRows "+cmd) - print(sqls) - for k in sqls.keys(): - sql = sqls[k] - if dry == 1: - #spark = self.getConnector() - #df = spark.sql(cmd) - pass - else: - print("select "+sql) - #self.comp.m.logInfo(cmd) - - def insertRows(self, table, rows, job): - """ method to insert rows into a database - the rows will be interpreted by the ddl of the component - """ - job = self.job # basic.program.Job.getInstance() - verify = -1+job.getDebugLevel("db_tool") - spark = self.getConnector() - df = spark.createDataFrame(rows) - - self.comp.m.logInfo("cmd") - - def getConnector(self): - """ add-on-method to get the connector - this method should only called by the class itself """ - job = self.job # basic.program.Job.getInstance() - attr = self.getDbAttributes(B.SVAL_NULL) - spark = None - if B.ATTR_DB_CONN_JAR in attr: - spark = pyspark.SparkSession\ - .builder\ - .appName("datest")\ - .config("sparkjar", "")\ - .getOrCreate() - return spark - - - - diff --git a/utils/dbshive_tool.py b/utils/dbshive_tool.py deleted file mode 100644 index 795d83b..0000000 --- a/utils/dbshive_tool.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------------------------------------- -# Author : Ulrich Carmesin -# Source : gitea.ucarmesin.de -# --------------------------------------------------------------------------------------------------------- -""" -This class is a technical implementation for Hive-connection with spark - typically used in a -Machine Learning environment for example in hadoop -""" -import json -import os - -import basic.program -import utils.config_tool -import utils.db_abstract -import pyspark -import basic.constants as B - -class DbFcts(utils.db_abstract.DbFcts): - """ - This interface defines each necessary connection to any kind of database. - The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. - """ - def __init__(self): - pass - - - def selectRows(self, table, job): - """ method to select rows from a database - statement written in sql """ - tdata = {} - dry = 0 - verify = -1+job.getDebugLevel("db_tool") - attr = self.getDbAttributes(B.SVAL_NULL) - cmd = "SELECT "+",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]) - cmd += " FROM "+table - sqls = self.comp.composeSqlClauses(job, cmd) - data = [] - for k in sqls.keys(): - sql = sqls[k] - if dry == 1: - try: - spark = self.getConnector() - df = spark.sql(sql) - dfj = df.toJSON() - for r in dfj.collect(): - data.append(json.loads(r)) - except: - self.comp.m.setError("Table couldnt read "+table) - else: - print("select "+sql) - self.comp.m.logInfo(sql) - tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER] - tdata[B.DATA_NODE_DATA] = data - return tdata - - def deleteRows(self, table, job): - """ method to delete rows from a database - statement written in sql """ - dry = 0 - verify = -1+job.getDebugLevel("db_tool") - cmd = "DELETE FROM "+table - print("deleteRows "+cmd) - sqls = self.comp.composeSqlClauses(job, cmd) - print("deleteRows "+cmd) - print(sqls) - for k in sqls.keys(): - sql = sqls[k] - if dry == 1: - #spark = self.getConnector() - #df = spark.sql(cmd) - pass - else: - print("select "+sql) - #self.comp.m.logInfo(cmd) - - def insertRows(self, table, rows, job): - """ method to insert rows into a database - the rows will be interpreted by the ddl of the component - """ - job = self.job # basic.program.Job.getInstance() - verify = -1+job.getDebugLevel("db_tool") - spark = self.getConnector() - df = spark.createDataFrame(rows) - - self.comp.m.logInfo("cmd") - - def getConnector(self): - """ add-on-method to get the connector - this method should only called by the class itself """ - job = self.job # basic.program.Job.getInstance() - attr = self.getDbAttributes(B.SVAL_NULL) - spark = None - if B.ATTR_DB_CONN_JAR in attr: - connectorJar = os.environ.get(attr[B.ATTR_DB_CONN_JAR]) - spark = pyspark.SparkSession\ - .builder\ - .appName("datest")\ - .config("spark.jars", f"{connectorJar}")\ - .getOrCreate() - return spark - - - - diff --git a/utils/dbspark_tool.py b/utils/dbspark_tool.py deleted file mode 100644 index 39fc77f..0000000 --- a/utils/dbspark_tool.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------------------------------------- -# Author : Ulrich Carmesin -# Source : gitea.ucarmesin.de -# --------------------------------------------------------------------------------------------------------- -import basic.program -import utils.config_tool -import utils.db_abstract -import pyspark -import basic.constants as B - -class DbFcts(utils.db_abstract.DbFcts): - """ - This interface defines each necessary connection to any kind of database. - The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. - """ - def __init__(self): - pass - - def getDbAttributes(self): - out = {} - - return out - - def selectRows(self, table, job): - """ method to select rows from a database - statement written in sql """ - tdata = {} - verify = -1+job.getDebugLevel("db_tool") - cmd = "SELECT "+",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]) - cmd += " FROM "+table+""+self.getWhere()+""+self.getOrder() - spark = self.getConnector() - df = spark.sql(cmd) - data = [] - for r in df: - data.append(r) - tdata[B.DATA_NODE_HEADER] = self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER] - tdata[B.DATA_NODE_DATA] = data - return tdata - - def deleteRows(self, table): - """ method to delete rows from a database - statement written in sql """ - job = self.job # basic.program.Job.getInstance() - verify = -1+job.getDebugLevel("db_tool") - cmd = "DELETE FROM "+table+";" - self.comp.m.logInfo(cmd) - - def insertRows(self, table, rows): - """ method to insert rows into a database - the rows will be interpreted by the ddl of the component - """ - job = self.job # basic.program.Job.getInstance() - verify = -1+job.getDebugLevel("db_tool") - spark = self.getConnector() - df = spark.createDataFrame(rows) - - self.comp.m.logInfo("cmd") - - def getConnector(self): - """ add-on-method to get the connector - this method should only called by the class itself """ - job = self.job # basic.program.Job.getInstance() - spark = pyspark.SparkSession\ - .builder\ - .appName("datest")\ - .getOrCreate() - return spark - - - -