From 419715d629b15301c130455798ea9a0abf56abb6 Mon Sep 17 00:00:00 2001 From: Ulrich Date: Mon, 3 Oct 2022 10:51:46 +0200 Subject: [PATCH] Testserver incl. DB-mysql --- basic/DATASTRUCTURE.yml | 214 ++++++++++++++++++++++++++++++ basic/Testserver.py | 87 +++++++++++++ basic/application.py | 264 ++++++++++++++++++++++++++++++++++++++ basic/constants.py | 11 +- basic/entity.py | 52 +++++++- basic/message.py | 5 +- basic/program.py | 30 +++-- basic/step.py | 6 +- test/test_09git.py | 13 +- test/test_10testserver.py | 91 +++++++++++++ utils/config_tool.py | 18 +++ utils/data_const.py | 1 + utils/date_tool.py | 6 +- utils/db_abstract.py | 23 +++- utils/dbmysql_tool.py | 55 ++++---- utils/dbrel_tool.py | 4 +- utils/git_tool.py | 19 ++- 17 files changed, 844 insertions(+), 55 deletions(-) create mode 100644 basic/DATASTRUCTURE.yml create mode 100644 basic/Testserver.py create mode 100644 basic/application.py create mode 100644 test/test_10testserver.py diff --git a/basic/DATASTRUCTURE.yml b/basic/DATASTRUCTURE.yml new file mode 100644 index 0000000..d1c1c0d --- /dev/null +++ b/basic/DATASTRUCTURE.yml @@ -0,0 +1,214 @@ +application: + _header: + - apid + - name + - description + - reference + - attributes + - inscommit + - insauthor + - instime + - updcommit + - updauthor + - updtime + - actual + apid: + field: apid + type: pk + name: + field: name + index: I + type: str + description: + field: description + type: string + reference: + field: reference + type: str + attributes: + field: attributes + type: string + insauthor: + field: insauthor + type: str + inscommit: + field: inscommit + type: str + instime: + field: instime + type: time + updauthor: + field: updauthor + type: str + updcommit: + field: updcommit + type: str + updtime: + field: updtime + type: time + actual: + field: actual + index: I + type: int +ap_component: + _header: + - apcomid + - apid + - component + apcomid: + field: apcomid + type: pk + apid: + field: apid + index: I + type: int + component: + field: component + index: I + type: str +ap_project: + _header: + - approid + - apid + - project + - description + - reference + approid: + field: apid + type: pk + apid: + field: apid + index: I + type: int + project: + field: project + index: I + type: str + description: + field: description + type: string + reference: + field: reference + type: str +component: + _header: + - coid + - name + - description + - reference + - attributes + - inscommit + - insauthor + - instime + - updcommit + - updauthor + - updtime + - actual + coid: + field: apid + type: pk + name: + field: name + index: I + type: str + description: + field: description + type: string + reference: + field: reference + type: str + attributes: + field: attributes + type: string + insauthor: + field: insauthor + type: str + inscommit: + field: inscommit + type: str + instime: + field: instime + type: time + updauthor: + field: updauthor + type: str + updcommit: + field: updcommit + type: str + updtime: + field: updtime + type: time + actual: + field: actual + index: I + type: int + +connection: + _header: + - coid + - environment + - component + - type + - ip + - port + - hostname + - dompath + - attributes + - inscommit + - insauthor + - instime + - updcommit + - updauthor + - updtime + - actual + cnid: + field: cnid + type: pk + environment: + field: environment + index: I + type: str + component: + field: component + index: I + type: string + type: + field: type + type: str + ip: + field: ip + type: str + port: + field: port + type: str + hostname: + field: hostname + type: str + dompath: + field: dompath + type: str + attributes: + field: attributes + type: string + insauthor: + field: insauthor + type: str + inscommit: + field: inscommit + type: str + instime: + field: instime + type: time + updauthor: + field: updauthor + type: str + updcommit: + field: updcommit + type: str + updtime: + field: updtime + type: time + actual: + field: actual + index: I + type: int diff --git a/basic/Testserver.py b/basic/Testserver.py new file mode 100644 index 0000000..f75b30b --- /dev/null +++ b/basic/Testserver.py @@ -0,0 +1,87 @@ +import basic.component +import basic.constants as B +import utils.config_tool +import utils.data_const as D +import utils.file_tool + +COMP_NAME = "testserver" +COMP_TABLES = ["application", "ap_component", "ap_project"] + +class Testserver(basic.component.Component): + def __init__(self, job): + print('init '+COMP_NAME) + self.m = job.m + self.conf = {} + if B.TOPIC_NODE_DB in job.conf.confs: + self.conf[B.SUBJECT_CONN] = {} + self.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB] = {} + for attr in B.LIST_DB_ATTR: + if attr in job.conf.confs[B.TOPIC_NODE_DB]: + self.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr] = job.conf.confs[B.TOPIC_NODE_DB][attr] + if not B.DATA_NODE_DDL in self.conf: + self.conf[B.DATA_NODE_DDL] = {} + for table in COMP_TABLES: + if table in B.LIST_DB_ATTR: + continue + ddl = utils.config_tool.getConfig(job, D.DDL_FILENAME, COMP_NAME, table) + path = "/home/ulrich/workspace/Datest/temp/DATASTRUCTURE.yml" + utils.file_tool.writeFileDict(job.m, job, path, ddl) + if B.DATA_NODE_TABLES in ddl and table in ddl[B.DATA_NODE_TABLES]: + self.conf[B.DATA_NODE_DDL][table] = ddl[B.DATA_NODE_TABLES][table] + elif table in ddl: + self.conf[B.DATA_NODE_DDL][table] = ddl[table] + else: + self.conf[B.DATA_NODE_DDL][table] = ddl + + def createDBTables(self, job): + if B.TOPIC_NODE_DB in job.conf.confs: + dbi = basic.toolHandling.getDbTool(job, self, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) + else: + return "No DB in job-config" + for t in COMP_TABLES: + sql = self.getDBSchema(job, dbi, t) + for s in sql.split(";\n"): + if len(s) < 3: + continue + try: + dbi.execStatement(s+";", job.conf.confs[B.TOPIC_NODE_DB]) + print("SQL executed: "+s) + except Exception as e: + raise Exception("Fehler bei createSchema "+s) + pass + + + def getDBSchema(self, job, dbi, table): + sqlTable = "" + sqlSub = "" + sqlTable += dbi.getCreateTable(table) + tableId = "" + for f in self.conf[B.DATA_NODE_DDL][table]: + if f[0:1] == "_": + continue + fo = self.conf[B.DATA_NODE_DDL][table][f] + if D.DDL_INDEX in fo and len(fo[D.DDL_INDEX]) > 0: + a = fo[D.DDL_INDEX].split(":") + if a[0] == "I": + sqlSub += dbi.getSchemaIndex(table, fo[D.DDL_FNAME]) + "\n" + elif a[0] == "S": + attrList = [] + attr = {"attr":fo[D.DDL_FNAME], "atype": fo[D.DDL_TYPE]} + attrList.append(attr) + for i in range(2, len(a)): + if i % 2 == 1: + continue + if a[i] == "attr": + attr = {"attr":"attributes", "atype": D.TYPE_TEXT} + elif i+1 < len(a): + attr = {"attr": a[i], "atype": a[i+1]} + attrList.append(attr) + sqlSub += dbi.getSchemaSubtable(a[1], attrList) + "\n" + sqlSub += dbi.getSchemaIndex(dbi.getSubTableName(a[1], fo[D.DDL_FNAME]), tableId)+"\n" + continue + sqlTable += dbi.getSchemaAttribut(fo[D.DDL_FNAME], fo[D.DDL_TYPE]) + "," + if fo[D.DDL_TYPE] == D.TYPE_PK: + tableId = fo[D.DDL_FNAME] + sql = sqlTable[0:-1]+");\n"+sqlSub + print(sql) + return sql diff --git a/basic/application.py b/basic/application.py new file mode 100644 index 0000000..2b88c17 --- /dev/null +++ b/basic/application.py @@ -0,0 +1,264 @@ +# --------------------------------------------------------------------------------------------------------- +# Author : Ulrich Carmesin +# Source : gitea.ucarmesin.de +# --------------------------------------------------------------------------------------------------------- +import os +import utils.db_abstract +import basic.toolHandling +import utils.data_const as D +import basic.constants as B +import basic.entity +import utils.path_const as P +import utils.config_tool +import utils.file_tool +import utils.git_tool + +TABLE_NAMES = ["application", "ap_project", "ap_component"] + +def getProjects(job): + """ + get all project which are configured for the workspace + with all environments where the application of the project are installed + :param job: + :return: + """ + appl = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS) + return searchProjects(job, appl) + +def searchProjects(job, appl): + """ + search all relevant projects from server-configuration + filtered by parameter --application , --project + :param job: + :return: + """ + projects = {} + if B.SUBJECT_PROJECTS in job.conf.confs: + for k in job.conf.confs[B.SUBJECT_PROJECTS]: + if k in B.LIST_SUBJECTS: + continue + if hasattr(job.par, B.PAR_PROJ) and k != getattr(job.par, B.PAR_PROJ): + continue + if hasattr(job.par, B.PAR_APP) \ + and k not in appl[B.SUBJECT_APPS][getattr(job.par, B.PAR_APP)][B.SUBJECT_PROJECTS]: + continue + projects[k] = appl[B.SUBJECT_PROJECTS][k] + projects[k][B.SUBJECT_ENV] = [] + else: + job.conf.confs[B.SUBJECT_PROJECTS] = appl[B.SUBJECT_PROJECTS] + return projects + +def getEnvironments(job, projectList): + """ + searches and gets environments in which the applications of the project are declared that these are installed + filtered by parameter --environment + :param job: + :return: + """ + projects = {} + path = job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ENV] + if os.path.exists(path): + raise Exception("Umgebungsverzeichnis existiert nicht "+path) + for envdir in os.listdir(path): + print ("-- "+envdir) + if not os.path.isdir(os.path.join(path, envdir)): + continue + if envdir[0:1] == "_": + continue + if hasattr(job.par, B.PAR_ENV) and envdir != getattr(job.par, B.PAR_ENV): + continue + for format in utils.config_tool.CONFIG_FORMAT: + pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_ENV), + envdir, P.VAL_CONFIG, P.KEY_TOOL + "_conn." + format) + if os.path.exists(pathname): + break + if os.path.exists(pathname): + doc = utils.file_tool.readFileDict(job, pathname, job.m) + print(str(doc)) + for proj in doc["env"]["general"][B.SUBJECT_PROJECTS]: + if proj in projectList: + projects[proj][B.SUBJECT_ENV].append(envdir) + return projects + +def getApplications(job, projectList): + """ + get all project which are configured for the workspace + with all environments where the application of the project are installed + :param job: + :return: + """ + appl = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS) + return searchApplications(job, projectList, appl) + +def searchApplications(job, projectList, appl): + appList = {} + for proj in projectList: + if hasattr(job.par, B.PAR_PROJ) and proj != getattr(job.par, B.PAR_PROJ): + continue + for app in appl[B.SUBJECT_PROJECTS][proj][B.SUBJECT_APPS]: + if hasattr(job.par, B.PAR_APP) and app != getattr(job.par, B.PAR_APP): + continue + appList[app] = appl[B.SUBJECT_APPS][app] + return appList + +def syncApplications(job): + """ + synchronize the configuration with the database + :param job: + :return: + """ + # get git-commit + apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "") + repopath = apppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:] + gitresult = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1) + if B.TOPIC_NODE_DB in job.conf.confs: + dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) + else: + return "No DB in job-config" + data = dbi.selectRows(TABLE_NAMES[0], job) + print(str(gitresult)) + print(str(data[B.DATA_NODE_DATA])) + if gitresult[0]["date"] == data[B.DATA_NODE_DATA][0]["updtime"]: + print("gleich") + if len(gitresult) > 0: + return + if len(data[B.DATA_NODE_DATA]) > 0: + for t in TABLE_NAMES: + dbi.deleteRows(t, job) + # insertRows + # get list of application + applData = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS) + + for app in applData[B.SUBJECT_APPS]: + ao = Application(job) + ao.readEntity(job, app) + rows = ao.getApplicationRows(job) + apid = dbi.insertRows(TABLE_NAMES[0], rows, job) + rows = ao.getAppProjectRows(job, apid) + dbi.insertRows(TABLE_NAMES[1], rows, job) + rows = ao.getAppComponentRows(job, apid) + dbi.insertRows(TABLE_NAMES[2], rows, job) + + +class Application(basic.entity.Entity): + table = "application" + name = "" + description = "" + reference = "" + component = [] + project = {} + + def __init__(self, job, name=""): + """ + to be initialized by readSpec + :param job: + """ + self.job = job + + if len(name) > 1: + self.getEntity(job, name) + + def getEntity(self, job, name): + if B.TOPIC_NODE_DB in job.conf.confs: + self.selectEntity(job, name) + #self.readEntity(job, name) + else: + self.readEntity(job, name) + + def readEntity(self, job, app): + apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "") + repopath = apppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:] + gitresult = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1) + applData = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS) + # main object + for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[0]][B.DATA_NODE_HEADER]: + if f == basic.entity.ENTITY_NAME: + setattr(self, f, app) + elif f == basic.entity.ENTITY_ATTRIBUTES: + setattr(self, f, {}) + elif f in applData[B.SUBJECT_APPS][app]: + setattr(self, f, applData[B.SUBJECT_APPS][app][f]) + elif f in basic.entity.ENTITY_FIELDS: + setattr(self, f, basic.entity.getEntityValue(job, f, gitresult[0])) + else: + setattr(self, f, "xx") + project = {} + if applData[B.SUBJECT_APPS][app][B.SUBJECT_PROJECTS] is not None: + for proj in applData[B.SUBJECT_APPS][app][B.SUBJECT_PROJECTS]: + project[proj] = {} + for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[1]][B.DATA_NODE_HEADER]: + if f == basic.entity.ENTITY_NAME: + project[proj][f] = proj + elif f == "project": + project[proj][f] = proj + elif f == basic.entity.ENTITY_ATTRIBUTES: + project[proj][f] = {} + elif f in applData[B.SUBJECT_PROJECTS][proj]: + project[proj][f] = applData[B.SUBJECT_PROJECTS][proj][f] + elif f in basic.entity.ENTITY_FIELDS: + project[proj][f] = basic.entity.getEntityValue(job, f, gitresult[0]) + else: + project[proj][f] = "xx" + setattr(self, "project", project) + component = [] + if applData[B.SUBJECT_APPS][app][B.SUBJECT_COMPS] is not None: + for comp in applData[B.SUBJECT_APPS][app][B.SUBJECT_COMPS]: + component.append(comp) + setattr(self, "component", component) + + def getApplicationRows(self, job): + rows = [] + row = {} + for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[0]][B.DATA_NODE_HEADER]: + row[f] = getattr(self, f) + rows.append(row) + return rows + + def getAppProjectRows(self, job, apid): + rows = [] + for proj in self.project: + row = {} + for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[1]][B.DATA_NODE_HEADER]: + if f == "apid": + row[f] = apid + elif f in self.project[proj]: + row[f] = self.project[proj][f] + rows.append(row) + return rows + + def getAppComponentRows(self, job, apid): + rows = [] + for comp in self.component: + row = {} + row["apid"] = apid + row["component"] = comp + rows.append(row) + return rows + + def selectEntity(self, job, app): + dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) + data = dbi.selectRows(TABLE_NAMES[0], job, "WHERE name = \'"+app+"\' AND actual = "+basic.entity.ENTITY_ACTUAL) + # main object + for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[0]][B.DATA_NODE_HEADER]: + if f == basic.entity.ENTITY_NAME: + setattr(self, f, app) + else: + setattr(self, f, str(data[B.DATA_NODE_DATA][0][f])) + apid = getattr(self, "apid") + data = dbi.selectRows(TABLE_NAMES[1], job, "WHERE apid = "+str(apid)) + project = {} + for row in data[B.DATA_NODE_DATA]: + project[row["project"]] = row + setattr(self, "project", project) + data = dbi.selectRows(TABLE_NAMES[2], job, "WHERE apid = " + str(apid)) + component = [] + for row in data[B.DATA_NODE_DATA]: + component.append(row["component"]) + setattr(self, "component", component) + + def getSchema(self): + """ + ersetzt durch testserver.createDB + :return: + """ + return "" \ No newline at end of file diff --git a/basic/constants.py b/basic/constants.py index 111dc94..f1b500f 100644 --- a/basic/constants.py +++ b/basic/constants.py @@ -30,6 +30,8 @@ SVAL_NULL = "null" # ------------------------------------------------------------- # parameter with arguments +PAR_PROJ = 'project' +""" definition of the project which will be tested """ PAR_APP = 'application' """ definition of the application which will be tested """ PAR_ENV = 'environment' @@ -210,6 +212,8 @@ ATTR_INST_SUBCOMP = SUBJECT_COMPS #SUBJECT_FCT = "function" # | | | | x | main-programs SUBJECT_ARTS = "artifact" # | | | | x | Component + + """ in this subject-node are each kind of result of any component with the structure: * topic (db, cli, api, ...) @@ -222,6 +226,8 @@ ATTR_ARTS_LOG = "log" ATTR_ARTS_LOB = "lob" ATTR_ARTS_FILE = "file" #SUBJECT_DB = "databases" # | | | | # | db*_tools, match_tool +SUBJECT_PROJECTS = "projects" +SUBJECT_ENV = PAR_ENV SUBJECT_CONN = "conn" # | | x | | | conn_tool, db*_tools, cli*_toold ATTR_TYPE = "type" # | x | x | | x | conn_tool, toolHandling, db*_tools @@ -233,8 +239,11 @@ ATTR_EXEC_REF = "_exec" ATTR_DATA_REF = "_nr" ATTR_DATA_COMP = "_comp" -SUBJECT_TOOL = "tool" +SUBJECT_DESCRIPTION = "description" +SUBJECT_REFERENCE = "reference" +SUBJECT_TOOL = "tool" +LIST_SUBJECTS = [SUBJECT_APPS, SUBJECT_ARTS, SUBJECT_CONN, SUBJECT_COMPS, SUBJECT_INST, SUBJECT_TOOL, SUBJECT_PROJECTS] # ------------------------------------------------------------- # exception texts EXP_NO_BASIS_FILE = "basis file cant be found" diff --git a/basic/entity.py b/basic/entity.py index b494af5..2e591c2 100644 --- a/basic/entity.py +++ b/basic/entity.py @@ -1,11 +1,46 @@ +import getpass import utils.db_abstract import basic.toolHandling import utils.data_const as D import basic.constants as B +import utils.date_tool + + +ENTITY_NAME = "name" +ENTITY_ATTRIBUTES = "attributes" +ENTITY_INS_COMMIT = "inscommit" +ENTITY_INS_AUTHOR = "insauthor" +ENTITY_INS_TIME = "instime" +ENTITY_UPD_COMMIT = "updcommit" +ENTITY_UPD_AUTHOR = "updauthor" +ENTITY_UPD_TIME = "updtime" +ENTITY_ACTUAL = "actual" +VAL_ACTUAL = 1 +ENTITY_FIELDS = [ENTITY_INS_COMMIT, ENTITY_INS_AUTHOR, ENTITY_INS_TIME, + ENTITY_UPD_COMMIT, ENTITY_UPD_AUTHOR, ENTITY_UPD_TIME, ENTITY_ACTUAL] + +def getEntityValue(job, field, gitcommit): + if field == ENTITY_INS_COMMIT: + return "" + if field == ENTITY_INS_AUTHOR: + return getpass.getuser() + if field == ENTITY_INS_TIME: + return utils.date_tool.getActdate(utils.date_tool.F_DIR) + if field == ENTITY_UPD_COMMIT: + return gitcommit["commit"] + if field == ENTITY_UPD_AUTHOR: + return gitcommit["author"] + if field == ENTITY_UPD_TIME: + return gitcommit["date"] + if field == ENTITY_ACTUAL: + return VAL_ACTUAL + class Entity: def __int__(self, job): self.job = job + self.table = "" + self.testserver = None def getDbAttr(self, job): out = {} @@ -26,16 +61,20 @@ class Entity: out[t][B.DATA_NODE_HEADER] = list(ddl[t].keys()) return out - def createSchema(self): + def createSchema(self, testserver): if B.TOPIC_NODE_DB in self.job.conf.confs: - dbi = basic.toolHandling.getDbTool(self.job, None, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) + dbi = basic.toolHandling.getDbTool(self.job, testserver, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) else: return "No DB in job-config" sql = self.getSchema() print(sql) for s in sql.split(";\n"): if len(s) < 3: continue - dbi.execStatement(self.job.conf.confs[B.TOPIC_NODE_DB], s+";") + try: + dbi.execStatement(s+";", self.job.conf.confs[B.TOPIC_NODE_DB]) + print("SQL executed: "+s) + except Exception as e: + raise Exception("Fehler bei createSchema "+s) def getSchema(self): raise Exception(B.EXCEPT_NOT_IMPLEMENT) @@ -52,6 +91,13 @@ class Entity: sql += dbi.getSchemaAttribut("actual", D.TYPE_INT) return sql + def selectHistoryFields(self): + if B.TOPIC_NODE_DB in self.job.conf.confs: + dbi = basic.toolHandling.getDbTool(self.job, self.testserver, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) + else: + return "No DB in job-config" + dbi.selectRows + def getHistoryIndex(self, table): dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE] dbi = basic.toolHandling.getDbTool(self.job, None, dbtype) diff --git a/basic/message.py b/basic/message.py index 7866c67..514aa9f 100644 --- a/basic/message.py +++ b/basic/message.py @@ -241,7 +241,10 @@ class Message: def log(self, prio, text): """ eigentliche Schreibroutine: hierin wird debug-Level beruecksichtgigt""" if (int(prio) <= int(self.level)) and (self.componente is None): # and self.logfile.closed == False: - self.logfile.write(text + "\n") + try: + self.logfile.write(text + "\n") + except: + pass elif (int(prio) <= int(self.level)): self.messages.append(text) else: diff --git a/basic/program.py b/basic/program.py index d7b1e7a..55446b8 100644 --- a/basic/program.py +++ b/basic/program.py @@ -14,7 +14,6 @@ import os from datetime import datetime import basic.constants as B import basic.message -import basic.message import basic.componentHandling import utils.date_tool import utils.path_tool @@ -260,9 +259,15 @@ class Job: except: pass # the special path is not necessary appl = utils.config_tool.getConfig(self, P.KEY_BASIC, B.SUBJECT_APPS) - print(appl) if appl is not None: self.conf.confs[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS] + if B.SUBJECT_PROJECTS in self.conf.confs: + for k in self.conf.confs[B.SUBJECT_PROJECTS]: + if k not in appl[B.SUBJECT_PROJECTS]: + raise Exception("Workspace has project "+k+" which is not configured") + self.conf.confs[B.SUBJECT_PROJECTS][k] = appl[B.SUBJECT_PROJECTS][k] + else: + self.conf.confs[B.SUBJECT_PROJECTS] = appl[B.SUBJECT_PROJECTS] par = Parameter(self, program, args) self.par = par logTime = utils.date_tool.getActdate(utils.date_tool.F_LOG) @@ -322,22 +327,25 @@ class Job: utils.job_tool.stopJobProcesses(self) self.ende = datetime.now() self.dumpParameter() - print("stopJob " + str(self.m.messages) + ", " + str(self.m.debugfile)) - self.m.logInfo("# # " + self.m.topmessage + " # # # ") - footer = "# # # Stop Job " + utils.date_tool.formatParsedDate(str(self.start), utils.date_tool.F_LOG) - footer += " # " + utils.date_tool.formatParsedDate(str(self.ende), utils.date_tool.F_LOG) + " # # # " - self.m.logInfo(footer) - self.m.debug(basic.message.LIMIT_INFO, "# # " + self.m.topmessage + " # # # ") - self.m.debug(basic.message.LIMIT_INFO, footer + " # # # RC: " + str(self.m.getFinalRc())) + footer1 = "# # " + self.m.topmessage + " # # # " + footer2 = "# # # Stop Job " + utils.date_tool.formatParsedDate(str(self.start), utils.date_tool.F_DE_TSTAMP) + footer2 += " # " + utils.date_tool.formatParsedDate(str(self.ende), utils.date_tool.F_DE_TSTAMP) + " # # # " + footer2 += " # # # RC: " + str(self.m.getFinalRc()) + self.m.logInfo(footer1) + self.m.logInfo(footer2) + self.m.debug(basic.message.LIMIT_INFO, footer1) + self.m.debug(basic.message.LIMIT_INFO, footer2) self.m.closeMessage() rc = self.m.getFinalRc() - print ("rc " + str(rc)) - print(footer) + print(footer1) + print(footer2) if reboot == 0: exit(rc) def dumpParameter(self): + if len(jobdef[self.program]["pfiletarget"]) < 2: + return parpath = utils.path_tool.composePath(self, jobdef[self.program]["pfiletarget"], None) output = {} cconf = basic.componentHandling.getComponentDict() diff --git a/basic/step.py b/basic/step.py index 7a303d5..2c3546c 100644 --- a/basic/step.py +++ b/basic/step.py @@ -53,6 +53,7 @@ def parseStep(job, fields): step.refLine = fields[D.STEP_REFNR_I] step.variante = fields[D.STEP_VARIANT_I] setattr(step, B.ATTR_DATA_REF, step.refLine) + i = 0 if D.STEP_ARGS_I == D.STEP_LIST_I: args = "" for i in range(D.STEP_ARGS_I, len(fields)): @@ -63,16 +64,19 @@ def parseStep(job, fields): args += "," + fields[i] args = args[1:] else: + i = D.STEP_ARGS_I args = fields[D.STEP_ARGS_I] + a = args.split(",") for arg in a: print("arg " + arg) b = arg.split(":") if len(b) < 2: - raise Exception(D.EXCP_MALFORMAT + "" + str(fields)) + raise Exception(D.EXCP_MALFORMAT + " in arg["+str(i)+ "] " + str(fields)) step.args[b[0]] = b[1] if b[0] in LIST_ARGS: setattr(step, b[0], b[1]) + i += 1 # data[B.DATA_NODE_STEPS].append(step) return step diff --git a/test/test_09git.py b/test/test_09git.py index 81877f4..120c1a5 100644 --- a/test/test_09git.py +++ b/test/test_09git.py @@ -28,7 +28,7 @@ class MyTestCase(unittest.TestCase): if actfunction not in TEST_FUNCTIONS: return job = test.testtools.getJob() - utils.git_tool.runGit(job, B.ATTR_PATH_PROGRAM, "git status") + result = utils.git_tool.runGit(job, B.ATTR_PATH_PROGRAM, "git status") MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) def test_02status(self): @@ -38,7 +38,7 @@ class MyTestCase(unittest.TestCase): if actfunction not in TEST_FUNCTIONS: return job = test.testtools.getJob() - utils.git_tool.gitStatus(job, B.ATTR_PATH_PROGRAM) + result = utils.git_tool.gitStatus(job, B.ATTR_PATH_PROGRAM) MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) def test_03log(self): @@ -48,7 +48,14 @@ class MyTestCase(unittest.TestCase): if actfunction not in TEST_FUNCTIONS: return job = test.testtools.getJob() - utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS) + result = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS) + self.assertGreaterEqual(10, len(result)) + result = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, cnt=1) + self.assertEqual(1, len(result)) + apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "") + repopath = apppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS])+1:] + result = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1) + self.assertEqual(1, len(result)) MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) def test_zzz(self): diff --git a/test/test_10testserver.py b/test/test_10testserver.py new file mode 100644 index 0000000..8842081 --- /dev/null +++ b/test/test_10testserver.py @@ -0,0 +1,91 @@ +""" +unit-test +""" +import unittest +import inspect +import utils.gen_tool +import basic.program +import basic.Testserver +import test.testtools +import basic.application +import basic.constants as B +import utils.path_const as P + +# the list of TEST_FUNCTIONS defines which function will be really tested. +# if you minimize the list you can check the specific test-function +TEST_FUNCTIONS = ["test_01createTestserver", "test_02getDBSchema", "test_11createDBTables", "test_11syncApplication"] +TEST_FUNCTIONS = ["test_02getDBSchema"] +# with this variable you can switch prints on and off +verbose = False + + + +class MyTestCase(unittest.TestCase): + mymsg = "--------------------------------------------------------------" + + def test_01createTestserver(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return + job = test.testtools.getJob() + testserver = basic.Testserver.Testserver(job) + self.assertIsNotNone(testserver) + cnttest += 1 + if B.TOPIC_NODE_DB in job.conf.confs: + self.assertIn(B.TOPIC_NODE_DB, testserver.conf[B.SUBJECT_CONN]) + self.assertIn(B.ATTR_DB_DATABASE, testserver.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) + self.assertIn(B.DATA_NODE_DDL, testserver.conf) + self.assertIn("application", testserver.conf[B.DATA_NODE_DDL]) + MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest) + + def test_02getDBSchema(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return + job = test.testtools.getJob() + testserver = basic.Testserver.Testserver(job) + if B.TOPIC_NODE_DB in job.conf.confs: + dbi = basic.toolHandling.getDbTool(job, self, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]) + else: + return "No DB in job-config" + sql = testserver.getDBSchema(job, dbi, "application") + print(sql+"##") + lines = sql.split("\n") + self.assertEqual(4, len(lines)) + self.assertIn("CREATE TABLE", lines[0]) + self.assertIn("CREATE INDEX", lines[1]) + sql = testserver.getDBSchema(job, dbi, "ap_project") + print(sql+"##") + lines = sql.split("\n") + self.assertEqual(4, len(lines)) + self.assertIn("CREATE TABLE", lines[0]) + self.assertIn("CREATE INDEX", lines[1]) + sql = testserver.getDBSchema(job, dbi, "ap_component") + print(sql+"##") + lines = sql.split("\n") + self.assertEqual(4, len(lines)) + self.assertIn("CREATE TABLE", lines[0]) + self.assertIn("CREATE INDEX", lines[1]) + + + def test_11createDBTables(self): + global mymsg + actfunction = str(inspect.currentframe().f_code.co_name) + cnttest = 0 + if actfunction not in TEST_FUNCTIONS: + return + job = test.testtools.getJob() + testserver = basic.Testserver.Testserver(job) + testserver.createDBTables(job) + + def test_zzz(self): + if verbose: print(MyTestCase.mymsg) + + +if __name__ == '__main__': + verbose = True + unittest.main() diff --git a/utils/config_tool.py b/utils/config_tool.py index 59d232b..b0009d1 100644 --- a/utils/config_tool.py +++ b/utils/config_tool.py @@ -24,6 +24,14 @@ import utils.path_const as P COMP_FILES = [D.DDL_FILENAME] CONFIG_FORMAT = [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV] +def getExistgetConfigPath(job, pathnames): + for p in pathnames: + for format in CONFIG_FORMAT: + pathname = p+"."+format + if os.path.exists(pathname): + return pathname + return None + def getConfigPath(job, modul, name, subname=""): """ gets the most specified configuration of different sources @@ -90,6 +98,16 @@ def getConfigPath(job, modul, name, subname=""): raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name) elif modul in COMP_FILES: # for example DATASTRUCURE or the table + pathnames = [] + pathnames.append(os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), + basic.componentHandling.getComponentFolder(name), modul)) + pathnames.append(os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), + basic.componentHandling.getComponentFolder(subname), modul)) + pathnames.append(os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_BASIC, modul)) + pathnames.append(os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_BASIC, subname)) + configpath = getExistgetConfigPath(job, pathnames) + if configpath is not None: + return configpath for format in CONFIG_FORMAT: pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS), basic.componentHandling.getComponentFolder(name), modul+"."+format) diff --git a/utils/data_const.py b/utils/data_const.py index e0834f8..9c47a97 100644 --- a/utils/data_const.py +++ b/utils/data_const.py @@ -23,6 +23,7 @@ DDL_FNAME = "field" DDL_ACCEPTANCE = "acceptance" DDL_KEY = "key" DDL_TYPE = "type" +DDL_INDEX = "index" DFILE_TYPE_YML = "yml" DFILE_TYPE_JSON = "json" diff --git a/utils/date_tool.py b/utils/date_tool.py index bfc80dc..a599f86 100644 --- a/utils/date_tool.py +++ b/utils/date_tool.py @@ -11,9 +11,11 @@ import utils.data_const as D F_DIR = "%Y-%m-%d_%H-%M-%S" F_DB_DATE = "%Y-%m-%d" +F_DB_TIME = "%Y-%m-%d %H:%M:%S" F_DE = "%d.%m.%Y" F_N8 = "%Y%m%d" F_LOG = "%Y%m%d_%H%M%S" +F_DE_TSTAMP = "%d.%m.%Y %H:%M:%S" MONTH_EN = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"] MONTH_DE = ["jan", "feb", "mar", "apr", "mai", "jun", "jul", "aug", "sep", "okt", "nov", "dez"] def getActdate(format): @@ -34,7 +36,7 @@ def getFormatDatetupel(dtupel, format): def formatParsedDate(instring, format): dtupel = parseDate(instring) - print ("---------------"+str(dtupel)) + #print ("---------------"+str(dtupel)) return getFormatDatetupel(dtupel, format) def parseFormula(instring): @@ -117,7 +119,7 @@ def parseDate(instring): hour = 0 min = 0 sec = 0 - print(instring) + #print(instring) if instring[0:2] == "{(" and instring[-2:] == ")}": return parseFormula(instring) if re.match(r"\d{8}_\d{6}", instring): diff --git a/utils/db_abstract.py b/utils/db_abstract.py index 08fab06..706428a 100644 --- a/utils/db_abstract.py +++ b/utils/db_abstract.py @@ -230,6 +230,10 @@ def formatDbVal(msg, val, dtyp): if not isinstance(val, str): msg.logError("field must be " + dtyp + ", " + str(val)) return utils.date_tool.getFormatDatetupel(utils.date_tool.parseDate(val), utils.date_tool.F_DB_DATE) + if dtyp == D.TYPE_TIME: + if not isinstance(val, str): + msg.logError("field must be " + dtyp + ", " + str(val)) + return utils.date_tool.getFormatDatetupel(utils.date_tool.parseDate(val), utils.date_tool.F_DB_TIME) if dtyp == D.TYPE_INT: if not (isinstance(val, int) or re.match(r"^\d+$", val)): msg.logError("field must be " + dtyp + ", " + str(val)) @@ -349,7 +353,7 @@ class DbFcts(): """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) - def execStatement(self, statement): + def execStatement(self, statement, conn=None): """ add-on-method to execute the statement this method should only called by the class itself """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) @@ -370,10 +374,11 @@ class DbFcts(): def getOrder(self): return "" - def getDbValue(self, fo, value): - value = str(formatDbField(self.comp, value, fo)) - if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES: - return self.getDbNull() + def getDbValue(self, fo, pvalue): + value = str(formatDbField(self.comp, pvalue, fo)) + if len(value.strip()) == 0: + if D.DDL_FNULLABLE not in fo or fo[D.DDL_FNULLABLE] == B.SVAL_YES: + return self.getDbNull() if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING or fo[D.DATA_NODE_TYPE] == D.TYPE_STR: return value.strip() elif fo[D.DATA_NODE_TYPE] == D.TYPE_INT: @@ -422,3 +427,11 @@ class DbFcts(): if attr in table: return "idx_"+table return "idx_"+table+"_"+attr + + def getInsertFields(self, ddl): + header = [] + for f in ddl[B.DATA_NODE_HEADER]: + if D.DDL_TYPE in ddl[f] and ddl[f][D.DDL_TYPE] == D.TYPE_PK: + continue + header.append(f) + return header \ No newline at end of file diff --git a/utils/dbmysql_tool.py b/utils/dbmysql_tool.py index f598793..3032f1a 100644 --- a/utils/dbmysql_tool.py +++ b/utils/dbmysql_tool.py @@ -10,6 +10,7 @@ import utils.dbrel_tool import mysql.connector import basic.constants as B import utils.data_const as D +import utils.date_tool class DbFcts(utils.dbrel_tool.DbFcts): """ @@ -43,7 +44,10 @@ class DbFcts(utils.dbrel_tool.DbFcts): r = {} i = 0 for f in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: - r[f] = x[i] + if self.comp.conf[B.DATA_NODE_DDL][table][f][D.DDL_TYPE] in [D.TYPE_TIME, D.TYPE_DATE]: + r[f] = utils.date_tool.getFormatdate(x[i], utils.date_tool.F_DIR) + else: + r[f] = str(x[i]) i += 1 tdata[B.DATA_NODE_DATA].append(r) self.comp.m.logInfo(str(tdata)) @@ -72,40 +76,44 @@ class DbFcts(utils.dbrel_tool.DbFcts): """ verify = -1+job.getDebugLevel("db_tool") attr = self.getDbAttributes(B.SVAL_NULL) + insheader = self.getInsertFields(self.comp.conf[B.DATA_NODE_DDL][table]) + if len(insheader) < len(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]): + lastid = 1 + else: + lastid = 0 sql = "INSERT INTO "+attr[B.ATTR_DB_DATABASE]+"."+table - sql += " ( "+",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]) + " ) " - sql += " VALUES ( " - for x in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: - sql += "%s, " - sql = sql[0:-2] + " )" + sql += " ( "+",".join(insheader) + " ) " + sql += " VALUES " self.comp.m.logInfo(sql) values = [] for r in rows: rowvalues = [] - for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]: - if (self.comp.conf[B.DATA_NODE_DDL][table][h] == D.TYPE_PK): - continue + for h in insheader: if (h in r): - rowvalues.append(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], r[h])) + rowvalues.append("\'"+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], r[h])+"\'") else: - rowvalues.append(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], "")) - values.append( tuple(rowvalues)) + rowvalues.append("\'"+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], "")+"\'") + sql += "("+",".join(rowvalues)+"), " + values.append(tuple(rowvalues)) + sql = sql[0:-2] self.comp.m.logInfo(str(values)) try: connector = self.getConnector() mycursor = connector.cursor() - mycursor.executemany(sql, values) + mycursor.execute(sql) + if lastid > 0: + lastid = mycursor.lastrowid connector.commit() except Exception as e: self.comp.m.setError("") - return + return 0 self.comp.m.setMsg(str(len(values))+" rows inserted into "+table) + return lastid - - def execStatement(self, statement): + def execStatement(self, statement, conn=None): """ add-on-method to execute the statement this method should only called by the class itself """ - connector = self.getConnector() + connector = self.getConnector(conn) cursor = connector.cursor() try: cursor.execute(statement) @@ -117,15 +125,16 @@ class DbFcts(utils.dbrel_tool.DbFcts): print("Statement executed "+statement) self.comp.m.setMsg("Statement executed") - def getConnector(self): + def getConnector(self, conn=None): """ add-on-method to get the connector this method should only called by the class itself """ job = self.job # basic.program.Job.getInstance() - attr = self.getDbAttributes(B.SVAL_NULL) + if conn is None: + conn = self.getDbAttributes(B.SVAL_NULL) cnx = mysql.connector.connect( - host=attr[B.ATTR_DB_HOST], - user=attr[B.ATTR_DB_USER], - password=attr[B.ATTR_DB_PASSWD], - database=attr[B.ATTR_DB_DATABASE] + host=conn[B.ATTR_DB_HOST], + user=conn[B.ATTR_DB_USER], + password=conn[B.ATTR_DB_PASSWD], + database=conn[B.ATTR_DB_DATABASE] ) return cnx diff --git a/utils/dbrel_tool.py b/utils/dbrel_tool.py index 458d3af..bcf776f 100644 --- a/utils/dbrel_tool.py +++ b/utils/dbrel_tool.py @@ -95,6 +95,8 @@ class DbFcts(utils.db_abstract.DbFcts): def getSchemaAttribut(self, attr, atype): if atype == "id": return attr + " INTEGER PRIMARY KEY AUTO_INCREMENT" + elif atype == D.TYPE_PK: + return attr + " INTEGER PRIMARY KEY AUTO_INCREMENT" elif atype == D.TYPE_STR: return attr + " varchar(50)" elif atype == D.TYPE_STRING: @@ -141,7 +143,7 @@ class DbFcts(utils.db_abstract.DbFcts): return mysql @staticmethod - def execStatement(self, comp, conn, statement): + def execStatement(self, statement, conn=None): """ add-on-method to execute the statement this method should only called by the class itself """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) diff --git a/utils/git_tool.py b/utils/git_tool.py index 0883b7e..c71a65c 100644 --- a/utils/git_tool.py +++ b/utils/git_tool.py @@ -21,6 +21,13 @@ COMMIT_DATE = "date" COMMIT_COMMENT = "comment" def runGit(job, repo, cmd): + """ + executes the command on the repository + :param job: + :param repo: + :param cmd: + :return: + """ cdpath = "" if os.path.isdir(repo): cdpath = repo @@ -37,6 +44,7 @@ def runGit(job, repo, cmd): def gitStatus(job, repo): text = runGit(job, repo, "git status") + return text def gitLog(job, repo, arg="", cnt=DEFAULT_CNT_COMMITS): """ @@ -46,15 +54,17 @@ def gitLog(job, repo, arg="", cnt=DEFAULT_CNT_COMMITS): :param cnt: :return: [ {commit: "", author: ] """ - text = runGit(job, repo, "git log") if len(arg) > 1: arg = " -- "+arg else: arg = "" - text = runGit(job, repo, "git log --pretty=format:\"%H | %cn | %cd | %s\""+arg) - print(text) + text = runGit(job, repo, "git log -n "+str(cnt)+" --pretty=format:\"%H | %cn | %cd | %s\""+arg) + #print(text) logs = [] + i = 0 for l in text.split("\n"): + if i == cnt: + break res = {} a = l.split("|") res[COMMIT_ID] = a[0].strip() @@ -63,8 +73,9 @@ def gitLog(job, repo, arg="", cnt=DEFAULT_CNT_COMMITS): res[COMMIT_DATE] = utils.date_tool.getFormatDatetupel(cdate, utils.date_tool.F_DIR) res[COMMIT_COMMENT] = a[3].strip() logs.append(res) + i += 1 print(str(logs)) - + return logs def gitCommits(job, repo, arg=""): if len(arg) > 1: