Testserver incl. DB-mysql

basic/DATASTRUCTURE.yml

@@ -0,0 +1,214 @@
+application:
+  _header:
+  - apid
+  - name
+  - description
+  - reference
+  - attributes
+  - inscommit
+  - insauthor
+  - instime
+  - updcommit
+  - updauthor
+  - updtime
+  - actual
+  apid:
+    field: apid
+    type: pk
+  name:
+    field: name
+    index: I
+    type: str
+  description:
+    field: description
+    type: string
+  reference:
+    field: reference
+    type: str
+  attributes:
+    field: attributes
+    type: string
+  insauthor:
+    field: insauthor
+    type: str
+  inscommit:
+    field: inscommit
+    type: str
+  instime:
+    field: instime
+    type: time
+  updauthor:
+    field: updauthor
+    type: str
+  updcommit:
+    field: updcommit
+    type: str
+  updtime:
+    field: updtime
+    type: time
+  actual:
+    field: actual
+    index: I
+    type: int
+ap_component:
+  _header:
+  - apcomid
+  - apid
+  - component
+  apcomid:
+    field: apcomid
+    type: pk
+  apid:
+    field: apid
+    index: I
+    type: int
+  component:
+    field: component
+    index: I
+    type: str
+ap_project:
+  _header:
+  - approid
+  - apid
+  - project
+  - description
+  - reference
+  approid:
+    field: apid
+    type: pk
+  apid:
+    field: apid
+    index: I
+    type: int
+  project:
+    field: project
+    index: I
+    type: str
+  description:
+    field: description
+    type: string
+  reference:
+    field: reference
+    type: str
+component:
+  _header:
+  - coid
+  - name
+  - description
+  - reference
+  - attributes
+  - inscommit
+  - insauthor
+  - instime
+  - updcommit
+  - updauthor
+  - updtime
+  - actual
+  coid:
+    field: apid
+    type: pk
+  name:
+    field: name
+    index: I
+    type: str
+  description:
+    field: description
+    type: string
+  reference:
+    field: reference
+    type: str
+  attributes:
+    field: attributes
+    type: string
+  insauthor:
+    field: insauthor
+    type: str
+  inscommit:
+    field: inscommit
+    type: str
+  instime:
+    field: instime
+    type: time
+  updauthor:
+    field: updauthor
+    type: str
+  updcommit:
+    field: updcommit
+    type: str
+  updtime:
+    field: updtime
+    type: time
+  actual:
+    field: actual
+    index: I
+    type: int
+
+connection:
+  _header:
+  - coid
+  - environment
+  - component
+  - type
+  - ip
+  - port
+  - hostname
+  - dompath
+  - attributes
+  - inscommit
+  - insauthor
+  - instime
+  - updcommit
+  - updauthor
+  - updtime
+  - actual
+  cnid:
+    field: cnid
+    type: pk
+  environment:
+    field: environment
+    index: I
+    type: str
+  component:
+    field: component
+    index: I
+    type: string
+  type:
+    field: type
+    type: str
+  ip:
+    field: ip
+    type: str
+  port:
+    field: port
+    type: str
+  hostname:
+    field: hostname
+    type: str
+  dompath:
+    field: dompath
+    type: str
+  attributes:
+    field: attributes
+    type: string
+  insauthor:
+    field: insauthor
+    type: str
+  inscommit:
+    field: inscommit
+    type: str
+  instime:
+    field: instime
+    type: time
+  updauthor:
+    field: updauthor
+    type: str
+  updcommit:
+    field: updcommit
+    type: str
+  updtime:
+    field: updtime
+    type: time
+  actual:
+    field: actual
+    index: I
+    type: int

basic/Testserver.py

@@ -0,0 +1,87 @@
+import basic.component
+import basic.constants as B
+import utils.config_tool
+import utils.data_const as D
+import utils.file_tool
+
+COMP_NAME = "testserver"
+COMP_TABLES = ["application", "ap_component", "ap_project"]
+
+class Testserver(basic.component.Component):
+    def __init__(self, job):
+        print('init '+COMP_NAME)
+        self.m = job.m
+        self.conf = {}
+        if B.TOPIC_NODE_DB in job.conf.confs:
+            self.conf[B.SUBJECT_CONN] = {}
+            self.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB] = {}
+            for attr in B.LIST_DB_ATTR:
+                if attr in job.conf.confs[B.TOPIC_NODE_DB]:
+                    self.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr] = job.conf.confs[B.TOPIC_NODE_DB][attr]
+        if not B.DATA_NODE_DDL in self.conf:
+            self.conf[B.DATA_NODE_DDL] = {}
+            for table in COMP_TABLES:
+                if table in B.LIST_DB_ATTR:
+                    continue
+                ddl = utils.config_tool.getConfig(job, D.DDL_FILENAME, COMP_NAME, table)
+                path = "/home/ulrich/workspace/Datest/temp/DATASTRUCTURE.yml"
+                utils.file_tool.writeFileDict(job.m, job, path, ddl)
+                if B.DATA_NODE_TABLES in ddl and table in ddl[B.DATA_NODE_TABLES]:
+                    self.conf[B.DATA_NODE_DDL][table] = ddl[B.DATA_NODE_TABLES][table]
+                elif table in ddl:
+                    self.conf[B.DATA_NODE_DDL][table] = ddl[table]
+                else:
+                    self.conf[B.DATA_NODE_DDL][table] = ddl
+
+    def createDBTables(self, job):
+        if B.TOPIC_NODE_DB in job.conf.confs:
+            dbi = basic.toolHandling.getDbTool(job, self, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE])
+        else:
+            return "No DB in job-config"
+        for t in COMP_TABLES:
+            sql = self.getDBSchema(job, dbi, t)
+            for s in sql.split(";\n"):
+                if len(s) < 3:
+                    continue
+                try:
+                    dbi.execStatement(s+";", job.conf.confs[B.TOPIC_NODE_DB])
+                    print("SQL executed: "+s)
+                except Exception as e:
+                    raise Exception("Fehler bei createSchema "+s)
+        pass
+
+
+    def getDBSchema(self, job, dbi, table):
+        sqlTable = ""
+        sqlSub = ""
+        sqlTable += dbi.getCreateTable(table)
+        tableId = ""
+        for f in self.conf[B.DATA_NODE_DDL][table]:
+            if f[0:1] == "_":
+                continue
+            fo = self.conf[B.DATA_NODE_DDL][table][f]
+            if D.DDL_INDEX in fo and len(fo[D.DDL_INDEX]) > 0:
+                a = fo[D.DDL_INDEX].split(":")
+                if a[0] == "I":
+                    sqlSub += dbi.getSchemaIndex(table, fo[D.DDL_FNAME]) + "\n"
+                elif a[0] == "S":
+                    attrList = []
+                    attr = {"attr":fo[D.DDL_FNAME], "atype": fo[D.DDL_TYPE]}
+                    attrList.append(attr)
+                    for i in range(2, len(a)):
+                        if i % 2 == 1:
+                            continue
+                        if a[i] == "attr":
+                            attr = {"attr":"attributes", "atype": D.TYPE_TEXT}
+                        elif i+1 < len(a):
+                            attr = {"attr": a[i], "atype": a[i+1]}
+                        attrList.append(attr)
+                    sqlSub += dbi.getSchemaSubtable(a[1], attrList) + "\n"
+                    sqlSub += dbi.getSchemaIndex(dbi.getSubTableName(a[1], fo[D.DDL_FNAME]), tableId)+"\n"
+                    continue
+            sqlTable += dbi.getSchemaAttribut(fo[D.DDL_FNAME], fo[D.DDL_TYPE]) + ","
+            if fo[D.DDL_TYPE] == D.TYPE_PK:
+                tableId = fo[D.DDL_FNAME]
+        sql = sqlTable[0:-1]+");\n"+sqlSub
+        print(sql)
+        return sql

basic/application.py

@@ -0,0 +1,264 @@
+# ---------------------------------------------------------------------------------------------------------
+# Author : Ulrich Carmesin
+# Source : gitea.ucarmesin.de
+# ---------------------------------------------------------------------------------------------------------
+import os
+import utils.db_abstract
+import basic.toolHandling
+import utils.data_const as D
+import basic.constants as B
+import basic.entity
+import utils.path_const as P
+import utils.config_tool
+import utils.file_tool
+import utils.git_tool
+
+TABLE_NAMES = ["application", "ap_project", "ap_component"]
+
+def getProjects(job):
+    """
+    get all project which are configured for the workspace
+    with all environments where the application of the project are installed
+    :param job:
+    :return:
+    """
+    appl = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS)
+    return searchProjects(job, appl)
+
+def searchProjects(job, appl):
+    """
+    search all relevant projects from server-configuration
+    filtered by parameter --application , --project
+    :param job:
+    :return:
+    """
+    projects = {}
+    if B.SUBJECT_PROJECTS in job.conf.confs:
+        for k in job.conf.confs[B.SUBJECT_PROJECTS]:
+            if k in B.LIST_SUBJECTS:
+                continue
+            if hasattr(job.par, B.PAR_PROJ) and k != getattr(job.par, B.PAR_PROJ):
+                continue
+            if hasattr(job.par, B.PAR_APP) \
+            and k not in appl[B.SUBJECT_APPS][getattr(job.par, B.PAR_APP)][B.SUBJECT_PROJECTS]:
+                continue
+            projects[k] = appl[B.SUBJECT_PROJECTS][k]
+            projects[k][B.SUBJECT_ENV] = []
+    else:
+        job.conf.confs[B.SUBJECT_PROJECTS] = appl[B.SUBJECT_PROJECTS]
+    return projects
+
+def getEnvironments(job, projectList):
+    """
+    searches and gets environments in which the applications of the project are declared that these are installed
+    filtered by parameter --environment
+    :param job:
+    :return:
+    """
+    projects = {}
+    path = job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_ENV]
+    if os.path.exists(path):
+        raise Exception("Umgebungsverzeichnis existiert nicht "+path)
+    for envdir in os.listdir(path):
+        print ("-- "+envdir)
+        if not os.path.isdir(os.path.join(path, envdir)):
+            continue
+        if envdir[0:1] == "_":
+            continue
+        if hasattr(job.par, B.PAR_ENV) and envdir != getattr(job.par, B.PAR_ENV):
+            continue
+        for format in utils.config_tool.CONFIG_FORMAT:
+            pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_ENV),
+                                    envdir, P.VAL_CONFIG, P.KEY_TOOL + "_conn." + format)
+            if os.path.exists(pathname):
+                break
+        if os.path.exists(pathname):
+            doc = utils.file_tool.readFileDict(job, pathname, job.m)
+        print(str(doc))
+        for proj in doc["env"]["general"][B.SUBJECT_PROJECTS]:
+            if proj in projectList:
+                projects[proj][B.SUBJECT_ENV].append(envdir)
+    return projects
+
+def getApplications(job, projectList):
+    """
+    get all project which are configured for the workspace
+    with all environments where the application of the project are installed
+    :param job:
+    :return:
+    """
+    appl = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS)
+    return searchApplications(job, projectList, appl)
+
+def searchApplications(job, projectList, appl):
+    appList = {}
+    for proj in projectList:
+        if hasattr(job.par, B.PAR_PROJ) and proj != getattr(job.par, B.PAR_PROJ):
+            continue
+        for app in appl[B.SUBJECT_PROJECTS][proj][B.SUBJECT_APPS]:
+            if hasattr(job.par, B.PAR_APP) and app != getattr(job.par, B.PAR_APP):
+                continue
+            appList[app] = appl[B.SUBJECT_APPS][app]
+    return appList
+
+def syncApplications(job):
+    """
+    synchronize the configuration with the database
+    :param job:
+    :return:
+    """
+    # get git-commit
+    apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "")
+    repopath = apppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:]
+    gitresult = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1)
+    if B.TOPIC_NODE_DB in job.conf.confs:
+        dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE])
+    else:
+        return "No DB in job-config"
+    data = dbi.selectRows(TABLE_NAMES[0], job)
+    print(str(gitresult))
+    print(str(data[B.DATA_NODE_DATA]))
+    if gitresult[0]["date"] == data[B.DATA_NODE_DATA][0]["updtime"]:
+        print("gleich")
+    if len(gitresult) > 0:
+        return
+    if len(data[B.DATA_NODE_DATA]) > 0:
+        for t in TABLE_NAMES:
+            dbi.deleteRows(t, job)
+    # insertRows
+    # get list of application
+    applData = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS)
+
+    for app in applData[B.SUBJECT_APPS]:
+        ao = Application(job)
+        ao.readEntity(job, app)
+        rows = ao.getApplicationRows(job)
+        apid = dbi.insertRows(TABLE_NAMES[0], rows, job)
+        rows = ao.getAppProjectRows(job, apid)
+        dbi.insertRows(TABLE_NAMES[1], rows, job)
+        rows = ao.getAppComponentRows(job, apid)
+        dbi.insertRows(TABLE_NAMES[2], rows, job)
+
+
+class Application(basic.entity.Entity):
+    table = "application"
+    name = ""
+    description = ""
+    reference = ""
+    component = []
+    project = {}
+
+    def __init__(self, job, name=""):
+        """
+        to be initialized by readSpec
+        :param job:
+        """
+        self.job = job
+
+        if len(name) > 1:
+            self.getEntity(job, name)
+
+    def getEntity(self, job, name):
+        if B.TOPIC_NODE_DB in job.conf.confs:
+            self.selectEntity(job, name)
+            #self.readEntity(job, name)
+        else:
+            self.readEntity(job, name)
+
+    def readEntity(self, job, app):
+        apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "")
+        repopath = apppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS]) + 1:]
+        gitresult = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1)
+        applData = utils.config_tool.getConfig(job, P.KEY_BASIC, B.SUBJECT_APPS)
+        # main object
+        for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[0]][B.DATA_NODE_HEADER]:
+            if f == basic.entity.ENTITY_NAME:
+                setattr(self, f, app)
+            elif f == basic.entity.ENTITY_ATTRIBUTES:
+                setattr(self, f, {})
+            elif f in applData[B.SUBJECT_APPS][app]:
+                setattr(self, f, applData[B.SUBJECT_APPS][app][f])
+            elif f in basic.entity.ENTITY_FIELDS:
+                setattr(self, f, basic.entity.getEntityValue(job, f, gitresult[0]))
+            else:
+                setattr(self, f, "xx")
+        project = {}
+        if applData[B.SUBJECT_APPS][app][B.SUBJECT_PROJECTS] is not None:
+            for proj in applData[B.SUBJECT_APPS][app][B.SUBJECT_PROJECTS]:
+                project[proj] = {}
+                for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[1]][B.DATA_NODE_HEADER]:
+                    if f == basic.entity.ENTITY_NAME:
+                        project[proj][f] = proj
+                    elif f == "project":
+                        project[proj][f] = proj
+                    elif f == basic.entity.ENTITY_ATTRIBUTES:
+                        project[proj][f] = {}
+                    elif f in applData[B.SUBJECT_PROJECTS][proj]:
+                        project[proj][f] = applData[B.SUBJECT_PROJECTS][proj][f]
+                    elif f in basic.entity.ENTITY_FIELDS:
+                        project[proj][f] = basic.entity.getEntityValue(job, f, gitresult[0])
+                    else:
+                        project[proj][f] = "xx"
+        setattr(self, "project", project)
+        component = []
+        if applData[B.SUBJECT_APPS][app][B.SUBJECT_COMPS] is not None:
+            for comp in applData[B.SUBJECT_APPS][app][B.SUBJECT_COMPS]:
+                component.append(comp)
+        setattr(self, "component", component)
+
+    def getApplicationRows(self, job):
+        rows = []
+        row = {}
+        for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[0]][B.DATA_NODE_HEADER]:
+            row[f] = getattr(self, f)
+        rows.append(row)
+        return rows
+
+    def getAppProjectRows(self, job, apid):
+        rows = []
+        for proj in self.project:
+            row = {}
+            for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[1]][B.DATA_NODE_HEADER]:
+                if f == "apid":
+                    row[f] = apid
+                elif f in self.project[proj]:
+                    row[f] = self.project[proj][f]
+            rows.append(row)
+        return rows
+
+    def getAppComponentRows(self, job, apid):
+        rows = []
+        for comp in self.component:
+            row = {}
+            row["apid"] = apid
+            row["component"] = comp
+            rows.append(row)
+        return rows
+
+    def selectEntity(self, job, app):
+        dbi = basic.toolHandling.getDbTool(job, job.testserver, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE])
+        data = dbi.selectRows(TABLE_NAMES[0], job, "WHERE name = \'"+app+"\' AND actual = "+basic.entity.ENTITY_ACTUAL)
+        # main object
+        for f in job.testserver.conf[B.DATA_NODE_DDL][TABLE_NAMES[0]][B.DATA_NODE_HEADER]:
+            if f == basic.entity.ENTITY_NAME:
+                setattr(self, f, app)
+            else:
+                setattr(self, f, str(data[B.DATA_NODE_DATA][0][f]))
+        apid = getattr(self, "apid")
+        data = dbi.selectRows(TABLE_NAMES[1], job, "WHERE apid = "+str(apid))
+        project = {}
+        for row in data[B.DATA_NODE_DATA]:
+            project[row["project"]] = row
+        setattr(self, "project", project)
+        data = dbi.selectRows(TABLE_NAMES[2], job, "WHERE apid = " + str(apid))
+        component = []
+        for row in data[B.DATA_NODE_DATA]:
+            component.append(row["component"])
+        setattr(self, "component", component)
+
+    def getSchema(self):
+        """
+        ersetzt durch testserver.createDB
+        :return:
+        """
+        return ""

basic/constants.py

@@ -30,6 +30,8 @@ SVAL_NULL = "null"
 
 # -------------------------------------------------------------
 # parameter with arguments
+PAR_PROJ = 'project'
+""" definition of the project which will be tested """
 PAR_APP = 'application'
 """ definition of the application which will be tested """
 PAR_ENV = 'environment'
@@ -210,6 +212,8 @@ ATTR_INST_SUBCOMP = SUBJECT_COMPS
 #SUBJECT_FCT  = "function"      #     |     |     |    | x  |         main-programs
 
 SUBJECT_ARTS = "artifact"      #     |     |     |    | x  |         Component
+
+
 """ 
 in this subject-node are each kind of result of any component with the structure:
   * topic (db, cli, api, ...)
@@ -222,6 +226,8 @@ ATTR_ARTS_LOG = "log"
 ATTR_ARTS_LOB = "lob"
 ATTR_ARTS_FILE = "file"
 #SUBJECT_DB   = "databases"     #     |     |     |    | #  |         db*_tools, match_tool
+SUBJECT_PROJECTS = "projects"
+SUBJECT_ENV = PAR_ENV
 
 SUBJECT_CONN = "conn"          #     |     | x   |    |    |         conn_tool, db*_tools, cli*_toold
 ATTR_TYPE = "type"    #     | x   | x   |    | x  |         conn_tool, toolHandling, db*_tools
@@ -233,8 +239,11 @@ ATTR_EXEC_REF = "_exec"
 ATTR_DATA_REF = "_nr"
 ATTR_DATA_COMP = "_comp"
 
-SUBJECT_TOOL = "tool"
+SUBJECT_DESCRIPTION = "description"
+SUBJECT_REFERENCE = "reference"
 
+SUBJECT_TOOL = "tool"
+LIST_SUBJECTS = [SUBJECT_APPS, SUBJECT_ARTS, SUBJECT_CONN, SUBJECT_COMPS, SUBJECT_INST, SUBJECT_TOOL, SUBJECT_PROJECTS]
 # -------------------------------------------------------------
 # exception texts
 EXP_NO_BASIS_FILE = "basis file cant be found"

basic/entity.py

@@ -1,11 +1,46 @@
+import getpass
 import utils.db_abstract
 import basic.toolHandling
 import utils.data_const as D
 import basic.constants as B
+import utils.date_tool
+
+
+ENTITY_NAME = "name"
+ENTITY_ATTRIBUTES = "attributes"
+ENTITY_INS_COMMIT = "inscommit"
+ENTITY_INS_AUTHOR = "insauthor"
+ENTITY_INS_TIME = "instime"
+ENTITY_UPD_COMMIT = "updcommit"
+ENTITY_UPD_AUTHOR = "updauthor"
+ENTITY_UPD_TIME = "updtime"
+ENTITY_ACTUAL = "actual"
+VAL_ACTUAL = 1
+ENTITY_FIELDS = [ENTITY_INS_COMMIT, ENTITY_INS_AUTHOR, ENTITY_INS_TIME,
+                 ENTITY_UPD_COMMIT, ENTITY_UPD_AUTHOR, ENTITY_UPD_TIME, ENTITY_ACTUAL]
+
+def getEntityValue(job, field, gitcommit):
+    if field == ENTITY_INS_COMMIT:
+        return ""
+    if field == ENTITY_INS_AUTHOR:
+        return getpass.getuser()
+    if field == ENTITY_INS_TIME:
+        return utils.date_tool.getActdate(utils.date_tool.F_DIR)
+    if field == ENTITY_UPD_COMMIT:
+        return gitcommit["commit"]
+    if field == ENTITY_UPD_AUTHOR:
+        return gitcommit["author"]
+    if field == ENTITY_UPD_TIME:
+        return gitcommit["date"]
+    if field == ENTITY_ACTUAL:
+        return VAL_ACTUAL
+
 
 class Entity:
     def __int__(self, job):
         self.job = job
+        self.table = ""
+        self.testserver = None
 
     def getDbAttr(self, job):
         out = {}
@@ -26,16 +61,20 @@ class Entity:
             out[t][B.DATA_NODE_HEADER] = list(ddl[t].keys())
         return out
 
-    def createSchema(self):
+    def createSchema(self, testserver):
         if B.TOPIC_NODE_DB in self.job.conf.confs:
-            dbi = basic.toolHandling.getDbTool(self.job, None, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE])
+            dbi = basic.toolHandling.getDbTool(self.job, testserver, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE])
         else:
             return "No DB in job-config"
         sql = self.getSchema()
         print(sql)
         for s in sql.split(";\n"):
             if len(s) < 3: continue
-            dbi.execStatement(self.job.conf.confs[B.TOPIC_NODE_DB], s+";")
+            try:
+                dbi.execStatement(s+";", self.job.conf.confs[B.TOPIC_NODE_DB])
+                print("SQL executed: "+s)
+            except Exception as e:
+                raise Exception("Fehler bei createSchema "+s)
 
     def getSchema(self):
         raise Exception(B.EXCEPT_NOT_IMPLEMENT)
@@ -52,6 +91,13 @@ class Entity:
         sql += dbi.getSchemaAttribut("actual", D.TYPE_INT)
         return sql
 
+    def selectHistoryFields(self):
+        if B.TOPIC_NODE_DB in self.job.conf.confs:
+            dbi = basic.toolHandling.getDbTool(self.job, self.testserver, self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE])
+        else:
+            return "No DB in job-config"
+        dbi.selectRows
+
     def getHistoryIndex(self, table):
         dbtype = self.job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE]
         dbi = basic.toolHandling.getDbTool(self.job, None, dbtype)

basic/message.py

@@ -241,7 +241,10 @@ class Message:
     def log(self, prio, text):
         """ eigentliche Schreibroutine: hierin wird debug-Level beruecksichtgigt"""
         if (int(prio) <= int(self.level)) and (self.componente is None): # and self.logfile.closed == False:
+            try:
                 self.logfile.write(text + "\n")
+            except:
+                pass
         elif (int(prio) <= int(self.level)):
             self.messages.append(text)
         else:

basic/program.py

@@ -14,7 +14,6 @@ import os
 from datetime import datetime
 import basic.constants as B
 import basic.message
-import basic.message
 import basic.componentHandling
 import utils.date_tool
 import utils.path_tool
@@ -260,9 +259,15 @@ class Job:
 		except:
 			pass # the special path is not necessary
 		appl = utils.config_tool.getConfig(self, P.KEY_BASIC, B.SUBJECT_APPS)
-		print(appl)
 		if appl is not None:
 			self.conf.confs[B.SUBJECT_APPS] = appl[B.SUBJECT_APPS]
+			if B.SUBJECT_PROJECTS in self.conf.confs:
+				for k in self.conf.confs[B.SUBJECT_PROJECTS]:
+					if k not in appl[B.SUBJECT_PROJECTS]:
+						raise Exception("Workspace has project "+k+" which is not configured")
+					self.conf.confs[B.SUBJECT_PROJECTS][k] = appl[B.SUBJECT_PROJECTS][k]
+			else:
+				self.conf.confs[B.SUBJECT_PROJECTS] = appl[B.SUBJECT_PROJECTS]
 		par = Parameter(self, program, args)
 		self.par = par
 		logTime = utils.date_tool.getActdate(utils.date_tool.F_LOG)
@@ -322,22 +327,25 @@ class Job:
 		utils.job_tool.stopJobProcesses(self)
 		self.ende = datetime.now()
 		self.dumpParameter()
-		print("stopJob " + str(self.m.messages) + ", " + str(self.m.debugfile))
-		self.m.logInfo("# # " + self.m.topmessage + " # # # ")
-		footer = "# # # Stop Job " + utils.date_tool.formatParsedDate(str(self.start), utils.date_tool.F_LOG)
-		footer += " # " + utils.date_tool.formatParsedDate(str(self.ende), utils.date_tool.F_LOG) + " # # # "
-		self.m.logInfo(footer)
-		self.m.debug(basic.message.LIMIT_INFO, "# # " + self.m.topmessage + " # # # ")
-		self.m.debug(basic.message.LIMIT_INFO, footer + " # # # RC: " + str(self.m.getFinalRc()))
+		footer1 = "# # " + self.m.topmessage + " # # # "
+		footer2 = "# # # Stop Job " + utils.date_tool.formatParsedDate(str(self.start), utils.date_tool.F_DE_TSTAMP)
+		footer2 += " # " + utils.date_tool.formatParsedDate(str(self.ende), utils.date_tool.F_DE_TSTAMP) + " # # # "
+		footer2 += " # # # RC: " + str(self.m.getFinalRc())
+		self.m.logInfo(footer1)
+		self.m.logInfo(footer2)
+		self.m.debug(basic.message.LIMIT_INFO, footer1)
+		self.m.debug(basic.message.LIMIT_INFO, footer2)
 		self.m.closeMessage()
 		rc = self.m.getFinalRc()
-		print ("rc " + str(rc))
-		print(footer)
+		print(footer1)
+		print(footer2)
 		if reboot == 0:
 			exit(rc)
 
 
 	def dumpParameter(self):
+		if len(jobdef[self.program]["pfiletarget"]) < 2:
+			return
 		parpath = utils.path_tool.composePath(self, jobdef[self.program]["pfiletarget"], None)
 		output = {}
 		cconf = basic.componentHandling.getComponentDict()

basic/step.py

@@ -53,6 +53,7 @@ def parseStep(job, fields):
     step.refLine = fields[D.STEP_REFNR_I]
     step.variante = fields[D.STEP_VARIANT_I]
     setattr(step, B.ATTR_DATA_REF, step.refLine)
+    i = 0
     if D.STEP_ARGS_I == D.STEP_LIST_I:
         args = ""
         for i in range(D.STEP_ARGS_I, len(fields)):
@@ -63,16 +64,19 @@ def parseStep(job, fields):
             args += "," + fields[i]
         args = args[1:]
     else:
+        i = D.STEP_ARGS_I
         args = fields[D.STEP_ARGS_I]
+
     a = args.split(",")
     for arg in a:
         print("arg " + arg)
         b = arg.split(":")
         if len(b) < 2:
-            raise Exception(D.EXCP_MALFORMAT + "" + str(fields))
+            raise Exception(D.EXCP_MALFORMAT + " in arg["+str(i)+ "] " + str(fields))
         step.args[b[0]] = b[1]
         if b[0] in LIST_ARGS:
             setattr(step, b[0], b[1])
+        i += 1
     # data[B.DATA_NODE_STEPS].append(step)
     return step
 

test/test_09git.py

@@ -28,7 +28,7 @@ class MyTestCase(unittest.TestCase):
         if actfunction not in TEST_FUNCTIONS:
             return
         job = test.testtools.getJob()
-        utils.git_tool.runGit(job, B.ATTR_PATH_PROGRAM, "git status")
+        result = utils.git_tool.runGit(job, B.ATTR_PATH_PROGRAM, "git status")
         MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
 
     def test_02status(self):
@@ -38,7 +38,7 @@ class MyTestCase(unittest.TestCase):
         if actfunction not in TEST_FUNCTIONS:
             return
         job = test.testtools.getJob()
-        utils.git_tool.gitStatus(job, B.ATTR_PATH_PROGRAM)
+        result = utils.git_tool.gitStatus(job, B.ATTR_PATH_PROGRAM)
         MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
 
     def test_03log(self):
@@ -48,7 +48,14 @@ class MyTestCase(unittest.TestCase):
         if actfunction not in TEST_FUNCTIONS:
             return
         job = test.testtools.getJob()
-        utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS)
+        result = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS)
+        self.assertGreaterEqual(10, len(result))
+        result = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, cnt=1)
+        self.assertEqual(1, len(result))
+        apppath = utils.config_tool.getConfigPath(job, P.KEY_BASIC, B.SUBJECT_APPS, "")
+        repopath = apppath[len(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_COMPS])+1:]
+        result = utils.git_tool.gitLog(job, B.ATTR_PATH_COMPS, repopath, 1)
+        self.assertEqual(1, len(result))
         MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
 
     def test_zzz(self):

test/test_10testserver.py

@@ -0,0 +1,91 @@
+"""
+unit-test
+"""
+import unittest
+import inspect
+import utils.gen_tool
+import basic.program
+import basic.Testserver
+import test.testtools
+import basic.application
+import basic.constants as B
+import utils.path_const as P
+
+# the list of TEST_FUNCTIONS defines which function will be really tested.
+# if you minimize the list you can check the specific test-function
+TEST_FUNCTIONS = ["test_01createTestserver", "test_02getDBSchema", "test_11createDBTables", "test_11syncApplication"]
+TEST_FUNCTIONS = ["test_02getDBSchema"]
+# with this variable you can switch prints on and off
+verbose = False
+
+
+
+class MyTestCase(unittest.TestCase):
+    mymsg = "--------------------------------------------------------------"
+
+    def test_01createTestserver(self):
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
+        job = test.testtools.getJob()
+        testserver = basic.Testserver.Testserver(job)
+        self.assertIsNotNone(testserver)
+        cnttest += 1
+        if B.TOPIC_NODE_DB in job.conf.confs:
+            self.assertIn(B.TOPIC_NODE_DB, testserver.conf[B.SUBJECT_CONN])
+            self.assertIn(B.ATTR_DB_DATABASE, testserver.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB])
+        self.assertIn(B.DATA_NODE_DDL, testserver.conf)
+        self.assertIn("application", testserver.conf[B.DATA_NODE_DDL])
+        MyTestCase.mymsg += "\n----- "+actfunction+" : "+str(cnttest)
+
+    def test_02getDBSchema(self):
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
+        job = test.testtools.getJob()
+        testserver = basic.Testserver.Testserver(job)
+        if B.TOPIC_NODE_DB in job.conf.confs:
+            dbi = basic.toolHandling.getDbTool(job, self, job.conf.confs[B.TOPIC_NODE_DB][B.ATTR_TYPE])
+        else:
+            return "No DB in job-config"
+        sql = testserver.getDBSchema(job, dbi, "application")
+        print(sql+"##")
+        lines = sql.split("\n")
+        self.assertEqual(4, len(lines))
+        self.assertIn("CREATE TABLE", lines[0])
+        self.assertIn("CREATE INDEX", lines[1])
+        sql = testserver.getDBSchema(job, dbi, "ap_project")
+        print(sql+"##")
+        lines = sql.split("\n")
+        self.assertEqual(4, len(lines))
+        self.assertIn("CREATE TABLE", lines[0])
+        self.assertIn("CREATE INDEX", lines[1])
+        sql = testserver.getDBSchema(job, dbi, "ap_component")
+        print(sql+"##")
+        lines = sql.split("\n")
+        self.assertEqual(4, len(lines))
+        self.assertIn("CREATE TABLE", lines[0])
+        self.assertIn("CREATE INDEX", lines[1])
+
+
+    def test_11createDBTables(self):
+        global mymsg
+        actfunction = str(inspect.currentframe().f_code.co_name)
+        cnttest = 0
+        if actfunction not in TEST_FUNCTIONS:
+            return
+        job = test.testtools.getJob()
+        testserver = basic.Testserver.Testserver(job)
+        testserver.createDBTables(job)
+
+    def test_zzz(self):
+        if verbose: print(MyTestCase.mymsg)
+
+
+if __name__ == '__main__':
+    verbose = True
+    unittest.main()

utils/config_tool.py

@@ -24,6 +24,14 @@ import utils.path_const as P
 COMP_FILES = [D.DDL_FILENAME]
 CONFIG_FORMAT = [D.DFILE_TYPE_YML, D.DFILE_TYPE_JSON, D.DFILE_TYPE_CSV]
 
+def getExistgetConfigPath(job, pathnames):
+    for p in pathnames:
+        for format in CONFIG_FORMAT:
+            pathname = p+"."+format
+            if os.path.exists(pathname):
+                return pathname
+    return None
+
 def getConfigPath(job, modul, name, subname=""):
     """
     gets the most specified configuration of different sources
@@ -90,6 +98,16 @@ def getConfigPath(job, modul, name, subname=""):
         raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name)
     elif modul in COMP_FILES:
         # for example DATASTRUCURE or the table
+        pathnames = []
+        pathnames.append(os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
+                         basic.componentHandling.getComponentFolder(name), modul))
+        pathnames.append(os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
+                         basic.componentHandling.getComponentFolder(subname), modul))
+        pathnames.append(os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_BASIC, modul))
+        pathnames.append(os.path.join(job.conf.getPath(P.ATTR_PATH_PROGRAM), P.VAL_BASIC, subname))
+        configpath = getExistgetConfigPath(job, pathnames)
+        if configpath is not None:
+            return configpath
         for format in CONFIG_FORMAT:
             pathname = os.path.join(job.conf.getPath(P.ATTR_PATH_COMPONENTS),
                                     basic.componentHandling.getComponentFolder(name), modul+"."+format)

utils/data_const.py

@@ -23,6 +23,7 @@ DDL_FNAME = "field"
 DDL_ACCEPTANCE = "acceptance"
 DDL_KEY = "key"
 DDL_TYPE = "type"
+DDL_INDEX = "index"
 
 DFILE_TYPE_YML = "yml"
 DFILE_TYPE_JSON = "json"

utils/date_tool.py

@@ -11,9 +11,11 @@ import utils.data_const as D
 
 F_DIR = "%Y-%m-%d_%H-%M-%S"
 F_DB_DATE = "%Y-%m-%d"
+F_DB_TIME = "%Y-%m-%d %H:%M:%S"
 F_DE = "%d.%m.%Y"
 F_N8 = "%Y%m%d"
 F_LOG = "%Y%m%d_%H%M%S"
+F_DE_TSTAMP = "%d.%m.%Y %H:%M:%S"
 MONTH_EN = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"]
 MONTH_DE = ["jan", "feb", "mar", "apr", "mai", "jun", "jul", "aug", "sep", "okt", "nov", "dez"]
 def getActdate(format):
@@ -34,7 +36,7 @@ def getFormatDatetupel(dtupel, format):
 
 def formatParsedDate(instring, format):
     dtupel = parseDate(instring)
-    print ("---------------"+str(dtupel))
+    #print ("---------------"+str(dtupel))
     return getFormatDatetupel(dtupel, format)
 
 def parseFormula(instring):
@@ -117,7 +119,7 @@ def parseDate(instring):
     hour = 0
     min = 0
     sec = 0
-    print(instring)
+    #print(instring)
     if instring[0:2] == "{(" and instring[-2:] == ")}":
         return parseFormula(instring)
     if re.match(r"\d{8}_\d{6}", instring):

utils/db_abstract.py

@@ -230,6 +230,10 @@ def formatDbVal(msg, val, dtyp):
         if not isinstance(val, str):
             msg.logError("field must be " + dtyp + ", " + str(val))
         return utils.date_tool.getFormatDatetupel(utils.date_tool.parseDate(val), utils.date_tool.F_DB_DATE)
+    if dtyp == D.TYPE_TIME:
+        if not isinstance(val, str):
+            msg.logError("field must be " + dtyp + ", " + str(val))
+        return utils.date_tool.getFormatDatetupel(utils.date_tool.parseDate(val), utils.date_tool.F_DB_TIME)
     if dtyp == D.TYPE_INT:
         if not (isinstance(val, int) or re.match(r"^\d+$", val)):
             msg.logError("field must be " + dtyp + ", " + str(val))
@@ -349,7 +353,7 @@ class DbFcts():
         """
         raise Exception(B.EXCEPT_NOT_IMPLEMENT)
 
-    def execStatement(self, statement):
+    def execStatement(self, statement, conn=None):
         """ add-on-method to execute the statement
         this method should only called by the class itself """
         raise Exception(B.EXCEPT_NOT_IMPLEMENT)
@@ -370,9 +374,10 @@ class DbFcts():
     def getOrder(self):
         return ""
 
-    def getDbValue(self, fo, value):
-        value = str(formatDbField(self.comp, value, fo))
-        if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES:
+    def getDbValue(self, fo, pvalue):
+        value = str(formatDbField(self.comp, pvalue, fo))
+        if len(value.strip()) == 0:
+            if D.DDL_FNULLABLE not in fo or fo[D.DDL_FNULLABLE] == B.SVAL_YES:
                 return self.getDbNull()
         if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING or fo[D.DATA_NODE_TYPE] == D.TYPE_STR:
             return value.strip()
@@ -422,3 +427,11 @@ class DbFcts():
         if attr in table:
             return "idx_"+table
         return "idx_"+table+"_"+attr
+
+    def getInsertFields(self, ddl):
+        header = []
+        for f in ddl[B.DATA_NODE_HEADER]:
+            if D.DDL_TYPE in ddl[f] and ddl[f][D.DDL_TYPE] == D.TYPE_PK:
+                continue
+            header.append(f)
+        return header

utils/dbmysql_tool.py

@@ -10,6 +10,7 @@ import utils.dbrel_tool
 import mysql.connector
 import basic.constants as B
 import utils.data_const as D
+import utils.date_tool
 
 class DbFcts(utils.dbrel_tool.DbFcts):
     """
@@ -43,7 +44,10 @@ class DbFcts(utils.dbrel_tool.DbFcts):
             r = {}
             i = 0
             for f in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
-                r[f] = x[i]
+                if self.comp.conf[B.DATA_NODE_DDL][table][f][D.DDL_TYPE] in [D.TYPE_TIME, D.TYPE_DATE]:
+                    r[f] = utils.date_tool.getFormatdate(x[i], utils.date_tool.F_DIR)
+                else:
+                    r[f] = str(x[i])
                 i += 1
             tdata[B.DATA_NODE_DATA].append(r)
         self.comp.m.logInfo(str(tdata))
@@ -72,40 +76,44 @@ class DbFcts(utils.dbrel_tool.DbFcts):
         """
         verify = -1+job.getDebugLevel("db_tool")
         attr = self.getDbAttributes(B.SVAL_NULL)
+        insheader = self.getInsertFields(self.comp.conf[B.DATA_NODE_DDL][table])
+        if len(insheader) < len(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]):
+            lastid = 1
+        else:
+            lastid = 0
         sql = "INSERT INTO "+attr[B.ATTR_DB_DATABASE]+"."+table
-        sql += " ( "+",".join(self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]) + " ) "
-        sql += " VALUES ( "
-        for x in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
-            sql += "%s, "
-        sql = sql[0:-2] + " )"
+        sql += " ( "+",".join(insheader) + " ) "
+        sql += " VALUES "
         self.comp.m.logInfo(sql)
         values = []
         for r in rows:
             rowvalues = []
-            for h in self.comp.conf[B.DATA_NODE_DDL][table][B.DATA_NODE_HEADER]:
-                if (self.comp.conf[B.DATA_NODE_DDL][table][h] == D.TYPE_PK):
-                    continue
+            for h in insheader:
                 if (h in r):
-                    rowvalues.append(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], r[h]))
+                    rowvalues.append("\'"+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], r[h])+"\'")
                 else:
-                    rowvalues.append(self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], ""))
-            values.append( tuple(rowvalues))
+                    rowvalues.append("\'"+self.getDbValue(self.comp.conf[B.DATA_NODE_DDL][table][h], "")+"\'")
+            sql += "("+",".join(rowvalues)+"), "
+            values.append(tuple(rowvalues))
+        sql = sql[0:-2]
         self.comp.m.logInfo(str(values))
         try:
             connector = self.getConnector()
             mycursor = connector.cursor()
-            mycursor.executemany(sql, values)
+            mycursor.execute(sql)
+            if lastid > 0:
+                lastid = mycursor.lastrowid
             connector.commit()
         except Exception as e:
             self.comp.m.setError("")
-            return
+            return 0
         self.comp.m.setMsg(str(len(values))+" rows inserted into "+table)
+        return lastid
 
-
-    def execStatement(self, statement):
+    def execStatement(self, statement, conn=None):
         """ add-on-method to execute the statement
         this method should only called by the class itself """
-        connector = self.getConnector()
+        connector = self.getConnector(conn)
         cursor = connector.cursor()
         try:
             cursor.execute(statement)
@@ -117,15 +125,16 @@ class DbFcts(utils.dbrel_tool.DbFcts):
         print("Statement executed "+statement)
         self.comp.m.setMsg("Statement executed")
 
-    def getConnector(self):
+    def getConnector(self, conn=None):
         """ add-on-method to get the connector
         this method should only called by the class itself """
         job = self.job # basic.program.Job.getInstance()
-        attr = self.getDbAttributes(B.SVAL_NULL)
+        if conn is None:
+            conn = self.getDbAttributes(B.SVAL_NULL)
         cnx = mysql.connector.connect(
-          host=attr[B.ATTR_DB_HOST],
-          user=attr[B.ATTR_DB_USER],
-          password=attr[B.ATTR_DB_PASSWD],
-          database=attr[B.ATTR_DB_DATABASE]
+          host=conn[B.ATTR_DB_HOST],
+          user=conn[B.ATTR_DB_USER],
+          password=conn[B.ATTR_DB_PASSWD],
+          database=conn[B.ATTR_DB_DATABASE]
         )
         return cnx

utils/dbrel_tool.py

@@ -95,6 +95,8 @@ class DbFcts(utils.db_abstract.DbFcts):
   def getSchemaAttribut(self, attr, atype):
     if atype == "id":
       return attr + " INTEGER PRIMARY KEY AUTO_INCREMENT"
+    elif atype == D.TYPE_PK:
+        return attr + " INTEGER PRIMARY KEY AUTO_INCREMENT"
     elif atype == D.TYPE_STR:
       return attr + " varchar(50)"
     elif atype == D.TYPE_STRING:
@@ -141,7 +143,7 @@ class DbFcts(utils.db_abstract.DbFcts):
     return mysql
 
   @staticmethod
-  def execStatement(self, comp, conn, statement):
+  def execStatement(self, statement, conn=None):
     """ add-on-method to execute the statement
     this method should only called by the class itself """
     raise Exception(B.EXCEPT_NOT_IMPLEMENT)

utils/git_tool.py

@@ -21,6 +21,13 @@ COMMIT_DATE = "date"
 COMMIT_COMMENT = "comment"
 
 def runGit(job, repo, cmd):
+    """
+    executes the command on the repository
+    :param job:
+    :param repo:
+    :param cmd:
+    :return:
+    """
     cdpath = ""
     if os.path.isdir(repo):
         cdpath = repo
@@ -37,6 +44,7 @@ def runGit(job, repo, cmd):
 
 def gitStatus(job, repo):
     text = runGit(job, repo, "git status")
+    return text
 
 def gitLog(job, repo, arg="", cnt=DEFAULT_CNT_COMMITS):
     """
@@ -46,15 +54,17 @@ def gitLog(job, repo, arg="", cnt=DEFAULT_CNT_COMMITS):
     :param cnt:
     :return: [ {commit: "", author: ]
     """
-    text = runGit(job, repo, "git log")
     if len(arg) > 1:
         arg = " -- "+arg
     else:
         arg = ""
-    text = runGit(job, repo, "git log --pretty=format:\"%H | %cn | %cd | %s\""+arg)
-    print(text)
+    text = runGit(job, repo, "git log -n "+str(cnt)+" --pretty=format:\"%H | %cn | %cd | %s\""+arg)
+    #print(text)
     logs = []
+    i = 0
     for l in text.split("\n"):
+        if i == cnt:
+            break
         res = {}
         a = l.split("|")
         res[COMMIT_ID] = a[0].strip()
@@ -63,8 +73,9 @@ def gitLog(job, repo, arg="", cnt=DEFAULT_CNT_COMMITS):
         res[COMMIT_DATE] = utils.date_tool.getFormatDatetupel(cdate, utils.date_tool.F_DIR)
         res[COMMIT_COMMENT] = a[3].strip()
         logs.append(res)
+        i += 1
     print(str(logs))
-
+    return logs
 
 def gitCommits(job, repo, arg=""):
     if len(arg) > 1: