#!/usr/bin/python # -*- coding: utf-8 -*- # --------------------------------------------------------------------------------------------------------- # Author : Ulrich Carmesin # Source : gitea.ucarmesin.de # --------------------------------------------------------------------------------------------------------- """ This abstract class DbFcts defines the interface for any function relating to any kind of a database-management. It uses the following configuration .a) COMP.conf->artifact->db->[table] : for structural attributes of the database, maybe specialized for tables \n .b) COMP.conf->artifact->ddl : for structural attributes of the database-tables \n .c) COMP.conf->conn->[db] : for connection-attributes and structural attributes, maybe specialized for the database The attribute db.type resolves which technique is used, implemented in a specific tool-class: * db2,mysql,pgsql,(s)hive ... for specific synchronous RDBs * spark,shive,sfile ... using spark-technology * csv,(s)file ... for data managed in files The main tasks are: \n .1. connecting to the datebase - with attributes * conn.ip, host, port, user, password, ... for synchronous db-connection * conn.root, ... for path-definitions for file-implementations (csv, ) .2. delete at the initialisation - with * db.reset for the context testcase, testsuite or never, on which the content will be deleted - default: testcase * db.character of the table if the content will be be deleted in a sequence of testcases * db.tabname -- if the tablename is differs from the node-name of the table - default: not necessary * db.filename -- if the filename is differs from the node-name of the table - default: not necessary * par.dbwhere which rows will be deleted - default empty .3. insert testdata at the initialisation * ddl.[fields] * db.tabname - if the tablename is differs from the node-name of the table - default: not necessary * db.filename - if the filename is differs from the node-name of the table - default: not necessary .4. select the data with * db.tabname - if the tablename is differs from the node-name of the table - default: not necessary * db.filename - if the filename is differs from the node-name of the table - default: not necessary * ddl._data.[fields].order for an order-clause * par.dbwhere which rows will be deleted - default empty SPECIAL CASES: * If the table is partitioned tables the functions delete/insert/select calls the callback-functions COMP.nextTable() resp. COMP.nextTdata(). """ import basic.program import utils.config_tool import basic.constants as B import utils.data_const as D import os def getDbAttributes(comp, table): """ this function collects all relevant db-attributes from any location where it can be set. The location could be * comp.artifact.db.[table].attr * comp.artifact.[db].[table].attr """ out = { B.ATTR_DB_DATABASE: "", B.ATTR_DB_SCHEMA: "", B.ATTR_DB_TABNAME: "", B.ATTR_DB_PARTITION: D.DEFAULT_DB_PARTITION, B.ATTR_DB_CONN_JAR: D.DEFAULT_DB_CONN_JAR } for attr in out.keys(): print(attr) if (table in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \ and (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]): out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr] print("a " + attr + " " + out[attr]) elif (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]): out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr] print("b " + attr + " " + out[attr]) elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \ and (table in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \ and (attr in comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]): out[attr] = comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr] print("c " + attr + " " + out[attr]) elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \ and (attr in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]): out[attr] = comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr] print("d " + attr + " " + out[attr]) elif (attr in comp.conf[B.SUBJECT_CONN]): out[attr] = comp.conf[B.SUBJECT_CONN][attr] print("e " + attr + " " + out[attr]) else: print("f " + attr + " " + out[attr]) return out def getStringIndex(text, intern): if intern in text: return text.index(intern) return 999 def parseCondition(condition): out = [] for operator in [">=", "<=", ">", "<", "=", " like ", " in "]: if operator in condition: i = condition.lower().index(operator) field = condition[0:i] attr = condition[i+len(operator):] out = [operator.strip(), field.strip(), attr.strip()] return out return out def parseConditions(condition): """ the functions parses the condition into their syntactical parts: parts = [ [ conjunctor , [ operator , field, attribute ], ... ] the condition should be a simple normalform either with "and" or with "or" """ rest = condition dbwhere = [] if (" and " in rest and " or " in rest): raise Exception("the sql-condition must contain only \"ands\" or \"ors\" "+rest) while (" and " in rest or " or " in rest): iand = getStringIndex(rest, " and ") ior = getStringIndex(rest, " or ") print("conjunctors " + str(iand) + " " + str(ior)) if iand == 999 and ior == 999: print("fertig") elif iand < ior: conjunctor = "and" condition = rest[0:iand] rest = rest[iand + 5:] cond = parseCondition(condition) cond.append(conjunctor) dbwhere.append(cond) elif iand > ior: conjunctor = "or" condition = rest[0:ior] rest = rest[ior + 4:] cond = parseCondition(condition) cond.append(conjunctor) dbwhere.append(cond) cond = parseCondition(rest) cond.append("end") dbwhere.append(cond) return dbwhere def parseSQLwhere(condition, ddl=None): parts = parseConditions(condition) conjunctor = "" dbwhere = "" for cond in parts: if cond[1] in ddl[B.DATA_NODE_HEADER]: dbwhere += " "+conjunctor+" "+cond[1]+" "+cond[0]+" "+cond[2] conjunctor = cond[3] return "WHERE "+dbwhere.strip() # --------------------------------------------------------------------------------------------------------------- class DbFcts(): """ This interface defines each necessary connection to any kind of database. The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool. """ def __init__(self): self.comp = None pass def setComp(self, comp): self.comp = comp def getDbAttributes(self, table): """ this function collects all relevant db-attributes from any location where it can be set. The location could be * comp.artifact.db.[table].attr * comp.artifact.[db].[table].attr """ return getDbAttributes(self.comp, table) def xxgetDbAttributes(self, table): out = { B.ATTR_DB_TABNAME: "", B.ATTR_DB_PARTITION: D.DFLT_DB_PARTITION, B.ATTR_DB_CONN_JAR: D.DFLT_DB_CONN_JAR } for attr in out.keys(): print(attr) if (table in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \ and (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]): out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr] print("a "+attr+" "+out[attr]) elif (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]): out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr] print("b "+attr+" "+out[attr]) elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \ and (table in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \ and (attr in self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]): out[attr] = self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr] print("c " + attr+" "+out[attr]) elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \ and (attr in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]): out[attr] = self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr] print("d "+attr+" "+out[attr]) elif (attr in self.comp.conf[B.SUBJECT_CONN]): out[attr] = self.comp.conf[B.SUBJECT_CONN][attr] print("e " + attr+" "+out[attr]) else: print("f " + attr+" "+out[attr]) return out def selectTables(self, subdir): """ method to delete rows from a database statement written in sql """ self.loadDdl() tdata = {} tdata[subdir] = {} for t in self.comp.conf[B.DATA_NODE_DDL]: tdata[subdir][t] = self.selectRows(t) return tdata def selectRows(self, statement): """ method to select rows from a database statement written in sql """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) def deleteTables(self): """ method to delete rows from a database statement written in sql """ self.loadDdl() for t in self.comp.conf[B.DATA_NODE_DDL]: print("zu loeschende Tabelle "+t) self.deleteRows(t) def deleteRows(self, table): """ method to delete rows from a database statement written in sql """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) def updateRows(self, statement): """ method to delete rows from a database statement written in sql """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) def getConnector(self): """ add-on-method to get the connector this method should only called by the class itself """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) def insertTables(self, tdata): """ method to insert rows into a database statement written in sql """ plainname = basic.componentHandling.getPlainCompname(self.comp.name) self.loadDdl() for t in self.comp.conf[B.DATA_NODE_DDL]: print("einzufuegende Tabelle "+t) if (t in tdata[plainname]): self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA]) def insertRows(self, rows): """ method to insert rows into a database the rows will be interpreted by the ddl of the component """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) def execStatement(self, statement): """ add-on-method to execute the statement this method should only called by the class itself """ raise Exception(B.EXCEPT_NOT_IMPLEMENT) def loadDdl(self): """" load the DDL for each database-table the ddl are mostly stored as csv in the component-folder """ job = basic.program.Job.getInstance() if (B.DATA_NODE_DDL in self.comp.conf): return conf = utils.config_tool.getConfig(D.DDL_FILENAME, self.comp.name) self.comp.conf[B.DATA_NODE_DDL] = {} for k in conf[self.comp.name]: self.comp.conf[B.DATA_NODE_DDL][k] = conf[self.comp.name][k] return conf[self.comp.name] def getWhere(self): return "" def getOrder(self): return "" def getDbValue(self, fo, value): if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES: return self.getDbNull() if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING: return "'"+value.strip()+"'" elif fo[D.DATA_NODE_TYPE] == D.TYPE_INT: return value.strip() elif fo[D.DATA_NODE_TYPE] == D.TYPE_DOUBLE: return self.getDbDouble(value) elif fo[D.DATA_NODE_TYPE] == D.TYPE_FLOAT: return self.getDbFloat(value) elif fo[D.DATA_NODE_TYPE] == D.TYPE_DATE: return self.getDbDate(value) elif fo[D.DATA_NODE_TYPE] == D.TYPE_TIME: return self.getDbTime(value) def getDbDouble(self, value): return value def getDbFloat(self, value): return value def getDbDate(self, value): return value def getDbTime(self, value): return value def getDbNull(self): return B.SVAL_NULL