You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
306 lines
12 KiB
306 lines
12 KiB
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
# ---------------------------------------------------------------------------------------------------------
|
|
# Author : Ulrich Carmesin
|
|
# Source : gitea.ucarmesin.de
|
|
# ---------------------------------------------------------------------------------------------------------
|
|
"""
|
|
This abstract class DbFcts defines the interface for any function relating to any kind of a database-management.
|
|
It uses the following configuration
|
|
.a) COMP.conf->artifact->db->[table] : for structural attributes of the database, maybe specialized for tables \n
|
|
.b) COMP.conf->artifact->ddl : for structural attributes of the database-tables \n
|
|
.c) COMP.conf->conn->[db] : for connection-attributes and structural attributes, maybe specialized for the database
|
|
|
|
The attribute db.type resolves which technique is used, implemented in a specific tool-class:
|
|
* db2,mysql,pgsql,(s)hive ... for specific synchronous RDBs
|
|
* spark,shive,sfile ... using spark-technology
|
|
* csv,(s)file ... for data managed in files
|
|
The main tasks are: \n
|
|
.1. connecting to the datebase - with attributes
|
|
* conn.ip, host, port, user, password, ... for synchronous db-connection
|
|
* conn.root, ... for path-definitions for file-implementations (csv, )
|
|
.2. delete at the initialisation - with
|
|
* db.reset for the context testcase, testsuite or never, on which the content will be deleted - default: testcase
|
|
* db.character of the table if the content will be be deleted in a sequence of testcases
|
|
* db.tabname -- if the tablename is differs from the node-name of the table - default: not necessary
|
|
* db.filename -- if the filename is differs from the node-name of the table - default: not necessary
|
|
* par.dbwhere which rows will be deleted - default empty
|
|
.3. insert testdata at the initialisation
|
|
* ddl.[fields]
|
|
* db.tabname - if the tablename is differs from the node-name of the table - default: not necessary
|
|
* db.filename - if the filename is differs from the node-name of the table - default: not necessary
|
|
.4. select the data with
|
|
* db.tabname - if the tablename is differs from the node-name of the table - default: not necessary
|
|
* db.filename - if the filename is differs from the node-name of the table - default: not necessary
|
|
* ddl._data.[fields].order for an order-clause
|
|
* par.dbwhere which rows will be deleted - default empty
|
|
SPECIAL CASES:
|
|
* If the table is partitioned tables the functions delete/insert/select calls the callback-functions
|
|
COMP.nextTable() resp. COMP.nextTdata().
|
|
"""
|
|
import basic.program
|
|
import utils.config_tool
|
|
import basic.constants as B
|
|
import utils.data_const as D
|
|
import os
|
|
|
|
|
|
|
|
def getDbAttributes(comp, table):
|
|
"""
|
|
this function collects all relevant db-attributes from any location where it can be set.
|
|
The location could be
|
|
* comp.artifact.db.[table].attr
|
|
* comp.artifact.[db].[table].attr
|
|
"""
|
|
out = {
|
|
B.ATTR_DB_DATABASE: "",
|
|
B.ATTR_DB_SCHEMA: "",
|
|
B.ATTR_DB_TABNAME: "",
|
|
B.ATTR_DB_PARTITION: D.DEFAULT_DB_PARTITION,
|
|
B.ATTR_DB_CONN_JAR: D.DEFAULT_DB_CONN_JAR
|
|
}
|
|
for attr in out.keys():
|
|
print(attr)
|
|
if (table in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \
|
|
and (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]):
|
|
out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr]
|
|
print("a " + attr + " " + out[attr])
|
|
elif (attr in comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]):
|
|
out[attr] = comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr]
|
|
print("b " + attr + " " + out[attr])
|
|
elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \
|
|
and (table in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \
|
|
and (attr in comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]):
|
|
out[attr] = comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr]
|
|
print("c " + attr + " " + out[attr])
|
|
elif (B.TOPIC_NODE_DB in comp.conf[B.SUBJECT_CONN]) \
|
|
and (attr in comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]):
|
|
out[attr] = comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr]
|
|
print("d " + attr + " " + out[attr])
|
|
elif (attr in comp.conf[B.SUBJECT_CONN]):
|
|
out[attr] = comp.conf[B.SUBJECT_CONN][attr]
|
|
print("e " + attr + " " + out[attr])
|
|
else:
|
|
print("f " + attr + " " + out[attr])
|
|
return out
|
|
|
|
def getStringIndex(text, intern):
|
|
if intern in text:
|
|
return text.index(intern)
|
|
return 999
|
|
|
|
def parseCondition(condition):
|
|
out = []
|
|
for operator in [">=", "<=", ">", "<", "=", " like ", " in "]:
|
|
if operator in condition:
|
|
i = condition.lower().index(operator)
|
|
field = condition[0:i]
|
|
attr = condition[i+len(operator):]
|
|
out = [operator.strip(), field.strip(), attr.strip()]
|
|
return out
|
|
return out
|
|
|
|
def parseConditions(condition):
|
|
""" the functions parses the condition into their syntactical parts:
|
|
parts = [ [ conjunctor , [ operator , field, attribute ], ... ]
|
|
the condition should be a simple normalform either with "and" or with "or"
|
|
"""
|
|
rest = condition
|
|
dbwhere = []
|
|
if (" and " in rest and " or " in rest):
|
|
raise Exception("the sql-condition must contain only \"ands\" or \"ors\" "+rest)
|
|
|
|
while (" and " in rest or " or " in rest):
|
|
iand = getStringIndex(rest, " and ")
|
|
ior = getStringIndex(rest, " or ")
|
|
print("conjunctors " + str(iand) + " " + str(ior))
|
|
if iand == 999 and ior == 999:
|
|
print("fertig")
|
|
elif iand < ior:
|
|
conjunctor = "and"
|
|
condition = rest[0:iand]
|
|
rest = rest[iand + 5:]
|
|
cond = parseCondition(condition)
|
|
cond.append(conjunctor)
|
|
dbwhere.append(cond)
|
|
elif iand > ior:
|
|
conjunctor = "or"
|
|
condition = rest[0:ior]
|
|
rest = rest[ior + 4:]
|
|
cond = parseCondition(condition)
|
|
cond.append(conjunctor)
|
|
dbwhere.append(cond)
|
|
cond = parseCondition(rest)
|
|
cond.append("end")
|
|
dbwhere.append(cond)
|
|
return dbwhere
|
|
|
|
def parseSQLwhere(condition, ddl=None):
|
|
parts = parseConditions(condition)
|
|
conjunctor = ""
|
|
dbwhere = ""
|
|
for cond in parts:
|
|
if cond[1] in ddl[B.DATA_NODE_HEADER]:
|
|
dbwhere += " "+conjunctor+" "+cond[1]+" "+cond[0]+" "+cond[2]
|
|
conjunctor = cond[3]
|
|
return "WHERE "+dbwhere.strip()
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------------------------
|
|
|
|
class DbFcts():
|
|
"""
|
|
This interface defines each necessary connection to any kind of database.
|
|
The specific technique how to connect to the concrete DBMS has to be implemented in the specific tool.
|
|
"""
|
|
def __init__(self):
|
|
self.comp = None
|
|
pass
|
|
|
|
def setComp(self, comp):
|
|
self.comp = comp
|
|
|
|
def getDbAttributes(self, table):
|
|
"""
|
|
this function collects all relevant db-attributes from any location where it can be set.
|
|
The location could be
|
|
* comp.artifact.db.[table].attr
|
|
* comp.artifact.[db].[table].attr
|
|
"""
|
|
return getDbAttributes(self.comp, table)
|
|
|
|
def xxgetDbAttributes(self, table):
|
|
out = {
|
|
B.ATTR_DB_TABNAME: "",
|
|
B.ATTR_DB_PARTITION: D.DFLT_DB_PARTITION,
|
|
B.ATTR_DB_CONN_JAR: D.DFLT_DB_CONN_JAR
|
|
}
|
|
for attr in out.keys():
|
|
print(attr)
|
|
if (table in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]) \
|
|
and (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table]):
|
|
out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][table][attr]
|
|
print("a "+attr+" "+out[attr])
|
|
elif (attr in self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB]):
|
|
out[attr] = self.comp.conf[B.SUBJECT_ARTS][B.TOPIC_NODE_DB][attr]
|
|
print("b "+attr+" "+out[attr])
|
|
elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \
|
|
and (table in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]) \
|
|
and (attr in self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB]):
|
|
out[attr] = self.comp.conf[B.SUBJECT_CONN][table][B.TOPIC_NODE_DB][attr]
|
|
print("c " + attr+" "+out[attr])
|
|
elif (B.TOPIC_NODE_DB in self.comp.conf[B.SUBJECT_CONN]) \
|
|
and (attr in self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB]):
|
|
out[attr] = self.comp.conf[B.SUBJECT_CONN][B.TOPIC_NODE_DB][attr]
|
|
print("d "+attr+" "+out[attr])
|
|
elif (attr in self.comp.conf[B.SUBJECT_CONN]):
|
|
out[attr] = self.comp.conf[B.SUBJECT_CONN][attr]
|
|
print("e " + attr+" "+out[attr])
|
|
else:
|
|
print("f " + attr+" "+out[attr])
|
|
|
|
return out
|
|
|
|
def selectTables(self, subdir):
|
|
""" method to delete rows from a database
|
|
statement written in sql """
|
|
self.loadDdl()
|
|
tdata = {}
|
|
tdata[subdir] = {}
|
|
for t in self.comp.conf[B.DATA_NODE_DDL]:
|
|
tdata[subdir][t] = self.selectRows(t)
|
|
return tdata
|
|
|
|
def selectRows(self, statement):
|
|
""" method to select rows from a database
|
|
statement written in sql """
|
|
raise Exception(B.EXCEPT_NOT_IMPLEMENT)
|
|
|
|
def deleteTables(self):
|
|
""" method to delete rows from a database
|
|
statement written in sql """
|
|
self.loadDdl()
|
|
for t in self.comp.conf[B.DATA_NODE_DDL]:
|
|
print("zu loeschende Tabelle "+t)
|
|
self.deleteRows(t)
|
|
|
|
def deleteRows(self, table):
|
|
""" method to delete rows from a database
|
|
statement written in sql """
|
|
raise Exception(B.EXCEPT_NOT_IMPLEMENT)
|
|
|
|
def updateRows(self, statement):
|
|
""" method to delete rows from a database
|
|
statement written in sql """
|
|
raise Exception(B.EXCEPT_NOT_IMPLEMENT)
|
|
|
|
def getConnector(self):
|
|
""" add-on-method to get the connector
|
|
this method should only called by the class itself """
|
|
raise Exception(B.EXCEPT_NOT_IMPLEMENT)
|
|
|
|
def insertTables(self, tdata):
|
|
""" method to insert rows into a database
|
|
statement written in sql """
|
|
plainname = basic.componentHandling.getPlainCompname(self.comp.name)
|
|
self.loadDdl()
|
|
for t in self.comp.conf[B.DATA_NODE_DDL]:
|
|
print("einzufuegende Tabelle "+t)
|
|
if (t in tdata[plainname]):
|
|
self.insertRows(t, tdata[plainname][t][B.DATA_NODE_DATA])
|
|
|
|
def insertRows(self, rows):
|
|
""" method to insert rows into a database
|
|
the rows will be interpreted by the ddl of the component
|
|
"""
|
|
raise Exception(B.EXCEPT_NOT_IMPLEMENT)
|
|
|
|
def execStatement(self, statement):
|
|
""" add-on-method to execute the statement
|
|
this method should only called by the class itself """
|
|
raise Exception(B.EXCEPT_NOT_IMPLEMENT)
|
|
|
|
def loadDdl(self):
|
|
"""" load the DDL for each database-table
|
|
the ddl are mostly stored as csv in the component-folder """
|
|
job = basic.program.Job.getInstance()
|
|
if (B.DATA_NODE_DDL in self.comp.conf):
|
|
return
|
|
conf = utils.config_tool.getConfig(D.DDL_FILENAME, self.comp.name)
|
|
self.comp.conf[B.DATA_NODE_DDL] = {}
|
|
for k in conf[self.comp.name]:
|
|
self.comp.conf[B.DATA_NODE_DDL][k] = conf[self.comp.name][k]
|
|
return conf[self.comp.name]
|
|
|
|
def getWhere(self):
|
|
return ""
|
|
def getOrder(self):
|
|
return ""
|
|
|
|
def getDbValue(self, fo, value):
|
|
if len(value.strip()) == 0 and fo[D.DDL_FNULLABLE] == B.SVAL_YES:
|
|
return self.getDbNull()
|
|
if fo[D.DATA_NODE_TYPE] == D.TYPE_STRING:
|
|
return "'"+value.strip()+"'"
|
|
elif fo[D.DATA_NODE_TYPE] == D.TYPE_INT:
|
|
return value.strip()
|
|
elif fo[D.DATA_NODE_TYPE] == D.TYPE_DOUBLE:
|
|
return self.getDbDouble(value)
|
|
elif fo[D.DATA_NODE_TYPE] == D.TYPE_FLOAT:
|
|
return self.getDbFloat(value)
|
|
elif fo[D.DATA_NODE_TYPE] == D.TYPE_DATE:
|
|
return self.getDbDate(value)
|
|
elif fo[D.DATA_NODE_TYPE] == D.TYPE_TIME:
|
|
return self.getDbTime(value)
|
|
|
|
def getDbDouble(self, value):
|
|
return value
|
|
def getDbFloat(self, value):
|
|
return value
|
|
def getDbDate(self, value):
|
|
return value
|
|
def getDbTime(self, value):
|
|
return value
|
|
def getDbNull(self):
|
|
return B.SVAL_NULL
|