You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
255 lines
9.4 KiB
255 lines
9.4 KiB
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
# ---------------------------------------------------------------------------------------------------------
|
|
# Author : Ulrich Carmesin
|
|
# Source : gitea.ucarmesin.de
|
|
# ---------------------------------------------------------------------------------------------------------
|
|
import json
|
|
import re
|
|
import basic.program
|
|
import tools.file_abstract
|
|
import basic.constants as B
|
|
import tools.data_const as D
|
|
import tools.file_tool
|
|
|
|
|
|
class FileFcts(tools.file_abstract.FileFcts):
|
|
|
|
def __init__(self):
|
|
pass
|
|
|
|
def loadFile(self, path):
|
|
"""
|
|
this function parses the text and translates it to dict
|
|
:param text:
|
|
:return:
|
|
"""
|
|
lines = tools.file_tool.readFileLines(self.job, path, self.getMsg())
|
|
return self.parseCsv(self.getMsg(), self.job, lines)
|
|
|
|
def parseCsv(self, msg, job, lines, ttype=""):
|
|
"""
|
|
|
|
:param msg:
|
|
:param lines:
|
|
:param type:
|
|
:param job:
|
|
:return:
|
|
"""
|
|
tdata = {}
|
|
status = "start"
|
|
verbose = False
|
|
tableAttr = {} # table
|
|
tableDict = {} # table
|
|
for l in lines:
|
|
if verbose: print("lines "+l)
|
|
fields = splitFields(l, D.CSV_DELIMITER, job)
|
|
# check empty line, comment
|
|
if (len(fields) < 1) or (len(l.strip().replace(D.CSV_DELIMITER,"")) < 1):
|
|
status = "start"
|
|
continue
|
|
if (fields[0][0:1] == "#"):
|
|
continue
|
|
a = fields[0].lower().split(":")
|
|
# keywords option, step, table
|
|
if verbose: print(str(a)+" -- "+str(fields))
|
|
tableAttr = setTableAttribute(tableAttr, a[0], fields[1], job)
|
|
if a[0].lower() in D.LIST_DATA_ATTR:
|
|
status = "TABLE_ALIAS"
|
|
if a[0].lower() == D.DATA_ATTR_KEY:
|
|
ttype = D.CSV_SPECTYPE_KEYS
|
|
continue
|
|
if (a[0].lower() in [D.CSV_BLOCK_HEAD]):
|
|
if verbose: print("head "+l)
|
|
setTdataLine(tdata, fields, D.CSV_BLOCK_HEAD, job)
|
|
status = "start"
|
|
continue
|
|
elif (a[0].lower() == D.CSV_BLOCK_OPTION):
|
|
if verbose: print("option " + l)
|
|
setTdataLine(tdata, fields, D.CSV_BLOCK_OPTION, job)
|
|
status = "start"
|
|
continue
|
|
elif (a[0].lower() == D.CSV_BLOCK_STEP):
|
|
if verbose: print("step "+l)
|
|
step = basic.step.parseStep(job, fields)
|
|
if D.CSV_BLOCK_STEP not in tdata:
|
|
tdata[D.CSV_BLOCK_STEP] = []
|
|
tdata[D.CSV_BLOCK_STEP].append(step)
|
|
status = "start"
|
|
continue
|
|
elif (a[0].lower() == D.CSV_BLOCK_IMPORT):
|
|
if verbose: print("includes " + l)
|
|
if D.CSV_BLOCK_IMPORT not in tdata:
|
|
tdata[D.CSV_BLOCK_IMPORT] = []
|
|
tdata[D.CSV_BLOCK_IMPORT].append(fields[1])
|
|
status = "start"
|
|
continue
|
|
elif (a[0].lower() == D.CSV_BLOCK_TABLES) or (a[0].lower() in D.CSV_HEADER_START):
|
|
if verbose: print("tables "+l)
|
|
h = a
|
|
h[0] = B.DATA_NODE_TABLES
|
|
if ttype == D.CSV_SPECTYPE_CONF:
|
|
del h[0]
|
|
tableDict = getTdataContent(msg, tdata, h)
|
|
setTableHeader(tableDict, tableAttr, fields, ttype, job)
|
|
status = D.CSV_SPECTYPE_DATA
|
|
elif (status == D.CSV_SPECTYPE_DATA):
|
|
tableDict = getTdataContent(msg, tdata, h)
|
|
if verbose: print("setTableData "+str(h)+" "+str(tableDict))
|
|
setTableData(tableDict, fields, ttype, job)
|
|
elif (status == "TABLE_ALIAS") and D.DATA_ATTR_ALIAS in tdata:
|
|
alias = tdata[D.DATA_ATTR_ALIAS]
|
|
b = alias.split(":")
|
|
h = [B.DATA_NODE_TABLES] + b
|
|
tableDict = getTdataContent(msg, tdata, h)
|
|
tableDict[D.DATA_ATTR_ALIAS] = alias
|
|
fields = [alias] + fields
|
|
setTableHeader(tableDict, tableAttr, fields, ttype, job)
|
|
status = D.CSV_SPECTYPE_DATA
|
|
if ttype == D.CSV_SPECTYPE_CONF:
|
|
header = []
|
|
for k in tdata:
|
|
if k in D.LIST_DATA_ATTR:
|
|
continue
|
|
if B.DATA_NODE_DATA in tdata[k]:
|
|
tdata[k].pop(B.DATA_NODE_DATA)
|
|
for f in tdata[k]:
|
|
if f in [B.DATA_NODE_HEADER, "_hit"] + D.LIST_DATA_ATTR:
|
|
continue
|
|
header.append(f)
|
|
tdata[k][B.DATA_NODE_HEADER] = header
|
|
header = []
|
|
if B.DATA_NODE_TABLES in tdata and B.DATA_NODE_TABLES in tdata[B.DATA_NODE_TABLES]:
|
|
for k in tdata[B.DATA_NODE_TABLES][B.DATA_NODE_TABLES]:
|
|
if k in tdata[B.DATA_NODE_TABLES]:
|
|
if verbose: print("Error")
|
|
else:
|
|
tdata[B.DATA_NODE_TABLES][k] = tdata[B.DATA_NODE_TABLES][B.DATA_NODE_TABLES][k]
|
|
tdata[B.DATA_NODE_TABLES].pop(B.DATA_NODE_TABLES)
|
|
return tdata
|
|
|
|
def splitFields(line, delimiter, job):
|
|
out = []
|
|
fields = line.split(delimiter)
|
|
for i in range(0, len(fields)):
|
|
if fields[i][0:1] == "#":
|
|
break
|
|
if re.match(r"^\"(.*)\"$", fields[i]):
|
|
fields[i] = fields[i][1:-1]
|
|
if fields[i].find("{\"") == 0:
|
|
if fields[i].find("{\"\"") == 0:
|
|
fields[i] = fields[i].replace("\"\"", "\"")
|
|
try:
|
|
val = json.loads(fields[i])
|
|
fields[i] = val
|
|
except Exception as e:
|
|
pass
|
|
out.append(fields[i])
|
|
return out
|
|
|
|
def setTableAttribute(tableAttr, key, val, job):
|
|
for attr in D.LIST_DATA_ATTR:
|
|
if (key.lower() == attr):
|
|
tableAttr[attr] = val.strip()
|
|
tableAttr["_hit"] = True
|
|
return tableAttr
|
|
tableAttr["_hit"] = False
|
|
return tableAttr
|
|
|
|
def setTdataLine(tdata, fields, block, job):
|
|
"""
|
|
sets field(s) into tdata as a key-value-pair
|
|
additional fields will be concatenate to a intern separated list
|
|
:param tdata:
|
|
:param fields:
|
|
:param block:
|
|
:param job:
|
|
:return:
|
|
"""
|
|
a = fields[0].lower().split(":")
|
|
a[0] = block # normalized key
|
|
val = ""
|
|
for i in range(1, len(fields)-1):
|
|
val += D.INTERNAL_DELIMITER+fields[i]
|
|
if len(val) > len(D.INTERNAL_DELIMITER):
|
|
val = val[len(D.INTERNAL_DELIMITER):]
|
|
setTdataContent(job.m, tdata, val, a)
|
|
return tdata
|
|
|
|
def setTdataContent(msg, data, tabledata, path):
|
|
setTdataStructure(msg, data, path)
|
|
if len(path) == 2:
|
|
data[path[0]][path[1]] = tabledata
|
|
elif len(path) == 3:
|
|
data[path[0]][path[1]][path[2]] = tabledata
|
|
elif len(path) == 4:
|
|
data[path[0]][path[1]][path[2]][path[3]] = tabledata
|
|
|
|
def setTdataStructure(msg, data, path):
|
|
if len(path) >= 1 and path[0] not in data:
|
|
data[path[0]] = {}
|
|
if len(path) >= 2 and path[1] not in data[path[0]]:
|
|
data[path[0]][path[1]] = {}
|
|
if len(path) >= 3 and path[2] not in data[path[0]][path[1]]:
|
|
data[path[0]][path[1]][path[2]] = {}
|
|
if len(path) >= 4 and path[3] not in data[path[0]][path[1]][path[2]]:
|
|
data[path[0]][path[1]][path[2]][path[3]] = {}
|
|
return data
|
|
|
|
def getTdataContent(msg, data, path):
|
|
setTdataStructure(msg, data, path)
|
|
if len(path) == 2:
|
|
return data[path[0]][path[1]]
|
|
elif len(path) == 3:
|
|
return data[path[0]][path[1]][path[2]]
|
|
elif len(path) == 4:
|
|
return data[path[0]][path[1]][path[2]][path[3]]
|
|
elif len(path) == 1:
|
|
return data[path[0]]
|
|
else:
|
|
return None
|
|
|
|
def setTableHeader(tableDict, tableAttr, fields, ttype, job):
|
|
header = []
|
|
for i in range(1, len(fields)):
|
|
header.append(fields[i].strip())
|
|
tableDict[B.DATA_NODE_HEADER] = header
|
|
for attr in tableAttr:
|
|
tableDict[attr] = tableAttr[attr]
|
|
# preparate the sub-structure for row-data
|
|
if ttype == D.CSV_SPECTYPE_TREE:
|
|
tableDict[B.DATA_NODE_DATA] = {}
|
|
elif ttype == D.CSV_SPECTYPE_KEYS:
|
|
tableDict[D.CSV_NODETYPE_KEYS] = {}
|
|
tableDict[D.DATA_ATTR_KEY] = 1
|
|
if D.DATA_ATTR_KEY in tableAttr:
|
|
tableDict[D.DATA_ATTR_KEY] = header.index(tableAttr[D.DATA_ATTR_KEY]) + 1
|
|
else:
|
|
tableDict[B.DATA_NODE_DATA] = []
|
|
return tableDict
|
|
|
|
def setTableData(tableDict, fields, ttype, job):
|
|
row = {}
|
|
if ttype == D.CSV_SPECTYPE_DATA and ":" not in fields[0] and D.DATA_ATTR_ALIAS in tableDict:
|
|
fields = [tableDict[D.DATA_ATTR_ALIAS]] + fields
|
|
i = 1
|
|
for f in tableDict[B.DATA_NODE_HEADER]:
|
|
row[f] = fields[i]
|
|
i += 1
|
|
if ttype == D.CSV_SPECTYPE_DATA:
|
|
if B.ATTR_DATA_COMP in tableDict:
|
|
tcomps = tableDict[B.ATTR_DATA_COMP]
|
|
else:
|
|
tcomps = {}
|
|
row[B.ATTR_DATA_COMP] = {}
|
|
for c in fields[0].split(","):
|
|
a = c.split(":")
|
|
tcomps[a[0]] = a[1]
|
|
row[B.ATTR_DATA_COMP][a[0]] = a[1].strip()
|
|
tableDict[B.DATA_NODE_DATA].append(row)
|
|
tableDict[B.ATTR_DATA_COMP] = tcomps
|
|
elif ttype == D.CSV_SPECTYPE_KEYS:
|
|
tableDict[D.CSV_NODETYPE_KEYS][fields[tableDict[D.DATA_ATTR_KEY]].strip()] = row
|
|
elif ttype == D.CSV_SPECTYPE_CONF:
|
|
tableDict[fields[1]] = row
|
|
return tableDict
|
|
|