@ -41,11 +41,14 @@ class DbFcts(utils.db_abstract.DbFcts):
for k in sqls.keys():
sql = sqls[k]
if dry == 1:
spark = self.getConnector()
df = spark.sql(sql)
dfj = df.toJSON()
for r in dfj.collect():
data.append(json.loads(r))
try:
except:
self.comp.m.setError("Table couldnt read "+table)
else:
print("select "+sql)
self.comp.m.logInfo(sql)
@ -38,6 +38,8 @@ def getFiles(msg, path, pattern, conn):
verify = int(job.getDebugLevel("file_tool"))
out = []
msg.debug(verify, "getFiles " + path + " , " + pattern)
if not os.path.exists(path):
return out
for f in os.listdir(path):
msg.debug(verify, "getFiles " + f)
if re.search(pattern, f):