Data-Test-Executer Framework speziell zum Test von Datenverarbeitungen mit Datengenerierung, Systemvorbereitungen, Einspielungen, ganzheitlicher diversifizierender Vergleich
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

318 lines
9.7 KiB

#!/usr/bin/python
# -*- coding: utf-8 -*-
# ---------------------------------------------------------------------------------------------------------
# Author : Ulrich Carmesin
# Source : gitea.ucarmesin.de
# ---------------------------------------------------------------------------------------------------------
import json
import os
import datetime
import re
import subprocess
import traceback
import yaml
INSTALLED = False
try:
import basic.program
INSTALLED = True
except:
INSTALLED = False
PROGRAM_NAME = "install_workspace"
CONFIG_FORMAT = "yml"
BASIS_FORMAT = "json"
REPO_NAME = "_name"
REPO_URL = "url"
REPO_BRANCH = "_branch"
job = None
# -----------------------------------------------------------------------------------------
# Miniimplementierung des Programmeahmens
class Logger:
"""
Kurzversion des Messages mit Standardfunktionen
* opel_logs()
* close_logs()
* log_info()
* log_error()
"""
def __init__(self, job, level, logTime, comp):
self.openLog(job, logTime)
def openLog(self, job, logTime):
# job, level, logTime, componente
home = getHome()
path = os.path.join(home, "log")
if not os.path.exists(path):
os.mkdir(path)
logpath = os.path.join(home, "log", job.program+"_"+logTime+".txt")
print("logpath "+logpath)
self.logfile = open(logpath, "w")
def logInfo(self, text):
self.logfile.write(text + "\n")
def setMsg(self, text):
self.logfile.write(text + "\n")
def logError(self, text):
self.logfile.write("ERROR:" + text + "\n")
print("ERROR:" + text)
def closeLog(self):
self.logfile.close()
class ActJob:
"""
Kurzversion des Jobs mit Standardfunktionen
* start_job() startet Job mit Messaging
* set_parameter() setzt Parameter aus args oder aus Aufruf
* stop_job() startet Job mit Messaging
"""
def __init__(self, program):
self.program = program
self.start = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
self.jobid = 100000
self.conf = {}
self.par = {}
def startJob(self):
self.m = Logger(self, "info", self.start, None) # job, level, logTime, componente
text = "# # # Start Job " + self.start + " # # # "
self.m.logInfo(text)
print(text)
def stopJob(self):
self.ende = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
text = "# # # Stop Job " + self.start + " - " + self.ende + " # # # "
self.m.logInfo(text)
self.m.closeLog()
print(text)
def getDebugLevel(self, tool):
return 0
def debug(self, verify, text):
self.m.logInfo(text)
def setParameter(self, args):
for k in args:
setattr(self, k, args[k])
# -----------------------------------------------------------------------------------------
# Standardsteuerung Hauptverarbeitung
def startPyJob(job):
"""
Steuerung der Hauptverarbeitung, aufrufbar vom Programm selbst oder aus job_tool
:param job:
:return:
"""
job.m.logInfo("startPyJob gestertet ")
try:
setParameter(job)
readConfig(job)
createFolders(job)
createGit(job)
createBasisConfig(job)
createDb(job)
except Exception as e:
job.m.logError("+++++++++++++++++++++++++++++++++++++++++++++")
job.m.logError(str(e))
job.m.logError("+++++++++++++++++++++++++++++++++++++++++++++")
job.m.logError("execpt "+traceback.format_exc())
job.m.logError("+++++++++++++++++++++++++++++++++++++++++++++")
# -----------------------------------------------------------------------------------------
# konkrete Verarbeitungsroutinen
def setParameter(job):
job.m.logInfo("--- setze Parameter ")
def readConfig(job):
job.m.logInfo("--- suche config-Datei ")
args = {}
args["home"] = getHome()
configPath = ""
for p in os.listdir(args["home"]):
print(p)
path = os.path.join(args["home"], p)
if os.path.isfile(path) and "workspace" in p:
configPath = path
break
if len(configPath) < 1:
raise Exception("Keine Konfiguration gefunden in "+args["home"])
with open(configPath, 'r') as file:
doc = yaml.full_load(file)
file.close()
for k in doc:
args[k] = doc[k]
job.conf[k] = doc[k]
home = getHome()
for k in job.conf["paths"]:
job.conf["paths"][k] = os.path.join(home, job.conf["paths"][k])
job.setParameter(args)
def createFolders(job):
job.m.logInfo("--- erstelle Verzeichnisse ")
for p in job.paths:
path = os.path.join(job.home, job.paths[p])
createFolder(job, path)
def createFolder(job, path):
if not os.path.exists(path):
os.mkdir(path)
job.m.logInfo("Verzeichnis angelegt: "+ path)
elif not os.path.isdir(path):
job.m.logError("Verzeichnisname existiert und ist kein Verzeichnis "+ path)
else:
job.m.logInfo("Verzeichnis existiert: " + path)
# --------------------------------------------------------------------------------------
# git_tool
# --------------------------------------------------------------------------------------
def createGit(job):
job.m.logInfo("--- erstelle und aktualisiere git-Repos ")
repos = {}
local = {}
attr = {
REPO_NAME: "",
REPO_BRANCH: ""
}
# erstelle Repoliste mit den Attributen: name, branch, url
for r in job.repos:
if r in attr:
attr[r] = job.repos[r]
else:
repo = {}
for a in job.repos[r]:
repo[a] = job.repos[r][a]
repos[r] = repo
for k in attr:
a = k
for r in repos:
if a not in repos[r]:
repos[r][a] = attr[k]
for r in repos:
repo = repos[r]
path = os.path.join(job.home, job.paths[r])
if os.path.exists(path):
local[REPO_URL] = os.path.join(job.home, job.paths[r])
local[REPO_BRANCH] = repo[REPO_BRANCH]
local[REPO_NAME] = repo[REPO_NAME]
rpath = os.path.join(local[REPO_URL], ".git")
if os.path.exists(rpath):
job.m.logInfo("Repo existiert bereits "+r)
else:
job.m.logInfo("Repo erzeugen "+r)
initGit(job, local, repo)
updateLocal(job, local, repo)
else:
job.m.logError("Verzeichnis existiert nicht: " + path)
def initGit(job, local, repo, bare=False):
job.m.logInfo("--- initialisiere git-Repo "+str(repo)+","+str(local))
os.chdir(local[REPO_URL])
cmd = "git init "
if bare:
cmd += " --bare"
execCmd(job, cmd)
cmd = "git checkout " + local[REPO_BRANCH]
execCmd(job, cmd)
cmd = "git remote add " + repo[REPO_NAME] + " " + repo[REPO_URL]
execCmd(job, cmd)
os.chdir(job.home)
def execCmd(job, cmd):
job.m.logInfo(cmd)
text = ""
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
btext = process.communicate()[0]
text = btext.decode('utf-8')
job.m.logInfo(text)
return text
def checkoutLocal(job, local):
os.chdir(local[REPO_URL])
cmd = "git checkout " + local[REPO_BRANCH]
text = execCmd(job, cmd)
return text
def updateLocal(job, local, repo):
job.m.logInfo("--- aktualisiere git-Repo "+str(repo)+","+str(local))
text = checkoutLocal(job, local)
# if len(text) > 0 and re.match(r"[MA]\s\w+", text):
match = re.search(r"([DMA])\s(\S+)", text)
if match is not None:
os.chdir(job.home)
job.m.logError("ERROR: lokales Repo " + local[REPO_URL] + ", " + local[REPO_BRANCH] + " hat uncommited Aenderungen")
print("regex gefunden")
return
cmd = "git pull " + repo[REPO_NAME] + " " + repo[REPO_BRANCH]
text = execCmd(job, cmd)
job.m.logInfo(text)
os.chdir(job.home)
def updateRemote(job, local, repo):
job.m.logInfo("--- aktualisiere git-Repo "+str(repo)+","+str(local))
text = checkoutLocal(job, local)
cmd = "git push " + repo[REPO_NAME] + " " + repo[REPO_BRANCH]
text = execCmd(job, cmd)
os.chdir(job.home)
def createBasisConfig(job):
job.m.logInfo("--- erstelle Basis-Koniguration ")
config = {}
config["basic"] = {}
config["basic"]["paths"] = {}
config["basic"]["paths"]["home"] = job.home
for p in job.paths:
path = os.path.join(job.home, job.paths[p])
config["basic"]["paths"][p] = path
for p in ["temp", "config"]:
path = os.path.join(job.home, p)
createFolder(job, path)
config["basic"]["paths"][p] = path
if BASIS_FORMAT == "yml":
path = os.path.join(job.home, "config", "basis.json")
with open(path, 'w', encoding="utf-8") as file:
doc = yaml.dump(config, file)
file.write(doc)
file.close()
elif BASIS_FORMAT == "json":
path = os.path.join(job.home, "config", "basis.json")
with open(path, 'w', encoding="utf-8") as file:
doc = json.dumps(config, indent=4)
file.write(doc)
file.close()
def createDb(job):
if "db" in job.conf:
import basic.connection
import basic.Testserver
testserver = basic.Testserver.Testserver(job)
testserver.createDBTables(job)
def getHome():
home = os.getcwd()
if home[-7:] == "program":
home = home[0:-8]
return home
# -----------------------------------------------------------------------------------------
# Pythonstandard Programmaufruf
# Job-Objekt erzeugen und beenden
if __name__ == '__main__':
if INSTALLED:
#job = basic.program.Job(PROGRAM_NAME)
job = ActJob(PROGRAM_NAME)
else:
job = ActJob(PROGRAM_NAME)
job.startJob()
startPyJob(job)
job.stopJob()