Browse Source

refactoring around job_tool

refactor
Ulrich 2 years ago
parent
commit
b5a55cc6a4
  1. 6
      catalog/programs.csv
  2. 15
      test/test_10job.py
  3. 2
      test/testtools.py
  4. 162
      tools/config_tool.py
  5. 2
      tools/path_const.py
  6. 2
      tools/path_tool.py
  7. 8
      tools/value_tool.py

6
catalog/programs.csv

@ -10,10 +10,10 @@ table:programs;name;objtype;objname;time;env;app;variant;pardef;pfilesource;pfil
;exec_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcdir;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt ;exec_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcdir;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt
;collect_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcdir;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt ;collect_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcdir;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt
;compare_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcdir;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt ;compare_testcase;tc;m;m;m;m;o;;tcparfile;tcparfile;tcdir;tcbase;info;{job.par.tcdir}/{log}/{job.program}_{job.start}.txt
;check_environment;env;;;m;;o;;;envparfile;envdir;envbase;info;{job.par.envdir}/{log}/log_{job.start}.txt ;check_environment;env;;;m;;o;"{""par"":{""application"":""service"",""environment"":""args"",""variant"":""""}}";;envparfile;envdir;envbase;info;{job.par.envdir}/{log}/log_{job.start}.txt
;check_specification;tp,ts,tc;o;;;;n;;;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt ;check_specification;tp,ts,tc;o;;;;n;;;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt
;check_configuration;ws;o;;;o;n;;;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt ;check_configuration;ws;o;;;o;n;"{""par"":{""application"":""args"",""environment"":""Testserver"",""variant"":""""}}";;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt
;make_workspace;ws;;;_;;n;;;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt ;make_workspace;ws;;;_;;n;"{""par"":{""application"":""service"",""environment"":""Testserver"",""variant"":""""}}";;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt
;unit_tester;ws;o;;d;;n;"{""par"":{""application"":""service"",""environment"":""Testserver"",""variant"":""""}}";;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt ;unit_tester;ws;o;;d;;n;"{""par"":{""application"":""service"",""environment"":""Testserver"",""variant"":""""}}";;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt
;declare_expection;ts,tc;m;m;m;m;o;;;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt ;declare_expection;ts,tc;m;m;m;m;o;;;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt
;clean_workspace;ws;o ;;_;;n;;;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt ;clean_workspace;ws;o ;;_;;n;;;;wsdir;wsbase;info;{job.par.wsdir}/{log}/log_{job.start}.txt

1 _key name
10 exec_testcase tc m m m m o tcparfile tcparfile tcdir tcbase info {job.par.tcdir}/{log}/{job.program}_{job.start}.txt
11 collect_testcase tc m m m m o tcparfile tcparfile tcdir tcbase info {job.par.tcdir}/{log}/{job.program}_{job.start}.txt
12 compare_testcase tc m m m m o tcparfile tcparfile tcdir tcbase info {job.par.tcdir}/{log}/{job.program}_{job.start}.txt
13 check_environment env m o {"par":{"application":"service","environment":"args","variant":""}} envparfile envdir envbase info {job.par.envdir}/{log}/log_{job.start}.txt
14 check_specification tp,ts,tc o n wsdir wsbase info {job.par.wsdir}/{log}/log_{job.start}.txt
15 check_configuration ws o o n {"par":{"application":"args","environment":"Testserver","variant":""}} wsdir wsbase info {job.par.wsdir}/{log}/log_{job.start}.txt
16 make_workspace ws _ n {"par":{"application":"service","environment":"Testserver","variant":""}} wsdir wsbase info {job.par.wsdir}/{log}/log_{job.start}.txt
17 unit_tester ws o d n {"par":{"application":"service","environment":"Testserver","variant":""}} wsdir wsbase info {job.par.wsdir}/{log}/log_{job.start}.txt
18 declare_expection ts,tc m m m m o wsdir wsbase info {job.par.wsdir}/{log}/log_{job.start}.txt
19 clean_workspace ws o _ n wsdir wsbase info {job.par.wsdir}/{log}/log_{job.start}.txt

15
test/test_10job.py

@ -4,6 +4,7 @@ import inspect
import shutil import shutil
import tools.path_tool import tools.path_tool
import tools.job_tool
import basic.program import basic.program
from basic.componentHandling import ComponentManager from basic.componentHandling import ComponentManager
import test.constants import test.constants
@ -14,7 +15,8 @@ import tools.file_tool
HOME_PATH = test.constants.HOME_PATH HOME_PATH = test.constants.HOME_PATH
PYTHON_CMD = "python" PYTHON_CMD = "python"
TEST_FUNCTIONS = ["test_00init"] TEST_FUNCTIONS = ["test_00init", "test_30startActJob"]
TEST_FUNCTIONS = ["test_30startActJob"]
PROGRAM_NAME = "clean_workspace" PROGRAM_NAME = "clean_workspace"
@ -44,6 +46,17 @@ class MyTestCase(unittest.TestCase):
self.assertIn("configpath", job.conf) self.assertIn("configpath", job.conf)
self.assertIn("paths", job.conf) self.assertIn("paths", job.conf)
def test_30startActJob(self):
global mymsg
actfunction = str(inspect.currentframe().f_code.co_name)
cnttest = 0
if actfunction not in TEST_FUNCTIONS:
return
# simple job instantiate - without parameter and only simple messaging
job = basic.program.Job(PROGRAM_NAME)
path = tools.path_tool.getActualJsonPath(job)
args = tools.file_tool.read_file_dict(job, path, job.m)
tools.job_tool.start_child_process(job, args)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

2
test/testtools.py

@ -76,7 +76,7 @@ def getJob(pgran="", papp="", penv="", ptstamp="", pmode=""):
path = DEFAULT_ARCHIV_DIR + "/testlauf/TST001_" + tstamp path = DEFAULT_ARCHIV_DIR + "/testlauf/TST001_" + tstamp
#job.conf[B.SUBJECT_PATH]["components"] = T.COMP_PATH #job.conf[B.SUBJECT_PATH]["components"] = T.COMP_PATH
args = {"application": app, "environment": env, "modus": mode, gran+"time": tstamp, args = {"application": app, "environment": env, "modus": mode, gran+"time": tstamp,
gran+"dir": path, gran+"dir": path, "project": "TESTPROJ",
"step": 2} "step": 2}
# "usecase": "TST001", "tstime": "2022-03-17_17-28"} # "usecase": "TST001", "tstime": "2022-03-17_17-28"}
job = basic.program.Job("unit_tester", "", args) job = basic.program.Job("unit_tester", "", args)

162
tools/config_tool.py

@ -36,7 +36,7 @@ def getExistingPath(job, pathnames):
return pathname return pathname
return None return None
def getConfigPath(job, modul, name, subname=""): def select_config_path(job, modul, name, subname=""):
""" """
gets the most specified configuration of different sources gets the most specified configuration of different sources
Parameter: Parameter:
@ -55,26 +55,10 @@ def getConfigPath(job, modul, name, subname=""):
if job is None: if job is None:
verify = False # job = basic.program.Job.getInstance() verify = False # job = basic.program.Job.getInstance()
else: else:
verify = job.getDebugLevel("config_tool")-4 verify = job.getDebugLevel("config_tool")
if verify: job.debug(verify, "getConfig " + modul + ", " + name) if verify: job.debug(verify, "getConfig " + modul + ", " + name)
if modul == P.KEY_TOOL: if modul == P.KEY_TOOL:
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS], return getToolPath(job, name, subname)
P.VAL_CONFIG, name))
if path is not None:
return path
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_HOME],
P.VAL_CONFIG, name))
if path is not None:
return path
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_PROGRAM],
P.VAL_UTIL, P.VAL_CONFIG, name))
if path is not None:
return path
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_ENV],
job.par.environment, P.VAL_CONFIG, P.KEY_TOOL+"_"+ name))
if path is not None:
return path
raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name)
elif modul == P.KEY_COMP: elif modul == P.KEY_COMP:
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_HOME], pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_HOME],
@ -91,20 +75,87 @@ def getConfigPath(job, modul, name, subname=""):
if verify: job.debug(verify, "6 " + pathname) if verify: job.debug(verify, "6 " + pathname)
raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name) raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name)
elif modul in COMP_FILES: elif modul in COMP_FILES:
return getCompPath(job, name, subname, modul)
elif modul == P.KEY_BASIC:
return getBasicPath(job, name)
elif modul == P.KEY_TESTCASE:
return getTestPath(job, name, D.DFILE_TESTCASE_NAME)
elif modul == P.KEY_TESTSUITE:
return getTestPath(job, name, D.DFILE_TESTSUITE_NAME)
elif modul == P.KEY_CATALOG:
return getCatalogPath(job, name)
else:
pathname = tools.path_tool.compose_path(job, P.P_TCPARFILE)
if verify: job.debug(verify, "7 " + pathname)
if os.path.exists(pathname):
return pathname
pathname = tools.path_tool.compose_path(job, P.P_TSPARFILE)
if verify: job.debug(verify, "8 " + pathname)
if os.path.exists(pathname):
return pathname
for format in CONFIG_FORMAT:
if len(subname) > 1:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_RELEASE],
P.VAL_CONFIG, "basis."+format)
if verify: job.debug(verify, "9 " + pathname)
if os.path.exists(pathname):
return pathname
for format in CONFIG_FORMAT:
if len(subname) > 1:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_ENV],
P.VAL_CONFIG, "basis."+format)
if verify: job.debug(verify, "9 " + pathname)
if os.path.exists(pathname):
return pathname
for format in CONFIG_FORMAT:
if len(subname) > 1:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_HOME],
P.VAL_CONFIG, "basis."+format)
if verify: job.debug(verify, "9 " + pathname)
if os.path.exists(pathname):
return pathname
raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name)
def getToolPath(job, name, subname):
if subname != "":
envdir = subname
elif hasattr(job.par, "environment"):
envdir = job.par.environment
else:
envdir = "xxxxx"
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_ENV],
envdir, P.VAL_CONFIG, P.KEY_TOOL + "_" + name))
if path is not None:
return path
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS],
P.VAL_CONFIG, name))
if path is not None:
return path
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_HOME],
P.VAL_CONFIG, name))
if path is not None:
return path
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_PROGRAM],
P.VAL_UTIL, P.VAL_CONFIG, name))
if path is not None:
return path
raise Exception(P.EXP_CONFIG_MISSING, envdir + ", " + name)
def getCompPath(job, name, subname, filename):
# for example DATASTRUCURE or the table # for example DATASTRUCURE or the table
pathnames = [] pathnames = []
pathnames.append(os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS], pathnames.append(os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS],
basic.componentHandling.getComponentFolder(name), modul)) basic.componentHandling.getComponentFolder(name), filename))
pathnames.append(os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS], pathnames.append(os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS],
basic.componentHandling.getComponentFolder(subname), modul)) basic.componentHandling.getComponentFolder(subname), filename))
pathnames.append(os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_PROGRAM], P.VAL_BASIC, modul)) pathnames.append(os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_PROGRAM], P.VAL_BASIC, filename))
pathnames.append(os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_PROGRAM], P.VAL_BASIC, subname)) pathnames.append(os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_PROGRAM], P.VAL_BASIC, subname))
configpath = getExistingPath(job, pathnames) configpath = getExistingPath(job, pathnames)
if configpath is not None: if configpath is not None:
return configpath return configpath
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS], pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS],
basic.componentHandling.getComponentFolder(name), modul+"."+format) basic.componentHandling.getComponentFolder(name), filename + "." + format)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
@ -113,8 +164,10 @@ def getConfigPath(job, modul, name, subname=""):
basic.componentHandling.getComponentFolder(name), subname + "." + format) basic.componentHandling.getComponentFolder(name), subname + "." + format)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name) raise Exception(P.EXP_CONFIG_MISSING, filename + ", " + name)
elif modul == P.KEY_BASIC:
def getBasicPath(job, name):
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS], path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_COMPONENTS],
P.VAL_CONFIG, name)) P.VAL_CONFIG, name))
if path is not None: if path is not None:
@ -123,21 +176,21 @@ def getConfigPath(job, modul, name, subname=""):
P.VAL_CONFIG, name)) P.VAL_CONFIG, name))
if path is not None: if path is not None:
return path return path
elif modul == P.KEY_TESTCASE:
for format in CONFIG_FORMAT: def getTestPath(job, name, filename):
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_TDATA],
name, D.DFILE_TESTCASE_NAME + "."+format)
if verify: job.debug(verify, "4 " + pathname)
if os.path.exists(pathname):
return pathname
elif modul == P.KEY_TESTSUITE:
for format in CONFIG_FORMAT: for format in CONFIG_FORMAT:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_TDATA], pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_TDATA],
name, D.DFILE_TESTSUITE_NAME + "." + format) name, filename + "." + format)
if verify: job.debug(verify, "4 " + pathname)
if os.path.exists(pathname): if os.path.exists(pathname):
return pathname return pathname
elif modul == P.KEY_CATALOG:
def getCatalogPath(job, name):
if hasattr(job, "par") and hasattr(job.par, "project"):
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_TDATA],
getattr(job.par, "project"), P.KEY_CATALOG, name))
if path is not None:
return path
path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_TDATA], path = getExistingPath(job, os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_TDATA],
P.KEY_CATALOG, name)) P.KEY_CATALOG, name))
if path is not None: if path is not None:
@ -151,37 +204,6 @@ def getConfigPath(job, modul, name, subname=""):
if path is not None: if path is not None:
return path return path
raise Exception(P.EXP_CONFIG_MISSING, name) raise Exception(P.EXP_CONFIG_MISSING, name)
else:
pathname = tools.path_tool.compose_path(job, P.P_TCPARFILE)
if verify: job.debug(verify, "7 " + pathname)
if os.path.exists(pathname):
return pathname
pathname = tools.path_tool.compose_path(job, P.P_TSPARFILE)
if verify: job.debug(verify, "8 " + pathname)
if os.path.exists(pathname):
return pathname
for format in CONFIG_FORMAT:
if len(subname) > 1:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_RELEASE],
P.VAL_CONFIG, "basis."+format)
if verify: job.debug(verify, "9 " + pathname)
if os.path.exists(pathname):
return pathname
for format in CONFIG_FORMAT:
if len(subname) > 1:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_ENV],
P.VAL_CONFIG, "basis."+format)
if verify: job.debug(verify, "9 " + pathname)
if os.path.exists(pathname):
return pathname
for format in CONFIG_FORMAT:
if len(subname) > 1:
pathname = os.path.join(job.conf[B.SUBJECT_PATH][P.ATTR_PATH_HOME],
P.VAL_CONFIG, "basis."+format)
if verify: job.debug(verify, "9 " + pathname)
if os.path.exists(pathname):
return pathname
raise Exception(P.EXP_CONFIG_MISSING, modul+", "+name)
def getConfValue(attribute, comp): def getConfValue(attribute, comp):
@ -223,12 +245,12 @@ def hasAttr(o, name):
def getConfig(job, modul, name, subname=""): def getConfig(job, modul, name, subname=""):
if job is None: if job is None:
verify = 24 verify = 0
else: else:
verify = job.getDebugLevel("config_tool")-4 verify = job.getDebugLevel("config_tool")
msg = None msg = None
if hasattr(job, "m"): msg = job.m if hasattr(job, "m"): msg = job.m
pathname = getConfigPath(job, modul, name, subname) pathname = select_config_path(job, modul, name, subname)
confs = {} confs = {}
job.debug(verify, "getConfig " + pathname) job.debug(verify, "getConfig " + pathname)
if len(pathname) < 1: if len(pathname) < 1:

2
tools/path_const.py

@ -108,6 +108,8 @@ P_TSSUM = "tssum"
P_XPBASE = "xpbase" P_XPBASE = "xpbase"
P_XPRESULT = "xpresult" P_XPRESULT = "xpresult"
P_XPBACKUP = "xpbackup" P_XPBACKUP = "xpbackup"
P_TDROOT = "tdroot"
P_TDBASE = "tdbase"
# ------------------------------------------------------------- # -------------------------------------------------------------
# exception texts # exception texts

2
tools/path_tool.py

@ -50,7 +50,7 @@ def getBasisConfigPath():
def getActualJsonPath(job): def getActualJsonPath(job):
username = getpass.getuser() username = getpass.getuser()
path = os.path.join(job.conf.confs[B.SUBJECT_PATH][B.ATTR_PATH_DEBUG], username+"Job.json") path = os.path.join(job.conf[B.SUBJECT_PATH][B.ATTR_PATH_DEBUG], username+"Job.json")
return path return path

8
tools/value_tool.py

@ -49,7 +49,7 @@ def get_key_value(job, key, comp=None):
:return: :return:
""" """
try: try:
verify = job.getDebugLevel(TOOL_NAME)-4 verify = job.getDebugLevel(TOOL_NAME)
except: except:
verify = False verify = False
#pt = PathConf.getInstance(job) #pt = PathConf.getInstance(job)
@ -80,9 +80,9 @@ def get_key_value(job, key, comp=None):
if tools.config_tool.hasAttr(comp, key[5:]): if tools.config_tool.hasAttr(comp, key[5:]):
return tools.config_tool.getAttr(comp, key[5:]) return tools.config_tool.getAttr(comp, key[5:])
return "" return ""
elif DOM_ENV in key: elif DOM_ENV + "." in key:
if key[4:] in comp.conf["conn"]: if key[4:] in comp["conn"]:
return comp.conf["conn"][key[4:]] return comp["conn"][key[4:]]
pass pass
elif DOM_CTLG in key: elif DOM_CTLG in key:
a = key.split(".") a = key.split(".")

Loading…
Cancel
Save