1import sys 

2import os 

3import yaml 

4import re 

5 

6from itertools import chain 

7 

8from control.utils import pick as G, serverprint, cap1, E, LOW, HYPHEN 

9 

10SERVER_PATH = os.path.split(os.path.realpath(__file__))[0] 

11 

12CONFIG_EXT = ".yaml" 

13CONFIG_DIR = f"{SERVER_PATH}/yaml" 

14TABLE_DIR = f"{SERVER_PATH}/tables" 

15 

16ALL = "all" 

17NAMES = "names" 

18 

19TERSE = True 

20 

21 

22class Config: 

23 pass 

24 

25 

26class Base: 

27 pass 

28 

29 

30class Mongo: 

31 pass 

32 

33 

34class Web: 

35 pass 

36 

37 

38class Perm: 

39 pass 

40 

41 

42class Workflow: 

43 pass 

44 

45 

46class Clean: 

47 pass 

48 

49 

50class Tables: 

51 @classmethod 

52 def showReferences(cls): 

53 reference = cls.reference 

54 serverprint("""\nREFERENCE FIELD DEPENDENCIES""") 

55 for (dep, tables) in sorted(reference.items()): 

56 serverprint(dep) 

57 for (table, fields) in tables.items(): 

58 serverprint(f"""\t{table:<20}: {", ".join(fields)}""") 

59 

60 

61class Names: 

62 @staticmethod 

63 def isName(val): 

64 return val.replace(LOW, E).replace(HYPHEN, E).isalnum() 

65 

66 @staticmethod 

67 def getNames(source, val, doString=True, inner=False): 

68 names = set() 

69 pureNames = set() 

70 good = True 

71 

72 if type(val) is str: 

73 names = {val} if doString and Names.isName(val) else set() 

74 elif type(val) is list: 

75 for v in val: 

76 if type(v) is str and Names.isName(v): 

77 names.add(v) 

78 elif type(v) is dict: 

79 names |= Names.getNames(source, v, doString=False, inner=True) 

80 elif type(val) is dict: 

81 for (k, v) in val.items(): 

82 if inner or k != NAMES: 

83 if type(k) is str and Names.isName(k): 

84 names.add(k) 

85 names |= Names.getNames(source, v, doString=False, inner=True) 

86 else: 

87 for val in v: 

88 if type(val) is not str: 88 ↛ 89line 88 didn't jump to line 89, because the condition on line 88 was never true

89 serverprint( 

90 f"NAMES in {source}: " 

91 f"WARNING: wrong type {type(val)} for {val}" 

92 ) 

93 good = False 

94 else: 

95 pureNames.add(val) 

96 if not good: 96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true

97 serverprint("EXIT because of FATAL ERROR") 

98 sys.exit() 

99 return names if inner else (pureNames, names) 

100 

101 @classmethod 

102 def getPureNames(settings): 

103 return set(G(settings, NAMES, default=[])) 

104 

105 @classmethod 

106 def setName(cls, name): 

107 nameRep = name.replace(HYPHEN, LOW) 

108 if not hasattr(cls, nameRep): 

109 setattr(cls, nameRep, name) 

110 

111 @classmethod 

112 def addNames(cls, source, settings): 

113 (pureNames, names) = cls.getNames(source, settings) 

114 for name in pureNames | names: 

115 Names.setName(name) 

116 return (pureNames, names) 

117 

118 @classmethod 

119 def showNames(cls): 

120 serverprint("""\nNAMES""") 

121 for (k, v) in sorted(cls.__dict__.items()): 

122 if callable(getattr(cls, k)): 

123 serverprint(f"""\t{k:<20} = {v}""") 

124 

125 @classmethod 

126 def getMethods(cls): 

127 return {method for method in cls.__dict__ if callable(getattr(cls, method))} 

128 

129 

130def main(): 

131 methodNames = Names.getMethods() 

132 allPureNames = set() 

133 allNames = set() 

134 

135 with os.scandir(CONFIG_DIR) as sd: 

136 files = tuple(e.name for e in sd if e.is_file() and e.name.endswith(CONFIG_EXT)) 

137 for configFile in files: 

138 section = os.path.splitext(configFile)[0] 

139 className = cap1(section) 

140 classObj = globals()[className] 

141 setattr(Config, section, classObj) 

142 

143 with open(f"""{CONFIG_DIR}/{section}{CONFIG_EXT}""") as fh: 

144 settings = yaml.load(fh, Loader=yaml.FullLoader) 

145 

146 for (subsection, subsettings) in settings.items(): 

147 if subsection != NAMES: 

148 setattr(classObj, subsection, subsettings) 

149 

150 (pureNames, names) = Names.addNames(configFile, settings) 

151 allPureNames |= pureNames 

152 allNames |= names 

153 

154 N = Names 

155 C = Config 

156 CP = C.perm 

157 

158 groupRank = {} 

159 for (r, group) in enumerate(CP.rolesOrder): 

160 groupRank[group] = r 

161 setattr(CP, "groupRank", groupRank) 

162 

163 CT = C.tables 

164 

165 masters = {} 

166 for (master, details) in CT.details.items(): 

167 for detail in details: 

168 masters.setdefault(detail, set()).add(master) 

169 setattr(CT, "masters", masters) 

170 

171 with os.scandir(TABLE_DIR) as sd: 

172 files = tuple(e.name for e in sd if e.is_file() and e.name.endswith(CONFIG_EXT)) 

173 for tableFile in files: 

174 with open(f"""{TABLE_DIR}/{tableFile}""") as fh: 

175 settings = yaml.load(fh, Loader=yaml.FullLoader) 

176 (pureNames, names) = Names.addNames(configFile, settings) 

177 allPureNames |= pureNames 

178 allNames |= names 

179 

180 spuriousNames = allPureNames & allNames 

181 if spuriousNames: 181 ↛ 182line 181 didn't jump to line 182, because the condition on line 181 was never true

182 serverprint(f"NAMES: {len(spuriousNames)} spurious names") 

183 serverprint(", ".join(sorted(spuriousNames))) 

184 else: 

185 if not TERSE: 185 ↛ 186line 185 didn't jump to line 186, because the condition on line 185 was never true

186 serverprint("NAMES: No spurious names") 

187 

188 NAME_RE = re.compile(r"""\bN\.[A-Za-z0-9_]+""") 

189 

190 usedNames = set() 

191 

192 for (top, subdirs, files) in os.walk(SERVER_PATH): 

193 for f in files: 

194 if not f.endswith(".py"): 

195 continue 

196 path = f"{top}/{f}" 

197 with open(path) as pf: 

198 text = pf.read() 

199 usedNames |= {name[2:] for name in set(NAME_RE.findall(text))} 

200 

201 unusedNames = allPureNames - usedNames 

202 if unusedNames: 202 ↛ 203line 202 didn't jump to line 203, because the condition on line 202 was never true

203 serverprint(f"NAMES: {len(unusedNames)} unused names") 

204 serverprint(", ".join(sorted(unusedNames))) 

205 else: 

206 if not TERSE: 206 ↛ 207line 206 didn't jump to line 207, because the condition on line 206 was never true

207 serverprint("NAMES: No unused names") 

208 

209 undefNames = usedNames - allPureNames - allNames - methodNames 

210 if undefNames: 210 ↛ 211line 210 didn't jump to line 211, because the condition on line 210 was never true

211 serverprint(f"NAMES: {len(undefNames)} undefined names") 

212 serverprint(", ".join(sorted(undefNames))) 

213 else: 

214 if not TERSE: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true

215 serverprint("NAMES: No undefined names") 

216 

217 if not TERSE: 217 ↛ 218line 217 didn't jump to line 218, because the condition on line 217 was never true

218 serverprint(f"NAMES: {len(allPureNames | allNames):>4} defined in yaml files") 

219 serverprint(f"NAMES: {len(usedNames):>4} used in python code") 

220 

221 if undefNames: 221 ↛ 222line 221 didn't jump to line 222, because the condition on line 221 was never true

222 serverprint("EXIT because of FATAL ERROR") 

223 sys.exit() 

224 

225 tables = set() 

226 

227 MAIN_TABLE = CT.userTables[0] 

228 USER_TABLES = set(CT.userTables) 

229 USER_ENTRY_TABLES = set(CT.userEntryTables) 

230 VALUE_TABLES = set(CT.valueTables) 

231 SYSTEM_TABLES = set(CT.systemTables) 

232 SCALAR_TYPES = CT.scalarTypes 

233 SCALAR_TYPE_SET = set(chain.from_iterable(SCALAR_TYPES.values())) 

234 PROV_SPECS = CT.prov 

235 VALUE_SPECS = CT.value 

236 CASCADE = CT.cascade 

237 

238 tables = tables | USER_TABLES | USER_ENTRY_TABLES | VALUE_TABLES | SYSTEM_TABLES 

239 sortedTables = ( 

240 [MAIN_TABLE] 

241 + sorted(USER_TABLES - {MAIN_TABLE}) 

242 + sorted(tables - USER_TABLES - {MAIN_TABLE}) 

243 ) 

244 

245 reference = {} 

246 cascade = {} 

247 

248 for table in tables: 

249 specs = {} 

250 tableFile = f"""{TABLE_DIR}/{table}{CONFIG_EXT}""" 

251 if os.path.exists(tableFile): 

252 with open(tableFile) as fh: 

253 specs.update(yaml.load(fh, Loader=yaml.FullLoader)) 

254 else: 

255 specs.update(VALUE_SPECS) 

256 specs.update(PROV_SPECS) 

257 

258 for (field, fieldSpecs) in specs.items(): 

259 fieldType = G(fieldSpecs, N.type) 

260 if fieldType and fieldType not in SCALAR_TYPE_SET: 

261 cascaded = set(G(CASCADE, fieldType, default=[])) 

262 if table in cascaded: 

263 cascade.setdefault(fieldType, {}).setdefault(table, set()).add(field) 

264 else: 

265 reference.setdefault(fieldType, {}).setdefault(table, set()).add(field) 

266 setattr(Tables, table, specs) 

267 tables.add(table) 

268 

269 Names.addNames(table, specs) 

270 

271 constrainedPre = {} 

272 for table in VALUE_TABLES: 

273 fieldSpecs = getattr(Tables, table, {}) 

274 for (field, fieldSpec) in fieldSpecs.items(): 

275 tp = G(fieldSpec, N.type) 

276 if tp in VALUE_TABLES and tp == field: 

277 constrainedPre[table] = field 

278 

279 constrained = {} 

280 for table in tables: 

281 fieldSpecs = getattr(Tables, table, {}) 

282 fields = set(fieldSpecs) 

283 for (ctable, mfield) in constrainedPre.items(): 

284 if ctable in fields and mfield in fields: 

285 ctp = G(fieldSpecs[ctable], N.type) 

286 if ctp == ctable: 

287 constrained[ctable] = mfield 

288 

289 setattr(Tables, ALL, tables) 

290 setattr(Tables, N.sorted, sortedTables) 

291 setattr(Tables, N.reference, reference) 

292 setattr(Tables, N.cascade, cascade) 

293 setattr(Tables, N.constrained, constrained) 

294 

295 CF = C.workflow 

296 

297 TASKS = CF.tasks 

298 

299 taskFields = {} 

300 

301 for taskInfo in TASKS.values(): 

302 if G(taskInfo, N.operator) == N.set: 

303 table = G(taskInfo, N.table) 

304 taskFields.setdefault(table, set()).add(G(taskInfo, N.field)) 

305 dateField = G(taskInfo, N.date) 

306 if dateField: 306 ↛ 301line 306 didn't jump to line 301, because the condition on line 306 was never false

307 taskFields[table].add(dateField) 

308 

309 setattr(Workflow, N.taskFields, taskFields) 

310 

311 

312main()