Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import sys 

2import os 

3import yaml 

4import re 

5 

6from itertools import chain 

7 

8from control.utils import pick as G, serverprint, cap1, E, LOW, HYPHEN 

9 

10SERVER_PATH = os.path.split(os.path.realpath(__file__))[0] 

11 

12CONFIG_EXT = ".yaml" 

13CONFIG_DIR = f"{SERVER_PATH}/yaml" 

14TABLE_DIR = f"{SERVER_PATH}/tables" 

15 

16ALL = "all" 

17NAMES = "names" 

18 

19TERSE = True 

20 

21 

22class Config: 

23 pass 

24 

25 

26class Base: 

27 pass 

28 

29 

30class Mongo: 

31 pass 

32 

33 

34class Web: 

35 pass 

36 

37 

38class Perm: 

39 pass 

40 

41 

42class Workflow: 

43 pass 

44 

45 

46class Clean: 

47 pass 

48 

49 

50class Tables: 

51 @classmethod 

52 def showReferences(cls): 

53 reference = cls.reference 

54 serverprint("""\nREFERENCE FIELD DEPENDENCIES""") 

55 for (dep, tables) in sorted(reference.items()): 

56 serverprint(dep) 

57 for (table, fields) in tables.items(): 

58 serverprint(f"""\t{table:<20}: {", ".join(fields)}""") 

59 

60 

61class Names: 

62 @staticmethod 

63 def isName(val): 

64 return val.replace(LOW, E).replace(HYPHEN, E).isalnum() 

65 

66 @staticmethod 

67 def getNames(source, val, doString=True, inner=False): 

68 names = set() 

69 pureNames = set() 

70 good = True 

71 

72 if type(val) is str: 

73 names = {val} if doString and Names.isName(val) else set() 

74 elif type(val) is list: 

75 for v in val: 

76 if type(v) is str and Names.isName(v): 

77 names.add(v) 

78 elif type(v) is dict: 

79 names |= Names.getNames(source, v, doString=False, inner=True) 

80 elif type(val) is dict: 

81 for (k, v) in val.items(): 

82 if inner or k != NAMES: 

83 if type(k) is str and Names.isName(k): 

84 names.add(k) 

85 names |= Names.getNames(source, v, doString=False, inner=True) 

86 else: 

87 for val in v: 

88 if type(val) is not str: 88 ↛ 89line 88 didn't jump to line 89, because the condition on line 88 was never true

89 serverprint( 

90 f"NAMES in {source}: " 

91 f"WARNING: wrong type {type(val)} for {val}" 

92 ) 

93 good = False 

94 else: 

95 pureNames.add(val) 

96 if not good: 96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true

97 serverprint("EXIT because of FATAL ERROR") 

98 sys.exit() 

99 return names if inner else (pureNames, names) 

100 

101 @classmethod 

102 def getPureNames(settings): 

103 return set(G(settings, NAMES, default=[])) 

104 

105 @classmethod 

106 def setName(cls, name): 

107 nameRep = name.replace(HYPHEN, LOW) 

108 if not hasattr(cls, nameRep): 

109 setattr(cls, nameRep, name) 

110 

111 @classmethod 

112 def addNames(cls, source, settings): 

113 (pureNames, names) = cls.getNames(source, settings) 

114 for name in pureNames | names: 

115 Names.setName(name) 

116 return (pureNames, names) 

117 

118 @classmethod 

119 def showNames(cls): 

120 serverprint("""\nNAMES""") 

121 for (k, v) in sorted(cls.__dict__.items()): 

122 if callable(getattr(cls, k)): 

123 serverprint(f"""\t{k:<20} = {v}""") 

124 

125 @classmethod 

126 def getMethods(cls): 

127 return {method for method in cls.__dict__ if callable(getattr(cls, method))} 

128 

129 

130def main(): 

131 methodNames = Names.getMethods() 

132 allPureNames = set() 

133 allNames = set() 

134 

135 with os.scandir(CONFIG_DIR) as sd: 

136 files = tuple(e.name for e in sd if e.is_file() and e.name.endswith(CONFIG_EXT)) 

137 for configFile in files: 

138 section = os.path.splitext(configFile)[0] 

139 className = cap1(section) 

140 classObj = globals()[className] 

141 setattr(Config, section, classObj) 

142 

143 with open(f"""{CONFIG_DIR}/{section}{CONFIG_EXT}""") as fh: 

144 settings = yaml.load(fh, Loader=yaml.FullLoader) 

145 

146 for (subsection, subsettings) in settings.items(): 

147 if subsection != NAMES: 

148 setattr(classObj, subsection, subsettings) 

149 

150 (pureNames, names) = Names.addNames(configFile, settings) 

151 allPureNames |= pureNames 

152 allNames |= names 

153 

154 N = Names 

155 C = Config 

156 CT = C.tables 

157 

158 masters = {} 

159 for (master, details) in CT.details.items(): 

160 for detail in details: 

161 masters.setdefault(detail, set()).add(master) 

162 setattr(CT, "masters", masters) 

163 

164 with os.scandir(TABLE_DIR) as sd: 

165 files = tuple(e.name for e in sd if e.is_file() and e.name.endswith(CONFIG_EXT)) 

166 for tableFile in files: 

167 with open(f"""{TABLE_DIR}/{tableFile}""") as fh: 

168 settings = yaml.load(fh, Loader=yaml.FullLoader) 

169 (pureNames, names) = Names.addNames(configFile, settings) 

170 allPureNames |= pureNames 

171 allNames |= names 

172 

173 spuriousNames = allPureNames & allNames 

174 if spuriousNames: 174 ↛ 175line 174 didn't jump to line 175, because the condition on line 174 was never true

175 serverprint(f"NAMES: {len(spuriousNames)} spurious names") 

176 serverprint(", ".join(sorted(spuriousNames))) 

177 else: 

178 if not TERSE: 178 ↛ 179line 178 didn't jump to line 179, because the condition on line 178 was never true

179 serverprint("NAMES: No spurious names") 

180 

181 NAME_RE = re.compile(r"""\bN\.[A-Za-z0-9_]+""") 

182 

183 usedNames = set() 

184 

185 for (top, subdirs, files) in os.walk(SERVER_PATH): 

186 for f in files: 

187 if not f.endswith(".py"): 

188 continue 

189 path = f"{top}/{f}" 

190 with open(path) as pf: 

191 text = pf.read() 

192 usedNames |= {name[2:] for name in set(NAME_RE.findall(text))} 

193 

194 unusedNames = allPureNames - usedNames 

195 if unusedNames: 195 ↛ 196line 195 didn't jump to line 196, because the condition on line 195 was never true

196 serverprint(f"NAMES: {len(unusedNames)} unused names") 

197 serverprint(", ".join(sorted(unusedNames))) 

198 else: 

199 if not TERSE: 199 ↛ 200line 199 didn't jump to line 200, because the condition on line 199 was never true

200 serverprint("NAMES: No unused names") 

201 

202 undefNames = usedNames - allPureNames - allNames - methodNames 

203 if undefNames: 203 ↛ 204line 203 didn't jump to line 204, because the condition on line 203 was never true

204 serverprint(f"NAMES: {len(undefNames)} undefined names") 

205 serverprint(", ".join(sorted(undefNames))) 

206 else: 

207 if not TERSE: 207 ↛ 208line 207 didn't jump to line 208, because the condition on line 207 was never true

208 serverprint("NAMES: No undefined names") 

209 

210 if not TERSE: 210 ↛ 211line 210 didn't jump to line 211, because the condition on line 210 was never true

211 serverprint(f"NAMES: {len(allPureNames | allNames):>4} defined in yaml files") 

212 serverprint(f"NAMES: {len(usedNames):>4} used in python code") 

213 

214 if undefNames: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true

215 serverprint("EXIT because of FATAL ERROR") 

216 sys.exit() 

217 

218 tables = set() 

219 

220 MAIN_TABLE = CT.userTables[0] 

221 USER_TABLES = set(CT.userTables) 

222 USER_ENTRY_TABLES = set(CT.userEntryTables) 

223 VALUE_TABLES = set(CT.valueTables) 

224 SYSTEM_TABLES = set(CT.systemTables) 

225 SCALAR_TYPES = CT.scalarTypes 

226 SCALAR_TYPE_SET = set(chain.from_iterable(SCALAR_TYPES.values())) 

227 PROV_SPECS = CT.prov 

228 VALUE_SPECS = CT.value 

229 CASCADE = CT.cascade 

230 

231 tables = tables | USER_TABLES | USER_ENTRY_TABLES | VALUE_TABLES | SYSTEM_TABLES 

232 sortedTables = ( 

233 [MAIN_TABLE] 

234 + sorted(USER_TABLES - {MAIN_TABLE}) 

235 + sorted(tables - USER_TABLES - {MAIN_TABLE}) 

236 ) 

237 

238 reference = {} 

239 cascade = {} 

240 

241 for table in tables: 

242 specs = {} 

243 tableFile = f"""{TABLE_DIR}/{table}{CONFIG_EXT}""" 

244 if os.path.exists(tableFile): 

245 with open(tableFile) as fh: 

246 specs.update(yaml.load(fh, Loader=yaml.FullLoader)) 

247 else: 

248 specs.update(VALUE_SPECS) 

249 specs.update(PROV_SPECS) 

250 

251 for (field, fieldSpecs) in specs.items(): 

252 fieldType = G(fieldSpecs, N.type) 

253 if fieldType and fieldType not in SCALAR_TYPE_SET: 

254 cascaded = set(G(CASCADE, fieldType, default=[])) 

255 if table in cascaded: 

256 cascade.setdefault(fieldType, {}).setdefault(table, set()).add(field) 

257 else: 

258 reference.setdefault(fieldType, {}).setdefault(table, set()).add(field) 

259 setattr(Tables, table, specs) 

260 tables.add(table) 

261 

262 Names.addNames(table, specs) 

263 

264 constrainedPre = {} 

265 for table in VALUE_TABLES: 

266 fieldSpecs = getattr(Tables, table, {}) 

267 for (field, fieldSpec) in fieldSpecs.items(): 

268 tp = G(fieldSpec, N.type) 

269 if tp in VALUE_TABLES and tp == field: 

270 constrainedPre[table] = field 

271 

272 constrained = {} 

273 for table in tables: 

274 fieldSpecs = getattr(Tables, table, {}) 

275 fields = set(fieldSpecs) 

276 for (ctable, mfield) in constrainedPre.items(): 

277 if ctable in fields and mfield in fields: 

278 ctp = G(fieldSpecs[ctable], N.type) 

279 if ctp == ctable: 

280 constrained[ctable] = mfield 

281 

282 setattr(Tables, ALL, tables) 

283 setattr(Tables, N.sorted, sortedTables) 

284 setattr(Tables, N.reference, reference) 

285 setattr(Tables, N.cascade, cascade) 

286 setattr(Tables, N.constrained, constrained) 

287 

288 CF = C.workflow 

289 

290 TASKS = CF.tasks 

291 

292 taskFields = {} 

293 

294 for taskInfo in TASKS.values(): 

295 if G(taskInfo, N.operator) == N.set: 

296 table = G(taskInfo, N.table) 

297 taskFields.setdefault(table, set()).add(G(taskInfo, N.field)) 

298 dateField = G(taskInfo, N.date) 

299 if dateField: 299 ↛ 294line 299 didn't jump to line 294, because the condition on line 299 was never false

300 taskFields[table].add(dateField) 

301 

302 setattr(Workflow, N.taskFields, taskFields) 

303 

304 

305main()