1"""Produces a clean test database. 

2 

3The clean slate database does not have contributions, assessments, reviews. 

4 

5It does have all value tables, but with simplified contents. 

6""" 

7 

8import sys 

9import collections 

10 

11# from datetime import datetime as dt 

12from pymongo import MongoClient 

13from bson.objectid import ObjectId 

14from hashlib import md5 

15 

16import magic # noqa 

17from config import Config as C, Names as N 

18from control.utils import now 

19 

20 

21CB = C.base 

22CC = C.clean 

23CT = C.tables 

24 

25CREATOR = CB.creator 

26DATABASE = CB.database["test"] 

27 

28COUNTRY = CC.country 

29GROUP = CC.group 

30USER = CC.user 

31VALUES = CC.values 

32DECISION = CC.decision 

33KEY_FIELD = CC.keyField 

34PROCEDURE = CC.procedure 

35 

36VALUE_TABLES = CT.valueTables 

37 

38 

39def info(x): 

40 sys.stdout.write("{}\n".format(x)) 

41 

42 

43def warning(x): 

44 sys.stderr.write("{}\n".format(x)) 

45 

46 

47def toHexName(name): 

48 return md5(bytes(name, "utf-8")).hexdigest()[:10] 

49 

50 

51def toHexNumber(number): 

52 return "{:0>6x}".format(number) 

53 

54 

55def toHexMongo(name, number): 

56 return "{:0>8x}{}{}".format(0, toHexName(name), toHexNumber(number)) 

57 

58 

59class IdIndex: 

60 def __init__(self): 

61 self._idFromName = {} 

62 self._nameFromId = {} 

63 

64 def getId(self, name): 

65 _id = self._idFromName.get(name, None) 

66 if _id is None: 66 ↛ 70line 66 didn't jump to line 70, because the condition on line 66 was never false

67 _id = ObjectId(name) 

68 self._idFromName[name] = _id 

69 self._nameFromId[_id] = name 

70 return _id 

71 

72 def getName(self, _id): 

73 return self._nameFromId[_id] 

74 

75 

76class MongoId(IdIndex): 

77 def __init__(self): 

78 super().__init__() 

79 self.cur = collections.Counter() 

80 

81 def newId(self, table): 

82 self.cur[table] += 1 

83 return self.getId(toHexMongo(table, self.cur[table])) 

84 

85 

86def clean(): 

87 db = MongoId() 

88 allData = collections.defaultdict(list) 

89 valueDict = collections.defaultdict(dict) 

90 countryMapping = {} 

91 userMapping = {} 

92 groupMapping = {} 

93 

94 def countryTable(): 

95 table = "country" 

96 for (iso, info) in sorted(COUNTRY.items()): 

97 _id = db.newId(table) 

98 countryMapping[iso] = _id 

99 allData[table].append( 

100 dict( 

101 _id=_id, 

102 iso=iso, 

103 name=info["name"], 

104 isMember=info["isMember"], 

105 latitude=info["latitude"], 

106 longitude=info["longitude"], 

107 ) 

108 ) 

109 

110 def groupTable(): 

111 table = "permissionGroup" 

112 for (name, description) in GROUP: 

113 _id = db.newId(table) 

114 groupMapping[name] = _id 

115 allData[table].append(dict(_id=_id, rep=name, description=description)) 

116 

117 def userTable(): 

118 table = "user" 

119 for user in USER: 

120 _id = db.newId(table) 

121 u = dict(x for x in user.items()) 

122 u["_id"] = _id 

123 userMapping[u["eppn"]] = _id 

124 u["group"] = groupMapping[u["group"]] 

125 if "country" in u: 

126 u["country"] = countryMapping[u["country"]] 

127 allData[table].append(u) 

128 

129 def relTables(): 

130 for (table, values) in VALUES.items(): 

131 for value in values: 

132 _id = db.newId(table) 

133 valueDict[table][value] = _id 

134 v = dict(_id=_id, rep=value) 

135 allData[table].append(v) 

136 

137 def yearTable(): 

138 table = "year" 

139 targetInterval = list(range(2010, 2030)) 

140 allData[table] = [ 

141 dict(_id=db.newId(table), rep=year) for year in targetInterval 

142 ] 

143 

144 def decisionTable(): 

145 table = "decision" 

146 allData[table] = [ 

147 dict(_id=db.newId(table), **DECISION["values"][decision]) 

148 for decision in DECISION["order"] 

149 ] 

150 

151 def backoffice(): 

152 relIndex = collections.defaultdict(dict) 

153 

154 for tableInfo in PROCEDURE: 

155 table = tableInfo["name"] 

156 rows = tableInfo["rows"] 

157 keyField = KEY_FIELD[table] 

158 

159 for row in rows: # deterministic order 

160 _id = db.newId(table) 

161 newRow = dict() 

162 newRow["_id"] = _id 

163 relIndex[table][row[keyField]] = _id 

164 for (field, value) in row.items(): 

165 if field in {"startDate", "endDate"}: 

166 # newRow[field] = dt.fromisoformat(value) 

167 newRow[field] = value # yaml has already converted the datetime 

168 elif field == "creator": 168 ↛ 169line 168 didn't jump to line 169, because the condition on line 168 was never true

169 newRow[field] = userMapping[value] 

170 elif ( 

171 table == N.package 

172 and field == N.typeContribution 

173 or table == N.criteria 

174 and field == N.typeContribution 

175 ): 

176 newRow[field] = [relIndex[field][val] for val in value] 

177 elif ( 

178 table == N.criteria 

179 and field == N.package 

180 or table == N.score 

181 and field == N.criteria 

182 ): 

183 newRow[field] = relIndex[field][value] 

184 else: 

185 newRow[field] = value 

186 allData[table].append(newRow) 

187 for tableInfo in PROCEDURE: 

188 table = tableInfo["name"] 

189 keyField = KEY_FIELD[table] 

190 

191 if keyField == "key": 

192 for row in allData[table]: 

193 del row["key"] 

194 

195 def importMongo(): 

196 client = MongoClient() 

197 sys.stdout.write(f"RESET the DATABASE {DATABASE} ... ") 

198 client.drop_database(DATABASE) 

199 db = client[DATABASE] 

200 for (table, rows) in allData.items(): 

201 db[table].insert_many(list(rows)) 

202 

203 justNow = now() 

204 for table in VALUE_TABLES: 

205 db.collect.update_one( 

206 {"table": table}, {"$set": {"dateCollected": justNow}}, upsert=True, 

207 ) 

208 sys.stdout.write("DONE\n") 

209 

210 countryTable() 

211 groupTable() 

212 userTable() 

213 relTables() 

214 yearTable() 

215 decisionTable() 

216 backoffice() 

217 importMongo() 

218 

219 

220if __name__ == "__main__": 220 ↛ 221line 220 didn't jump to line 221, because the condition on line 220 was never true

221 clean()