Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1"""All access to the database. 

2 

3* MongoDb 

4* Create/Read/Update/Delete 

5* Caching values 

6""" 

7 

8import sys 

9from itertools import chain 

10from pymongo import MongoClient 

11 

12from config import Config as C, Names as N 

13from control.utils import ( 

14 pick as G, 

15 serverprint, 

16 now, 

17 filterModified, 

18 isIterable, 

19 E, 

20 ON, 

21 ONE, 

22 MINONE, 

23 COMMA, 

24) 

25from control.typ.related import castObjectId 

26 

27CB = C.base 

28CM = C.mongo 

29CP = C.perm 

30CT = C.tables 

31CF = C.workflow 

32CW = C.web 

33 

34DATABASE = CB.database 

35DEBUG = CB.debug 

36DEBUG_MONGO = G(DEBUG, N.mongo) 

37DEBUG_SYNCH = G(DEBUG, N.synch) 

38CREATOR = CB.creator 

39 

40M_SET = CM.set 

41M_UNSET = CM.unset 

42M_LTE = CM.lte 

43M_GTE = CM.gte 

44M_OR = CM.OR 

45M_IN = CM.IN 

46M_EX = CM.ex 

47M_MATCH = CM.match 

48M_PROJ = CM.project 

49M_LOOKUP = CM.lookup 

50M_ELEM = CM.elem 

51 

52SHOW_ARGS = set(CM.showArgs) 

53OTHER_COMMANDS = set(CM.otherCommands) 

54M_COMMANDS = SHOW_ARGS | OTHER_COMMANDS 

55 

56GROUP_RANK = CP.groupRank 

57 

58ACTUAL_TABLES = set(CT.actualTables) 

59VALUE_TABLES = set(CT.valueTables) 

60REFERENCE_SPECS = CT.reference 

61CASCADE_SPECS = CT.cascade 

62 

63RECOLLECT_SPECS = CT.recollect 

64RECOLLECT_TABLE = RECOLLECT_SPECS[N.table] 

65RECOLLECT_NAME = RECOLLECT_SPECS[N.tableField] 

66RECOLLECT_DATE = RECOLLECT_SPECS[N.dateField] 

67 

68WORKFLOW_FIELDS = CF.fields 

69FIELD_PROJ = {field: True for field in WORKFLOW_FIELDS} 

70 

71OVERVIEW_FIELDS = CT.overviewFields 

72OVERVIEW_FIELDS_WF = CT.overviewFieldsWorkflow 

73 

74OPTIONS = CW.options 

75 

76MOD_FMT = """{} on {}""" 

77 

78 

79class Db: 

80 """All access to the MongoDb will happen through this class. 

81 

82 It will read all content of all value tables and keep it cached. 

83 

84 The data in the user tables will be cached by the higher level 

85 `control.context.Context`, but only per request. 

86 

87 !!! caution 

88 We start without a Mongo connection. 

89 We make connection the first time we need it, and then keep the 

90 connection in the `mongo` attribute. 

91 This way, we have a single Mongo connection per worker process, 

92 as recommended in 

93 [PyMongo](https://api.mongodb.com/python/current/faq.html#is-pymongo-fork-safe). 

94 """ 

95 

96 def __init__(self, regime, test=False): 

97 """## Initialization 

98 

99 Pick up the connection to MongoDb. 

100 

101 !!! note 

102 

103 Parameters 

104 ---------- 

105 regime: {"production", "development"} 

106 See below 

107 test: boolean 

108 See below. 

109 """ 

110 

111 self.regime = regime 

112 """*string* Whether the app runs in production or in development.""" 

113 

114 self.test = test 

115 """*boolean* Whether to connect to the test database.""" 

116 

117 database = G(DATABASE, N.test) if test else G(DATABASE, regime) 

118 self.database = database 

119 

120 mode = f"""regime = {regime} {"test" if test else E}""" 

121 if not self.database: 121 ↛ 122line 121 didn't jump to line 122, because the condition on line 121 was never true

122 serverprint(f"""MONGO: no database configured for {mode}""") 

123 sys.exit(1) 

124 

125 self.client = None 

126 """*object* The MongoDb client.""" 

127 

128 self.mongo = None 

129 """*object* The connection to the MongoDb database. 

130 

131 The connnection exists before the Db singleton is initialized. 

132 """ 

133 

134 self.collected = {} 

135 """*dict* For each value table, the time that this worker last collected it. 

136 

137 In the database there is a table which holds the last time for each value 

138 table that a worker updated a value in it. 

139 """ 

140 self.collect() 

141 

142 creator = [ 

143 G(record, N._id) 

144 for record in self.user.values() 

145 if G(record, N.eppn) == CREATOR 

146 ] 

147 if not creator: 147 ↛ 148line 147 didn't jump to line 148, because the condition on line 147 was never true

148 serverprint(f"""DATABASE: no creator user found in {database}.user""") 

149 sys.exit(1) 

150 

151 self.creatorId = creator[0] 

152 """*ObjectId* System user. 

153 

154 There is a userId, fixed by configuration, that represents the system. 

155 It is only used when user records are created: those records will said 

156 to be created by the system. 

157 """ 

158 

159 def mongoOpen(self): 

160 """Open connection with MongoDb. 

161 

162 Which database we open, depends on `Db.regime` and `Db.test`. 

163 """ 

164 

165 client = self.client 

166 mongo = self.mongo 

167 database = self.database 

168 

169 if not mongo: 

170 client = MongoClient() 

171 mongo = client[database] 

172 self.client = client 

173 self.mongo = mongo 

174 serverprint(f"""MONGO: new connection to {database}""") 

175 

176 def mongoClose(self): 

177 """Close connection with MongoDb. 

178 

179 We need this, because before we fork the process to workers, 

180 all MongoDb connections should be closed. 

181 """ 

182 

183 client = self.client 

184 

185 if client: 185 ↛ exitline 185 didn't return from function 'mongoClose', because the condition on line 185 was never false

186 client.close() 

187 self.client = None 

188 self.mongo = None 

189 serverprint("""MONGO: connection closed""") 

190 

191 def mongoCmd(self, label, table, command, *args, **kwargs): 

192 """Wrapper around calls to MongoDb. 

193 

194 All commands fired at the NongoDb go through this wrapper. 

195 It will spit out debug information if mongo debugging is True. 

196 

197 Parameters 

198 ---------- 

199 label: string 

200 A key to be mentioned in debug messages. 

201 Very convenient to put here the name of the method that calls mongoCmd. 

202 table: string 

203 The table in MongoDB that is targeted by the command. 

204 If the table does not exists, no command will be fired. 

205 command: string 

206 The Mongo command to execute. 

207 The command must be listed in the mongo.yaml config file. 

208 *args: iterable 

209 Additional arguments will be passed straight to the Mongo command. 

210 

211 Returns 

212 ------- 

213 mixed 

214 Whatever the the MongoDb returns. 

215 """ 

216 

217 self.mongoOpen() 

218 mongo = self.mongo 

219 

220 method = getattr(mongo[table], command, None) if command in M_COMMANDS else None 

221 warning = """!UNDEFINED""" if method is None else E 

222 if DEBUG_MONGO: 222 ↛ 223line 222 didn't jump to line 223, because the condition on line 222 was never true

223 argRep = args[0] if args and args[0] and command in SHOW_ARGS else E 

224 kwargRep = COMMA.join(f"{k}={v}" for (k, v) in kwargs.items()) 

225 serverprint( 

226 f"""MONGO<<{label}>>.{table}.{command}{warning}({argRep} {kwargRep})""" 

227 ) 

228 if method: 228 ↛ 230line 228 didn't jump to line 230, because the condition on line 228 was never false

229 return method(*args, **kwargs) 

230 return None 

231 

232 def cacheValueTable(self, valueTable): 

233 """Caches the contents of a value table. 

234 

235 The tables will be cached under two attributes: 

236 

237 the name of the table 

238 : dictionary keyed by id and valued by the corresponding record 

239 

240 the name of the table + `Inv` 

241 : dictionary keyed by a key field and valued by the corresponding id. 

242 

243 Parameters 

244 ---------- 

245 valueTable: string 

246 The value table to be cached. 

247 """ 

248 

249 valueList = list(self.mongoCmd(N.collect, valueTable, N.find)) 

250 repField = ( 

251 N.iso 

252 if valueTable == N.country 

253 else N.eppn 

254 if valueTable == N.user 

255 else N.rep 

256 ) 

257 

258 setattr( 

259 self, 

260 valueTable, 

261 {G(record, N._id): record for record in valueList}, 

262 ) 

263 setattr( 

264 self, 

265 f"""{valueTable}Inv""", 

266 {G(record, repField): G(record, N._id) for record in valueList}, 

267 ) 

268 if valueTable == N.permissionGroup: 

269 setattr( 

270 self, 

271 f"""{valueTable}Desc""", 

272 {G(record, repField): G(record, N.description) for record in valueList}, 

273 ) 

274 

275 def collect(self): 

276 """Collect the contents of the value tables. 

277 

278 Value tables have content that is needed almost all the time. 

279 All value tables will be completely cached within Db. 

280 

281 !!! note 

282 This is meant to run at start up, before the workers start. 

283 After that, this worker will not execute it again. 

284 See also `recollect`. 

285 

286 !!! warning 

287 We must take other workers into account. They need a signal 

288 to recollect. See `recollect`. 

289 We store the time that this worker has collected each table 

290 in attribute `collected`. 

291 

292 !!! caution 

293 If you change the MongoDb from without, an you forget to 

294 put an appropriate time stamp, the app will not see it until it 

295 is restarted. 

296 See for example how `root.makeUserRoot` handles this. 

297 

298 !!! warning 

299 This is a complicated app. 

300 Some tables have records that specify whether other records are "actual". 

301 After collecting a value table, the "actual" items will be recomputed. 

302 """ 

303 

304 collected = self.collected 

305 

306 for valueTable in VALUE_TABLES: 

307 self.cacheValueTable(valueTable) 

308 justNow = now() 

309 collected[valueTable] = justNow 

310 self.mongoCmd( 

311 N.recollect, 

312 N.collect, 

313 N.update_one, 

314 {RECOLLECT_NAME: valueTable}, 

315 {M_SET: {RECOLLECT_DATE: justNow}}, 

316 upsert=True, 

317 ) 

318 

319 self.collectActualItems() 

320 if DEBUG_SYNCH: 320 ↛ 321line 320 didn't jump to line 321, because the condition on line 320 was never true

321 serverprint(f"""COLLECTED {COMMA.join(sorted(VALUE_TABLES))}""") 

322 

323 def recollect(self, table=None): 

324 """Collect the contents of the value tables if they have changed. 

325 

326 For each value table it will be checked if they have been 

327 collected (by another worker) after this worker has started and if so, 

328 those tables and those tables only will be recollected. 

329 

330 !!! caution 

331 Although the initial `collect` is done before workers start 

332 (`gunicorn --preload`), individual workers will end up with their 

333 own copy of the value table cache. 

334 So when we need to recollect values for our cache, we must notify 

335 in some way that other workers also have to recollect this table. 

336 

337 ### Global recollection 

338 

339 Whenever we (re)collect a value table, we insert the time of recollection 

340 in a record in the MongoDb. 

341 

342 Somewhere at the start of each request, these records will be checked, 

343 and if needed, recollections will be done before the request processing. 

344 

345 There is a table `collect`, with records having fields `table` and 

346 `dateCollected`. After each (re)collect of a table, the `dateCollected` of 

347 the appropriate record will be set to the current time. 

348 

349 !!! note "recollect()" 

350 A `recollect()` without arguments should be done at the start of each 

351 request. 

352 

353 !!! note "recollect(table)" 

354 A `recollect(table)` should be done whenever this worker has changed 

355 something in that value table. 

356 

357 Parameters 

358 ---------- 

359 table: string, optional `None` 

360 A recollect() without arguments collects *all* value tables that need 

361 collecting based on the times of change as recorded in the `collect` 

362 table. 

363 

364 A recollect of a single table means that this worker has made a change. 

365 After the recollect, a timestamp will go into the `collect` table, 

366 so that other workers can pick it up. 

367 

368 If table is `True`, all timestamps in the `collect` table will be set 

369 to now, so that each worker will refresh its value cache. 

370 """ 

371 

372 collected = self.collected 

373 

374 if table is None: 

375 affected = set() 

376 for valueTable in VALUE_TABLES: 

377 record = self.mongoCmd( 

378 N.recollect, N.collect, N.find_one, {RECOLLECT_NAME: valueTable} 

379 ) 

380 lastChangedGlobally = G(record, RECOLLECT_DATE) 

381 lastChangedHere = G(collected, valueTable) 

382 if lastChangedGlobally and ( 

383 not lastChangedHere or lastChangedHere < lastChangedGlobally 

384 ): 

385 self.cacheValueTable(valueTable) 

386 collected[valueTable] = now() 

387 affected.add(valueTable) 

388 elif table is True: 

389 affected = set() 

390 for valueTable in VALUE_TABLES: 

391 self.cacheValueTable(valueTable) 

392 collected[valueTable] = now() 

393 affected.add(valueTable) 

394 else: 

395 self.cacheValueTable(table) 

396 collected[table] = now() 

397 affected = {table} 

398 if affected: 398 ↛ 410line 398 didn't jump to line 410, because the condition on line 398 was never false

399 justNow = now() 

400 for aTable in affected: 

401 self.mongoCmd( 

402 N.recollect, 

403 N.collect, 

404 N.update_one, 

405 {RECOLLECT_NAME: aTable}, 

406 {M_SET: {RECOLLECT_DATE: justNow}}, 

407 upsert=True, 

408 ) 

409 

410 self.collectActualItems(tables=affected) 

411 

412 if affected: 412 ↛ exitline 412 didn't return from function 'recollect', because the condition on line 412 was never false

413 if DEBUG_SYNCH: 413 ↛ 414line 413 didn't jump to line 414, because the condition on line 413 was never true

414 serverprint(f"""COLLECTED {COMMA.join(sorted(affected))}""") 

415 

416 def collectActualItems(self, tables=None): 

417 """Determines which items are "actual". 

418 

419 Actual items are those types and criteria that are specified in a 

420 package record that is itself actual. 

421 A package record is actual if the current data is between its start 

422 and end days. 

423 

424 !!! caution 

425 If only value table needs to be collected that are not 

426 involved in the concept of "actual", nothing will be done. 

427 

428 Parameters 

429 ---------- 

430 tables: set of string, optional `None` 

431 """ 

432 if tables is not None and not (tables & ACTUAL_TABLES): 

433 return 

434 

435 justNow = now() 

436 

437 packageActual = { 

438 G(record, N._id) 

439 for record in self.mongoCmd( 

440 N.collectActualItems, 

441 N.package, 

442 N.find, 

443 {N.startDate: {M_LTE: justNow}, N.endDate: {M_GTE: justNow}}, 

444 ) 

445 } 

446 for record in self.package.values(): 

447 record[N.actual] = G(record, N._id) in packageActual 

448 

449 typeActual = set( 

450 chain.from_iterable( 

451 G(record, N.typeContribution) or [] 

452 for (_id, record) in self.package.items() 

453 if _id in packageActual 

454 ) 

455 ) 

456 for record in self.typeContribution.values(): 

457 record[N.actual] = G(record, N._id) in typeActual 

458 

459 criteriaActual = { 

460 _id 

461 for (_id, record) in self.criteria.items() 

462 if G(record, N.package) in packageActual 

463 } 

464 for record in self.criteria.values(): 

465 record[N.actual] = G(record, N._id) in criteriaActual 

466 

467 self.typeCriteria = {} 

468 for (_id, record) in self.criteria.items(): 

469 if _id in criteriaActual: 

470 for tp in G(record, N.typeContribution) or []: 

471 self.typeCriteria.setdefault(tp, set()).add(_id) 

472 

473 if DEBUG_SYNCH: 473 ↛ 474line 473 didn't jump to line 474, because the condition on line 473 was never true

474 serverprint(f"""UPDATED {", ".join(ACTUAL_TABLES)}""") 

475 

476 def bulkContribWorkflow(self, countryId, bulk): 

477 """Collects workflow information in bulk. 

478 

479 When overviews are being produced, workflow info is needed for a lot 

480 of records. We do not fetch them one by one, but all in one. 

481 

482 We use the MongoDB aggregation pipeline to collect the 

483 contrib ids from the contrib table and to lookup the workflow 

484 information from the workflow table, and to flatten the nested documents 

485 to simple key-value pair. 

486 

487 Parameters 

488 ---------- 

489 countryId: ObjectId 

490 If `None`, all workflow items will be fetched. 

491 Otherwise, this should be 

492 the id of a countryId, and only the workflow 

493 for items belonging to this country are fetched. 

494 bulk: boolean 

495 If `True`, fetches only records that have been bulk-imported. 

496 Those records are marked by the presence of the field `import`. 

497 """ 

498 crit = {} if countryId is None else {"country": countryId} 

499 if bulk: 499 ↛ 500line 499 didn't jump to line 500, because the condition on line 499 was never true

500 crit["import"] = {M_EX: True} 

501 

502 project = { 

503 field: f"${fieldTrans}" for (field, fieldTrans) in OVERVIEW_FIELDS.items() 

504 } 

505 project.update( 

506 { 

507 field: {M_ELEM: [f"${N.workflow}.{fieldTrans}", 0]} 

508 for (field, fieldTrans) in OVERVIEW_FIELDS_WF.items() 

509 } 

510 ) 

511 records = self.mongoCmd( 

512 N.bulkContribWorkflow, 

513 N.contrib, 

514 N.aggregate, 

515 [ 

516 {M_MATCH: crit}, 

517 { 

518 M_LOOKUP: { 

519 "from": N.workflow, 

520 N.localField: N._id, 

521 N.foreignField: N._id, 

522 "as": N.workflow, 

523 } 

524 }, 

525 {M_PROJ: project}, 

526 ], 

527 ) 

528 return records 

529 

530 def makeCrit(self, mainTable, conditions): 

531 """Translate conditons into a MongoDb criterion. 

532 

533 The conditions come from the options on the interface: 

534 whether to constrain to items that have assessments and or reviews. 

535 

536 The result can be fed into an other Mongo query. 

537 It can also be used to filter a list of record that has already been fetched. 

538 

539 !!! hint 

540 `{'assessment': '1'}` means: only those things that have an assessment. 

541 

542 `'-1'`: means: not having an assessment. 

543 

544 `'0'`: means: don't care. 

545 

546 !!! hint 

547 See also `Db.getList`. 

548 

549 Parameters 

550 ---------- 

551 mainTable: string 

552 The name of the table that is being filtered. 

553 conditions: dict 

554 keyed by a table name (such as assessment or review) 

555 and valued by -1, 0 or 1 (as strings). 

556 

557 Result 

558 ------ 

559 dict 

560 keyed by the same table name as `conditions` and valued by a set of 

561 mongo ids of items that satisfy the criterion. 

562 Only for the criteria that do care! 

563 """ 

564 activeOptions = { 

565 G(G(OPTIONS, cond), N.table): crit == ONE 

566 for (cond, crit) in conditions.items() 

567 if crit == ONE or crit == MINONE 

568 } 

569 if None in activeOptions: 569 ↛ 570line 569 didn't jump to line 570, because the condition on line 569 was never true

570 del activeOptions[None] 

571 

572 criterion = {} 

573 for (table, crit) in activeOptions.items(): 573 ↛ 574line 573 didn't jump to line 574, because the loop on line 573 never started

574 eids = { 

575 G(record, mainTable) 

576 for record in self.mongoCmd( 

577 N.makeCrit, 

578 table, 

579 N.find, 

580 {mainTable: {M_EX: True}}, 

581 {mainTable: True}, 

582 ) 

583 } 

584 if crit in criterion: 

585 criterion[crit] |= eids 

586 else: 

587 criterion[crit] = eids 

588 return criterion 

589 

590 def getList( 

591 self, 

592 table, 

593 titleSort=None, 

594 my=None, 

595 our=None, 

596 assign=False, 

597 review=None, 

598 selectable=None, 

599 unfinished=False, 

600 select=False, 

601 **conditions, 

602 ): 

603 """Fetch a list of records from a table. 

604 

605 It fetches all records of a table, but you can constrain 

606 what is fetched and what is returned in several ways, as specified 

607 by the optional arguments. 

608 

609 Some constraints need to fetch more from Mongo than will be returned: 

610 post-filtering may be needed. 

611 

612 

613 !!! note 

614 All records have a field `editors` which contains the ids of users 

615 that are allowed to edit it besides the creator. 

616 

617 !!! note 

618 Assessment records have fields `reviewerE` and `reviewerF` that 

619 point to the expert reviewer and the final reviewer. 

620 

621 !!! hint 

622 `select` and `**conditions` below are used as a consequence of 

623 the filtering on the interface by the options `assessed` and `reviewed`. 

624 See also `Db.makeCrit` and `Db.satisfies`. 

625 

626 Parameters 

627 ---------- 

628 table: string 

629 The table from which the record are fetched. 

630 titleSort: function, optional `None` 

631 The sort key by which the resulting list of records will be sorted. 

632 It must be a function that takes a record and returns a key, for example 

633 the title string of that record. 

634 If absent or None, records will not be sorted. 

635 my: ObjectId, optional `None` 

636 **Task: produce a list of "my" records.** 

637 If passed, it should be the id of a user (typically the one that is 

638 logged in). 

639 Only records that are created/edited by this user will pass through. 

640 our: ObjectId, optional `None` 

641 **Task: produce a list of "our" records (coming from my country).** 

642 If passed, it should be the id of a user (typically the one that is 

643 logged in). 

644 Only records that have a country field containing this country id pass 

645 through. 

646 unfinished: boolean, optional `False` 

647 **Task: produce a list of "my" assessments that are unfinished.** 

648 assign: boolean, optional `False` 

649 **Task: produce a list of assessments that need reviewers.** 

650 Only meaningful if the table is `assessment`. 

651 If `True`, only records that are submitted and who lack at least one 

652 reviewer pass through. 

653 review: ObjectId, optional `None` 

654 **Task: produce a list of assessments that "I" am reviewing or have reviewed.** 

655 Only meaningful if the table is `assessment`. 

656 If passed, it should be the id of a user (typically the one that is 

657 logged in). 

658 Only records pass that have this user in either their `reviewerE` 

659 or in their 

660 `reviewerF` field. 

661 selectable: ObjectId, optional `None` 

662 **Task: produce a list of contribs that the current user can select** 

663 as a DARIAH contribution. 

664 Only meaningful if the table is `contribution`. 

665 Pick those contribs whose `selected` field is not yet filled in. 

666 The value of `selectable` should be an id of a country. 

667 Typically, this is the country of the currently logged in user, 

668 and typically, that user is a National Coordinator. 

669 select: boolean, optional `False` 

670 **Task: trigger addtional filtering by custom `conditions`.** 

671 **conditions: dict 

672 **Task: produce a list of records filtered by custom conditions.** 

673 If `select`, carry out filtering on the retrieved records, where 

674 **conditions specify the filtering 

675 (through `Db.makeCrit` and `Db.satisfies`). 

676 

677 Returns 

678 ------- 

679 list 

680 The result is a sorted list of records. 

681 """ 

682 

683 crit = {} 

684 if my: 

685 crit.update({M_OR: [{N.creator: my}, {N.editors: my}]}) 

686 if our: 

687 crit.update({N.country: our}) 

688 if assign: 

689 crit.update( 

690 {N.submitted: True, M_OR: [{N.reviewerE: None}, {N.reviewerF: None}]} 

691 ) 

692 if review: 

693 crit.update({M_OR: [{N.reviewerE: review}, {N.reviewerF: review}]}) 

694 if selectable: 

695 crit.update({N.country: selectable, N.selected: None}) 

696 

697 if table in VALUE_TABLES: 

698 records = ( 

699 record 

700 for record in getattr(self, table, {}).values() 

701 if ( 

702 ( 

703 my is None 

704 or G(record, N.creator) == my 

705 or my in G(record, N.editors, default=[]) 

706 ) 

707 and (our is None or G(record, N.country) == our) 

708 ) 

709 ) 

710 else: 

711 records = self.mongoCmd(N.getList, table, N.find, crit) 

712 if select: 

713 criterion = self.makeCrit(table, conditions) 

714 records = (record for record in records if Db.satisfies(record, criterion)) 

715 return records if titleSort is None else sorted(records, key=titleSort) 

716 

717 def getItem(self, table, eid): 

718 """Fetch a single record from a table. 

719 

720 Parameters 

721 ---------- 

722 table: string 

723 The table from which the record is fetched. 

724 eid: ObjectId 

725 (Entity) ID of the particular record. 

726 

727 Returns 

728 ------- 

729 dict 

730 """ 

731 if not eid: 731 ↛ 732line 731 didn't jump to line 732, because the condition on line 731 was never true

732 return {} 

733 

734 oid = castObjectId(eid) 

735 

736 if table in VALUE_TABLES: 

737 return G(getattr(self, table, {}), oid, default={}) 

738 

739 records = list(self.mongoCmd(N.getItem, table, N.find, {N._id: oid})) 

740 record = records[0] if len(records) else {} 

741 return record 

742 

743 def getWorkflowItem(self, contribId): 

744 """Fetch a single workflow record. 

745 

746 Parameters 

747 ---------- 

748 contribId: ObjectId 

749 The id of the workflow item to be fetched. 

750 

751 Returns 

752 ------- 

753 dict 

754 The record wrapped in a `control.workflow.apply.WorkflowItem` object. 

755 """ 

756 

757 if contribId is None: 757 ↛ 758line 757 didn't jump to line 758, because the condition on line 757 was never true

758 return {} 

759 

760 crit = {N._id: contribId} 

761 entries = list(self.mongoCmd(N.getWorkflowItem, N.workflow, N.find, crit)) 

762 return entries[0] if entries else {} 

763 

764 def getDetails(self, table, masterField, eids, sortKey=None): 

765 """Fetch the detail records connected to one or more master records. 

766 

767 Parameters 

768 ---------- 

769 table: string 

770 The table from which to fetch the detail records. 

771 masterField: string 

772 The field in the detail records that points to the master record. 

773 eids: ObjectId | iterable of ObjectId 

774 The id(s) of the master record(s). 

775 sortKey: function, optional `None` 

776 A function to sort the resulting records. 

777 """ 

778 if table in VALUE_TABLES: 778 ↛ 779line 778 didn't jump to line 779, because the condition on line 778 was never true

779 crit = eids if isIterable(eids) else [eids] 

780 details = [ 

781 record 

782 for record in getattr(self, table, {}).values() 

783 if G(record, masterField) in crit 

784 ] 

785 else: 

786 crit = {masterField: {M_IN: list(eids)} if isIterable(eids) else eids} 

787 details = list(self.mongoCmd(N.getDetails, table, N.find, crit)) 

788 

789 return sorted(details, key=sortKey) if sortKey else details 

790 

791 def getValueRecords(self, valueTable, constrain=None, upper=None): 

792 """Fetch records from a value table. 

793 

794 It will apply some standard and custom constraints. 

795 

796 The standard constraints are: if the valueTable is 

797 

798 * `country`: only the DARIAH member countries will be delivered 

799 * `user`: only the non-legacy users will be returned. 

800 

801 !!! note 

802 See the tables.yaml configuration has a key, `constrained`, 

803 which is generated by `config.py` from the field specs of the value tables. 

804 This collects the cases where the valid choices for a value are not all 

805 available values in the table, but only those that are linked to a certain 

806 master record. 

807 

808 !!! hint 

809 If you want to pick a score for an assessment criterion, only those scores 

810 that are linked to that criterion record are eligible. 

811 

812 Parameters 

813 ---------- 

814 valueTable: string 

815 The table from which fetch the records. 

816 constrain: 2-tuple, optional `None` 

817 A custom constraint. If present, it should be a tuple `(fieldName, value)`. 

818 Only records with that value in that field will be delivered. 

819 upper: string 

820 The name of a permission group. 

821 If the valueTable is permissionGroup, not all values will be shown, 

822 only the values that are not more powerful than this group. 

823 This is needed to prevent users to make somebody more powerful 

824 then themselves. 

825 

826 Returns 

827 ------- 

828 list 

829 """ 

830 

831 records = getattr(self, valueTable, {}).values() 

832 result = ( 

833 (r for r in records if G(r, N.isMember) or False) 

834 if valueTable == N.country 

835 else (r for r in records if G(r, N.authority) != N.legacy) 

836 if valueTable == N.user 

837 else (r for r in records if G(r, constrain[0]) == constrain[1]) 

838 if constrain 

839 else records 

840 ) 

841 if valueTable == N.permissionGroup: 841 ↛ 842line 841 didn't jump to line 842, because the condition on line 841 was never true

842 result = ( 

843 r for r in result if G(r, N.rep, "") not in {N.edit, N.own, N.nobody} 

844 ) 

845 if upper is not None: 

846 upperRank = G(GROUP_RANK, upper, 0) 

847 result = ( 

848 r 

849 for r in result 

850 if G(GROUP_RANK, G(r, N.rep, ""), 100) <= upperRank 

851 ) 

852 return sorted(result, key=lambda r: G(GROUP_RANK, G(r, N.rep, ""), 100)) 

853 return tuple(result) 

854 

855 def getValueInv(self, valueTable, constrain): 

856 """Fetch a mapping from values to ids from a value table. 

857 

858 The mapping is like the *valueTable*`Inv` attribute of `Db`, 

859 but with members restricted by a constraint. 

860 

861 !!! caution 

862 This only works properly if the valueTable has a field `rep`. 

863 

864 Parameters 

865 ---------- 

866 valueTable: string 

867 The table that contains the records. 

868 constrain: 2-tuple, optional `None` 

869 A custom constraint. If present, it should be a tuple `(fieldName, value)`. 

870 Only records with that value in that field will be delivered. 

871 

872 Returns 

873 ------- 

874 dict 

875 Keyed by values, valued by ids. 

876 """ 

877 

878 records = ( 

879 r 

880 for r in getattr(self, valueTable, {}).values() 

881 if G(r, constrain[0]) == constrain[1] 

882 ) 

883 eids = {G(r, N._id) for r in records} 

884 return { 

885 value: eid 

886 for (value, eid) in getattr(self, f"""{valueTable}Inv""", {}).items() 

887 if eid in eids 

888 } 

889 

890 def getValueIds(self, valueTable, constrain): 

891 """Fetch a set of ids from a value table. 

892 

893 The ids are taken from the value reocrds that satisfy a constraint. 

894 but with members restricted by a constraint. 

895 

896 Parameters 

897 ---------- 

898 valueTable: string 

899 The table that contains the records. 

900 constrain: 2-tuple, optional `None` 

901 A custom constraint. If present, it should be a tuple `(fieldName, value)`. 

902 Only records with that value in that field will be delivered. 

903 

904 Returns 

905 ------- 

906 set of ObjectId 

907 """ 

908 

909 records = ( 

910 r 

911 for r in getattr(self, valueTable, {}).values() 

912 if G(r, constrain[0]) == constrain[1] 

913 ) 

914 return {G(r, N._id) for r in records} 

915 

916 def insertItem(self, table, uid, eppn, onlyIfNew, **fields): 

917 """Inserts a new record in a table, possibly only if it is new. 

918 

919 The record will be filled with the specified fields, but also with 

920 provenance fields. 

921 

922 The provenance fields are the creation date, the creator, 

923 and the start of the trail of modifiers. 

924 

925 Parameters 

926 ---------- 

927 table: string 

928 The table in which the record will be inserted. 

929 uid: ObjectId 

930 The user that creates the record, typically the logged in user. 

931 onlyIfNew: boolean 

932 If `True`, it will be checked whether a record with the specified fields 

933 already exists. If so, no record will be inserted. 

934 eppn: string 

935 The eppn of that same user. This is the unique identifier that comes from 

936 the DARIAH authentication service. 

937 **fields: dict 

938 The field names and their contents to populate the new record with. 

939 

940 Returns 

941 ------- 

942 ObjectId 

943 The id of the newly inserted record, or the id of the first existing 

944 record found, if `onlyIfNew` is true. 

945 """ 

946 

947 if onlyIfNew: 

948 existing = [ 

949 G(rec, N._id) 

950 for rec in getattr(self, table, {}).values() 

951 if all(G(rec, k) == v for (k, v) in fields.items()) 

952 ] 

953 if existing: 953 ↛ 954line 953 didn't jump to line 954, because the condition on line 953 was never true

954 return existing[0] 

955 

956 justNow = now() 

957 newRecord = { 

958 N.dateCreated: justNow, 

959 N.creator: uid, 

960 N.modified: [MOD_FMT.format(eppn, justNow)], 

961 **fields, 

962 } 

963 result = self.mongoCmd(N.insertItem, table, N.insert_one, newRecord) 

964 if table in VALUE_TABLES: 

965 self.recollect(table) 

966 return result.inserted_id 

967 

968 def insertMany(self, table, uid, eppn, records): 

969 """Insert several records at once. 

970 

971 Typically used for inserting criteriaEntry en reviewEntry records. 

972 

973 Parameters 

974 ---------- 

975 table: string 

976 The table in which the record will be inserted. 

977 uid: ObjectId 

978 The user that creates the record, typically the logged in user. 

979 eppn: string 

980 The `eppn` of that same user. This is the unique identifier that comes from 

981 the DARIAH authentication service. 

982 records: iterable of dict 

983 The records (as dicts) to insert. 

984 """ 

985 

986 justNow = now() 

987 newRecords = [ 

988 { 

989 N.dateCreated: justNow, 

990 N.creator: uid, 

991 N.modified: [MOD_FMT.format(eppn, justNow)], 

992 **record, 

993 } 

994 for record in records 

995 ] 

996 self.mongoCmd(N.insertMany, table, N.insert_many, newRecords) 

997 

998 def insertUser(self, record): 

999 """Insert a user record, i.e. a record corresponding to a user. 

1000 

1001 NB: the creator of this record is the system, by name of the 

1002 `creatorId` attribute. 

1003 

1004 Parameters 

1005 ---------- 

1006 record: dict 

1007 The user information to be stored, as a dictionary. 

1008 

1009 Returns 

1010 ------- 

1011 None 

1012 But note that the new _id and the generated field values are added to the 

1013 record. 

1014 """ 

1015 

1016 creatorId = self.creatorId 

1017 

1018 justNow = now() 

1019 record.update( 

1020 { 

1021 N.dateLastLogin: justNow, 

1022 N.statusLastLogin: N.Approved, 

1023 N.mayLogin: True, 

1024 N.creator: creatorId, 

1025 N.dateCreated: justNow, 

1026 N.modified: [MOD_FMT.format(CREATOR, justNow)], 

1027 } 

1028 ) 

1029 result = self.mongoCmd(N.insertUser, N.user, N.insert_one, record) 

1030 self.recollect(N.user) 

1031 record[N._id] = result.inserted_id 

1032 

1033 def deleteItem(self, table, eid): 

1034 """Delete a record. 

1035 

1036 Parameters 

1037 ---------- 

1038 table: string 

1039 The table which holds the record to be deleted. 

1040 eid: ObjectId 

1041 (Entity) id of the record to be deleted. 

1042 

1043 Returns 

1044 ------- 

1045 boolean 

1046 Whether the MongoDB operation was successful 

1047 """ 

1048 

1049 oid = castObjectId(eid) 

1050 if oid is None: 1050 ↛ 1051line 1050 didn't jump to line 1051, because the condition on line 1050 was never true

1051 return False 

1052 status = self.mongoCmd(N.deleteItem, table, N.delete_one, {N._id: oid}) 

1053 if table in VALUE_TABLES: 

1054 self.recollect(table) 

1055 return G(status.raw_result, N.ok, default=False) 

1056 

1057 def deleteMany(self, table, crit): 

1058 """Delete a several records. 

1059 

1060 Typically used to delete all detail records of another record. 

1061 

1062 Parameters 

1063 ---------- 

1064 table: string 

1065 The table which holds the records to be deleted. 

1066 crit: dict 

1067 A criterion that specfifies which records must be deleted. 

1068 Given as a dict. 

1069 """ 

1070 

1071 self.mongoCmd(N.deleteMany, table, N.delete_many, crit) 

1072 

1073 def updateField( 

1074 self, 

1075 table, 

1076 eid, 

1077 field, 

1078 data, 

1079 actor, 

1080 modified, 

1081 nowFields=[], 

1082 ): 

1083 """Update a single field in a single record. 

1084 

1085 !!! hint 

1086 Whenever a field is updated in a record which has the field `isPristine`, 

1087 this field will be deleted from the record. 

1088 The rule is that pristine records are the ones that originate from the 

1089 legacy data and have not changed since then. 

1090 

1091 Parameters 

1092 ---------- 

1093 table: string 

1094 The table which holds the record to be updated. 

1095 eid: ObjectId 

1096 (Entity) id of the record to be updated. 

1097 data: mixed 

1098 The new value of for the updated field. 

1099 actor: ObjectId 

1100 The user that has triggered the update action. 

1101 modified: list of string 

1102 The current provenance trail of the record, which is a list of 

1103 strings of the form "person on date". 

1104 Here "person" is not an ID but a consolidated string representing 

1105 the name of that person. 

1106 The provenance trail will be trimmed in order to prevent excessively long 

1107 trails. On each day, only the last action by each person will be recorded. 

1108 nowFields: iterable of string, optional `[]` 

1109 The names of additional fields in which the current datetime will be stored. 

1110 For exampe, if `submitted` is modified, the current datetime will be saved in 

1111 `dateSubmitted`. 

1112 

1113 Returns 

1114 ------- 

1115 dict | boolean 

1116 The updated record, if the MongoDb operation was successful, else False 

1117 """ 

1118 

1119 oid = castObjectId(eid) 

1120 if oid is None: 1120 ↛ 1121line 1120 didn't jump to line 1121, because the condition on line 1120 was never true

1121 return False 

1122 

1123 justNow = now() 

1124 newModified = filterModified((modified or []) + [f"""{actor}{ON}{justNow}"""]) 

1125 criterion = {N._id: oid} 

1126 nowItems = {nowField: justNow for nowField in nowFields} 

1127 update = { 

1128 field: data, 

1129 N.modified: newModified, 

1130 **nowItems, 

1131 } 

1132 delete = {N.isPristine: E} 

1133 instructions = { 

1134 M_SET: update, 

1135 M_UNSET: delete, 

1136 } 

1137 

1138 status = self.mongoCmd( 

1139 N.updateField, table, N.update_one, criterion, instructions 

1140 ) 

1141 if not G(status.raw_result, N.ok, default=False): 1141 ↛ 1142line 1141 didn't jump to line 1142, because the condition on line 1141 was never true

1142 return False 

1143 

1144 if table in VALUE_TABLES: 

1145 self.recollect(table) 

1146 return ( 

1147 update, 

1148 set(delete.keys()), 

1149 ) 

1150 

1151 def updateUser(self, record): 

1152 """Updates user information. 

1153 

1154 When users log in, or when they are assigned an other status, 

1155 some of their attributes will change. 

1156 

1157 Parameters 

1158 ---------- 

1159 record: dict 

1160 The new user information as a dict. 

1161 """ 

1162 

1163 if N.isPristine in record: 

1164 del record[N.isPristine] 

1165 justNow = now() 

1166 record.update( 

1167 { 

1168 N.dateLastLogin: justNow, 

1169 N.statusLastLogin: N.Approved, 

1170 N.modified: [MOD_FMT.format(CREATOR, justNow)], 

1171 } 

1172 ) 

1173 criterion = {N._id: G(record, N._id)} 

1174 updates = {k: v for (k, v) in record.items() if k != N._id} 

1175 instructions = {M_SET: updates, M_UNSET: {N.isPristine: E}} 

1176 self.mongoCmd(N.updateUser, N.user, N.update_one, criterion, instructions) 

1177 self.recollect(N.user) 

1178 

1179 def dependencies(self, table, record): 

1180 """Computes the number of dependent records of a record. 

1181 

1182 A record is dependent on another record if one of the fields of the 

1183 dependent record contains an id of that other record. 

1184 

1185 Detail records are dependent on master records. 

1186 Also, records that specify a choice in a value table, are dependent on 

1187 the chosen value record. 

1188 

1189 Parameters 

1190 ---------- 

1191 table: string 

1192 The table in which the record resides of which we want to know the 

1193 dependencies. 

1194 record: dict 

1195 The record, given as dict, of which we want to know the dependencies. 

1196 

1197 Returns 

1198 ------- 

1199 int 

1200 """ 

1201 

1202 eid = G(record, N._id) 

1203 if eid is None: 

1204 return {} 

1205 

1206 depSpecs = dict( 

1207 reference=G(REFERENCE_SPECS, table, default={}), 

1208 cascade=G(CASCADE_SPECS, table, default={}), 

1209 ) 

1210 depResult = {} 

1211 for (depKind, depSpec) in depSpecs.items(): 

1212 nDep = 0 

1213 for (referringTable, referringFields) in depSpec.items(): 

1214 if not len(referringFields): 1214 ↛ 1215line 1214 didn't jump to line 1215, because the condition on line 1214 was never true

1215 continue 

1216 fields = list(referringFields) 

1217 crit = ( 

1218 {fields[0]: eid} 

1219 if len(fields) == 1 

1220 else {M_OR: [{field: eid} for field in fields]} 

1221 ) 

1222 

1223 nDep += self.mongoCmd(depKind, referringTable, N.count_documents, crit) 

1224 depResult[depKind] = nDep 

1225 

1226 return depResult 

1227 

1228 def dropWorkflow(self): 

1229 """Drop the entire workflow table. 

1230 

1231 This happens at startup of the server. 

1232 All workflow information will be computed from scratch before the server starts 

1233 serving pages. 

1234 """ 

1235 

1236 self.mongoCmd(N.dropWorkflow, N.workflow, N.drop) 

1237 

1238 def clearWorkflow(self): 

1239 """Clear the entire workflow table. 

1240 

1241 The table is not deleted, but all of its records are. 

1242 This happens when the workflow information is reinitialized while the 

1243 webserver remains running, e.g. by command of a sysadmin or office user. 

1244 (Currently this function is not used). 

1245 """ 

1246 

1247 self.mongoCmd(N.clearWorkflow, N.workflow, N.delete_many, {}) 

1248 

1249 def entries(self, table, crit={}): 

1250 """Get relevant records from a table as a dictionary of entries. 

1251 

1252 Parameters 

1253 ---------- 

1254 table: string 

1255 Table from which the entries are taken. 

1256 crit: dict, optional `{}` 

1257 Criteria to select which records should be used. 

1258 

1259 !!! hint 

1260 This function is used to collect the records that carry user 

1261 content in order to compute workflow information. 

1262 

1263 Its more targeted use is to fetch assessment and review records 

1264 that are relevant to a single contribution. 

1265 

1266 Returns 

1267 ------- 

1268 dict 

1269 Keyed by the ids of the selected records. The records themselves 

1270 are the values. 

1271 """ 

1272 

1273 entries = {} 

1274 for record in list(self.mongoCmd(N.entries, table, N.find, crit, FIELD_PROJ)): 

1275 entries[G(record, N._id)] = record 

1276 

1277 return entries 

1278 

1279 def insertWorkflowMany(self, records): 

1280 """Bulk insert records into the workflow table. 

1281 

1282 Parameters 

1283 ---------- 

1284 records: iterable of dict 

1285 The records to be inserted. 

1286 """ 

1287 

1288 self.mongoCmd(N.insertWorkflowMany, N.workflow, N.insert_many, records) 

1289 

1290 def insertWorkflow(self, record): 

1291 """Insert a single workflow record. 

1292 

1293 Parameters 

1294 ---------- 

1295 record: dict 

1296 The record to be inserted, as a dict. 

1297 """ 

1298 

1299 self.mongoCmd(N.insertWorkflow, N.workflow, N.insert_one, record) 

1300 

1301 def updateWorkflow(self, contribId, record): 

1302 """Replace a workflow record by an other one. 

1303 

1304 !!! note 

1305 Workflow records have an id that is identical to the id of the contribution 

1306 they are about. 

1307 

1308 Parameters 

1309 ---------- 

1310 contribId: ObjectId 

1311 The id of the workflow record that has to be replaced with new information. 

1312 record: dict 

1313 The new record which acts as replacement. 

1314 """ 

1315 

1316 crit = {N._id: contribId} 

1317 self.mongoCmd(N.updateWorkflow, N.workflow, N.replace_one, crit, record) 

1318 

1319 def deleteWorkflow(self, contribId): 

1320 """Delete a workflow record. 

1321 

1322 Parameters 

1323 ---------- 

1324 contribId: ObjectId 

1325 The id of the workflow item to be deleted. 

1326 """ 

1327 

1328 crit = {N._id: contribId} 

1329 self.mongoCmd(N.deleteWorkflow, N.workflow, N.delete_one, crit) 

1330 

1331 @staticmethod 

1332 def satisfies(record, criterion): 

1333 """Test whether a record satifies a criterion. 

1334 

1335 !!! hint 

1336 See also `Db.getList`. 

1337 

1338 Parameters 

1339 ---------- 

1340 record: dict 

1341 A dict of fields. 

1342 criterion: dict 

1343 A dict keyed by a boolean and valued by sets of ids. 

1344 The ids under `True` are the ones that must contain the id of the 

1345 record in question. 

1346 The ids under `False` are the onse that may not contain the id of 

1347 that record. 

1348 

1349 Returns 

1350 ------- 

1351 boolean 

1352 """ 

1353 

1354 eid = G(record, N._id) 

1355 for (crit, eids) in criterion.items(): 1355 ↛ 1356line 1355 didn't jump to line 1356, because the loop on line 1355 never started

1356 if crit and eid not in eids or not crit and eid in eids: 

1357 return False 

1358 return True 

1359 

1360 @staticmethod 

1361 def inCrit(items): 

1362 """Compiles a list of items into a Monngo DB `$in` criterion. 

1363 

1364 Parameters 

1365 ---------- 

1366 items: iterable of mixed 

1367 Typically ObjectIds. 

1368 

1369 Returns 

1370 ------- 

1371 dict 

1372 A MongoDB criterion that tests whether the thing in question is one 

1373 of the items given. 

1374 """ 

1375 

1376 return {M_IN: list(items)}