Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1"""All access to the database. 

2 

3* MongoDb 

4* Create/Read/Update/Delete 

5* Caching values 

6""" 

7 

8import sys 

9from itertools import chain 

10from pymongo import MongoClient 

11 

12from config import Config as C, Names as N 

13from control.utils import ( 

14 pick as G, 

15 serverprint, 

16 now, 

17 filterModified, 

18 isIterable, 

19 E, 

20 ON, 

21 ONE, 

22 MINONE, 

23 COMMA, 

24) 

25from control.typ.related import castObjectId 

26 

27CB = C.base 

28CM = C.mongo 

29CT = C.tables 

30CF = C.workflow 

31CW = C.web 

32 

33DATABASE = CB.database 

34DEBUG = CB.debug 

35DEBUG_MONGO = G(DEBUG, N.mongo) 

36DEBUG_SYNCH = G(DEBUG, N.synch) 

37CREATOR = CB.creator 

38 

39M_SET = CM.set 

40M_UNSET = CM.unset 

41M_LTE = CM.lte 

42M_GTE = CM.gte 

43M_OR = CM.OR 

44M_IN = CM.IN 

45M_EX = CM.ex 

46M_MATCH = CM.match 

47M_PROJ = CM.project 

48M_LOOKUP = CM.lookup 

49M_ELEM = CM.elem 

50 

51SHOW_ARGS = set(CM.showArgs) 

52OTHER_COMMANDS = set(CM.otherCommands) 

53M_COMMANDS = SHOW_ARGS | OTHER_COMMANDS 

54 

55ACTUAL_TABLES = set(CT.actualTables) 

56VALUE_TABLES = set(CT.valueTables) 

57REFERENCE_SPECS = CT.reference 

58CASCADE_SPECS = CT.cascade 

59 

60RECOLLECT_SPECS = CT.recollect 

61RECOLLECT_TABLE = RECOLLECT_SPECS[N.table] 

62RECOLLECT_NAME = RECOLLECT_SPECS[N.tableField] 

63RECOLLECT_DATE = RECOLLECT_SPECS[N.dateField] 

64 

65WORKFLOW_FIELDS = CF.fields 

66FIELD_PROJ = {field: True for field in WORKFLOW_FIELDS} 

67 

68OVERVIEW_FIELDS = CT.overviewFields 

69OVERVIEW_FIELDS_WF = CT.overviewFieldsWorkflow 

70 

71OPTIONS = CW.options 

72 

73MOD_FMT = """{} on {}""" 

74 

75 

76class Db: 

77 """All access to the MongoDb will happen through this class. 

78 

79 It will read all content of all value tables and keep it cached. 

80 

81 The data in the user tables will be cached by the higher level 

82 `control.context.Context`, but only per request. 

83 

84 !!! caution 

85 We start without a Mongo connection. 

86 We make connection the first time we need it, and then keep the 

87 connection in the `mongo` attribute. 

88 This way, we have a single Mongo connection per worker process, 

89 as recommended in 

90 [PyMongo](https://api.mongodb.com/python/current/faq.html#is-pymongo-fork-safe). 

91 """ 

92 

93 def __init__(self, regime, test=False): 

94 """## Initialization 

95 

96 Pick up the connection to MongoDb. 

97 

98 !!! note 

99 

100 Parameters 

101 ---------- 

102 regime: {"production", "development"} 

103 See below 

104 test: boolean 

105 See below. 

106 """ 

107 

108 self.regime = regime 

109 """*string* Whether the app runs in production or in development.""" 

110 

111 self.test = test 

112 """*boolean* Whether to connect to the test database.""" 

113 

114 database = G(DATABASE, N.test) if test else G(DATABASE, regime) 

115 self.database = database 

116 

117 mode = f"""regime = {regime} {"test" if test else E}""" 

118 if not self.database: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 serverprint(f"""MONGO: no database configured for {mode}""") 

120 sys.exit(1) 

121 

122 self.client = None 

123 """*object* The MongoDb client.""" 

124 

125 self.mongo = None 

126 """*object* The connection to the MongoDb database. 

127 

128 The connnection exists before the Db singleton is initialized. 

129 """ 

130 

131 self.collected = {} 

132 """*dict* For each value table, the time that this worker last collected it. 

133 

134 In the database there is a table which holds the last time for each value 

135 table that a worker updated a value in it. 

136 """ 

137 self.collect() 

138 

139 creator = [ 

140 G(record, N._id) 

141 for record in self.user.values() 

142 if G(record, N.eppn) == CREATOR 

143 ] 

144 if not creator: 144 ↛ 145line 144 didn't jump to line 145, because the condition on line 144 was never true

145 serverprint(f"""DATABASE: no creator user found in {database}.user""") 

146 sys.exit(1) 

147 

148 self.creatorId = creator[0] 

149 """*ObjectId* System user. 

150 

151 There is a userId, fixed by configuration, that represents the system. 

152 It is only used when user records are created: those records will said 

153 to be created by the system. 

154 """ 

155 

156 def mongoOpen(self): 

157 """Open connection with MongoDb. 

158 

159 Which database we open, depends on `Db.regime` and `Db.test`. 

160 """ 

161 

162 client = self.client 

163 mongo = self.mongo 

164 database = self.database 

165 

166 if not mongo: 

167 client = MongoClient() 

168 mongo = client[database] 

169 self.client = client 

170 self.mongo = mongo 

171 serverprint(f"""MONGO: new connection to {database}""") 

172 

173 def mongoClose(self): 

174 """Close connection with MongoDb. 

175 

176 We need this, because before we fork the process to workers, 

177 all MongoDb connections should be closed. 

178 """ 

179 

180 client = self.client 

181 

182 if client: 182 ↛ exitline 182 didn't return from function 'mongoClose', because the condition on line 182 was never false

183 client.close() 

184 self.client = None 

185 self.mongo = None 

186 serverprint("""MONGO: connection closed""") 

187 

188 def mongoCmd(self, label, table, command, *args, **kwargs): 

189 """Wrapper around calls to MongoDb. 

190 

191 All commands fired at the NongoDb go through this wrapper. 

192 It will spit out debug information if mongo debugging is True. 

193 

194 Parameters 

195 ---------- 

196 label: string 

197 A key to be mentioned in debug messages. 

198 Very convenient to put here the name of the method that calls mongoCmd. 

199 table: string 

200 The table in MongoDB that is targeted by the command. 

201 If the table does not exists, no command will be fired. 

202 command: string 

203 The Mongo command to execute. 

204 The command must be listed in the mongo.yaml config file. 

205 *args: iterable 

206 Additional arguments will be passed straight to the Mongo command. 

207 

208 Returns 

209 ------- 

210 mixed 

211 Whatever the the MongoDb returns. 

212 """ 

213 

214 self.mongoOpen() 

215 mongo = self.mongo 

216 

217 method = getattr(mongo[table], command, None) if command in M_COMMANDS else None 

218 warning = """!UNDEFINED""" if method is None else E 

219 if DEBUG_MONGO: 219 ↛ 220line 219 didn't jump to line 220, because the condition on line 219 was never true

220 argRep = args[0] if args and args[0] and command in SHOW_ARGS else E 

221 kwargRep = COMMA.join(f"{k}={v}" for (k, v) in kwargs.items()) 

222 serverprint( 

223 f"""MONGO<<{label}>>.{table}.{command}{warning}({argRep} {kwargRep})""" 

224 ) 

225 if method: 225 ↛ 227line 225 didn't jump to line 227, because the condition on line 225 was never false

226 return method(*args, **kwargs) 

227 return None 

228 

229 def cacheValueTable(self, valueTable): 

230 """Caches the contents of a value table. 

231 

232 The tables will be cached under two attributes: 

233 

234 the name of the table 

235 : dictionary keyed by id and valued by the corresponding record 

236 

237 the name of the table + `Inv` 

238 : dictionary keyed by a key field and valued by the corresponding id. 

239 

240 Parameters 

241 ---------- 

242 valueTable: string 

243 The value table to be cached. 

244 """ 

245 

246 valueList = list(self.mongoCmd(N.collect, valueTable, N.find)) 

247 repField = ( 

248 N.iso 

249 if valueTable == N.country 

250 else N.eppn 

251 if valueTable == N.user 

252 else N.rep 

253 ) 

254 

255 setattr( 

256 self, valueTable, {G(record, N._id): record for record in valueList}, 

257 ) 

258 setattr( 

259 self, 

260 f"""{valueTable}Inv""", 

261 {G(record, repField): G(record, N._id) for record in valueList}, 

262 ) 

263 if valueTable == N.permissionGroup: 

264 setattr( 

265 self, 

266 f"""{valueTable}Desc""", 

267 {G(record, repField): G(record, N.description) for record in valueList}, 

268 ) 

269 

270 def collect(self): 

271 """Collect the contents of the value tables. 

272 

273 Value tables have content that is needed almost all the time. 

274 All value tables will be completely cached within Db. 

275 

276 !!! note 

277 This is meant to run at start up, before the workers start. 

278 After that, this worker will not execute it again. 

279 See also `recollect`. 

280 

281 !!! warning 

282 We must take other workers into account. They need a signal 

283 to recollect. See `recollect`. 

284 We store the time that this worker has collected each table 

285 in attribute `collected`. 

286 

287 !!! caution 

288 If you change the MongoDb from without, an you forget to 

289 put an appropriate time stamp, the app will not see it untill it 

290 is restarted. 

291 See for example how `root.makeUserRoot` handles this. 

292 

293 !!! warning 

294 This is a complicated app. 

295 Some tables have records that specify whether other records are "actual". 

296 After collecting a value table, the "actual" items will be recomputed. 

297 """ 

298 

299 collected = self.collected 

300 

301 for valueTable in VALUE_TABLES: 

302 self.cacheValueTable(valueTable) 

303 collected[valueTable] = now() 

304 

305 self.collectActualItems() 

306 if DEBUG_SYNCH: 306 ↛ 307line 306 didn't jump to line 307, because the condition on line 306 was never true

307 serverprint(f"""COLLECTED {COMMA.join(sorted(VALUE_TABLES))}""") 

308 

309 def recollect(self, table=None): 

310 """Collect the contents of the value tables if they have changed. 

311 

312 For each value table it will be checked if they have been 

313 collected (by another worker) after this worker has started and if so, 

314 those tables and those tables only will be recollected. 

315 

316 !!! caution 

317 Although the initial `collect` is done before workers start 

318 (`gunicorn --preload`), individual workers will end up with their 

319 own copy of the value table cache. 

320 So when we need to recollect values for our cache, we must notify 

321 in some way that other workers also have to recollect this table. 

322 

323 ### Global recollection 

324 

325 Whenever we recollect a value table, we insert the time of recollection 

326 in a record in the MongoDb. 

327 

328 Somewhere at the start of each request, these records will be checked, 

329 and if needed, recollections will be done before the request processing. 

330 

331 There is a table `collect`, with records having fields `table` and 

332 `dateCollected`. After each (re)collect of a table, the `dateCollected` of 

333 the appropriate record will be set to the current time. 

334 

335 !!! note "recollect()" 

336 A `recollect()` without arguments should be done at the start of each 

337 request. 

338 

339 !!! note "recollect(table)" 

340 A `recollect(table)` should be done whenever this worker has changed 

341 something in that value table. 

342 

343 Parameters 

344 ---------- 

345 table: string, optional `None` 

346 A recollect() without arguments collects *all* value tables that need 

347 collecting based on the times of change as recorded in the `collect` 

348 table. 

349 

350 A recollect of a single table means that this worker has made a change. 

351 After the recollect, a timestamp will go into the `collect` table, 

352 so that other workers can pick it up. 

353 

354 If table is `True`, all timestamps in the `collect` table will be set 

355 to now, so that each worker will refresh its value cache. 

356 """ 

357 

358 collected = self.collected 

359 

360 if table is None: 

361 affected = set() 

362 for valueTable in VALUE_TABLES: 

363 record = self.mongoCmd( 

364 N.recollect, N.collect, N.find_one, {RECOLLECT_NAME: valueTable} 

365 ) 

366 lastChangedGlobally = G(record, RECOLLECT_DATE) 

367 lastChangedHere = G(collected, valueTable) 

368 if lastChangedGlobally and ( 

369 not lastChangedHere or lastChangedHere < lastChangedGlobally 

370 ): 

371 self.cacheValueTable(valueTable) 

372 collected[valueTable] = now() 

373 affected.add(valueTable) 

374 elif table is True: 

375 affected = set() 

376 for valueTable in VALUE_TABLES: 

377 self.cacheValueTable(valueTable) 

378 collected[valueTable] = now() 

379 affected.add(valueTable) 

380 else: 

381 self.cacheValueTable(table) 

382 collected[table] = now() 

383 affected = {table} 

384 if affected: 384 ↛ 396line 384 didn't jump to line 396, because the condition on line 384 was never false

385 justNow = now() 

386 for aTable in affected: 

387 self.mongoCmd( 

388 N.collect, 

389 N.collect, 

390 N.update_one, 

391 {RECOLLECT_NAME: aTable}, 

392 {M_SET: {RECOLLECT_DATE: justNow}}, 

393 upsert=True, 

394 ) 

395 

396 self.collectActualItems(tables=affected) 

397 

398 if affected: 398 ↛ exitline 398 didn't return from function 'recollect', because the condition on line 398 was never false

399 if DEBUG_SYNCH: 399 ↛ 400line 399 didn't jump to line 400, because the condition on line 399 was never true

400 serverprint(f"""COLLECTED {COMMA.join(sorted(affected))}""") 

401 

402 def collectActualItems(self, tables=None): 

403 """Determines which items are "actual". 

404 

405 Actual items are those types and criteria that are specified in a 

406 package record that is itself actual. 

407 A package record is actual if the current data is between its start 

408 and end days. 

409 

410 !!! caution 

411 If only value table needs to be collected that are not 

412 involved in the concept of "actual", nothing will be done. 

413 

414 Parameters 

415 ---------- 

416 tables: set of string, optional `None` 

417 """ 

418 if tables is not None and not (tables & ACTUAL_TABLES): 

419 return 

420 

421 justNow = now() 

422 

423 packageActual = { 

424 G(record, N._id) 

425 for record in self.mongoCmd( 

426 N.collectActualItems, 

427 N.package, 

428 N.find, 

429 {N.startDate: {M_LTE: justNow}, N.endDate: {M_GTE: justNow}}, 

430 ) 

431 } 

432 for record in self.package.values(): 

433 record[N.actual] = G(record, N._id) in packageActual 

434 

435 typeActual = set( 

436 chain.from_iterable( 

437 G(record, N.typeContribution) or [] 

438 for (_id, record) in self.package.items() 

439 if _id in packageActual 

440 ) 

441 ) 

442 for record in self.typeContribution.values(): 

443 record[N.actual] = G(record, N._id) in typeActual 

444 

445 criteriaActual = { 

446 _id 

447 for (_id, record) in self.criteria.items() 

448 if G(record, N.package) in packageActual 

449 } 

450 for record in self.criteria.values(): 

451 record[N.actual] = G(record, N._id) in criteriaActual 

452 

453 self.typeCriteria = {} 

454 for (_id, record) in self.criteria.items(): 

455 if _id in criteriaActual: 

456 for tp in G(record, N.typeContribution) or []: 

457 self.typeCriteria.setdefault(tp, set()).add(_id) 

458 

459 if DEBUG_SYNCH: 459 ↛ 460line 459 didn't jump to line 460, because the condition on line 459 was never true

460 serverprint(f"""UPDATED {", ".join(ACTUAL_TABLES)}""") 

461 

462 def bulkContribWorkflow(self, countryId, bulk): 

463 """Collects workflow information in bulk. 

464 

465 When overviews are being produced, workflow info is needed for a lot 

466 of records. We do not fetch them one by one, but all in one. 

467 

468 We use the MongoDB aggregation pipeline to collect the 

469 contrib ids from the contrib table and to lookup the workflow 

470 information from the workflow table, and to flatten the nested documents 

471 to simple key-value pair. 

472 

473 Parameters 

474 ---------- 

475 countryId: ObjectId 

476 If `None`, all workflow items will be fetched. 

477 Otherwise, this should be 

478 the id of a countryId, and only the workflow 

479 for items belonging to this country are fetched. 

480 bulk: boolean 

481 If `True`, fetches only records that have been bulk-imported. 

482 Those records are marked by the presence of the field `import`. 

483 """ 

484 crit = {} if countryId is None else {"country": countryId} 

485 if bulk: 485 ↛ 486line 485 didn't jump to line 486, because the condition on line 485 was never true

486 crit["import"] = {M_EX: True} 

487 

488 project = { 

489 field: f"${fieldTrans}" for (field, fieldTrans) in OVERVIEW_FIELDS.items() 

490 } 

491 project.update( 

492 { 

493 field: {M_ELEM: [f"${N.workflow}.{fieldTrans}", 0]} 

494 for (field, fieldTrans) in OVERVIEW_FIELDS_WF.items() 

495 } 

496 ) 

497 records = self.mongoCmd( 

498 N.bulkContribWorkflow, 

499 N.contrib, 

500 N.aggregate, 

501 [ 

502 {M_MATCH: crit}, 

503 { 

504 M_LOOKUP: { 

505 "from": N.workflow, 

506 N.localField: N._id, 

507 N.foreignField: N._id, 

508 "as": N.workflow, 

509 } 

510 }, 

511 {M_PROJ: project}, 

512 ], 

513 ) 

514 return records 

515 

516 def makeCrit(self, mainTable, conditions): 

517 """Translate conditons into a MongoDb criterion. 

518 

519 The conditions come from the options on the interface: 

520 whether to constrain to items that have assessments and or reviews. 

521 

522 The result can be fed into an other Mongo query. 

523 It can also be used to filter a list of record that has already been fetched. 

524 

525 !!! hint 

526 `{'assessment': '1'}` means: only those things that have an assessment. 

527 

528 `'-1'`: means: not having an assessment. 

529 

530 `'0'`: means: don't care. 

531 

532 !!! hint 

533 See also `Db.getList`. 

534 

535 Parameters 

536 ---------- 

537 mainTable: string 

538 The name of the table that is being filtered. 

539 conditions: dict 

540 keyed by a table name (such as assessment or review) 

541 and valued by -1, 0 or 1 (as strings). 

542 

543 Result 

544 ------ 

545 dict 

546 keyed by the same table name as `conditions` and valued by a set of 

547 mongo ids of items that satisfy the criterion. 

548 Only for the criteria that do care! 

549 """ 

550 activeOptions = { 

551 G(G(OPTIONS, cond), N.table): crit == ONE 

552 for (cond, crit) in conditions.items() 

553 if crit == ONE or crit == MINONE 

554 } 

555 if None in activeOptions: 555 ↛ 556line 555 didn't jump to line 556, because the condition on line 555 was never true

556 del activeOptions[None] 

557 

558 criterion = {} 

559 for (table, crit) in activeOptions.items(): 559 ↛ 560line 559 didn't jump to line 560, because the loop on line 559 never started

560 eids = { 

561 G(record, mainTable) 

562 for record in self.mongoCmd( 

563 N.makeCrit, 

564 table, 

565 N.find, 

566 {mainTable: {M_EX: True}}, 

567 {mainTable: True}, 

568 ) 

569 } 

570 if crit in criterion: 

571 criterion[crit] |= eids 

572 else: 

573 criterion[crit] = eids 

574 return criterion 

575 

576 def getList( 

577 self, 

578 table, 

579 titleSort=None, 

580 my=None, 

581 our=None, 

582 assign=False, 

583 review=None, 

584 selectable=None, 

585 unfinished=False, 

586 select=False, 

587 **conditions, 

588 ): 

589 """Fetch a list of records from a table. 

590 

591 It fetches all records of a table, but you can constrain 

592 what is fetched and what is returned in several ways, as specified 

593 by the optional arguments. 

594 

595 Some constraints need to fetch more from Mongo than will be returned: 

596 post-filtering may be needed. 

597 

598 

599 !!! note 

600 All records have a field `editors` which contains the ids of users 

601 that are allowed to edit it besides the creator. 

602 

603 !!! note 

604 Assessment records have fields `reviewerE` and `reviewerF` that 

605 point to the expert reviewer and the final reviewer. 

606 

607 !!! hint 

608 `select` and `**conditions` below are used as a consequence of 

609 the filtering on the interface by the options `assessed` and `reviewed`. 

610 See also `Db.makeCrit` and `Db.satisfies`. 

611 

612 Parameters 

613 ---------- 

614 table: string 

615 The table from which the record are fetched. 

616 titleSort: function, optional `None` 

617 The sort key by which the resulting list of records will be sorted. 

618 It must be a function that takes a record and returns a key, for example 

619 the title string of that record. 

620 If absent or None, records will not be sorted. 

621 my: ObjectId, optional `None` 

622 **Task: produce a list of "my" records.** 

623 If passed, it should be the id of a user (typically the one that is 

624 logged in). 

625 Only records that are created/edited by this user will pass through. 

626 our: ObjectId, optional `None` 

627 **Task: produce a list of "our" records (coming from my country).** 

628 If passed, it should be the id of a user (typically the one that is 

629 logged in). 

630 Only records that have a country field containing this country id pass 

631 through. 

632 unfinished: boolean, optional `False` 

633 **Task: produce a list of "my" assessments that are unfinished.** 

634 assign: boolean, optional `False` 

635 **Task: produce a list of assessments that need reviewers.** 

636 Only meaningful if the table is `assessment`. 

637 If `True`, only records that are submitted and who lack at least one 

638 reviewer pass through. 

639 review: ObjectId, optional `None` 

640 **Task: produce a list of assessments that "I" am reviewing or have reviewed.** 

641 Only meaningful if the table is `assessment`. 

642 If passed, it should be the id of a user (typically the one that is 

643 logged in). 

644 Only records pass that have this user in either their `reviewerE` 

645 or in their 

646 `reviewerF` field. 

647 selectable: ObjectId, optional `None` 

648 **Task: produce a list of contribs that the current user can select** 

649 as a DARIAH contribution. 

650 Only meaningful if the table is `contribution`. 

651 Pick those contribs whose `selected` field is not yet filled in. 

652 The value of `selectable` should be an id of a country. 

653 Typically, this is the country of the currently logged in user, 

654 and typically, that user is a National Coordinator. 

655 select: boolean, optional `False` 

656 **Task: trigger addtional filtering by custom `conditions`.** 

657 **conditions: dict 

658 **Task: produce a list of records filtered by custom conditions.** 

659 If `select`, carry out filtering on the retrieved records, where 

660 **conditions specify the filtering 

661 (through `Db.makeCrit` and `Db.satisfies`). 

662 

663 Returns 

664 ------- 

665 list 

666 The result is a sorted list of records. 

667 """ 

668 

669 crit = {} 

670 if my: 

671 crit.update({M_OR: [{N.creator: my}, {N.editors: my}]}) 

672 if our: 

673 crit.update({N.country: our}) 

674 if assign: 

675 crit.update( 

676 {N.submitted: True, M_OR: [{N.reviewerE: None}, {N.reviewerF: None}]} 

677 ) 

678 if review: 

679 crit.update({M_OR: [{N.reviewerE: review}, {N.reviewerF: review}]}) 

680 if selectable: 

681 crit.update({N.country: selectable, N.selected: None}) 

682 

683 if table in VALUE_TABLES: 

684 records = ( 

685 record 

686 for record in getattr(self, table, {}).values() 

687 if ( 

688 ( 

689 my is None 

690 or G(record, N.creator) == my 

691 or my in G(record, N.editors, default=[]) 

692 ) 

693 and (our is None or G(record, N.country) == our) 

694 ) 

695 ) 

696 else: 

697 records = self.mongoCmd(N.getList, table, N.find, crit) 

698 if select: 

699 criterion = self.makeCrit(table, conditions) 

700 records = (record for record in records if Db.satisfies(record, criterion)) 

701 return records if titleSort is None else sorted(records, key=titleSort) 

702 

703 def getItem(self, table, eid): 

704 """Fetch a single record from a table. 

705 

706 Parameters 

707 ---------- 

708 table: string 

709 The table from which the record is fetched. 

710 eid: ObjectId 

711 (Entity) ID of the particular record. 

712 

713 Returns 

714 ------- 

715 dict 

716 """ 

717 if not eid: 717 ↛ 718line 717 didn't jump to line 718, because the condition on line 717 was never true

718 return {} 

719 

720 oid = castObjectId(eid) 

721 

722 if table in VALUE_TABLES: 

723 return G(getattr(self, table, {}), oid, default={}) 

724 

725 records = list(self.mongoCmd(N.getItem, table, N.find, {N._id: oid})) 

726 record = records[0] if len(records) else {} 

727 return record 

728 

729 def getWorkflowItem(self, contribId): 

730 """Fetch a single workflow record. 

731 

732 Parameters 

733 ---------- 

734 contribId: ObjectId 

735 The id of the workflow item to be fetched. 

736 

737 Returns 

738 ------- 

739 dict 

740 The record wrapped in a `control.workflow.apply.WorkflowItem` object. 

741 """ 

742 

743 if contribId is None: 743 ↛ 744line 743 didn't jump to line 744, because the condition on line 743 was never true

744 return {} 

745 

746 crit = {N._id: contribId} 

747 entries = list(self.mongoCmd(N.getWorkflowItem, N.workflow, N.find, crit)) 

748 return entries[0] if entries else {} 

749 

750 def getDetails(self, table, masterField, eids, sortKey=None): 

751 """Fetch the detail records connected to one or more master records. 

752 

753 Parameters 

754 ---------- 

755 table: string 

756 The table from which to fetch the detail records. 

757 masterField: string 

758 The field in the detail records that points to the master record. 

759 eids: ObjectId | iterable of ObjectId 

760 The id(s) of the master record(s). 

761 sortKey: function, optional `None` 

762 A function to sort the resulting records. 

763 """ 

764 if table in VALUE_TABLES: 764 ↛ 765line 764 didn't jump to line 765, because the condition on line 764 was never true

765 crit = eids if isIterable(eids) else [eids] 

766 details = [ 

767 record 

768 for record in getattr(self, table, {}).values() 

769 if G(record, masterField) in crit 

770 ] 

771 else: 

772 crit = {masterField: {M_IN: list(eids)} if isIterable(eids) else eids} 

773 details = list(self.mongoCmd(N.getDetails, table, N.find, crit)) 

774 

775 return sorted(details, key=sortKey) if sortKey else details 

776 

777 def getValueRecords(self, valueTable, constrain=None): 

778 """Fetch records from a value table. 

779 

780 It will apply some standard and custom constraints. 

781 

782 The standard constraints are: if the valueTable is 

783 

784 * `country`: only the DARIAH member countries will be delivered 

785 * `user`: only the non-legacy users will be returned. 

786 

787 !!! note 

788 See the tables.yaml configuration has a key, `constrained`, 

789 which is generated by `config.py` from the field specs of the value tables. 

790 This collects the cases where the valid choices for a value are not all 

791 available values in the table, but only those that are linked to a certain 

792 master record. 

793 

794 !!! hint 

795 If you want to pick a score for an assessment criterion, only those scores 

796 that are linked to that criterion record are eligible. 

797 

798 Parameters 

799 ---------- 

800 valueTable: string 

801 The table from which fetch the records. 

802 constrain: 2-tuple, optional `None` 

803 A custom constraint. If present, it should be a tuple `(fieldName, value)`. 

804 Only records with that value in that field will be delivered. 

805 

806 Returns 

807 ------- 

808 list 

809 """ 

810 

811 records = getattr(self, valueTable, {}).values() 

812 return list( 

813 (r for r in records if G(r, N.isMember) or False) 

814 if valueTable == N.country 

815 else (r for r in records if G(r, N.authority) != N.legacy) 

816 if valueTable == N.user 

817 else (r for r in records if G(r, constrain[0]) == constrain[1]) 

818 if constrain 

819 else records 

820 ) 

821 

822 def getValueInv(self, valueTable, constrain): 

823 """Fetch a mapping from values to ids from a value table. 

824 

825 The mapping is like the *valueTable*`Inv` attribute of `Db`, 

826 but with members restricted by a constraint. 

827 

828 !!! caution 

829 This only works properly if the valueTable has a field `rep`. 

830 

831 Parameters 

832 ---------- 

833 valueTable: string 

834 The table that contains the records. 

835 constrain: 2-tuple, optional `None` 

836 A custom constraint. If present, it should be a tuple `(fieldName, value)`. 

837 Only records with that value in that field will be delivered. 

838 

839 Returns 

840 ------- 

841 dict 

842 Keyed by values, valued by ids. 

843 """ 

844 

845 records = ( 

846 r 

847 for r in getattr(self, valueTable, {}).values() 

848 if G(r, constrain[0]) == constrain[1] 

849 ) 

850 eids = {G(r, N._id) for r in records} 

851 return { 

852 value: eid 

853 for (value, eid) in getattr(self, f"""{valueTable}Inv""", {}).items() 

854 if eid in eids 

855 } 

856 

857 def getValueIds(self, valueTable, constrain): 

858 """Fetch a set of ids from a value table. 

859 

860 The ids are taken from the value reocrds that satisfy a constraint. 

861 but with members restricted by a constraint. 

862 

863 Parameters 

864 ---------- 

865 valueTable: string 

866 The table that contains the records. 

867 constrain: 2-tuple, optional `None` 

868 A custom constraint. If present, it should be a tuple `(fieldName, value)`. 

869 Only records with that value in that field will be delivered. 

870 

871 Returns 

872 ------- 

873 set of ObjectId 

874 """ 

875 

876 records = ( 

877 r 

878 for r in getattr(self, valueTable, {}).values() 

879 if G(r, constrain[0]) == constrain[1] 

880 ) 

881 return {G(r, N._id) for r in records} 

882 

883 def insertItem(self, table, uid, eppn, onlyIfNew, **fields): 

884 """Inserts a new record in a table, possibly only if it is new. 

885 

886 The record will be filled with the specified fields, but also with 

887 provenance fields. 

888 

889 The provenance fields are the creation date, the creator, 

890 and the start of the trail of modifiers. 

891 

892 Parameters 

893 ---------- 

894 table: string 

895 The table in which the record will be inserted. 

896 uid: ObjectId 

897 The user that creates the record, typically the logged in user. 

898 onlyIfNew: boolean 

899 If `True`, it will be checked whether a record with the specified fields 

900 already exists. If so, no record will be inserted. 

901 eppn: string 

902 The eppn of that same user. This is the unique identifier that comes from 

903 the DARIAH authentication service. 

904 **fields: dict 

905 The field names and their contents to populate the new record with. 

906 

907 Returns 

908 ------- 

909 ObjectId 

910 The id of the newly inserted record, or the id of the first existing 

911 record found, if `onlyIfNew` is true. 

912 """ 

913 

914 if onlyIfNew: 

915 existing = [ 

916 G(rec, N._id) 

917 for rec in getattr(self, table, {}).values() 

918 if all(G(rec, k) == v for (k, v) in fields.items()) 

919 ] 

920 if existing: 920 ↛ 921line 920 didn't jump to line 921, because the condition on line 920 was never true

921 return existing[0] 

922 

923 justNow = now() 

924 newRecord = { 

925 N.dateCreated: justNow, 

926 N.creator: uid, 

927 N.modified: [MOD_FMT.format(eppn, justNow)], 

928 **fields, 

929 } 

930 result = self.mongoCmd(N.insertItem, table, N.insert_one, newRecord) 

931 if table in VALUE_TABLES: 

932 self.recollect(table) 

933 return result.inserted_id 

934 

935 def insertMany(self, table, uid, eppn, records): 

936 """Insert several records at once. 

937 

938 Typically used for inserting criteriaEntry en reviewEntry records. 

939 

940 Parameters 

941 ---------- 

942 table: string 

943 The table in which the record will be inserted. 

944 uid: ObjectId 

945 The user that creates the record, typically the logged in user. 

946 eppn: string 

947 The `eppn` of that same user. This is the unique identifier that comes from 

948 the DARIAH authentication service. 

949 records: iterable of dict 

950 The records (as dicts) to insert. 

951 """ 

952 

953 justNow = now() 

954 newRecords = [ 

955 { 

956 N.dateCreated: justNow, 

957 N.creator: uid, 

958 N.modified: [MOD_FMT.format(eppn, justNow)], 

959 **record, 

960 } 

961 for record in records 

962 ] 

963 self.mongoCmd(N.insertMany, table, N.insert_many, newRecords) 

964 

965 def insertUser(self, record): 

966 """Insert a user record, i.e. a record corresponding to a user. 

967 

968 NB: the creator of this record is the system, by name of the 

969 `creatorId` attribute. 

970 

971 Parameters 

972 ---------- 

973 record: dict 

974 The user information to be stored, as a dictionary. 

975 

976 Returns 

977 ------- 

978 None 

979 But note that the new _id and the generated field values are added to the 

980 record. 

981 """ 

982 

983 creatorId = self.creatorId 

984 

985 justNow = now() 

986 record.update( 

987 { 

988 N.dateLastLogin: justNow, 

989 N.statusLastLogin: N.Approved, 

990 N.mayLogin: True, 

991 N.creator: creatorId, 

992 N.dateCreated: justNow, 

993 N.modified: [MOD_FMT.format(CREATOR, justNow)], 

994 } 

995 ) 

996 result = self.mongoCmd(N.insertUser, N.user, N.insert_one, record) 

997 self.recollect(N.user) 

998 record[N._id] = result.inserted_id 

999 

1000 def deleteItem(self, table, eid): 

1001 """Delete a record. 

1002 

1003 Parameters 

1004 ---------- 

1005 table: string 

1006 The table which holds the record to be deleted. 

1007 eid: ObjectId 

1008 (Entity) id of the record to be deleted. 

1009 

1010 Returns 

1011 ------- 

1012 boolean 

1013 Whether the MongoDB operation was successful 

1014 """ 

1015 

1016 oid = castObjectId(eid) 

1017 if oid is None: 1017 ↛ 1018line 1017 didn't jump to line 1018, because the condition on line 1017 was never true

1018 return False 

1019 status = self.mongoCmd(N.deleteItem, table, N.delete_one, {N._id: oid}) 

1020 if table in VALUE_TABLES: 

1021 self.recollect(table) 

1022 return G(status.raw_result, N.ok, default=False) 

1023 

1024 def deleteMany(self, table, crit): 

1025 """Delete a several records. 

1026 

1027 Typically used to delete all detail records of another record. 

1028 

1029 Parameters 

1030 ---------- 

1031 table: string 

1032 The table which holds the records to be deleted. 

1033 crit: dict 

1034 A criterion that specfifies which records must be deleted. 

1035 Given as a dict. 

1036 """ 

1037 

1038 self.mongoCmd(N.deleteMany, table, N.delete_many, crit) 

1039 

1040 def updateField( 

1041 self, table, eid, field, data, actor, modified, nowFields=[], 

1042 ): 

1043 """Update a single field in a single record. 

1044 

1045 !!! hint 

1046 Whenever a field is updated in a record which has the field `isPristine`, 

1047 this field will be deleted from the record. 

1048 The rule is that pristine records are the ones that originate from the 

1049 legacy data and have not changed since then. 

1050 

1051 Parameters 

1052 ---------- 

1053 table: string 

1054 The table which holds the record to be updated. 

1055 eid: ObjectId 

1056 (Entity) id of the record to be updated. 

1057 data: mixed 

1058 The new value of for the updated field. 

1059 actor: ObjectId 

1060 The user that has triggered the update action. 

1061 modified: list of string 

1062 The current provenance trail of the record, which is a list of 

1063 strings of the form "person on date". 

1064 Here "person" is not an ID but a consolidated string representing 

1065 the name of that person. 

1066 The provenance trail will be trimmed in order to prevent excessively long 

1067 trails. On each day, only the last action by each person will be recorded. 

1068 nowFields: iterable of string, optional `[]` 

1069 The names of additional fields in which the current datetime will be stored. 

1070 For exampe, if `submitted` is modified, the current datetime will be saved in 

1071 `dateSubmitted`. 

1072 

1073 Returns 

1074 ------- 

1075 dict | boolean 

1076 The updated record, if the MongoDb operation was successful, else False 

1077 """ 

1078 

1079 oid = castObjectId(eid) 

1080 if oid is None: 1080 ↛ 1081line 1080 didn't jump to line 1081, because the condition on line 1080 was never true

1081 return False 

1082 

1083 justNow = now() 

1084 newModified = filterModified((modified or []) + [f"""{actor}{ON}{justNow}"""]) 

1085 criterion = {N._id: oid} 

1086 nowItems = {nowField: justNow for nowField in nowFields} 

1087 update = { 

1088 field: data, 

1089 N.modified: newModified, 

1090 **nowItems, 

1091 } 

1092 delete = {N.isPristine: E} 

1093 instructions = { 

1094 M_SET: update, 

1095 M_UNSET: delete, 

1096 } 

1097 

1098 status = self.mongoCmd( 

1099 N.updateField, table, N.update_one, criterion, instructions 

1100 ) 

1101 if not G(status.raw_result, N.ok, default=False): 1101 ↛ 1102line 1101 didn't jump to line 1102, because the condition on line 1101 was never true

1102 return False 

1103 

1104 if table in VALUE_TABLES: 

1105 self.recollect(table) 

1106 return ( 

1107 update, 

1108 set(delete.keys()), 

1109 ) 

1110 

1111 def updateUser(self, record): 

1112 """Updates user information. 

1113 

1114 When users log in, or when they are assigned an other status, 

1115 some of their attributes will change. 

1116 

1117 Parameters 

1118 ---------- 

1119 record: dict 

1120 The new user information as a dict. 

1121 """ 

1122 

1123 if N.isPristine in record: 

1124 del record[N.isPristine] 

1125 justNow = now() 

1126 record.update( 

1127 { 

1128 N.dateLastLogin: justNow, 

1129 N.statusLastLogin: N.Approved, 

1130 N.modified: [MOD_FMT.format(CREATOR, justNow)], 

1131 } 

1132 ) 

1133 criterion = {N._id: G(record, N._id)} 

1134 updates = {k: v for (k, v) in record.items() if k != N._id} 

1135 instructions = {M_SET: updates, M_UNSET: {N.isPristine: E}} 

1136 self.mongoCmd(N.updateUser, N.user, N.update_one, criterion, instructions) 

1137 self.recollect(N.user) 

1138 

1139 def dependencies(self, table, record): 

1140 """Computes the number of dependent records of a record. 

1141 

1142 A record is dependent on another record if one of the fields of the 

1143 dependent record contains an id of that other record. 

1144 

1145 Detail records are dependent on master records. 

1146 Also, records that specify a choice in a value table, are dependent on 

1147 the chosen value record. 

1148 

1149 Parameters 

1150 ---------- 

1151 table: string 

1152 The table in which the record resides of which we want to know the 

1153 dependencies. 

1154 record: dict 

1155 The record, given as dict, of which we want to know the dependencies. 

1156 

1157 Returns 

1158 ------- 

1159 int 

1160 """ 

1161 

1162 eid = G(record, N._id) 

1163 if eid is None: 

1164 return {} 

1165 

1166 depSpecs = dict( 

1167 reference=G(REFERENCE_SPECS, table, default={}), 

1168 cascade=G(CASCADE_SPECS, table, default={}), 

1169 ) 

1170 depResult = {} 

1171 for (depKind, depSpec) in depSpecs.items(): 

1172 nDep = 0 

1173 for (referringTable, referringFields) in depSpec.items(): 

1174 if not len(referringFields): 1174 ↛ 1175line 1174 didn't jump to line 1175, because the condition on line 1174 was never true

1175 continue 

1176 fields = list(referringFields) 

1177 crit = ( 

1178 {fields[0]: eid} 

1179 if len(fields) == 1 

1180 else {M_OR: [{field: eid} for field in fields]} 

1181 ) 

1182 

1183 nDep += self.mongoCmd(depKind, referringTable, N.count_documents, crit) 

1184 depResult[depKind] = nDep 

1185 

1186 return depResult 

1187 

1188 def dropWorkflow(self): 

1189 """Drop the entire workflow table. 

1190 

1191 This happens at startup of the server. 

1192 All workflow information will be computed from scratch before the server starts 

1193 serving pages. 

1194 """ 

1195 

1196 self.mongoCmd(N.dropWorkflow, N.workflow, N.drop) 

1197 

1198 def clearWorkflow(self): 

1199 """Clear the entire workflow table. 

1200 

1201 The table is not deleted, but all of its records are. 

1202 This happens when the workflow information is reinitialized while the 

1203 webserver remains running, e.g. by command of a sysadmin or office user. 

1204 (Currently this function is not used). 

1205 """ 

1206 

1207 self.mongoCmd(N.clearWorkflow, N.workflow, N.delete_many, {}) 

1208 

1209 def entries(self, table, crit={}): 

1210 """Get relevant records from a table as a dictionary of entries. 

1211 

1212 Parameters 

1213 ---------- 

1214 table: string 

1215 Table from which the entries are taken. 

1216 crit: dict, optional `{}` 

1217 Criteria to select which records should be used. 

1218 

1219 !!! hint 

1220 This function is used to collect the records that carry user 

1221 content in order to compute workflow information. 

1222 

1223 Its more targeted use is to fetch assessment and review records 

1224 that are relevant to a single contribution. 

1225 

1226 Returns 

1227 ------- 

1228 dict 

1229 Keyed by the ids of the selected records. The records themselves 

1230 are the values. 

1231 """ 

1232 

1233 entries = {} 

1234 for record in list(self.mongoCmd(N.entries, table, N.find, crit, FIELD_PROJ)): 

1235 entries[G(record, N._id)] = record 

1236 

1237 return entries 

1238 

1239 def insertWorkflowMany(self, records): 

1240 """Bulk insert records into the workflow table. 

1241 

1242 Parameters 

1243 ---------- 

1244 records: iterable of dict 

1245 The records to be inserted. 

1246 """ 

1247 

1248 self.mongoCmd(N.insertWorkflowMany, N.workflow, N.insert_many, records) 

1249 

1250 def insertWorkflow(self, record): 

1251 """Insert a single workflow record. 

1252 

1253 Parameters 

1254 ---------- 

1255 record: dict 

1256 The record to be inserted, as a dict. 

1257 """ 

1258 

1259 self.mongoCmd(N.insertWorkflow, N.workflow, N.insert_one, record) 

1260 

1261 def updateWorkflow(self, contribId, record): 

1262 """Replace a workflow record by an other one. 

1263 

1264 !!! note 

1265 Workflow records have an id that is identical to the id of the contribution 

1266 they are about. 

1267 

1268 Parameters 

1269 ---------- 

1270 contribId: ObjectId 

1271 The id of the workflow record that has to be replaced with new information. 

1272 record: dict 

1273 The new record which acts as replacement. 

1274 """ 

1275 

1276 crit = {N._id: contribId} 

1277 self.mongoCmd(N.updateWorkflow, N.workflow, N.replace_one, crit, record) 

1278 

1279 def deleteWorkflow(self, contribId): 

1280 """Delete a workflow record. 

1281 

1282 Parameters 

1283 ---------- 

1284 contribId: ObjectId 

1285 The id of the workflow item to be deleted. 

1286 """ 

1287 

1288 crit = {N._id: contribId} 

1289 self.mongoCmd(N.deleteWorkflow, N.workflow, N.delete_one, crit) 

1290 

1291 @staticmethod 

1292 def satisfies(record, criterion): 

1293 """Test whether a record satifies a criterion. 

1294 

1295 !!! hint 

1296 See also `Db.getList`. 

1297 

1298 Parameters 

1299 ---------- 

1300 record: dict 

1301 A dict of fields. 

1302 criterion: dict 

1303 A dict keyed by a boolean and valued by sets of ids. 

1304 The ids under `True` are the ones that must contain the id of the 

1305 record in question. 

1306 The ids under `False` are the onse that may not contain the id of 

1307 that record. 

1308 

1309 Returns 

1310 ------- 

1311 boolean 

1312 """ 

1313 

1314 eid = G(record, N._id) 

1315 for (crit, eids) in criterion.items(): 1315 ↛ 1316line 1315 didn't jump to line 1316, because the loop on line 1315 never started

1316 if crit and eid not in eids or not crit and eid in eids: 

1317 return False 

1318 return True 

1319 

1320 @staticmethod 

1321 def inCrit(items): 

1322 """Compiles a list of items into a Monngo DB `$in` criterion. 

1323 

1324 Parameters 

1325 ---------- 

1326 items: iterable of mixed 

1327 Typically ObjectIds. 

1328 

1329 Returns 

1330 ------- 

1331 dict 

1332 A MongoDB criterion that tests whether the thing in question is one 

1333 of the items given. 

1334 """ 

1335 

1336 return {M_IN: list(items)}