Coverage for control/db.py : 82%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""All access to the database.
3* MongoDb
4* Create/Read/Update/Delete
5* Caching values
6"""
8import sys
9from itertools import chain
10from pymongo import MongoClient
12from config import Config as C, Names as N
13from control.utils import (
14 pick as G,
15 serverprint,
16 now,
17 filterModified,
18 isIterable,
19 E,
20 ON,
21 ONE,
22 MINONE,
23 COMMA,
24)
25from control.typ.related import castObjectId
27CB = C.base
28CM = C.mongo
29CP = C.perm
30CT = C.tables
31CF = C.workflow
32CW = C.web
34DATABASE = CB.database
35DEBUG = CB.debug
36DEBUG_MONGO = G(DEBUG, N.mongo)
37DEBUG_SYNCH = G(DEBUG, N.synch)
38CREATOR = CB.creator
40M_SET = CM.set
41M_UNSET = CM.unset
42M_LTE = CM.lte
43M_GTE = CM.gte
44M_OR = CM.OR
45M_IN = CM.IN
46M_EX = CM.ex
47M_MATCH = CM.match
48M_PROJ = CM.project
49M_LOOKUP = CM.lookup
50M_ELEM = CM.elem
52SHOW_ARGS = set(CM.showArgs)
53OTHER_COMMANDS = set(CM.otherCommands)
54M_COMMANDS = SHOW_ARGS | OTHER_COMMANDS
56GROUP_RANK = CP.groupRank
58ACTUAL_TABLES = set(CT.actualTables)
59VALUE_TABLES = set(CT.valueTables)
60REFERENCE_SPECS = CT.reference
61CASCADE_SPECS = CT.cascade
63RECOLLECT_SPECS = CT.recollect
64RECOLLECT_TABLE = RECOLLECT_SPECS[N.table]
65RECOLLECT_NAME = RECOLLECT_SPECS[N.tableField]
66RECOLLECT_DATE = RECOLLECT_SPECS[N.dateField]
68WORKFLOW_FIELDS = CF.fields
69FIELD_PROJ = {field: True for field in WORKFLOW_FIELDS}
71OVERVIEW_FIELDS = CT.overviewFields
72OVERVIEW_FIELDS_WF = CT.overviewFieldsWorkflow
74OPTIONS = CW.options
76MOD_FMT = """{} on {}"""
79class Db:
80 """All access to the MongoDb will happen through this class.
82 It will read all content of all value tables and keep it cached.
84 The data in the user tables will be cached by the higher level
85 `control.context.Context`, but only per request.
87 !!! caution
88 We start without a Mongo connection.
89 We make connection the first time we need it, and then keep the
90 connection in the `mongo` attribute.
91 This way, we have a single Mongo connection per worker process,
92 as recommended in
93 [PyMongo](https://api.mongodb.com/python/current/faq.html#is-pymongo-fork-safe).
94 """
96 def __init__(self, regime, test=False):
97 """## Initialization
99 Pick up the connection to MongoDb.
101 !!! note
103 Parameters
104 ----------
105 regime: {"production", "development"}
106 See below
107 test: boolean
108 See below.
109 """
111 self.regime = regime
112 """*string* Whether the app runs in production or in development."""
114 self.test = test
115 """*boolean* Whether to connect to the test database."""
117 database = G(DATABASE, N.test) if test else G(DATABASE, regime)
118 self.database = database
120 mode = f"""regime = {regime} {"test" if test else E}"""
121 if not self.database: 121 ↛ 122line 121 didn't jump to line 122, because the condition on line 121 was never true
122 serverprint(f"""MONGO: no database configured for {mode}""")
123 sys.exit(1)
125 self.client = None
126 """*object* The MongoDb client."""
128 self.mongo = None
129 """*object* The connection to the MongoDb database.
131 The connnection exists before the Db singleton is initialized.
132 """
134 self.collected = {}
135 """*dict* For each value table, the time that this worker last collected it.
137 In the database there is a table which holds the last time for each value
138 table that a worker updated a value in it.
139 """
140 self.collect()
142 creator = [
143 G(record, N._id)
144 for record in self.user.values()
145 if G(record, N.eppn) == CREATOR
146 ]
147 if not creator: 147 ↛ 148line 147 didn't jump to line 148, because the condition on line 147 was never true
148 serverprint(f"""DATABASE: no creator user found in {database}.user""")
149 sys.exit(1)
151 self.creatorId = creator[0]
152 """*ObjectId* System user.
154 There is a userId, fixed by configuration, that represents the system.
155 It is only used when user records are created: those records will said
156 to be created by the system.
157 """
159 def mongoOpen(self):
160 """Open connection with MongoDb.
162 Which database we open, depends on `Db.regime` and `Db.test`.
163 """
165 client = self.client
166 mongo = self.mongo
167 database = self.database
169 if not mongo:
170 client = MongoClient()
171 mongo = client[database]
172 self.client = client
173 self.mongo = mongo
174 serverprint(f"""MONGO: new connection to {database}""")
176 def mongoClose(self):
177 """Close connection with MongoDb.
179 We need this, because before we fork the process to workers,
180 all MongoDb connections should be closed.
181 """
183 client = self.client
185 if client: 185 ↛ exitline 185 didn't return from function 'mongoClose', because the condition on line 185 was never false
186 client.close()
187 self.client = None
188 self.mongo = None
189 serverprint("""MONGO: connection closed""")
191 def mongoCmd(self, label, table, command, *args, **kwargs):
192 """Wrapper around calls to MongoDb.
194 All commands fired at the NongoDb go through this wrapper.
195 It will spit out debug information if mongo debugging is True.
197 Parameters
198 ----------
199 label: string
200 A key to be mentioned in debug messages.
201 Very convenient to put here the name of the method that calls mongoCmd.
202 table: string
203 The table in MongoDB that is targeted by the command.
204 If the table does not exists, no command will be fired.
205 command: string
206 The Mongo command to execute.
207 The command must be listed in the mongo.yaml config file.
208 *args: iterable
209 Additional arguments will be passed straight to the Mongo command.
211 Returns
212 -------
213 mixed
214 Whatever the the MongoDb returns.
215 """
217 self.mongoOpen()
218 mongo = self.mongo
220 method = getattr(mongo[table], command, None) if command in M_COMMANDS else None
221 warning = """!UNDEFINED""" if method is None else E
222 if DEBUG_MONGO: 222 ↛ 223line 222 didn't jump to line 223, because the condition on line 222 was never true
223 argRep = args[0] if args and args[0] and command in SHOW_ARGS else E
224 kwargRep = COMMA.join(f"{k}={v}" for (k, v) in kwargs.items())
225 serverprint(
226 f"""MONGO<<{label}>>.{table}.{command}{warning}({argRep} {kwargRep})"""
227 )
228 if method: 228 ↛ 230line 228 didn't jump to line 230, because the condition on line 228 was never false
229 return method(*args, **kwargs)
230 return None
232 def cacheValueTable(self, valueTable):
233 """Caches the contents of a value table.
235 The tables will be cached under two attributes:
237 the name of the table
238 : dictionary keyed by id and valued by the corresponding record
240 the name of the table + `Inv`
241 : dictionary keyed by a key field and valued by the corresponding id.
243 Parameters
244 ----------
245 valueTable: string
246 The value table to be cached.
247 """
249 valueList = list(self.mongoCmd(N.collect, valueTable, N.find))
250 repField = (
251 N.iso
252 if valueTable == N.country
253 else N.eppn
254 if valueTable == N.user
255 else N.rep
256 )
258 setattr(
259 self,
260 valueTable,
261 {G(record, N._id): record for record in valueList},
262 )
263 setattr(
264 self,
265 f"""{valueTable}Inv""",
266 {G(record, repField): G(record, N._id) for record in valueList},
267 )
268 if valueTable == N.permissionGroup:
269 setattr(
270 self,
271 f"""{valueTable}Desc""",
272 {G(record, repField): G(record, N.description) for record in valueList},
273 )
275 def collect(self):
276 """Collect the contents of the value tables.
278 Value tables have content that is needed almost all the time.
279 All value tables will be completely cached within Db.
281 !!! note
282 This is meant to run at start up, before the workers start.
283 After that, this worker will not execute it again.
284 See also `recollect`.
286 !!! warning
287 We must take other workers into account. They need a signal
288 to recollect. See `recollect`.
289 We store the time that this worker has collected each table
290 in attribute `collected`.
292 !!! caution
293 If you change the MongoDb from without, an you forget to
294 put an appropriate time stamp, the app will not see it until it
295 is restarted.
296 See for example how `root.makeUserRoot` handles this.
298 !!! warning
299 This is a complicated app.
300 Some tables have records that specify whether other records are "actual".
301 After collecting a value table, the "actual" items will be recomputed.
302 """
304 collected = self.collected
306 for valueTable in VALUE_TABLES:
307 self.cacheValueTable(valueTable)
308 justNow = now()
309 collected[valueTable] = justNow
310 self.mongoCmd(
311 N.recollect,
312 N.collect,
313 N.update_one,
314 {RECOLLECT_NAME: valueTable},
315 {M_SET: {RECOLLECT_DATE: justNow}},
316 upsert=True,
317 )
319 self.collectActualItems()
320 if DEBUG_SYNCH: 320 ↛ 321line 320 didn't jump to line 321, because the condition on line 320 was never true
321 serverprint(f"""COLLECTED {COMMA.join(sorted(VALUE_TABLES))}""")
323 def recollect(self, table=None):
324 """Collect the contents of the value tables if they have changed.
326 For each value table it will be checked if they have been
327 collected (by another worker) after this worker has started and if so,
328 those tables and those tables only will be recollected.
330 !!! caution
331 Although the initial `collect` is done before workers start
332 (`gunicorn --preload`), individual workers will end up with their
333 own copy of the value table cache.
334 So when we need to recollect values for our cache, we must notify
335 in some way that other workers also have to recollect this table.
337 ### Global recollection
339 Whenever we (re)collect a value table, we insert the time of recollection
340 in a record in the MongoDb.
342 Somewhere at the start of each request, these records will be checked,
343 and if needed, recollections will be done before the request processing.
345 There is a table `collect`, with records having fields `table` and
346 `dateCollected`. After each (re)collect of a table, the `dateCollected` of
347 the appropriate record will be set to the current time.
349 !!! note "recollect()"
350 A `recollect()` without arguments should be done at the start of each
351 request.
353 !!! note "recollect(table)"
354 A `recollect(table)` should be done whenever this worker has changed
355 something in that value table.
357 Parameters
358 ----------
359 table: string, optional `None`
360 A recollect() without arguments collects *all* value tables that need
361 collecting based on the times of change as recorded in the `collect`
362 table.
364 A recollect of a single table means that this worker has made a change.
365 After the recollect, a timestamp will go into the `collect` table,
366 so that other workers can pick it up.
368 If table is `True`, all timestamps in the `collect` table will be set
369 to now, so that each worker will refresh its value cache.
370 """
372 collected = self.collected
374 if table is None:
375 affected = set()
376 for valueTable in VALUE_TABLES:
377 record = self.mongoCmd(
378 N.recollect, N.collect, N.find_one, {RECOLLECT_NAME: valueTable}
379 )
380 lastChangedGlobally = G(record, RECOLLECT_DATE)
381 lastChangedHere = G(collected, valueTable)
382 if lastChangedGlobally and (
383 not lastChangedHere or lastChangedHere < lastChangedGlobally
384 ):
385 self.cacheValueTable(valueTable)
386 collected[valueTable] = now()
387 affected.add(valueTable)
388 elif table is True:
389 affected = set()
390 for valueTable in VALUE_TABLES:
391 self.cacheValueTable(valueTable)
392 collected[valueTable] = now()
393 affected.add(valueTable)
394 else:
395 self.cacheValueTable(table)
396 collected[table] = now()
397 affected = {table}
398 if affected: 398 ↛ 410line 398 didn't jump to line 410, because the condition on line 398 was never false
399 justNow = now()
400 for aTable in affected:
401 self.mongoCmd(
402 N.recollect,
403 N.collect,
404 N.update_one,
405 {RECOLLECT_NAME: aTable},
406 {M_SET: {RECOLLECT_DATE: justNow}},
407 upsert=True,
408 )
410 self.collectActualItems(tables=affected)
412 if affected: 412 ↛ exitline 412 didn't return from function 'recollect', because the condition on line 412 was never false
413 if DEBUG_SYNCH: 413 ↛ 414line 413 didn't jump to line 414, because the condition on line 413 was never true
414 serverprint(f"""COLLECTED {COMMA.join(sorted(affected))}""")
416 def collectActualItems(self, tables=None):
417 """Determines which items are "actual".
419 Actual items are those types and criteria that are specified in a
420 package record that is itself actual.
421 A package record is actual if the current data is between its start
422 and end days.
424 !!! caution
425 If only value table needs to be collected that are not
426 involved in the concept of "actual", nothing will be done.
428 Parameters
429 ----------
430 tables: set of string, optional `None`
431 """
432 if tables is not None and not (tables & ACTUAL_TABLES):
433 return
435 justNow = now()
437 packageActual = {
438 G(record, N._id)
439 for record in self.mongoCmd(
440 N.collectActualItems,
441 N.package,
442 N.find,
443 {N.startDate: {M_LTE: justNow}, N.endDate: {M_GTE: justNow}},
444 )
445 }
446 for record in self.package.values():
447 record[N.actual] = G(record, N._id) in packageActual
449 typeActual = set(
450 chain.from_iterable(
451 G(record, N.typeContribution) or []
452 for (_id, record) in self.package.items()
453 if _id in packageActual
454 )
455 )
456 for record in self.typeContribution.values():
457 record[N.actual] = G(record, N._id) in typeActual
459 criteriaActual = {
460 _id
461 for (_id, record) in self.criteria.items()
462 if G(record, N.package) in packageActual
463 }
464 for record in self.criteria.values():
465 record[N.actual] = G(record, N._id) in criteriaActual
467 self.typeCriteria = {}
468 for (_id, record) in self.criteria.items():
469 if _id in criteriaActual:
470 for tp in G(record, N.typeContribution) or []:
471 self.typeCriteria.setdefault(tp, set()).add(_id)
473 if DEBUG_SYNCH: 473 ↛ 474line 473 didn't jump to line 474, because the condition on line 473 was never true
474 serverprint(f"""UPDATED {", ".join(ACTUAL_TABLES)}""")
476 def bulkContribWorkflow(self, countryId, bulk):
477 """Collects workflow information in bulk.
479 When overviews are being produced, workflow info is needed for a lot
480 of records. We do not fetch them one by one, but all in one.
482 We use the MongoDB aggregation pipeline to collect the
483 contrib ids from the contrib table and to lookup the workflow
484 information from the workflow table, and to flatten the nested documents
485 to simple key-value pair.
487 Parameters
488 ----------
489 countryId: ObjectId
490 If `None`, all workflow items will be fetched.
491 Otherwise, this should be
492 the id of a countryId, and only the workflow
493 for items belonging to this country are fetched.
494 bulk: boolean
495 If `True`, fetches only records that have been bulk-imported.
496 Those records are marked by the presence of the field `import`.
497 """
498 crit = {} if countryId is None else {"country": countryId}
499 if bulk: 499 ↛ 500line 499 didn't jump to line 500, because the condition on line 499 was never true
500 crit["import"] = {M_EX: True}
502 project = {
503 field: f"${fieldTrans}" for (field, fieldTrans) in OVERVIEW_FIELDS.items()
504 }
505 project.update(
506 {
507 field: {M_ELEM: [f"${N.workflow}.{fieldTrans}", 0]}
508 for (field, fieldTrans) in OVERVIEW_FIELDS_WF.items()
509 }
510 )
511 records = self.mongoCmd(
512 N.bulkContribWorkflow,
513 N.contrib,
514 N.aggregate,
515 [
516 {M_MATCH: crit},
517 {
518 M_LOOKUP: {
519 "from": N.workflow,
520 N.localField: N._id,
521 N.foreignField: N._id,
522 "as": N.workflow,
523 }
524 },
525 {M_PROJ: project},
526 ],
527 )
528 return records
530 def makeCrit(self, mainTable, conditions):
531 """Translate conditons into a MongoDb criterion.
533 The conditions come from the options on the interface:
534 whether to constrain to items that have assessments and or reviews.
536 The result can be fed into an other Mongo query.
537 It can also be used to filter a list of record that has already been fetched.
539 !!! hint
540 `{'assessment': '1'}` means: only those things that have an assessment.
542 `'-1'`: means: not having an assessment.
544 `'0'`: means: don't care.
546 !!! hint
547 See also `Db.getList`.
549 Parameters
550 ----------
551 mainTable: string
552 The name of the table that is being filtered.
553 conditions: dict
554 keyed by a table name (such as assessment or review)
555 and valued by -1, 0 or 1 (as strings).
557 Result
558 ------
559 dict
560 keyed by the same table name as `conditions` and valued by a set of
561 mongo ids of items that satisfy the criterion.
562 Only for the criteria that do care!
563 """
564 activeOptions = {
565 G(G(OPTIONS, cond), N.table): crit == ONE
566 for (cond, crit) in conditions.items()
567 if crit == ONE or crit == MINONE
568 }
569 if None in activeOptions: 569 ↛ 570line 569 didn't jump to line 570, because the condition on line 569 was never true
570 del activeOptions[None]
572 criterion = {}
573 for (table, crit) in activeOptions.items(): 573 ↛ 574line 573 didn't jump to line 574, because the loop on line 573 never started
574 eids = {
575 G(record, mainTable)
576 for record in self.mongoCmd(
577 N.makeCrit,
578 table,
579 N.find,
580 {mainTable: {M_EX: True}},
581 {mainTable: True},
582 )
583 }
584 if crit in criterion:
585 criterion[crit] |= eids
586 else:
587 criterion[crit] = eids
588 return criterion
590 def getList(
591 self,
592 table,
593 titleSort=None,
594 my=None,
595 our=None,
596 assign=False,
597 review=None,
598 selectable=None,
599 unfinished=False,
600 select=False,
601 **conditions,
602 ):
603 """Fetch a list of records from a table.
605 It fetches all records of a table, but you can constrain
606 what is fetched and what is returned in several ways, as specified
607 by the optional arguments.
609 Some constraints need to fetch more from Mongo than will be returned:
610 post-filtering may be needed.
613 !!! note
614 All records have a field `editors` which contains the ids of users
615 that are allowed to edit it besides the creator.
617 !!! note
618 Assessment records have fields `reviewerE` and `reviewerF` that
619 point to the expert reviewer and the final reviewer.
621 !!! hint
622 `select` and `**conditions` below are used as a consequence of
623 the filtering on the interface by the options `assessed` and `reviewed`.
624 See also `Db.makeCrit` and `Db.satisfies`.
626 Parameters
627 ----------
628 table: string
629 The table from which the record are fetched.
630 titleSort: function, optional `None`
631 The sort key by which the resulting list of records will be sorted.
632 It must be a function that takes a record and returns a key, for example
633 the title string of that record.
634 If absent or None, records will not be sorted.
635 my: ObjectId, optional `None`
636 **Task: produce a list of "my" records.**
637 If passed, it should be the id of a user (typically the one that is
638 logged in).
639 Only records that are created/edited by this user will pass through.
640 our: ObjectId, optional `None`
641 **Task: produce a list of "our" records (coming from my country).**
642 If passed, it should be the id of a user (typically the one that is
643 logged in).
644 Only records that have a country field containing this country id pass
645 through.
646 unfinished: boolean, optional `False`
647 **Task: produce a list of "my" assessments that are unfinished.**
648 assign: boolean, optional `False`
649 **Task: produce a list of assessments that need reviewers.**
650 Only meaningful if the table is `assessment`.
651 If `True`, only records that are submitted and who lack at least one
652 reviewer pass through.
653 review: ObjectId, optional `None`
654 **Task: produce a list of assessments that "I" am reviewing or have reviewed.**
655 Only meaningful if the table is `assessment`.
656 If passed, it should be the id of a user (typically the one that is
657 logged in).
658 Only records pass that have this user in either their `reviewerE`
659 or in their
660 `reviewerF` field.
661 selectable: ObjectId, optional `None`
662 **Task: produce a list of contribs that the current user can select**
663 as a DARIAH contribution.
664 Only meaningful if the table is `contribution`.
665 Pick those contribs whose `selected` field is not yet filled in.
666 The value of `selectable` should be an id of a country.
667 Typically, this is the country of the currently logged in user,
668 and typically, that user is a National Coordinator.
669 select: boolean, optional `False`
670 **Task: trigger addtional filtering by custom `conditions`.**
671 **conditions: dict
672 **Task: produce a list of records filtered by custom conditions.**
673 If `select`, carry out filtering on the retrieved records, where
674 **conditions specify the filtering
675 (through `Db.makeCrit` and `Db.satisfies`).
677 Returns
678 -------
679 list
680 The result is a sorted list of records.
681 """
683 crit = {}
684 if my:
685 crit.update({M_OR: [{N.creator: my}, {N.editors: my}]})
686 if our:
687 crit.update({N.country: our})
688 if assign:
689 crit.update(
690 {N.submitted: True, M_OR: [{N.reviewerE: None}, {N.reviewerF: None}]}
691 )
692 if review:
693 crit.update({M_OR: [{N.reviewerE: review}, {N.reviewerF: review}]})
694 if selectable:
695 crit.update({N.country: selectable, N.selected: None})
697 if table in VALUE_TABLES:
698 records = (
699 record
700 for record in getattr(self, table, {}).values()
701 if (
702 (
703 my is None
704 or G(record, N.creator) == my
705 or my in G(record, N.editors, default=[])
706 )
707 and (our is None or G(record, N.country) == our)
708 )
709 )
710 else:
711 records = self.mongoCmd(N.getList, table, N.find, crit)
712 if select:
713 criterion = self.makeCrit(table, conditions)
714 records = (record for record in records if Db.satisfies(record, criterion))
715 return records if titleSort is None else sorted(records, key=titleSort)
717 def getItem(self, table, eid):
718 """Fetch a single record from a table.
720 Parameters
721 ----------
722 table: string
723 The table from which the record is fetched.
724 eid: ObjectId
725 (Entity) ID of the particular record.
727 Returns
728 -------
729 dict
730 """
731 if not eid: 731 ↛ 732line 731 didn't jump to line 732, because the condition on line 731 was never true
732 return {}
734 oid = castObjectId(eid)
736 if table in VALUE_TABLES:
737 return G(getattr(self, table, {}), oid, default={})
739 records = list(self.mongoCmd(N.getItem, table, N.find, {N._id: oid}))
740 record = records[0] if len(records) else {}
741 return record
743 def getWorkflowItem(self, contribId):
744 """Fetch a single workflow record.
746 Parameters
747 ----------
748 contribId: ObjectId
749 The id of the workflow item to be fetched.
751 Returns
752 -------
753 dict
754 The record wrapped in a `control.workflow.apply.WorkflowItem` object.
755 """
757 if contribId is None: 757 ↛ 758line 757 didn't jump to line 758, because the condition on line 757 was never true
758 return {}
760 crit = {N._id: contribId}
761 entries = list(self.mongoCmd(N.getWorkflowItem, N.workflow, N.find, crit))
762 return entries[0] if entries else {}
764 def getDetails(self, table, masterField, eids, sortKey=None):
765 """Fetch the detail records connected to one or more master records.
767 Parameters
768 ----------
769 table: string
770 The table from which to fetch the detail records.
771 masterField: string
772 The field in the detail records that points to the master record.
773 eids: ObjectId | iterable of ObjectId
774 The id(s) of the master record(s).
775 sortKey: function, optional `None`
776 A function to sort the resulting records.
777 """
778 if table in VALUE_TABLES: 778 ↛ 779line 778 didn't jump to line 779, because the condition on line 778 was never true
779 crit = eids if isIterable(eids) else [eids]
780 details = [
781 record
782 for record in getattr(self, table, {}).values()
783 if G(record, masterField) in crit
784 ]
785 else:
786 crit = {masterField: {M_IN: list(eids)} if isIterable(eids) else eids}
787 details = list(self.mongoCmd(N.getDetails, table, N.find, crit))
789 return sorted(details, key=sortKey) if sortKey else details
791 def getValueRecords(self, valueTable, constrain=None, upper=None):
792 """Fetch records from a value table.
794 It will apply some standard and custom constraints.
796 The standard constraints are: if the valueTable is
798 * `country`: only the DARIAH member countries will be delivered
799 * `user`: only the non-legacy users will be returned.
801 !!! note
802 See the tables.yaml configuration has a key, `constrained`,
803 which is generated by `config.py` from the field specs of the value tables.
804 This collects the cases where the valid choices for a value are not all
805 available values in the table, but only those that are linked to a certain
806 master record.
808 !!! hint
809 If you want to pick a score for an assessment criterion, only those scores
810 that are linked to that criterion record are eligible.
812 Parameters
813 ----------
814 valueTable: string
815 The table from which fetch the records.
816 constrain: 2-tuple, optional `None`
817 A custom constraint. If present, it should be a tuple `(fieldName, value)`.
818 Only records with that value in that field will be delivered.
819 upper: string
820 The name of a permission group.
821 If the valueTable is permissionGroup, not all values will be shown,
822 only the values that are not more powerful than this group.
823 This is needed to prevent users to make somebody more powerful
824 then themselves.
826 Returns
827 -------
828 list
829 """
831 records = getattr(self, valueTable, {}).values()
832 result = (
833 (r for r in records if G(r, N.isMember) or False)
834 if valueTable == N.country
835 else (r for r in records if G(r, N.authority) != N.legacy)
836 if valueTable == N.user
837 else (r for r in records if G(r, constrain[0]) == constrain[1])
838 if constrain
839 else records
840 )
841 if valueTable == N.permissionGroup: 841 ↛ 842line 841 didn't jump to line 842, because the condition on line 841 was never true
842 result = (
843 r for r in result if G(r, N.rep, "") not in {N.edit, N.own, N.nobody}
844 )
845 if upper is not None:
846 upperRank = G(GROUP_RANK, upper, 0)
847 result = (
848 r
849 for r in result
850 if G(GROUP_RANK, G(r, N.rep, ""), 100) <= upperRank
851 )
852 return sorted(result, key=lambda r: G(GROUP_RANK, G(r, N.rep, ""), 100))
853 return tuple(result)
855 def getValueInv(self, valueTable, constrain):
856 """Fetch a mapping from values to ids from a value table.
858 The mapping is like the *valueTable*`Inv` attribute of `Db`,
859 but with members restricted by a constraint.
861 !!! caution
862 This only works properly if the valueTable has a field `rep`.
864 Parameters
865 ----------
866 valueTable: string
867 The table that contains the records.
868 constrain: 2-tuple, optional `None`
869 A custom constraint. If present, it should be a tuple `(fieldName, value)`.
870 Only records with that value in that field will be delivered.
872 Returns
873 -------
874 dict
875 Keyed by values, valued by ids.
876 """
878 records = (
879 r
880 for r in getattr(self, valueTable, {}).values()
881 if G(r, constrain[0]) == constrain[1]
882 )
883 eids = {G(r, N._id) for r in records}
884 return {
885 value: eid
886 for (value, eid) in getattr(self, f"""{valueTable}Inv""", {}).items()
887 if eid in eids
888 }
890 def getValueIds(self, valueTable, constrain):
891 """Fetch a set of ids from a value table.
893 The ids are taken from the value reocrds that satisfy a constraint.
894 but with members restricted by a constraint.
896 Parameters
897 ----------
898 valueTable: string
899 The table that contains the records.
900 constrain: 2-tuple, optional `None`
901 A custom constraint. If present, it should be a tuple `(fieldName, value)`.
902 Only records with that value in that field will be delivered.
904 Returns
905 -------
906 set of ObjectId
907 """
909 records = (
910 r
911 for r in getattr(self, valueTable, {}).values()
912 if G(r, constrain[0]) == constrain[1]
913 )
914 return {G(r, N._id) for r in records}
916 def insertItem(self, table, uid, eppn, onlyIfNew, **fields):
917 """Inserts a new record in a table, possibly only if it is new.
919 The record will be filled with the specified fields, but also with
920 provenance fields.
922 The provenance fields are the creation date, the creator,
923 and the start of the trail of modifiers.
925 Parameters
926 ----------
927 table: string
928 The table in which the record will be inserted.
929 uid: ObjectId
930 The user that creates the record, typically the logged in user.
931 onlyIfNew: boolean
932 If `True`, it will be checked whether a record with the specified fields
933 already exists. If so, no record will be inserted.
934 eppn: string
935 The eppn of that same user. This is the unique identifier that comes from
936 the DARIAH authentication service.
937 **fields: dict
938 The field names and their contents to populate the new record with.
940 Returns
941 -------
942 ObjectId
943 The id of the newly inserted record, or the id of the first existing
944 record found, if `onlyIfNew` is true.
945 """
947 if onlyIfNew:
948 existing = [
949 G(rec, N._id)
950 for rec in getattr(self, table, {}).values()
951 if all(G(rec, k) == v for (k, v) in fields.items())
952 ]
953 if existing: 953 ↛ 954line 953 didn't jump to line 954, because the condition on line 953 was never true
954 return existing[0]
956 justNow = now()
957 newRecord = {
958 N.dateCreated: justNow,
959 N.creator: uid,
960 N.modified: [MOD_FMT.format(eppn, justNow)],
961 **fields,
962 }
963 result = self.mongoCmd(N.insertItem, table, N.insert_one, newRecord)
964 if table in VALUE_TABLES:
965 self.recollect(table)
966 return result.inserted_id
968 def insertMany(self, table, uid, eppn, records):
969 """Insert several records at once.
971 Typically used for inserting criteriaEntry en reviewEntry records.
973 Parameters
974 ----------
975 table: string
976 The table in which the record will be inserted.
977 uid: ObjectId
978 The user that creates the record, typically the logged in user.
979 eppn: string
980 The `eppn` of that same user. This is the unique identifier that comes from
981 the DARIAH authentication service.
982 records: iterable of dict
983 The records (as dicts) to insert.
984 """
986 justNow = now()
987 newRecords = [
988 {
989 N.dateCreated: justNow,
990 N.creator: uid,
991 N.modified: [MOD_FMT.format(eppn, justNow)],
992 **record,
993 }
994 for record in records
995 ]
996 self.mongoCmd(N.insertMany, table, N.insert_many, newRecords)
998 def insertUser(self, record):
999 """Insert a user record, i.e. a record corresponding to a user.
1001 NB: the creator of this record is the system, by name of the
1002 `creatorId` attribute.
1004 Parameters
1005 ----------
1006 record: dict
1007 The user information to be stored, as a dictionary.
1009 Returns
1010 -------
1011 None
1012 But note that the new _id and the generated field values are added to the
1013 record.
1014 """
1016 creatorId = self.creatorId
1018 justNow = now()
1019 record.update(
1020 {
1021 N.dateLastLogin: justNow,
1022 N.statusLastLogin: N.Approved,
1023 N.mayLogin: True,
1024 N.creator: creatorId,
1025 N.dateCreated: justNow,
1026 N.modified: [MOD_FMT.format(CREATOR, justNow)],
1027 }
1028 )
1029 result = self.mongoCmd(N.insertUser, N.user, N.insert_one, record)
1030 self.recollect(N.user)
1031 record[N._id] = result.inserted_id
1033 def deleteItem(self, table, eid):
1034 """Delete a record.
1036 Parameters
1037 ----------
1038 table: string
1039 The table which holds the record to be deleted.
1040 eid: ObjectId
1041 (Entity) id of the record to be deleted.
1043 Returns
1044 -------
1045 boolean
1046 Whether the MongoDB operation was successful
1047 """
1049 oid = castObjectId(eid)
1050 if oid is None: 1050 ↛ 1051line 1050 didn't jump to line 1051, because the condition on line 1050 was never true
1051 return False
1052 status = self.mongoCmd(N.deleteItem, table, N.delete_one, {N._id: oid})
1053 if table in VALUE_TABLES:
1054 self.recollect(table)
1055 return G(status.raw_result, N.ok, default=False)
1057 def deleteMany(self, table, crit):
1058 """Delete a several records.
1060 Typically used to delete all detail records of another record.
1062 Parameters
1063 ----------
1064 table: string
1065 The table which holds the records to be deleted.
1066 crit: dict
1067 A criterion that specfifies which records must be deleted.
1068 Given as a dict.
1069 """
1071 self.mongoCmd(N.deleteMany, table, N.delete_many, crit)
1073 def updateField(
1074 self,
1075 table,
1076 eid,
1077 field,
1078 data,
1079 actor,
1080 modified,
1081 nowFields=[],
1082 ):
1083 """Update a single field in a single record.
1085 !!! hint
1086 Whenever a field is updated in a record which has the field `isPristine`,
1087 this field will be deleted from the record.
1088 The rule is that pristine records are the ones that originate from the
1089 legacy data and have not changed since then.
1091 Parameters
1092 ----------
1093 table: string
1094 The table which holds the record to be updated.
1095 eid: ObjectId
1096 (Entity) id of the record to be updated.
1097 data: mixed
1098 The new value of for the updated field.
1099 actor: ObjectId
1100 The user that has triggered the update action.
1101 modified: list of string
1102 The current provenance trail of the record, which is a list of
1103 strings of the form "person on date".
1104 Here "person" is not an ID but a consolidated string representing
1105 the name of that person.
1106 The provenance trail will be trimmed in order to prevent excessively long
1107 trails. On each day, only the last action by each person will be recorded.
1108 nowFields: iterable of string, optional `[]`
1109 The names of additional fields in which the current datetime will be stored.
1110 For exampe, if `submitted` is modified, the current datetime will be saved in
1111 `dateSubmitted`.
1113 Returns
1114 -------
1115 dict | boolean
1116 The updated record, if the MongoDb operation was successful, else False
1117 """
1119 oid = castObjectId(eid)
1120 if oid is None: 1120 ↛ 1121line 1120 didn't jump to line 1121, because the condition on line 1120 was never true
1121 return False
1123 justNow = now()
1124 newModified = filterModified((modified or []) + [f"""{actor}{ON}{justNow}"""])
1125 criterion = {N._id: oid}
1126 nowItems = {nowField: justNow for nowField in nowFields}
1127 update = {
1128 field: data,
1129 N.modified: newModified,
1130 **nowItems,
1131 }
1132 delete = {N.isPristine: E}
1133 instructions = {
1134 M_SET: update,
1135 M_UNSET: delete,
1136 }
1138 status = self.mongoCmd(
1139 N.updateField, table, N.update_one, criterion, instructions
1140 )
1141 if not G(status.raw_result, N.ok, default=False): 1141 ↛ 1142line 1141 didn't jump to line 1142, because the condition on line 1141 was never true
1142 return False
1144 if table in VALUE_TABLES:
1145 self.recollect(table)
1146 return (
1147 update,
1148 set(delete.keys()),
1149 )
1151 def updateUser(self, record):
1152 """Updates user information.
1154 When users log in, or when they are assigned an other status,
1155 some of their attributes will change.
1157 Parameters
1158 ----------
1159 record: dict
1160 The new user information as a dict.
1161 """
1163 if N.isPristine in record:
1164 del record[N.isPristine]
1165 justNow = now()
1166 record.update(
1167 {
1168 N.dateLastLogin: justNow,
1169 N.statusLastLogin: N.Approved,
1170 N.modified: [MOD_FMT.format(CREATOR, justNow)],
1171 }
1172 )
1173 criterion = {N._id: G(record, N._id)}
1174 updates = {k: v for (k, v) in record.items() if k != N._id}
1175 instructions = {M_SET: updates, M_UNSET: {N.isPristine: E}}
1176 self.mongoCmd(N.updateUser, N.user, N.update_one, criterion, instructions)
1177 self.recollect(N.user)
1179 def dependencies(self, table, record):
1180 """Computes the number of dependent records of a record.
1182 A record is dependent on another record if one of the fields of the
1183 dependent record contains an id of that other record.
1185 Detail records are dependent on master records.
1186 Also, records that specify a choice in a value table, are dependent on
1187 the chosen value record.
1189 Parameters
1190 ----------
1191 table: string
1192 The table in which the record resides of which we want to know the
1193 dependencies.
1194 record: dict
1195 The record, given as dict, of which we want to know the dependencies.
1197 Returns
1198 -------
1199 int
1200 """
1202 eid = G(record, N._id)
1203 if eid is None:
1204 return {}
1206 depSpecs = dict(
1207 reference=G(REFERENCE_SPECS, table, default={}),
1208 cascade=G(CASCADE_SPECS, table, default={}),
1209 )
1210 depResult = {}
1211 for (depKind, depSpec) in depSpecs.items():
1212 nDep = 0
1213 for (referringTable, referringFields) in depSpec.items():
1214 if not len(referringFields): 1214 ↛ 1215line 1214 didn't jump to line 1215, because the condition on line 1214 was never true
1215 continue
1216 fields = list(referringFields)
1217 crit = (
1218 {fields[0]: eid}
1219 if len(fields) == 1
1220 else {M_OR: [{field: eid} for field in fields]}
1221 )
1223 nDep += self.mongoCmd(depKind, referringTable, N.count_documents, crit)
1224 depResult[depKind] = nDep
1226 return depResult
1228 def dropWorkflow(self):
1229 """Drop the entire workflow table.
1231 This happens at startup of the server.
1232 All workflow information will be computed from scratch before the server starts
1233 serving pages.
1234 """
1236 self.mongoCmd(N.dropWorkflow, N.workflow, N.drop)
1238 def clearWorkflow(self):
1239 """Clear the entire workflow table.
1241 The table is not deleted, but all of its records are.
1242 This happens when the workflow information is reinitialized while the
1243 webserver remains running, e.g. by command of a sysadmin or office user.
1244 (Currently this function is not used).
1245 """
1247 self.mongoCmd(N.clearWorkflow, N.workflow, N.delete_many, {})
1249 def entries(self, table, crit={}):
1250 """Get relevant records from a table as a dictionary of entries.
1252 Parameters
1253 ----------
1254 table: string
1255 Table from which the entries are taken.
1256 crit: dict, optional `{}`
1257 Criteria to select which records should be used.
1259 !!! hint
1260 This function is used to collect the records that carry user
1261 content in order to compute workflow information.
1263 Its more targeted use is to fetch assessment and review records
1264 that are relevant to a single contribution.
1266 Returns
1267 -------
1268 dict
1269 Keyed by the ids of the selected records. The records themselves
1270 are the values.
1271 """
1273 entries = {}
1274 for record in list(self.mongoCmd(N.entries, table, N.find, crit, FIELD_PROJ)):
1275 entries[G(record, N._id)] = record
1277 return entries
1279 def insertWorkflowMany(self, records):
1280 """Bulk insert records into the workflow table.
1282 Parameters
1283 ----------
1284 records: iterable of dict
1285 The records to be inserted.
1286 """
1288 self.mongoCmd(N.insertWorkflowMany, N.workflow, N.insert_many, records)
1290 def insertWorkflow(self, record):
1291 """Insert a single workflow record.
1293 Parameters
1294 ----------
1295 record: dict
1296 The record to be inserted, as a dict.
1297 """
1299 self.mongoCmd(N.insertWorkflow, N.workflow, N.insert_one, record)
1301 def updateWorkflow(self, contribId, record):
1302 """Replace a workflow record by an other one.
1304 !!! note
1305 Workflow records have an id that is identical to the id of the contribution
1306 they are about.
1308 Parameters
1309 ----------
1310 contribId: ObjectId
1311 The id of the workflow record that has to be replaced with new information.
1312 record: dict
1313 The new record which acts as replacement.
1314 """
1316 crit = {N._id: contribId}
1317 self.mongoCmd(N.updateWorkflow, N.workflow, N.replace_one, crit, record)
1319 def deleteWorkflow(self, contribId):
1320 """Delete a workflow record.
1322 Parameters
1323 ----------
1324 contribId: ObjectId
1325 The id of the workflow item to be deleted.
1326 """
1328 crit = {N._id: contribId}
1329 self.mongoCmd(N.deleteWorkflow, N.workflow, N.delete_one, crit)
1331 @staticmethod
1332 def satisfies(record, criterion):
1333 """Test whether a record satifies a criterion.
1335 !!! hint
1336 See also `Db.getList`.
1338 Parameters
1339 ----------
1340 record: dict
1341 A dict of fields.
1342 criterion: dict
1343 A dict keyed by a boolean and valued by sets of ids.
1344 The ids under `True` are the ones that must contain the id of the
1345 record in question.
1346 The ids under `False` are the onse that may not contain the id of
1347 that record.
1349 Returns
1350 -------
1351 boolean
1352 """
1354 eid = G(record, N._id)
1355 for (crit, eids) in criterion.items(): 1355 ↛ 1356line 1355 didn't jump to line 1356, because the loop on line 1355 never started
1356 if crit and eid not in eids or not crit and eid in eids:
1357 return False
1358 return True
1360 @staticmethod
1361 def inCrit(items):
1362 """Compiles a list of items into a Monngo DB `$in` criterion.
1364 Parameters
1365 ----------
1366 items: iterable of mixed
1367 Typically ObjectIds.
1369 Returns
1370 -------
1371 dict
1372 A MongoDB criterion that tests whether the thing in question is one
1373 of the items given.
1374 """
1376 return {M_IN: list(items)}