From 935cd42306383cb46228a4da216ff57d0bf1cf55 Mon Sep 17 00:00:00 2001 From: Ivan <8611739+IRBorisov@users.noreply.github.com> Date: Tue, 5 Aug 2025 15:27:52 +0300 Subject: [PATCH] R: Refactoring propagation mechanism --- .../apps/oss/models/OperationSchema.py | 78 +-- .../apps/oss/models/OperationSchemaCached.py | 508 +----------------- rsconcept/backend/apps/oss/models/OssCache.py | 188 +++++++ .../apps/oss/models/PropagationEngine.py | 316 +++++++++++ rsconcept/backend/apps/oss/models/utils.py | 79 +++ 5 files changed, 607 insertions(+), 562 deletions(-) create mode 100644 rsconcept/backend/apps/oss/models/OssCache.py create mode 100644 rsconcept/backend/apps/oss/models/PropagationEngine.py create mode 100644 rsconcept/backend/apps/oss/models/utils.py diff --git a/rsconcept/backend/apps/oss/models/OperationSchema.py b/rsconcept/backend/apps/oss/models/OperationSchema.py index 69819eab..d02b7922 100644 --- a/rsconcept/backend/apps/oss/models/OperationSchema.py +++ b/rsconcept/backend/apps/oss/models/OperationSchema.py @@ -1,21 +1,10 @@ ''' Models: OSS API. ''' # pylint: disable=duplicate-code -from typing import Optional - -from cctext import extract_entities from django.db.models import QuerySet from apps.library.models import Editor, LibraryItem, LibraryItemType -from apps.rsform.models import ( - DELETED_ALIAS, - Constituenta, - OrderManager, - RSFormCached, - extract_globals, - replace_entities, - replace_globals -) +from apps.rsform.models import Constituenta, OrderManager, RSFormCached from .Argument import Argument from .Block import Block @@ -25,54 +14,6 @@ from .Operation import Operation, OperationType from .Reference import Reference from .Substitution import Substitution -CstMapping = dict[str, Optional[Constituenta]] -CstSubstitution = list[tuple[Constituenta, Constituenta]] - - -def cst_mapping_to_alias(mapping: CstMapping) -> dict[str, str]: - ''' Convert constituenta mapping to alias mapping. ''' - result: dict[str, str] = {} - for alias, cst in mapping.items(): - if cst is None: - result[alias] = DELETED_ALIAS - else: - result[alias] = cst.alias - return result - - -def map_cst_update_data(cst: Constituenta, data: dict, old_data: dict, mapping: dict[str, str]) -> dict: - ''' Map data for constituenta update. ''' - new_data = {} - if 'term_forms' in data: - if old_data['term_forms'] == cst.term_forms: - new_data['term_forms'] = data['term_forms'] - if 'convention' in data: - new_data['convention'] = data['convention'] - if 'definition_formal' in data: - new_data['definition_formal'] = replace_globals(data['definition_formal'], mapping) - if 'term_raw' in data: - if replace_entities(old_data['term_raw'], mapping) == cst.term_raw: - new_data['term_raw'] = replace_entities(data['term_raw'], mapping) - if 'definition_raw' in data: - if replace_entities(old_data['definition_raw'], mapping) == cst.definition_raw: - new_data['definition_raw'] = replace_entities(data['definition_raw'], mapping) - return new_data - - -def extract_data_references(data: dict, old_data: dict) -> set[str]: - ''' Extract references from data. ''' - result: set[str] = set() - if 'definition_formal' in data: - result.update(extract_globals(data['definition_formal'])) - result.update(extract_globals(old_data['definition_formal'])) - if 'term_raw' in data: - result.update(extract_entities(data['term_raw'])) - result.update(extract_entities(old_data['term_raw'])) - if 'definition_raw' in data: - result.update(extract_entities(data['definition_raw'])) - result.update(extract_entities(old_data['definition_raw'])) - return result - class OperationSchema: ''' Operations schema API wrapper. No caching, propagation and minimal side effects. ''' @@ -101,23 +42,6 @@ class OperationSchema: ''' OSS layout. ''' return Layout.objects.get(oss_id=itemID) - @staticmethod - def create_dependant_mapping(source: RSFormCached, cst_list: list[Constituenta]) -> CstMapping: - ''' Create mapping for dependant Constituents. ''' - if len(cst_list) == len(source.cache.constituents): - return {c.alias: c for c in source.cache.constituents} - inserted_aliases = [cst.alias for cst in cst_list] - depend_aliases: set[str] = set() - for item in cst_list: - depend_aliases.update(item.extract_references()) - depend_aliases.difference_update(inserted_aliases) - alias_mapping: CstMapping = {} - for alias in depend_aliases: - cst = source.cache.by_alias.get(alias) - if cst is not None: - alias_mapping[alias] = cst - return alias_mapping - @staticmethod def create_input(oss: LibraryItem, operation: Operation) -> RSFormCached: ''' Create input RSForm for given Operation. ''' diff --git a/rsconcept/backend/apps/oss/models/OperationSchemaCached.py b/rsconcept/backend/apps/oss/models/OperationSchemaCached.py index 82c0a698..01f310d5 100644 --- a/rsconcept/backend/apps/oss/models/OperationSchemaCached.py +++ b/rsconcept/backend/apps/oss/models/OperationSchemaCached.py @@ -3,25 +3,17 @@ from typing import Optional -from rest_framework.serializers import ValidationError - from apps.library.models import LibraryItem -from apps.rsform.graph import Graph -from apps.rsform.models import INSERT_LAST, Constituenta, CstType, OrderManager, RSFormCached +from apps.rsform.models import Constituenta, CstType, OrderManager, RSFormCached from .Argument import Argument from .Inheritance import Inheritance -from .Operation import Operation, OperationType -from .OperationSchema import ( - CstMapping, - CstSubstitution, - OperationSchema, - cst_mapping_to_alias, - extract_data_references, - map_cst_update_data -) -from .Reference import Reference +from .Operation import Operation +from .OperationSchema import OperationSchema +from .OssCache import OssCache +from .PropagationEngine import PropagationEngine from .Substitution import Substitution +from .utils import CstMapping, CstSubstitution, create_dependant_mapping, extract_data_references class OperationSchemaCached: @@ -29,7 +21,8 @@ class OperationSchemaCached: def __init__(self, model: LibraryItem): self.model = model - self.cache = OssCache(self) + self.cache = OssCache(model.pk) + self.engine = PropagationEngine(self.cache) def delete_reference(self, target: int, keep_connections: bool = False, keep_constituents: bool = False): ''' Delete Reference Operation. ''' @@ -55,7 +48,7 @@ class OperationSchemaCached: if operation.result is not None and len(children) > 0: ids = list(Constituenta.objects.filter(schema=operation.result).values_list('pk', flat=True)) if not keep_constituents: - self._cascade_delete_inherited(operation.pk, ids) + self.engine.on_delete_inherited(operation.pk, ids) else: inheritance_to_delete: list[Inheritance] = [] for child_id in children: @@ -63,7 +56,7 @@ class OperationSchemaCached: child_schema = self.cache.get_schema(child_operation) if child_schema is None: continue - self._undo_substitutions_cst(ids, child_operation, child_schema) + self.engine.undo_substitutions_cst(ids, child_operation, child_schema) for item in self.cache.inheritance[child_id]: if item.parent_id in ids: inheritance_to_delete.append(item) @@ -148,7 +141,7 @@ class OperationSchemaCached: if len(deleted) > 0: if schema is not None: for sub in deleted: - self._undo_substitution(schema, sub) + self.engine.undo_substitution(schema, sub) else: for sub in deleted: self.cache.remove_substitution(sub) @@ -163,7 +156,7 @@ class OperationSchemaCached: substitution=sub_item['substitution'] ) added.append(new_sub) - self._process_added_substitutions(schema, added) + self._on_add_substitutions(schema, added) def execute_operation(self, operation: Operation) -> bool: ''' Execute target Operation. ''' @@ -223,7 +216,7 @@ class OperationSchemaCached: self.cache.insert_schema(source) self.cache.insert_schema(destination) operation = self.cache.get_operation(destination.model.pk) - self._undo_substitutions_cst(items, operation, destination) + self.engine.undo_substitutions_cst(items, operation, destination) inheritance_to_delete = [item for item in self.cache.inheritance[operation.pk] if item.parent_id in items] for item in inheritance_to_delete: self.cache.remove_inheritance(item) @@ -263,14 +256,14 @@ class OperationSchemaCached: ) -> None: ''' Trigger cascade resolutions when new Constituenta is created. ''' self.cache.insert_schema(source) - alias_mapping = OperationSchema.create_dependant_mapping(source, cst_list) + alias_mapping = create_dependant_mapping(source, cst_list) operation = self.cache.get_operation(source.model.pk) - self._cascade_inherit_cst(operation.pk, source, cst_list, alias_mapping, exclude) + self.engine.on_inherit_cst(operation.pk, source, cst_list, alias_mapping, exclude) def after_change_cst_type(self, schemaID: int, target: int, new_type: CstType) -> None: ''' Trigger cascade resolutions when Constituenta type is changed. ''' operation = self.cache.get_operation(schemaID) - self._cascade_change_cst_type(operation.pk, target, new_type) + self.engine.on_change_cst_type(operation.pk, target, new_type) def after_update_cst(self, source: RSFormCached, target: int, data: dict, old_data: dict) -> None: ''' Trigger cascade resolutions when Constituenta data is changed. ''' @@ -282,7 +275,7 @@ class OperationSchemaCached: cst = source.cache.by_alias.get(alias) if cst is not None: alias_mapping[alias] = cst - self._cascade_update_cst( + self.engine.on_update_cst( operation=operation.pk, cst_id=target, data=data, @@ -293,12 +286,12 @@ class OperationSchemaCached: def before_delete_cst(self, sourceID: int, target: list[int]) -> None: ''' Trigger cascade resolutions before Constituents are deleted. ''' operation = self.cache.get_operation(sourceID) - self._cascade_delete_inherited(operation.pk, target) + self.engine.on_delete_inherited(operation.pk, target) def before_substitute(self, schemaID: int, substitutions: CstSubstitution) -> None: ''' Trigger cascade resolutions before Constituents are substituted. ''' operation = self.cache.get_operation(schemaID) - self._cascade_before_substitute(substitutions, operation) + self.engine.on_before_substitute(substitutions, operation) def before_delete_arguments(self, target: Operation, arguments: list[Operation]) -> None: ''' Trigger cascade resolutions before arguments are deleted. ''' @@ -307,7 +300,7 @@ class OperationSchemaCached: for argument in arguments: parent_schema = self.cache.get_schema(argument) if parent_schema is not None: - self._execute_delete_inherited(target.pk, [cst.pk for cst in parent_schema.cache.constituents]) + self.engine.delete_inherited(target.pk, [cst.pk for cst in parent_schema.cache.constituents]) def after_create_arguments(self, target: Operation, arguments: list[Operation]) -> None: ''' Trigger cascade resolutions after arguments are created. ''' @@ -318,294 +311,15 @@ class OperationSchemaCached: parent_schema = self.cache.get_schema(argument) if parent_schema is None: continue - self._execute_inherit_cst( + self.engine.inherit_cst( target_operation=target.pk, source=parent_schema, items=list(parent_schema.constituentsQ().order_by('order')), mapping={} ) - # pylint: disable=too-many-arguments, too-many-positional-arguments - def _cascade_inherit_cst( - self, target_operation: int, - source: RSFormCached, - items: list[Constituenta], - mapping: CstMapping, - exclude: Optional[list[int]] = None - ) -> None: - children = self.cache.extend_graph.outputs[target_operation] - if len(children) == 0: - return - for child_id in children: - if not exclude or child_id not in exclude: - self._execute_inherit_cst(child_id, source, items, mapping) - - def _execute_inherit_cst( - self, - target_operation: int, - source: RSFormCached, - items: list[Constituenta], - mapping: CstMapping - ) -> None: - operation = self.cache.operation_by_id[target_operation] - destination = self.cache.get_schema(operation) - if destination is None: - return - - self.cache.ensure_loaded_subs() - new_mapping = self._transform_mapping(mapping, operation, destination) - alias_mapping = cst_mapping_to_alias(new_mapping) - insert_where = self._determine_insert_position(items[0].pk, operation, source, destination) - new_cst_list = destination.insert_copy(items, insert_where, alias_mapping) - for index, cst in enumerate(new_cst_list): - new_inheritance = Inheritance.objects.create( - operation=operation, - child=cst, - parent=items[index] - ) - self.cache.insert_inheritance(new_inheritance) - new_mapping = {alias_mapping[alias]: cst for alias, cst in new_mapping.items()} - self._cascade_inherit_cst(operation.pk, destination, new_cst_list, new_mapping) - - def _cascade_change_cst_type(self, operation_id: int, cst_id: int, ctype: CstType) -> None: - children = self.cache.extend_graph.outputs[operation_id] - if len(children) == 0: - return - self.cache.ensure_loaded_subs() - for child_id in children: - child_operation = self.cache.operation_by_id[child_id] - successor_id = self.cache.get_inheritor(cst_id, child_id) - if successor_id is None: - continue - child_schema = self.cache.get_schema(child_operation) - if child_schema is None: - continue - if child_schema.change_cst_type(successor_id, ctype): - self._cascade_change_cst_type(child_id, successor_id, ctype) - - # pylint: disable=too-many-arguments, too-many-positional-arguments - def _cascade_update_cst( - self, - operation: int, - cst_id: int, - data: dict, old_data: dict, - mapping: CstMapping - ) -> None: - children = self.cache.extend_graph.outputs[operation] - if len(children) == 0: - return - self.cache.ensure_loaded_subs() - for child_id in children: - child_operation = self.cache.operation_by_id[child_id] - successor_id = self.cache.get_inheritor(cst_id, child_id) - if successor_id is None: - continue - child_schema = self.cache.get_schema(child_operation) - assert child_schema is not None - new_mapping = self._transform_mapping(mapping, child_operation, child_schema) - alias_mapping = cst_mapping_to_alias(new_mapping) - successor = child_schema.cache.by_id.get(successor_id) - if successor is None: - continue - new_data = map_cst_update_data(successor, data, old_data, alias_mapping) - if len(new_data) == 0: - continue - new_old_data = child_schema.update_cst(successor.pk, new_data) - if len(new_old_data) == 0: - continue - new_mapping = {alias_mapping[alias]: cst for alias, cst in new_mapping.items()} - self._cascade_update_cst( - operation=child_id, - cst_id=successor_id, - data=new_data, - old_data=new_old_data, - mapping=new_mapping - ) - - def _cascade_delete_inherited(self, operation: int, target: list[int]) -> None: - children = self.cache.extend_graph.outputs[operation] - if len(children) == 0: - return - self.cache.ensure_loaded_subs() - for child_id in children: - self._execute_delete_inherited(child_id, target) - - def _execute_delete_inherited(self, operation_id: int, parent_ids: list[int]) -> None: - operation = self.cache.operation_by_id[operation_id] - schema = self.cache.get_schema(operation) - if schema is None: - return - self._undo_substitutions_cst(parent_ids, operation, schema) - target_ids = self.cache.get_inheritors_list(parent_ids, operation_id) - self._cascade_delete_inherited(operation_id, target_ids) - if len(target_ids) > 0: - self.cache.remove_cst(operation_id, target_ids) - schema.delete_cst(target_ids) - - def _cascade_before_substitute(self, substitutions: CstSubstitution, operation: Operation) -> None: - children = self.cache.extend_graph.outputs[operation.pk] - if len(children) == 0: - return - self.cache.ensure_loaded_subs() - for child_id in children: - child_operation = self.cache.operation_by_id[child_id] - child_schema = self.cache.get_schema(child_operation) - if child_schema is None: - continue - new_substitutions = self._transform_substitutions(substitutions, child_id, child_schema) - if len(new_substitutions) == 0: - continue - self._cascade_before_substitute(new_substitutions, child_operation) - child_schema.substitute(new_substitutions) - - def _cascade_partial_mapping( - self, - mapping: CstMapping, - target: list[int], - operation: int, - schema: RSFormCached - ) -> None: - alias_mapping = cst_mapping_to_alias(mapping) - schema.apply_partial_mapping(alias_mapping, target) - children = self.cache.extend_graph.outputs[operation] - if len(children) == 0: - return - self.cache.ensure_loaded_subs() - for child_id in children: - child_operation = self.cache.operation_by_id[child_id] - child_schema = self.cache.get_schema(child_operation) - if child_schema is None: - continue - new_mapping = self._transform_mapping(mapping, child_operation, child_schema) - if not new_mapping: - continue - new_target = self.cache.get_inheritors_list(target, child_id) - if len(new_target) == 0: - continue - self._cascade_partial_mapping(new_mapping, new_target, child_id, child_schema) - - def _transform_mapping(self, mapping: CstMapping, operation: Operation, schema: RSFormCached) -> CstMapping: - if len(mapping) == 0: - return mapping - result: CstMapping = {} - for alias, cst in mapping.items(): - if cst is None: - result[alias] = None - continue - successor_id = self.cache.get_successor(cst.pk, operation.pk) - if successor_id is None: - continue - successor = schema.cache.by_id.get(successor_id) - if successor is None: - continue - result[alias] = successor - return result - - def _determine_insert_position( - self, prototype_id: int, - operation: Operation, - source: RSFormCached, - destination: RSFormCached - ) -> int: - ''' Determine insert_after for new constituenta. ''' - prototype = source.cache.by_id[prototype_id] - prototype_index = source.cache.constituents.index(prototype) - if prototype_index == 0: - return 0 - prev_cst = source.cache.constituents[prototype_index - 1] - inherited_prev_id = self.cache.get_successor(prev_cst.pk, operation.pk) - if inherited_prev_id is None: - return INSERT_LAST - prev_cst = destination.cache.by_id[inherited_prev_id] - prev_index = destination.cache.constituents.index(prev_cst) - return prev_index + 1 - - def _transform_substitutions( - self, - target: CstSubstitution, - operation: int, - schema: RSFormCached - ) -> CstSubstitution: - result: CstSubstitution = [] - for current_sub in target: - sub_replaced = False - new_substitution_id = self.cache.get_inheritor(current_sub[1].pk, operation) - if new_substitution_id is None: - for sub in self.cache.substitutions[operation]: - if sub.original_id == current_sub[1].pk: - sub_replaced = True - new_substitution_id = self.cache.get_inheritor(sub.original_id, operation) - break - - new_original_id = self.cache.get_inheritor(current_sub[0].pk, operation) - original_replaced = False - if new_original_id is None: - for sub in self.cache.substitutions[operation]: - if sub.original_id == current_sub[0].pk: - original_replaced = True - sub.original_id = current_sub[1].pk - sub.save() - new_original_id = new_substitution_id - new_substitution_id = self.cache.get_inheritor(sub.substitution_id, operation) - break - - if sub_replaced and original_replaced: - raise ValidationError({'propagation': 'Substitution breaks OSS substitutions.'}) - - for sub in self.cache.substitutions[operation]: - if sub.substitution_id == current_sub[0].pk: - sub.substitution_id = current_sub[1].pk - sub.save() - - if new_original_id is not None and new_substitution_id is not None: - result.append((schema.cache.by_id[new_original_id], schema.cache.by_id[new_substitution_id])) - return result - - def _undo_substitutions_cst(self, target_ids: list[int], operation: Operation, schema: RSFormCached) -> None: - to_process = [] - for sub in self.cache.substitutions[operation.pk]: - if sub.original_id in target_ids or sub.substitution_id in target_ids: - to_process.append(sub) - for sub in to_process: - self._undo_substitution(schema, sub, target_ids) - - def _undo_substitution( - self, - schema: RSFormCached, - target: Substitution, - ignore_parents: Optional[list[int]] = None - ) -> None: - if ignore_parents is None: - ignore_parents = [] - operation_id = target.operation_id - original_schema = self.cache.get_schema_by_id(target.original.schema_id) - dependant = [] - for cst_id in original_schema.get_dependant([target.original_id]): - if cst_id not in ignore_parents: - inheritor_id = self.cache.get_inheritor(cst_id, operation_id) - if inheritor_id is not None: - dependant.append(inheritor_id) - - self.cache.substitutions[operation_id].remove(target) - target.delete() - - new_original: Optional[Constituenta] = None - if target.original_id not in ignore_parents: - full_cst = Constituenta.objects.get(pk=target.original_id) - cst_mapping = OperationSchema.create_dependant_mapping(original_schema, [full_cst]) - self._execute_inherit_cst(operation_id, original_schema, [full_cst], cst_mapping) - new_original_id = self.cache.get_inheritor(target.original_id, operation_id) - assert new_original_id is not None - new_original = schema.cache.by_id[new_original_id] - - if len(dependant) > 0: - substitution_id = self.cache.get_inheritor(target.substitution_id, operation_id) - assert substitution_id is not None - substitution_inheritor = schema.cache.by_id[substitution_id] - mapping = {substitution_inheritor.alias: new_original} - self._cascade_partial_mapping(mapping, dependant, operation_id, schema) - - def _process_added_substitutions(self, schema: Optional[RSFormCached], added: list[Substitution]) -> None: + def _on_add_substitutions(self, schema: Optional[RSFormCached], added: list[Substitution]) -> None: + ''' Trigger cascade resolutions when Constituenta substitution is added. ''' if len(added) == 0: return if schema is None: @@ -626,179 +340,3 @@ class OperationSchemaCached: schema.substitute(cst_mapping) for sub in added: self.cache.insert_substitution(sub) - - -class OssCache: - ''' Cache for OSS data. ''' - - def __init__(self, oss: OperationSchemaCached): - self._oss = oss - self._schemas: list[RSFormCached] = [] - self._schema_by_id: dict[int, RSFormCached] = {} - - self.operations = list(Operation.objects.filter(oss=oss.model).only('result_id', 'operation_type')) - self.operation_by_id = {operation.pk: operation for operation in self.operations} - self.graph = Graph[int]() - self.extend_graph = Graph[int]() - for operation in self.operations: - self.graph.add_node(operation.pk) - self.extend_graph.add_node(operation.pk) - - references = Reference.objects.filter(reference__oss=self._oss.model).only('reference_id', 'target_id') - self.reference_target = {ref.reference_id: ref.target_id for ref in references} - arguments = Argument.objects \ - .filter(operation__oss=self._oss.model) \ - .only('operation_id', 'argument_id') \ - .order_by('order') - for argument in arguments: - self.graph.add_edge(argument.argument_id, argument.operation_id) - self.extend_graph.add_edge(argument.argument_id, argument.operation_id) - target = self.reference_target.get(argument.argument_id) - if target is not None: - self.extend_graph.add_edge(target, argument.operation_id) - - self.is_loaded_subs = False - self.substitutions: dict[int, list[Substitution]] = {} - self.inheritance: dict[int, list[Inheritance]] = {} - - def ensure_loaded_subs(self) -> None: - ''' Ensure cache is fully loaded. ''' - if self.is_loaded_subs: - return - self.is_loaded_subs = True - for operation in self.operations: - self.inheritance[operation.pk] = [] - self.substitutions[operation.pk] = [] - for sub in Substitution.objects.filter(operation__oss=self._oss.model).only( - 'operation_id', 'original_id', 'substitution_id', 'original__schema_id'): - self.substitutions[sub.operation_id].append(sub) - for item in Inheritance.objects.filter(operation__oss=self._oss.model).only( - 'operation_id', 'parent_id', 'child_id'): - self.inheritance[item.operation_id].append(item) - - def get_schema(self, operation: Operation) -> Optional[RSFormCached]: - ''' Get schema by Operation. ''' - if operation.result_id is None: - return None - if operation.result_id in self._schema_by_id: - return self._schema_by_id[operation.result_id] - else: - schema = RSFormCached.from_id(operation.result_id) - schema.cache.ensure_loaded() - self._insert_new(schema) - return schema - - def get_schema_by_id(self, target: int) -> RSFormCached: - ''' Get schema by Operation. ''' - if target in self._schema_by_id: - return self._schema_by_id[target] - else: - schema = RSFormCached.from_id(target) - schema.cache.ensure_loaded() - self._insert_new(schema) - return schema - - def get_operation(self, schemaID: int) -> Operation: - ''' Get operation by schema. ''' - for operation in self.operations: - if operation.result_id == schemaID and operation.operation_type != OperationType.REFERENCE: - return operation - raise ValueError(f'Operation for schema {schemaID} not found') - - def get_inheritor(self, parent_cst: int, operation: int) -> Optional[int]: - ''' Get child for parent inside target RSFrom. ''' - for item in self.inheritance[operation]: - if item.parent_id == parent_cst: - return item.child_id - return None - - def get_inheritors_list(self, target: list[int], operation: int) -> list[int]: - ''' Get child for parent inside target RSFrom. ''' - result = [] - for item in self.inheritance[operation]: - if item.parent_id in target: - result.append(item.child_id) - return result - - def get_successor(self, parent_cst: int, operation: int) -> Optional[int]: - ''' Get child for parent inside target RSFrom including substitutions. ''' - for sub in self.substitutions[operation]: - if sub.original_id == parent_cst: - return self.get_inheritor(sub.substitution_id, operation) - return self.get_inheritor(parent_cst, operation) - - def insert_schema(self, schema: RSFormCached) -> None: - ''' Insert new schema. ''' - if not self._schema_by_id.get(schema.model.pk): - schema.cache.ensure_loaded() - self._insert_new(schema) - - def insert_argument(self, argument: Argument) -> None: - ''' Insert new argument. ''' - self.graph.add_edge(argument.argument_id, argument.operation_id) - self.extend_graph.add_edge(argument.argument_id, argument.operation_id) - target = self.reference_target.get(argument.argument_id) - if target is not None: - self.extend_graph.add_edge(target, argument.operation_id) - - def insert_inheritance(self, inheritance: Inheritance) -> None: - ''' Insert new inheritance. ''' - self.inheritance[inheritance.operation_id].append(inheritance) - - def insert_substitution(self, sub: Substitution) -> None: - ''' Insert new substitution. ''' - self.substitutions[sub.operation_id].append(sub) - - def remove_cst(self, operation: int, target: list[int]) -> None: - ''' Remove constituents from operation. ''' - subs_to_delete = [ - sub for sub in self.substitutions[operation] - if sub.original_id in target or sub.substitution_id in target - ] - for sub in subs_to_delete: - self.substitutions[operation].remove(sub) - inherit_to_delete = [item for item in self.inheritance[operation] if item.child_id in target] - for item in inherit_to_delete: - self.inheritance[operation].remove(item) - - def remove_schema(self, schema: RSFormCached) -> None: - ''' Remove schema from cache. ''' - self._schemas.remove(schema) - del self._schema_by_id[schema.model.pk] - - def remove_operation(self, operation: int) -> None: - ''' Remove operation from cache. ''' - target = self.operation_by_id[operation] - self.graph.remove_node(operation) - self.extend_graph.remove_node(operation) - if target.result_id in self._schema_by_id: - self._schemas.remove(self._schema_by_id[target.result_id]) - del self._schema_by_id[target.result_id] - self.operations.remove(self.operation_by_id[operation]) - del self.operation_by_id[operation] - if operation in self.reference_target: - del self.reference_target[operation] - if self.is_loaded_subs: - del self.substitutions[operation] - del self.inheritance[operation] - - def remove_argument(self, argument: Argument) -> None: - ''' Remove argument from cache. ''' - self.graph.remove_edge(argument.argument_id, argument.operation_id) - self.extend_graph.remove_edge(argument.argument_id, argument.operation_id) - target = self.reference_target.get(argument.argument_id) - if target is not None: - if not Argument.objects.filter(argument_id=target, operation_id=argument.operation_id).exists(): - self.extend_graph.remove_edge(target, argument.operation_id) - - def remove_substitution(self, target: Substitution) -> None: - ''' Remove substitution from cache. ''' - self.substitutions[target.operation_id].remove(target) - - def remove_inheritance(self, target: Inheritance) -> None: - ''' Remove inheritance from cache. ''' - self.inheritance[target.operation_id].remove(target) - - def _insert_new(self, schema: RSFormCached) -> None: - self._schemas.append(schema) - self._schema_by_id[schema.model.pk] = schema diff --git a/rsconcept/backend/apps/oss/models/OssCache.py b/rsconcept/backend/apps/oss/models/OssCache.py new file mode 100644 index 00000000..cd5a2f0e --- /dev/null +++ b/rsconcept/backend/apps/oss/models/OssCache.py @@ -0,0 +1,188 @@ +''' Models: OSS API. ''' + +from typing import Optional + +from apps.rsform.graph import Graph +from apps.rsform.models import RSFormCached + +from .Argument import Argument +from .Inheritance import Inheritance +from .Operation import Operation, OperationType +from .Reference import Reference +from .Substitution import Substitution + + +class OssCache: + ''' Cache for OSS data. ''' + + def __init__(self, item_id: int): + self._item_id = item_id + self._schemas: list[RSFormCached] = [] + self._schema_by_id: dict[int, RSFormCached] = {} + + self.operations = list(Operation.objects.filter(oss_id=item_id).only('result_id', 'operation_type')) + self.operation_by_id = {operation.pk: operation for operation in self.operations} + self.graph = Graph[int]() + self.extend_graph = Graph[int]() + for operation in self.operations: + self.graph.add_node(operation.pk) + self.extend_graph.add_node(operation.pk) + + references = Reference.objects.filter(reference__oss_id=self._item_id).only('reference_id', 'target_id') + self.reference_target = {ref.reference_id: ref.target_id for ref in references} + arguments = Argument.objects \ + .filter(operation__oss_id=self._item_id) \ + .only('operation_id', 'argument_id') \ + .order_by('order') + for argument in arguments: + self.graph.add_edge(argument.argument_id, argument.operation_id) + self.extend_graph.add_edge(argument.argument_id, argument.operation_id) + target = self.reference_target.get(argument.argument_id) + if target is not None: + self.extend_graph.add_edge(target, argument.operation_id) + + self.is_loaded_subs = False + self.substitutions: dict[int, list[Substitution]] = {} + self.inheritance: dict[int, list[Inheritance]] = {} + + def ensure_loaded_subs(self) -> None: + ''' Ensure cache is fully loaded. ''' + if self.is_loaded_subs: + return + self.is_loaded_subs = True + for operation in self.operations: + self.inheritance[operation.pk] = [] + self.substitutions[operation.pk] = [] + for sub in Substitution.objects.filter(operation__oss_id=self._item_id).only( + 'operation_id', 'original_id', 'substitution_id', 'original__schema_id'): + self.substitutions[sub.operation_id].append(sub) + for item in Inheritance.objects.filter(operation__oss_id=self._item_id).only( + 'operation_id', 'parent_id', 'child_id'): + self.inheritance[item.operation_id].append(item) + + def get_schema(self, operation: Operation) -> Optional[RSFormCached]: + ''' Get schema by Operation. ''' + if operation.result_id is None: + return None + if operation.result_id in self._schema_by_id: + return self._schema_by_id[operation.result_id] + else: + schema = RSFormCached.from_id(operation.result_id) + schema.cache.ensure_loaded() + self._insert_new(schema) + return schema + + def get_schema_by_id(self, target: int) -> RSFormCached: + ''' Get schema by Operation. ''' + if target in self._schema_by_id: + return self._schema_by_id[target] + else: + schema = RSFormCached.from_id(target) + schema.cache.ensure_loaded() + self._insert_new(schema) + return schema + + def get_operation(self, schemaID: int) -> Operation: + ''' Get operation by schema. ''' + for operation in self.operations: + if operation.result_id == schemaID and operation.operation_type != OperationType.REFERENCE: + return operation + raise ValueError(f'Operation for schema {schemaID} not found') + + def get_inheritor(self, parent_cst: int, operation: int) -> Optional[int]: + ''' Get child for parent inside target RSFrom. ''' + for item in self.inheritance[operation]: + if item.parent_id == parent_cst: + return item.child_id + return None + + def get_inheritors_list(self, target: list[int], operation: int) -> list[int]: + ''' Get child for parent inside target RSFrom. ''' + result = [] + for item in self.inheritance[operation]: + if item.parent_id in target: + result.append(item.child_id) + return result + + def get_successor(self, parent_cst: int, operation: int) -> Optional[int]: + ''' Get child for parent inside target RSFrom including substitutions. ''' + for sub in self.substitutions[operation]: + if sub.original_id == parent_cst: + return self.get_inheritor(sub.substitution_id, operation) + return self.get_inheritor(parent_cst, operation) + + def insert_schema(self, schema: RSFormCached) -> None: + ''' Insert new schema. ''' + if not self._schema_by_id.get(schema.model.pk): + schema.cache.ensure_loaded() + self._insert_new(schema) + + def insert_argument(self, argument: Argument) -> None: + ''' Insert new argument. ''' + self.graph.add_edge(argument.argument_id, argument.operation_id) + self.extend_graph.add_edge(argument.argument_id, argument.operation_id) + target = self.reference_target.get(argument.argument_id) + if target is not None: + self.extend_graph.add_edge(target, argument.operation_id) + + def insert_inheritance(self, inheritance: Inheritance) -> None: + ''' Insert new inheritance. ''' + self.inheritance[inheritance.operation_id].append(inheritance) + + def insert_substitution(self, sub: Substitution) -> None: + ''' Insert new substitution. ''' + self.substitutions[sub.operation_id].append(sub) + + def remove_cst(self, operation: int, target: list[int]) -> None: + ''' Remove constituents from operation. ''' + subs_to_delete = [ + sub for sub in self.substitutions[operation] + if sub.original_id in target or sub.substitution_id in target + ] + for sub in subs_to_delete: + self.substitutions[operation].remove(sub) + inherit_to_delete = [item for item in self.inheritance[operation] if item.child_id in target] + for item in inherit_to_delete: + self.inheritance[operation].remove(item) + + def remove_schema(self, schema: RSFormCached) -> None: + ''' Remove schema from cache. ''' + self._schemas.remove(schema) + del self._schema_by_id[schema.model.pk] + + def remove_operation(self, operation: int) -> None: + ''' Remove operation from cache. ''' + target = self.operation_by_id[operation] + self.graph.remove_node(operation) + self.extend_graph.remove_node(operation) + if target.result_id in self._schema_by_id: + self._schemas.remove(self._schema_by_id[target.result_id]) + del self._schema_by_id[target.result_id] + self.operations.remove(self.operation_by_id[operation]) + del self.operation_by_id[operation] + if operation in self.reference_target: + del self.reference_target[operation] + if self.is_loaded_subs: + del self.substitutions[operation] + del self.inheritance[operation] + + def remove_argument(self, argument: Argument) -> None: + ''' Remove argument from cache. ''' + self.graph.remove_edge(argument.argument_id, argument.operation_id) + self.extend_graph.remove_edge(argument.argument_id, argument.operation_id) + target = self.reference_target.get(argument.argument_id) + if target is not None: + if not Argument.objects.filter(argument_id=target, operation_id=argument.operation_id).exists(): + self.extend_graph.remove_edge(target, argument.operation_id) + + def remove_substitution(self, target: Substitution) -> None: + ''' Remove substitution from cache. ''' + self.substitutions[target.operation_id].remove(target) + + def remove_inheritance(self, target: Inheritance) -> None: + ''' Remove inheritance from cache. ''' + self.inheritance[target.operation_id].remove(target) + + def _insert_new(self, schema: RSFormCached) -> None: + self._schemas.append(schema) + self._schema_by_id[schema.model.pk] = schema diff --git a/rsconcept/backend/apps/oss/models/PropagationEngine.py b/rsconcept/backend/apps/oss/models/PropagationEngine.py new file mode 100644 index 00000000..64982c68 --- /dev/null +++ b/rsconcept/backend/apps/oss/models/PropagationEngine.py @@ -0,0 +1,316 @@ +''' Models: Change propagation engine. ''' +from typing import Optional + +from rest_framework.serializers import ValidationError + +from apps.rsform.models import INSERT_LAST, Constituenta, CstType, RSFormCached + +from .Inheritance import Inheritance +from .Operation import Operation +from .OssCache import OssCache +from .Substitution import Substitution +from .utils import ( + CstMapping, + CstSubstitution, + create_dependant_mapping, + cst_mapping_to_alias, + map_cst_update_data +) + + +class PropagationEngine: + ''' OSS changes propagation engine. ''' + + def __init__(self, cache: OssCache): + self.cache = cache + + def on_change_cst_type(self, operation_id: int, cst_id: int, ctype: CstType) -> None: + ''' Trigger cascade resolutions when Constituenta type is changed. ''' + children = self.cache.extend_graph.outputs[operation_id] + if len(children) == 0: + return + self.cache.ensure_loaded_subs() + for child_id in children: + child_operation = self.cache.operation_by_id[child_id] + successor_id = self.cache.get_inheritor(cst_id, child_id) + if successor_id is None: + continue + child_schema = self.cache.get_schema(child_operation) + if child_schema is None: + continue + if child_schema.change_cst_type(successor_id, ctype): + self.on_change_cst_type(child_id, successor_id, ctype) + + # pylint: disable=too-many-arguments, too-many-positional-arguments + def on_inherit_cst( + self, + target_operation: int, + source: RSFormCached, + items: list[Constituenta], + mapping: CstMapping, + exclude: Optional[list[int]] = None + ) -> None: + ''' Trigger cascade resolutions when Constituenta is inherited. ''' + children = self.cache.extend_graph.outputs[target_operation] + if len(children) == 0: + return + for child_id in children: + if not exclude or child_id not in exclude: + self.inherit_cst(child_id, source, items, mapping) + + def inherit_cst( + self, + target_operation: int, + source: RSFormCached, + items: list[Constituenta], + mapping: CstMapping + ) -> None: + ''' Execute inheritance of Constituenta. ''' + operation = self.cache.operation_by_id[target_operation] + destination = self.cache.get_schema(operation) + if destination is None: + return + + self.cache.ensure_loaded_subs() + new_mapping = self._transform_mapping(mapping, operation, destination) + alias_mapping = cst_mapping_to_alias(new_mapping) + insert_where = self._determine_insert_position(items[0].pk, operation, source, destination) + new_cst_list = destination.insert_copy(items, insert_where, alias_mapping) + for index, cst in enumerate(new_cst_list): + new_inheritance = Inheritance.objects.create( + operation=operation, + child=cst, + parent=items[index] + ) + self.cache.insert_inheritance(new_inheritance) + new_mapping = {alias_mapping[alias]: cst for alias, cst in new_mapping.items()} + self.on_inherit_cst(operation.pk, destination, new_cst_list, new_mapping) + + # pylint: disable=too-many-arguments, too-many-positional-arguments + def on_update_cst( + self, + operation: int, + cst_id: int, + data: dict, old_data: dict, + mapping: CstMapping + ) -> None: + ''' Trigger cascade resolutions when Constituenta data is changed. ''' + children = self.cache.extend_graph.outputs[operation] + if len(children) == 0: + return + self.cache.ensure_loaded_subs() + for child_id in children: + child_operation = self.cache.operation_by_id[child_id] + successor_id = self.cache.get_inheritor(cst_id, child_id) + if successor_id is None: + continue + child_schema = self.cache.get_schema(child_operation) + assert child_schema is not None + new_mapping = self._transform_mapping(mapping, child_operation, child_schema) + alias_mapping = cst_mapping_to_alias(new_mapping) + successor = child_schema.cache.by_id.get(successor_id) + if successor is None: + continue + new_data = map_cst_update_data(successor, data, old_data, alias_mapping) + if len(new_data) == 0: + continue + new_old_data = child_schema.update_cst(successor.pk, new_data) + if len(new_old_data) == 0: + continue + new_mapping = {alias_mapping[alias]: cst for alias, cst in new_mapping.items()} + self.on_update_cst( + operation=child_id, + cst_id=successor_id, + data=new_data, + old_data=new_old_data, + mapping=new_mapping + ) + + def on_before_substitute(self, substitutions: CstSubstitution, operation: Operation) -> None: + ''' Trigger cascade resolutions when Constituenta substitution is executed. ''' + children = self.cache.extend_graph.outputs[operation.pk] + if len(children) == 0: + return + self.cache.ensure_loaded_subs() + for child_id in children: + child_operation = self.cache.operation_by_id[child_id] + child_schema = self.cache.get_schema(child_operation) + if child_schema is None: + continue + new_substitutions = self._transform_substitutions(substitutions, child_id, child_schema) + if len(new_substitutions) == 0: + continue + self.on_before_substitute(new_substitutions, child_operation) + child_schema.substitute(new_substitutions) + + def on_delete_inherited(self, operation: int, target: list[int]) -> None: + ''' Trigger cascade resolutions when Constituenta inheritance is deleted. ''' + children = self.cache.extend_graph.outputs[operation] + if len(children) == 0: + return + self.cache.ensure_loaded_subs() + for child_id in children: + self.delete_inherited(child_id, target) + + def delete_inherited(self, operation_id: int, parent_ids: list[int]) -> None: + ''' Execute deletion of Constituenta inheritance. ''' + operation = self.cache.operation_by_id[operation_id] + schema = self.cache.get_schema(operation) + if schema is None: + return + self.undo_substitutions_cst(parent_ids, operation, schema) + target_ids = self.cache.get_inheritors_list(parent_ids, operation_id) + self.on_delete_inherited(operation_id, target_ids) + if len(target_ids) > 0: + self.cache.remove_cst(operation_id, target_ids) + schema.delete_cst(target_ids) + + def undo_substitutions_cst(self, target_ids: list[int], operation: Operation, schema: RSFormCached) -> None: + ''' Undo substitutions for Constituents. ''' + to_process = [] + for sub in self.cache.substitutions[operation.pk]: + if sub.original_id in target_ids or sub.substitution_id in target_ids: + to_process.append(sub) + for sub in to_process: + self.undo_substitution(schema, sub, target_ids) + + def undo_substitution( + self, + schema: RSFormCached, + target: Substitution, + ignore_parents: Optional[list[int]] = None + ) -> None: + ''' Undo target substitution. ''' + if ignore_parents is None: + ignore_parents = [] + operation_id = target.operation_id + original_schema = self.cache.get_schema_by_id(target.original.schema_id) + dependant = [] + for cst_id in original_schema.get_dependant([target.original_id]): + if cst_id not in ignore_parents: + inheritor_id = self.cache.get_inheritor(cst_id, operation_id) + if inheritor_id is not None: + dependant.append(inheritor_id) + + self.cache.substitutions[operation_id].remove(target) + target.delete() + + new_original: Optional[Constituenta] = None + if target.original_id not in ignore_parents: + full_cst = Constituenta.objects.get(pk=target.original_id) + cst_mapping = create_dependant_mapping(original_schema, [full_cst]) + self.inherit_cst(operation_id, original_schema, [full_cst], cst_mapping) + new_original_id = self.cache.get_inheritor(target.original_id, operation_id) + assert new_original_id is not None + new_original = schema.cache.by_id[new_original_id] + + if len(dependant) > 0: + substitution_id = self.cache.get_inheritor(target.substitution_id, operation_id) + assert substitution_id is not None + substitution_inheritor = schema.cache.by_id[substitution_id] + mapping = {substitution_inheritor.alias: new_original} + self._on_partial_mapping(mapping, dependant, operation_id, schema) + + def _determine_insert_position( + self, prototype_id: int, + operation: Operation, + source: RSFormCached, + destination: RSFormCached + ) -> int: + ''' Determine insert_after for new constituenta. ''' + prototype = source.cache.by_id[prototype_id] + prototype_index = source.cache.constituents.index(prototype) + if prototype_index == 0: + return 0 + prev_cst = source.cache.constituents[prototype_index - 1] + inherited_prev_id = self.cache.get_successor(prev_cst.pk, operation.pk) + if inherited_prev_id is None: + return INSERT_LAST + prev_cst = destination.cache.by_id[inherited_prev_id] + prev_index = destination.cache.constituents.index(prev_cst) + return prev_index + 1 + + def _transform_mapping(self, mapping: CstMapping, operation: Operation, schema: RSFormCached) -> CstMapping: + if len(mapping) == 0: + return mapping + result: CstMapping = {} + for alias, cst in mapping.items(): + if cst is None: + result[alias] = None + continue + successor_id = self.cache.get_successor(cst.pk, operation.pk) + if successor_id is None: + continue + successor = schema.cache.by_id.get(successor_id) + if successor is None: + continue + result[alias] = successor + return result + + def _transform_substitutions( + self, + target: CstSubstitution, + operation: int, + schema: RSFormCached + ) -> CstSubstitution: + result: CstSubstitution = [] + for current_sub in target: + sub_replaced = False + new_substitution_id = self.cache.get_inheritor(current_sub[1].pk, operation) + if new_substitution_id is None: + for sub in self.cache.substitutions[operation]: + if sub.original_id == current_sub[1].pk: + sub_replaced = True + new_substitution_id = self.cache.get_inheritor(sub.original_id, operation) + break + + new_original_id = self.cache.get_inheritor(current_sub[0].pk, operation) + original_replaced = False + if new_original_id is None: + for sub in self.cache.substitutions[operation]: + if sub.original_id == current_sub[0].pk: + original_replaced = True + sub.original_id = current_sub[1].pk + sub.save() + new_original_id = new_substitution_id + new_substitution_id = self.cache.get_inheritor(sub.substitution_id, operation) + break + + if sub_replaced and original_replaced: + raise ValidationError({'propagation': 'Substitution breaks OSS substitutions.'}) + + for sub in self.cache.substitutions[operation]: + if sub.substitution_id == current_sub[0].pk: + sub.substitution_id = current_sub[1].pk + sub.save() + + if new_original_id is not None and new_substitution_id is not None: + result.append((schema.cache.by_id[new_original_id], schema.cache.by_id[new_substitution_id])) + return result + + def _on_partial_mapping( + self, + mapping: CstMapping, + target: list[int], + operation: int, + schema: RSFormCached + ) -> None: + ''' Trigger cascade resolutions when Constituents are partially mapped. ''' + alias_mapping = cst_mapping_to_alias(mapping) + schema.apply_partial_mapping(alias_mapping, target) + children = self.cache.extend_graph.outputs[operation] + if len(children) == 0: + return + self.cache.ensure_loaded_subs() + for child_id in children: + child_operation = self.cache.operation_by_id[child_id] + child_schema = self.cache.get_schema(child_operation) + if child_schema is None: + continue + new_mapping = self._transform_mapping(mapping, child_operation, child_schema) + if not new_mapping: + continue + new_target = self.cache.get_inheritors_list(target, child_id) + if len(new_target) == 0: + continue + self._on_partial_mapping(new_mapping, new_target, child_id, child_schema) diff --git a/rsconcept/backend/apps/oss/models/utils.py b/rsconcept/backend/apps/oss/models/utils.py new file mode 100644 index 00000000..cf922e55 --- /dev/null +++ b/rsconcept/backend/apps/oss/models/utils.py @@ -0,0 +1,79 @@ +''' Utils for OSS models. ''' + +from typing import Optional + +from cctext import extract_entities + +from apps.rsform.models import ( + DELETED_ALIAS, + Constituenta, + RSFormCached, + extract_globals, + replace_entities, + replace_globals +) + +CstMapping = dict[str, Optional[Constituenta]] +CstSubstitution = list[tuple[Constituenta, Constituenta]] + + +def cst_mapping_to_alias(mapping: CstMapping) -> dict[str, str]: + ''' Convert constituenta mapping to alias mapping. ''' + result: dict[str, str] = {} + for alias, cst in mapping.items(): + if cst is None: + result[alias] = DELETED_ALIAS + else: + result[alias] = cst.alias + return result + + +def map_cst_update_data(cst: Constituenta, data: dict, old_data: dict, mapping: dict[str, str]) -> dict: + ''' Map data for constituenta update. ''' + new_data = {} + if 'term_forms' in data: + if old_data['term_forms'] == cst.term_forms: + new_data['term_forms'] = data['term_forms'] + if 'convention' in data: + new_data['convention'] = data['convention'] + if 'definition_formal' in data: + new_data['definition_formal'] = replace_globals(data['definition_formal'], mapping) + if 'term_raw' in data: + if replace_entities(old_data['term_raw'], mapping) == cst.term_raw: + new_data['term_raw'] = replace_entities(data['term_raw'], mapping) + if 'definition_raw' in data: + if replace_entities(old_data['definition_raw'], mapping) == cst.definition_raw: + new_data['definition_raw'] = replace_entities(data['definition_raw'], mapping) + return new_data + + +def extract_data_references(data: dict, old_data: dict) -> set[str]: + ''' Extract references from data. ''' + result: set[str] = set() + if 'definition_formal' in data: + result.update(extract_globals(data['definition_formal'])) + result.update(extract_globals(old_data['definition_formal'])) + if 'term_raw' in data: + result.update(extract_entities(data['term_raw'])) + result.update(extract_entities(old_data['term_raw'])) + if 'definition_raw' in data: + result.update(extract_entities(data['definition_raw'])) + result.update(extract_entities(old_data['definition_raw'])) + return result + + +def create_dependant_mapping(source: RSFormCached, cst_list: list[Constituenta]) -> CstMapping: + ''' Create mapping for dependant Constituents. ''' + if len(cst_list) == len(source.cache.constituents): + return {c.alias: c for c in source.cache.constituents} + inserted_aliases = [cst.alias for cst in cst_list] + depend_aliases: set[str] = set() + for item in cst_list: + depend_aliases.update(item.extract_references()) + depend_aliases.difference_update(inserted_aliases) + alias_mapping: CstMapping = {} + for alias in depend_aliases: + cst = source.cache.by_alias.get(alias) + if cst is not None: + alias_mapping[alias] = cst + return alias_mapping