From bf8af6317a3a170f9cac1bc444c935e134a62199 Mon Sep 17 00:00:00 2001 From: IRBorisov <8611739+IRBorisov@users.noreply.github.com> Date: Mon, 21 Aug 2023 00:25:12 +0300 Subject: [PATCH] Add minimal text resolution capabilities to API --- rsconcept/backend/apps/rsform/models.py | 62 +++++++++++++++++-- rsconcept/backend/apps/rsform/serializers.py | 31 +++++----- .../backend/apps/rsform/tests/t_views.py | 22 +++++-- rsconcept/backend/apps/rsform/views.py | 24 ++----- rsconcept/backend/cctext/context.py | 14 ++--- rsconcept/backend/cctext/tests/t_context.py | 6 +- 6 files changed, 106 insertions(+), 53 deletions(-) diff --git a/rsconcept/backend/apps/rsform/models.py b/rsconcept/backend/apps/rsform/models.py index 40a5aded..e2787239 100644 --- a/rsconcept/backend/apps/rsform/models.py +++ b/rsconcept/backend/apps/rsform/models.py @@ -1,5 +1,6 @@ ''' Models: RSForms for conceptual schemas. ''' import json +from typing import Optional import pyconcept from django.db import transaction from django.db.models import ( @@ -10,6 +11,7 @@ from django.core.validators import MinValueValidator from django.core.exceptions import ValidationError from django.urls import reverse from apps.users.models import User +from cctext import Resolver, Entity class CstType(TextChoices): @@ -73,10 +75,34 @@ class RSForm(Model): verbose_name = 'Схема' verbose_name_plural = 'Схемы' - def constituents(self) -> QuerySet: + def constituents(self) -> QuerySet['Constituenta']: ''' Get QuerySet containing all constituents of current RSForm ''' return Constituenta.objects.filter(schema=self) + def resolver(self) -> Resolver: + ''' Create resolver for text references based on schema terms. ''' + result = Resolver({}) + for cst in self.constituents(): + entity = Entity(alias=cst.alias, nominal=cst.term_resolved, manual_forms=cst.term_forms) + result.context[cst.alias] = entity + return result + + @transaction.atomic + def on_term_change(self, alias: str): + ''' Trigger cascade resolutions when term changes. ''' + pass + # void Thesaurus::OnTermChange(const EntityUID target) { + # auto expansion = TermGraph().ExpandOutputs({ target }); + # const auto ordered = TermGraph().Sort(expansion); + # for (const auto entity : ordered) { + # storage.at(entity).term.UpdateFrom(Context()); + # } + # expansion = DefGraph().ExpandOutputs(expansion); + # for (const auto entity : expansion) { + # storage.at(entity).definition.UpdateFrom(Context()); + # } + + @transaction.atomic def insert_at(self, position: int, alias: str, insert_type: CstType) -> 'Constituenta': ''' Insert new constituenta at given position. All following constituents order is shifted by 1 position ''' @@ -147,6 +173,30 @@ class RSForm(Model): self._update_from_core() self.save() + @transaction.atomic + def create_cst(self, data: dict, insert_after: Optional[str]=None) -> 'Constituenta': + ''' Create new cst from data. ''' + resolver = self.resolver() + cst = self._insert_new(data, insert_after) + cst.convention = data.get('convention', '') + cst.definition_formal = data.get('definition_formal', '') + cst.term_raw = data.get('term_raw', '') + if cst.term_raw != '': + cst.term_resolved = resolver.resolve(cst.term_raw) + cst.definition_raw = data.get('definition_raw', '') + if cst.definition_raw != '': + cst.definition_resolved = resolver.resolve(cst.definition_raw) + cst.save() + self.on_term_change(cst.alias) + return cst + + def _insert_new(self, data: dict, insert_after: Optional[str]=None) -> 'Constituenta': + if insert_after is not None: + cstafter = Constituenta.objects.get(pk=insert_after) + return self.insert_at(cstafter.order + 1, data['alias'], data['cst_type']) + else: + return self.insert_last(data['alias'], data['cst_type']) + @transaction.atomic def load_trs(self, data: dict, sync_metadata: bool, skip_update: bool): if sync_metadata: @@ -170,7 +220,7 @@ class RSForm(Model): loaded_ids.add(uid) order += 1 for prev_cst in prev_constituents: - if prev_cst.id not in loaded_ids: + if prev_cst.pk not in loaded_ids: prev_cst.delete() if not skip_update: self._update_from_core() @@ -214,8 +264,9 @@ class RSForm(Model): } @transaction.atomic - def _update_from_core(self) -> dict: - checked: dict = json.loads(pyconcept.check_schema(json.dumps(self.to_trs()))) + def _update_from_core(self): + # TODO: resolve text refs + checked = json.loads(pyconcept.check_schema(json.dumps(self.to_trs()))) update_list = self.constituents().only('id', 'order') if len(checked['items']) != update_list.count(): raise ValidationError('Invalid constituents count') @@ -223,12 +274,11 @@ class RSForm(Model): for cst in checked['items']: cst_id = cst['entityUID'] for oldCst in update_list: - if oldCst.id == cst_id: + if oldCst.pk == cst_id: oldCst.order = order order += 1 break Constituenta.objects.bulk_update(update_list, ['order']) - return checked @transaction.atomic def _create_items_from_trs(self, items): diff --git a/rsconcept/backend/apps/rsform/serializers.py b/rsconcept/backend/apps/rsform/serializers.py index 11b16fac..afa647b5 100644 --- a/rsconcept/backend/apps/rsform/serializers.py +++ b/rsconcept/backend/apps/rsform/serializers.py @@ -1,5 +1,6 @@ ''' Serializers for conceptual schema API. ''' import json +from typing import Optional from rest_framework import serializers import pyconcept @@ -93,16 +94,25 @@ class ConstituentaSerializer(serializers.ModelSerializer): ''' serializer metadata. ''' model = Constituenta fields = '__all__' - read_only_fields = ('id', 'order', 'alias', 'cst_type') + read_only_fields = ('id', 'order', 'alias', 'cst_type', 'definition_resolved', 'term_resolved') def update(self, instance: Constituenta, validated_data) -> Constituenta: - if 'term_raw' in validated_data: - validated_data['term_resolved'] = validated_data['term_raw'] - if 'definition_raw' in validated_data: - validated_data['definition_resolved'] = validated_data['definition_raw'] - + schema: RSForm = instance.schema + definition: Optional[str] = validated_data['definition_raw'] if 'definition_raw' in validated_data else None + term: Optional[str] = validated_data['term_raw'] if 'term_raw' in validated_data else None + term_changed = False + if definition is not None and definition != instance.definition_raw : + validated_data['definition_resolved'] = schema.resolver().resolve(definition) + if term is not None and term != instance.term_raw: + validated_data['term_resolved'] = schema.resolver().resolve(term) + if validated_data['term_resolved'] != instance.term_resolved: + validated_data['term_forms'] = [] + term_changed = validated_data['term_resolved'] != instance.term_resolved result: Constituenta = super().update(instance, validated_data) - instance.schema.save() + if term_changed: + schema.on_term_change(result.alias) + result.refresh_from_db() + schema.save() return result @@ -132,13 +142,6 @@ class CstCreateSerializer(serializers.ModelSerializer): model = Constituenta fields = 'alias', 'cst_type', 'convention', 'term_raw', 'definition_raw', 'definition_formal', 'insert_after' - def validate(self, attrs): - if 'term_raw' in attrs: - attrs['term_resolved'] = attrs['term_raw'] - if 'definition_raw' in attrs: - attrs['definition_resolved'] = attrs['definition_raw'] - return attrs - class CstListSerlializer(serializers.Serializer): ''' Serializer: List of constituents from one origin. ''' diff --git a/rsconcept/backend/apps/rsform/tests/t_views.py b/rsconcept/backend/apps/rsform/tests/t_views.py index 301074d3..3ad65cbb 100644 --- a/rsconcept/backend/apps/rsform/tests/t_views.py +++ b/rsconcept/backend/apps/rsform/tests/t_views.py @@ -28,12 +28,15 @@ class TestConstituentaAPI(APITestCase): self.rsform_owned = RSForm.objects.create(title='Test', alias='T1', owner=self.user) self.rsform_unowned = RSForm.objects.create(title='Test2', alias='T2') self.cst1 = Constituenta.objects.create( - alias='X1', schema=self.rsform_owned, order=1, convention='Test') + alias='X1', schema=self.rsform_owned, order=1, convention='Test', + term_raw='Test1', term_resolved='Test1R', + term_forms=[{'text':'form1', 'tags':'sing,datv'}]) self.cst2 = Constituenta.objects.create( - alias='X2', schema=self.rsform_unowned, order=1, convention='Test1') + alias='X2', schema=self.rsform_unowned, order=1, convention='Test1', + term_raw='Test2', term_resolved='Test2R') self.cst3 = Constituenta.objects.create( alias='X3', schema=self.rsform_owned, order=2, - term_raw='Test1', term_resolved='Test1', + term_raw='Test3', term_resolved='Test3', definition_raw='Test1', definition_resolved='Test2') def test_retrieve(self): @@ -61,7 +64,7 @@ class TestConstituentaAPI(APITestCase): response = self.client.patch(f'/api/constituents/{self.cst1.id}/', data, content_type='application/json') self.assertEqual(response.status_code, 200) - def test_partial_update_update_resolved(self): + def test_update_resolved_norefs(self): data = json.dumps({ 'term_raw': 'New term', 'definition_raw': 'New def' @@ -72,6 +75,17 @@ class TestConstituentaAPI(APITestCase): self.assertEqual(self.cst3.term_resolved, 'New term') self.assertEqual(self.cst3.definition_resolved, 'New def') + def test_update_resolved_refs(self): + data = json.dumps({ + 'term_raw': '@{X1|nomn,sing}', + 'definition_raw': '@{X1|nomn,sing} @{X1|sing,datv}' + }) + response = self.client.patch(f'/api/constituents/{self.cst3.id}/', data, content_type='application/json') + self.assertEqual(response.status_code, 200) + self.cst3.refresh_from_db() + self.assertEqual(self.cst3.term_resolved, self.cst1.term_resolved) + self.assertEqual(self.cst3.definition_resolved, f'{self.cst1.term_resolved} form1') + def test_readonly_cst_fields(self): data = json.dumps({'alias': 'X33', 'order': 10}) response = self.client.patch(f'/api/constituents/{self.cst1.id}/', data, content_type='application/json') diff --git a/rsconcept/backend/apps/rsform/views.py b/rsconcept/backend/apps/rsform/views.py index b7b9a4b6..36ba729e 100644 --- a/rsconcept/backend/apps/rsform/views.py +++ b/rsconcept/backend/apps/rsform/views.py @@ -77,27 +77,15 @@ class RSFormViewSet(viewsets.ModelViewSet): schema = self._get_schema() serializer = serializers.CstCreateSerializer(data=request.data) serializer.is_valid(raise_exception=True) - if ('insert_after' in serializer.validated_data and serializer.validated_data['insert_after'] is not None): - cstafter = models.Constituenta.objects.get(pk=serializer.validated_data['insert_after']) - constituenta = schema.insert_at(cstafter.order + 1, - serializer.validated_data['alias'], - serializer.validated_data['cst_type']) - else: - constituenta = schema.insert_last(serializer.validated_data['alias'], serializer.validated_data['cst_type']) - - constituenta.convention = serializer.validated_data.get('convention', '') - constituenta.term_raw = serializer.validated_data.get('term_raw', '') - constituenta.term_resolved = serializer.validated_data.get('term_resolved', '') - constituenta.definition_formal = serializer.validated_data.get('definition_formal', '') - constituenta.definition_raw = serializer.validated_data.get('definition_raw', '') - constituenta.definition_resolved = serializer.validated_data.get('definition_resolved', '') - constituenta.save() + data = serializer.validated_data + new_cst = schema.create_cst(data, data['insert_after'] if 'insert_after' in data else None) schema.refresh_from_db() outSerializer = serializers.RSFormDetailsSerlializer(schema) response = Response(status=201, data={ - 'new_cst': serializers.ConstituentaSerializer(constituenta).data, - 'schema': outSerializer.data}) - response['Location'] = constituenta.get_absolute_url() + 'new_cst': serializers.ConstituentaSerializer(new_cst).data, + 'schema': outSerializer.data + }) + response['Location'] = new_cst.get_absolute_url() return response @action(detail=True, methods=['patch'], url_path='cst-multidelete') diff --git a/rsconcept/backend/cctext/context.py b/rsconcept/backend/cctext/context.py index fd658f71..b421f89c 100644 --- a/rsconcept/backend/cctext/context.py +++ b/rsconcept/backend/cctext/context.py @@ -1,21 +1,19 @@ ''' Term context for reference resolution. ''' -from typing import Iterable, Dict, Optional -from dataclasses import dataclass +from typing import Iterable, Dict, Optional, TypedDict from .conceptapi import inflect -@dataclass -class TermForm: +class TermForm(TypedDict): ''' Term in a specific form. ''' text: str - form: str + tags: str def _search_form(query: str, data: Iterable[TermForm]) -> Optional[str]: for tf in data: - if tf.form == query: - return tf.text + if tf['tags'] == query: + return tf['text'] return None @@ -55,7 +53,7 @@ class Entity: text = inflect(self._nominal, form) except ValueError as error: text = f'!{error}!'.replace('Unknown grammeme', 'Неизвестная граммема') - self._cached.append(TermForm(text=text, form=form)) + self._cached.append({'text': text, 'tags': form}) return text # Term context for resolving entity references. diff --git a/rsconcept/backend/cctext/tests/t_context.py b/rsconcept/backend/cctext/tests/t_context.py index 0a2c68e4..e1e8f540 100644 --- a/rsconcept/backend/cctext/tests/t_context.py +++ b/rsconcept/backend/cctext/tests/t_context.py @@ -1,7 +1,7 @@ ''' Unit tests: context. ''' import unittest -from cctext.context import TermForm, Entity, TermContext +from cctext.context import Entity, TermContext class TestEntity(unittest.TestCase): '''Test Entity termform access.''' @@ -10,12 +10,12 @@ class TestEntity(unittest.TestCase): self.nominal = 'человек' self.text1 = 'test1' self.form1 = 'sing,datv' - self.entity = Entity(self.alias, self.nominal, [TermForm(self.text1, self.form1)]) + self.entity = Entity(self.alias, self.nominal, [{'text': self.text1, 'tags': self.form1}]) def test_attributes(self): self.assertEqual(self.entity.alias, self.alias) self.assertEqual(self.entity.get_nominal(), self.nominal) - self.assertEqual(self.entity.manual, [TermForm(self.text1, self.form1)]) + self.assertEqual(self.entity.manual, [{'text': self.text1, 'tags': self.form1}]) def test_get_form(self): self.assertEqual(self.entity.get_form(''), self.nominal)