From 2716d1088ea3b2f4e340e5f06bd216e5bba1ed78 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 15 Feb 2022 16:30:43 +0000 Subject: [PATCH 1/3] MNT-22410 Update FTS report to use correct field. Also fix syntax of facet query and always set to outdated when updated content is discovered. --- .../alfresco/solr/SolrInformationServer.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index d5e33597d..e31e1864a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -716,7 +716,7 @@ public class SolrInformationServer implements InformationServer .set(CommonParams.FQ, FIELD_DOC_TYPE + ":" + DOC_TYPE_NODE) .set(CommonParams.ROWS, 0) .set(FacetParams.FACET, true) - .add(FacetParams.FACET_QUERY, "{!key='OUTDATED'}LATEST_APPLIED_CONTENT_VERSION_ID:{-10 TO -10}"); + .add(FacetParams.FACET_QUERY, "{!key='OUTDATED'}LAST_INCOMING_CONTENT_VERSION_ID:\"-10\""); SolrQueryResponse response = cloud.getResponse(nativeRequestHandler, request, params); @@ -2625,7 +2625,7 @@ public class SolrInformationServer implements InformationServer * - Do we have an entry in the local content store for the document X? * - If we didn't have that entry then FTSSTATUS was set to New * - * FTSSTatus = DIRTY + * FTSSTATUS = DIRTY * * When a node, which requires content, arrived here for being indexed, the "Dirty" status was set if * @@ -2637,7 +2637,7 @@ public class SolrInformationServer implements InformationServer * something is different between the local document and the document which is in Solr". * * The content store allowed to workaround the problem: the code was deserializing the entry corresponding to the - * incoming node (which teoretically corresponded to the document indexed in Solr) so both versions were available + * incoming node (which theoretically corresponded to the document indexed in Solr) so both versions were available * for making decisions * * ------------------ @@ -2646,7 +2646,7 @@ public class SolrInformationServer implements InformationServer * about * * - the "INSERT" or "UPDATE" nature of the indexing operation that is going to be executed - * - the values of fields of a given document indeded in Solr + * - the values of fields of a given document indexed in Solr * * In order to indicate to the ContentTracker which documents will require the content update, we added two * additional fields in the schema: @@ -2656,8 +2656,8 @@ public class SolrInformationServer implements InformationServer * LATEST_APPLIED_CONTENT_VERSION_ID: as the name suggests, this is the latest DOCID applied to this document (again, not the lucene docid) * *
  • - * LAST_INCOMING_CONTENT_VERSION_ID: a field that will contains "-10" if the content is outdated, otherwise it will have the same value - * of LATEST_APPLIED_CONTENT_VERSION_ID. + * LAST_INCOMING_CONTENT_VERSION_ID: this call will set the field to "-10" as the content is outdated. The content tracker will then set it + * to have same value as LATEST_APPLIED_CONTENT_VERSION_ID. *
    • * * @@ -2667,11 +2667,8 @@ public class SolrInformationServer implements InformationServer { ofNullable(value) .map(ContentPropertyValue::getId) - .ifPresentOrElse( - id -> { - document.setField(LATEST_APPLIED_CONTENT_VERSION_ID, id); - document.setField(LAST_INCOMING_CONTENT_VERSION_ID, Map.of("removeregex", "^(?!"+id+"$).*$"));}, - () -> document.setField(LAST_INCOMING_CONTENT_VERSION_ID, CONTENT_OUTDATED_MARKER)); + .ifPresent(id -> document.setField(LATEST_APPLIED_CONTENT_VERSION_ID, id)); + document.setField(LAST_INCOMING_CONTENT_VERSION_ID, CONTENT_OUTDATED_MARKER); } private void addContentProperty( From 2255d1e5e5aec8ab5ca94645609be72011f0b569 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Wed, 16 Feb 2022 12:30:23 +0000 Subject: [PATCH 2/3] Unit test for calculation when creating FTS report. --- .../alfresco/solr/SolrInformationServer.java | 4 +- .../solr/SolrInformationServerTest.java | 60 ++++++++++++++++--- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index e31e1864a..149039837 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -373,11 +373,11 @@ public class SolrInformationServer implements InformationServer public static final String AND = " AND "; public static final String OR = " OR "; - private static final String REQUEST_HANDLER_NATIVE = "/native"; + static final String REQUEST_HANDLER_NATIVE = "/native"; static final String REQUEST_HANDLER_GET = "/get"; static final String RESPONSE_DEFAULT_ID = "doc"; - private static final String RESPONSE_DEFAULT_IDS = "response"; + static final String RESPONSE_DEFAULT_IDS = "response"; static final String PREFIX_ERROR = "ERROR-"; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java index 2e4ba4f21..e5cc646c1 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java @@ -27,6 +27,7 @@ package org.alfresco.solr; import static java.util.Optional.ofNullable; + import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ANY; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ASSOC_REF; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.BOOLEAN; @@ -45,6 +46,10 @@ import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.PATH; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.PERIOD; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.QNAME; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.TEXT; +import static org.alfresco.solr.SolrInformationServer.REQUEST_HANDLER_GET; +import static org.alfresco.solr.SolrInformationServer.REQUEST_HANDLER_NATIVE; +import static org.alfresco.solr.SolrInformationServer.RESPONSE_DEFAULT_ID; +import static org.alfresco.solr.SolrInformationServer.RESPONSE_DEFAULT_IDS; import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_DAY_FIELD_SUFFIX; import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_HOUR_FIELD_SUFFIX; import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_MINUTE_FIELD_SUFFIX; @@ -57,6 +62,8 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -76,12 +83,15 @@ import org.alfresco.solr.client.SOLRAPIClient; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.response.BasicResultContext; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.DocList; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -124,6 +134,8 @@ public class SolrInformationServerTest public void setUp() { when(core.getResourceLoader()).thenReturn(resourceLoader); + when(core.getRequestHandler(REQUEST_HANDLER_GET)).thenReturn(handler); + when(core.getRequestHandler(REQUEST_HANDLER_NATIVE)).thenReturn(handler); when(resourceLoader.getCoreProperties()).thenReturn(new Properties()); infoServer = new SolrInformationServer(adminHandler, core, client) { @@ -145,15 +157,14 @@ public class SolrInformationServerTest SolrDocument state = new SolrDocument(); SimpleOrderedMap responseContent = new SimpleOrderedMap<>(); - responseContent.add(SolrInformationServer.RESPONSE_DEFAULT_ID, state); + responseContent.add(RESPONSE_DEFAULT_ID, state); when(response.getValues()).thenReturn(responseContent); - when(core.getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET)).thenReturn(handler); SolrDocument document = infoServer.getState(core, request, id); assertEquals(id, request.getParams().get(CommonParams.ID)); - verify(core).getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET); + verify(core).getRequestHandler(REQUEST_HANDLER_GET); verify(response).getValues(); assertSame(state, document); @@ -313,15 +324,14 @@ public class SolrInformationServerTest String id = String.valueOf(System.currentTimeMillis()); SimpleOrderedMap responseContent = new SimpleOrderedMap<>(); - responseContent.add(SolrInformationServer.RESPONSE_DEFAULT_ID, null); + responseContent.add(RESPONSE_DEFAULT_ID, null); when(response.getValues()).thenReturn(responseContent); - when(core.getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET)).thenReturn(handler); SolrDocument document = infoServer.getState(core, request, id); assertEquals(id, request.getParams().get(CommonParams.ID)); - verify(core).getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET); + verify(core).getRequestHandler(REQUEST_HANDLER_GET); verify(response).getValues(); assertNull(document); @@ -417,5 +427,39 @@ public class SolrInformationServerTest }); } - -} \ No newline at end of file + + /** Check that the FTS report is derived from the correct parts of the Solr response. */ + @Test + public void testAddContentOutdatedAndUpdatedCounts() + { + // Pretend that there are three documents in total. + NamedList responseContent = new SimpleOrderedMap<>(); + DocList docList = mock(DocList.class); + when(docList.matches()).thenReturn(3); + BasicResultContext basicResultContext = mock(BasicResultContext.class); + when(basicResultContext.getDocList()).thenReturn(docList); + responseContent.add(RESPONSE_DEFAULT_IDS, basicResultContext); + + // Set the facet to say one document is outdated. + NamedList facetQueries = new SimpleOrderedMap<>(); + facetQueries.add("OUTDATED", 1); + NamedList> facetCounts = new SimpleOrderedMap<>(); + facetCounts.add("facet_queries", facetQueries); + responseContent.add("facet_counts", facetCounts); + + // Set up the request handler to return the fake response. + doAnswer(invocation -> { + SolrQueryResponse solrQueryResponse = invocation.getArgument(1); + solrQueryResponse.setAllValues(responseContent); + return null; + }).when(handler).handleRequest(any(SolrQueryRequest.class), any(SolrQueryResponse.class)); + + // Call the method under test. + NamedList report = new NamedList<>(); + infoServer.addContentOutdatedAndUpdatedCounts(report); + + // Check the report. + assertEquals("Expected two content nodes to be in sync.", report.get("Node count whose content is in sync"), 2L); + assertEquals("Expected one content node to need an update.", report.get("Node count whose content needs to be updated"), 1L); + } +} From 2a6ab0f1dd2e05f32bb51c50e5e2fbe33978c1c2 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 17 Feb 2022 13:27:16 +0000 Subject: [PATCH 3/3] MNT-22410 Further fix to ensure that the correct results are included in the results. --- .../main/java/org/alfresco/solr/SolrInformationServer.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 149039837..7006d50d6 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -712,11 +712,10 @@ public class SolrInformationServer implements InformationServer { ModifiableSolrParams params = new ModifiableSolrParams(request.getParams()) - .set(CommonParams.Q, "*:*") - .set(CommonParams.FQ, FIELD_DOC_TYPE + ":" + DOC_TYPE_NODE) + .set(CommonParams.Q, FIELD_DOC_TYPE + ":" + DOC_TYPE_NODE + " AND " + LATEST_APPLIED_CONTENT_VERSION_ID + ":*") .set(CommonParams.ROWS, 0) - .set(FacetParams.FACET, true) - .add(FacetParams.FACET_QUERY, "{!key='OUTDATED'}LAST_INCOMING_CONTENT_VERSION_ID:\"-10\""); + .set(FacetParams.FACET, "on") + .add(FacetParams.FACET_QUERY, "{!key='OUTDATED'}" + LAST_INCOMING_CONTENT_VERSION_ID + ":\"" + CONTENT_OUTDATED_MARKER + "\""); SolrQueryResponse response = cloud.getResponse(nativeRequestHandler, request, params);