diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2ESummaryTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2ESummaryTest.java new file mode 100644 index 000000000..ba22ca305 --- /dev/null +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2ESummaryTest.java @@ -0,0 +1,102 @@ +/* + * #%L + * Alfresco Search Services E2E Test + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.test.search.functional.searchServices.solr.admin; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; + +import java.util.Random; +import java.util.UUID; +import java.util.stream.IntStream; + +import org.alfresco.rest.core.RestResponse; +import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; +import org.alfresco.utility.Utility; +import org.alfresco.utility.model.FileModel; +import org.alfresco.utility.model.FileType; +import org.testng.annotations.Test; + +/** E2E tests for the SUMMARY admin report. */ +public class SolrE2ESummaryTest extends AbstractE2EFunctionalTest +{ + /** The maximum time to wait for a report to update (in ms). */ + private static final int MAX_TIME = 60 * 1000; + /** The frequency to check the report (in ms). */ + private static final int RETRY_INTERVAL = 100; + + /** Check the FTS section of the admin summary contains the expected fields. */ + @Test + public void testFTSReport() throws Exception + { + RestResponse response = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY"); + + int toUpdate = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content needs to be updated'"); + assertTrue(toUpdate >= 0, "Expecting non-negative pieces of content to need updating."); + + int inSync = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content is in sync'"); + assertTrue(inSync >= 0, "Expecting non-negative pieces of content to need updating."); + } + + /** Check that we can spot a document updating by using the SUMMARY report. */ + @Test + public void testFTSReport_contentUpdate() throws Exception + { + RestResponse response2 = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY"); + int previousInSync = response2.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content is in sync'"); + + FileModel file = new FileModel("file.txt", "file.txt", "", FileType.TEXT_PLAIN, "file.txt"); + FileModel content = dataContent.usingUser(adminUserModel).usingSite(testSite).createContent(file); + + // Wait for the number of "in-sync" documents to increase (i.e. when the document is indexed). + Utility.sleep(RETRY_INTERVAL, MAX_TIME, () -> { + RestResponse response = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY"); + int inSync = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content is in sync'"); + assertTrue(inSync > previousInSync, "Expected a document to be indexed."); + }); + + // Wait for the number of outdated documents to become zero. + Utility.sleep(RETRY_INTERVAL, MAX_TIME, () -> + { + RestResponse response = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY"); + int toUpdate = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content needs to be updated'"); + assertEquals(toUpdate, 0, "Expected number of outdated documents to drop to zero."); + }); + + // Update the document's content with a large amount of text. + StringBuilder largeText = new StringBuilder("Big update"); + IntStream.range(0, 100000).forEach((i) -> largeText.append(" ").append(UUID.randomUUID().toString())); + dataContent.usingUser(adminUserModel).usingResource(content).updateContent(largeText.toString()); + + // Expect to spot the number of outdated documents increase beyond zero. + Utility.sleep(RETRY_INTERVAL, MAX_TIME, () -> + { + RestResponse response = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY"); + int toUpdate = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content needs to be updated'"); + assertNotEquals(toUpdate, 0, "Expected number of outdated documents to be greater than zero."); + }); + } +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index c77048a63..87aeba0c8 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -373,11 +373,11 @@ public class SolrInformationServer implements InformationServer public static final String AND = " AND "; public static final String OR = " OR "; - private static final String REQUEST_HANDLER_NATIVE = "/native"; + static final String REQUEST_HANDLER_NATIVE = "/native"; static final String REQUEST_HANDLER_GET = "/get"; static final String RESPONSE_DEFAULT_ID = "doc"; - private static final String RESPONSE_DEFAULT_IDS = "response"; + static final String RESPONSE_DEFAULT_IDS = "response"; static final String PREFIX_ERROR = "ERROR-"; @@ -712,11 +712,10 @@ public class SolrInformationServer implements InformationServer { ModifiableSolrParams params = new ModifiableSolrParams(request.getParams()) - .set(CommonParams.Q, "*:*") - .set(CommonParams.FQ, FIELD_DOC_TYPE + ":" + DOC_TYPE_NODE) + .set(CommonParams.Q, FIELD_DOC_TYPE + ":" + DOC_TYPE_NODE + " AND " + FIELD_TYPE + ":\"{http://www.alfresco.org/model/content/1.0}content\"") .set(CommonParams.ROWS, 0) - .set(FacetParams.FACET, true) - .add(FacetParams.FACET_QUERY, "{!key='OUTDATED'}LATEST_APPLIED_CONTENT_VERSION_ID:{-10 TO -10}"); + .set(FacetParams.FACET, "on") + .add(FacetParams.FACET_QUERY, "{!key='OUTDATED'}" + LAST_INCOMING_CONTENT_VERSION_ID + ":\"" + CONTENT_OUTDATED_MARKER + "\""); SolrQueryResponse response = cloud.getResponse(nativeRequestHandler, request, params); @@ -2625,7 +2624,7 @@ public class SolrInformationServer implements InformationServer * - Do we have an entry in the local content store for the document X? * - If we didn't have that entry then FTSSTATUS was set to New * - * FTSSTatus = DIRTY + * FTSSTATUS = DIRTY * * When a node, which requires content, arrived here for being indexed, the "Dirty" status was set if * @@ -2637,7 +2636,7 @@ public class SolrInformationServer implements InformationServer * something is different between the local document and the document which is in Solr". * * The content store allowed to workaround the problem: the code was deserializing the entry corresponding to the - * incoming node (which teoretically corresponded to the document indexed in Solr) so both versions were available + * incoming node (which theoretically corresponded to the document indexed in Solr) so both versions were available * for making decisions * * ------------------ @@ -2646,7 +2645,7 @@ public class SolrInformationServer implements InformationServer * about * * - the "INSERT" or "UPDATE" nature of the indexing operation that is going to be executed - * - the values of fields of a given document indeded in Solr + * - the values of fields of a given document indexed in Solr * * In order to indicate to the ContentTracker which documents will require the content update, we added two * additional fields in the schema: @@ -2656,8 +2655,8 @@ public class SolrInformationServer implements InformationServer * LATEST_APPLIED_CONTENT_VERSION_ID: as the name suggests, this is the latest DOCID applied to this document (again, not the lucene docid) * *
  • - * LAST_INCOMING_CONTENT_VERSION_ID: a field that will contains "-10" if the content is outdated, otherwise it will have the same value - * of LATEST_APPLIED_CONTENT_VERSION_ID. + * LAST_INCOMING_CONTENT_VERSION_ID: this call will set the field to "-10" as the content is outdated. The content tracker will then set it + * to have same value as LATEST_APPLIED_CONTENT_VERSION_ID. *
    • * * @@ -2667,11 +2666,8 @@ public class SolrInformationServer implements InformationServer { ofNullable(value) .map(ContentPropertyValue::getId) - .ifPresentOrElse( - id -> { - document.setField(LATEST_APPLIED_CONTENT_VERSION_ID, id); - document.setField(LAST_INCOMING_CONTENT_VERSION_ID, Map.of("removeregex", "^(?!"+id+"$).*$"));}, - () -> document.setField(LAST_INCOMING_CONTENT_VERSION_ID, CONTENT_OUTDATED_MARKER)); + .ifPresent(id -> document.setField(LATEST_APPLIED_CONTENT_VERSION_ID, id)); + document.setField(LAST_INCOMING_CONTENT_VERSION_ID, CONTENT_OUTDATED_MARKER); } private void addContentProperty( diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java index 2e4ba4f21..e5cc646c1 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java @@ -27,6 +27,7 @@ package org.alfresco.solr; import static java.util.Optional.ofNullable; + import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ANY; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ASSOC_REF; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.BOOLEAN; @@ -45,6 +46,10 @@ import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.PATH; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.PERIOD; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.QNAME; import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.TEXT; +import static org.alfresco.solr.SolrInformationServer.REQUEST_HANDLER_GET; +import static org.alfresco.solr.SolrInformationServer.REQUEST_HANDLER_NATIVE; +import static org.alfresco.solr.SolrInformationServer.RESPONSE_DEFAULT_ID; +import static org.alfresco.solr.SolrInformationServer.RESPONSE_DEFAULT_IDS; import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_DAY_FIELD_SUFFIX; import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_HOUR_FIELD_SUFFIX; import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_MINUTE_FIELD_SUFFIX; @@ -57,6 +62,8 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -76,12 +83,15 @@ import org.alfresco.solr.client.SOLRAPIClient; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.response.BasicResultContext; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.DocList; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -124,6 +134,8 @@ public class SolrInformationServerTest public void setUp() { when(core.getResourceLoader()).thenReturn(resourceLoader); + when(core.getRequestHandler(REQUEST_HANDLER_GET)).thenReturn(handler); + when(core.getRequestHandler(REQUEST_HANDLER_NATIVE)).thenReturn(handler); when(resourceLoader.getCoreProperties()).thenReturn(new Properties()); infoServer = new SolrInformationServer(adminHandler, core, client) { @@ -145,15 +157,14 @@ public class SolrInformationServerTest SolrDocument state = new SolrDocument(); SimpleOrderedMap responseContent = new SimpleOrderedMap<>(); - responseContent.add(SolrInformationServer.RESPONSE_DEFAULT_ID, state); + responseContent.add(RESPONSE_DEFAULT_ID, state); when(response.getValues()).thenReturn(responseContent); - when(core.getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET)).thenReturn(handler); SolrDocument document = infoServer.getState(core, request, id); assertEquals(id, request.getParams().get(CommonParams.ID)); - verify(core).getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET); + verify(core).getRequestHandler(REQUEST_HANDLER_GET); verify(response).getValues(); assertSame(state, document); @@ -313,15 +324,14 @@ public class SolrInformationServerTest String id = String.valueOf(System.currentTimeMillis()); SimpleOrderedMap responseContent = new SimpleOrderedMap<>(); - responseContent.add(SolrInformationServer.RESPONSE_DEFAULT_ID, null); + responseContent.add(RESPONSE_DEFAULT_ID, null); when(response.getValues()).thenReturn(responseContent); - when(core.getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET)).thenReturn(handler); SolrDocument document = infoServer.getState(core, request, id); assertEquals(id, request.getParams().get(CommonParams.ID)); - verify(core).getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET); + verify(core).getRequestHandler(REQUEST_HANDLER_GET); verify(response).getValues(); assertNull(document); @@ -417,5 +427,39 @@ public class SolrInformationServerTest }); } - -} \ No newline at end of file + + /** Check that the FTS report is derived from the correct parts of the Solr response. */ + @Test + public void testAddContentOutdatedAndUpdatedCounts() + { + // Pretend that there are three documents in total. + NamedList responseContent = new SimpleOrderedMap<>(); + DocList docList = mock(DocList.class); + when(docList.matches()).thenReturn(3); + BasicResultContext basicResultContext = mock(BasicResultContext.class); + when(basicResultContext.getDocList()).thenReturn(docList); + responseContent.add(RESPONSE_DEFAULT_IDS, basicResultContext); + + // Set the facet to say one document is outdated. + NamedList facetQueries = new SimpleOrderedMap<>(); + facetQueries.add("OUTDATED", 1); + NamedList> facetCounts = new SimpleOrderedMap<>(); + facetCounts.add("facet_queries", facetQueries); + responseContent.add("facet_counts", facetCounts); + + // Set up the request handler to return the fake response. + doAnswer(invocation -> { + SolrQueryResponse solrQueryResponse = invocation.getArgument(1); + solrQueryResponse.setAllValues(responseContent); + return null; + }).when(handler).handleRequest(any(SolrQueryRequest.class), any(SolrQueryResponse.class)); + + // Call the method under test. + NamedList report = new NamedList<>(); + infoServer.addContentOutdatedAndUpdatedCounts(report); + + // Check the report. + assertEquals("Expected two content nodes to be in sync.", report.get("Node count whose content is in sync"), 2L); + assertEquals("Expected one content node to need an update.", report.get("Node count whose content needs to be updated"), 1L); + } +}