inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1364 from Alfresco/feature/MNT-22410_FixFTSReport_20x

Browse Source 
MNT-22410 Fix FTS report (release/V2.0.x).
This commit is contained in:
Tom Page
2022-02-22 10:16:10 +00:00
committed by GitHub
parent c63b6960ec e98a7b22fd
commit f3a136cbcc
3 changed files with 166 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2ESummaryTest.java
+102
View File
@@ -0,0 +1,102 @@
/*
* #%L
* Alfresco Search Services E2E Test
* %%
* Copyright (C) 2005 - 2020 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.test.search.functional.searchServices.solr.admin;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertTrue;
import java.util.Random;
import java.util.UUID;
import java.util.stream.IntStream;
import org.alfresco.rest.core.RestResponse;
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
import org.alfresco.utility.Utility;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FileType;
import org.testng.annotations.Test;
/** E2E tests for the SUMMARY admin report. */
public class SolrE2ESummaryTest extends AbstractE2EFunctionalTest
{
/** The maximum time to wait for a report to update (in ms). */
private static final int MAX_TIME = 60 * 1000;
/** The frequency to check the report (in ms). */
private static final int RETRY_INTERVAL = 100;
/** Check the FTS section of the admin summary contains the expected fields. */
@Test
public void testFTSReport() throws Exception
{
RestResponse response = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY");
int toUpdate = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content needs to be updated'");
assertTrue(toUpdate >= 0, "Expecting non-negative pieces of content to need updating.");
int inSync = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content is in sync'");
assertTrue(inSync >= 0, "Expecting non-negative pieces of content to need updating.");
}
/** Check that we can spot a document updating by using the SUMMARY report. */
@Test
public void testFTSReport_contentUpdate() throws Exception
{
RestResponse response2 = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY");
int previousInSync = response2.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content is in sync'");
FileModel file = new FileModel("file.txt", "file.txt", "", FileType.TEXT_PLAIN, "file.txt");
FileModel content = dataContent.usingUser(adminUserModel).usingSite(testSite).createContent(file);
// Wait for the number of "in-sync" documents to increase (i.e. when the document is indexed).
Utility.sleep(RETRY_INTERVAL, MAX_TIME, () -> {
RestResponse response = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY");
int inSync = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content is in sync'");
assertTrue(inSync > previousInSync, "Expected a document to be indexed.");
});
// Wait for the number of outdated documents to become zero.
Utility.sleep(RETRY_INTERVAL, MAX_TIME, () ->
{
RestResponse response = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY");
int toUpdate = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content needs to be updated'");
assertEquals(toUpdate, 0, "Expected number of outdated documents to drop to zero.");
});
// Update the document's content with a large amount of text.
StringBuilder largeText = new StringBuilder("Big update");
IntStream.range(0, 100000).forEach((i) -> largeText.append(" ").append(UUID.randomUUID().toString()));
dataContent.usingUser(adminUserModel).usingResource(content).updateContent(largeText.toString());
// Expect to spot the number of outdated documents increase beyond zero.
Utility.sleep(RETRY_INTERVAL, MAX_TIME, () ->
{
RestResponse response = restClient.withParams("core=alfresco").withSolrAdminAPI().getAction("SUMMARY");
int toUpdate = response.getResponse().body().jsonPath().get("Summary.alfresco.FTS.'Node count whose content needs to be updated'");
assertNotEquals(toUpdate, 0, "Expected number of outdated documents to be greater than zero.");
});
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java
+12 -16
View File
@@ -373,11 +373,11 @@ public class SolrInformationServer implements InformationServer
public static final String AND = " AND ";
public static final String OR = " OR ";
private static final String REQUEST_HANDLER_NATIVE = "/native";
static final String REQUEST_HANDLER_NATIVE = "/native";
static final String REQUEST_HANDLER_GET = "/get";
static final String RESPONSE_DEFAULT_ID = "doc";
private static final String RESPONSE_DEFAULT_IDS = "response";
static final String RESPONSE_DEFAULT_IDS = "response";
static final String PREFIX_ERROR = "ERROR-";
@@ -712,11 +712,10 @@ public class SolrInformationServer implements InformationServer
{
ModifiableSolrParams params =
new ModifiableSolrParams(request.getParams())
.set(CommonParams.Q, "*:*")
.set(CommonParams.FQ, FIELD_DOC_TYPE + ":" + DOC_TYPE_NODE)
.set(CommonParams.Q, FIELD_DOC_TYPE + ":" + DOC_TYPE_NODE + " AND " + FIELD_TYPE + ":\"{http://www.alfresco.org/model/content/1.0}content\"")
.set(CommonParams.ROWS, 0)
.set(FacetParams.FACET, true)
.add(FacetParams.FACET_QUERY, "{!key='OUTDATED'}LATEST_APPLIED_CONTENT_VERSION_ID:{-10 TO -10}");
.set(FacetParams.FACET, "on")
.add(FacetParams.FACET_QUERY, "{!key='OUTDATED'}" + LAST_INCOMING_CONTENT_VERSION_ID + ":\"" + CONTENT_OUTDATED_MARKER + "\"");
SolrQueryResponse response = cloud.getResponse(nativeRequestHandler, request, params);
@@ -2625,7 +2624,7 @@ public class SolrInformationServer implements InformationServer
* - Do we have an entry in the local content store for the document X?
* - If we didn't have that entry then FTSSTATUS was set to New
*
* FTSSTatus = DIRTY
* FTSSTATUS = DIRTY
*
* When a node, which requires content, arrived here for being indexed, the "Dirty" status was set if
*
@@ -2637,7 +2636,7 @@ public class SolrInformationServer implements InformationServer
* something is different between the local document and the document which is in Solr".
*
* The content store allowed to workaround the problem: the code was deserializing the entry corresponding to the
* incoming node (which teoretically corresponded to the document indexed in Solr) so both versions were available
* incoming node (which theoretically corresponded to the document indexed in Solr) so both versions were available
* for making decisions
*
* ------------------
@@ -2646,7 +2645,7 @@ public class SolrInformationServer implements InformationServer
* about
*
* - the "INSERT" or "UPDATE" nature of the indexing operation that is going to be executed
* - the values of fields of a given document indeded in Solr
* - the values of fields of a given document indexed in Solr
*
* In order to indicate to the ContentTracker which documents will require the content update, we added two
* additional fields in the schema:
@@ -2656,8 +2655,8 @@ public class SolrInformationServer implements InformationServer
* LATEST_APPLIED_CONTENT_VERSION_ID: as the name suggests, this is the latest DOCID applied to this document (again, not the lucene docid)
* </li>
* <li>
* LAST_INCOMING_CONTENT_VERSION_ID: a field that will contains "-10" if the content is outdated, otherwise it will have the same value
* of LATEST_APPLIED_CONTENT_VERSION_ID.
* LAST_INCOMING_CONTENT_VERSION_ID: this call will set the field to "-10" as the content is outdated. The content tracker will then set it
* to have same value as LATEST_APPLIED_CONTENT_VERSION_ID.
* </li>
* </ul>
*
@@ -2667,11 +2666,8 @@ public class SolrInformationServer implements InformationServer
{
ofNullable(value)
.map(ContentPropertyValue::getId)
.ifPresentOrElse(
id -> {
document.setField(LATEST_APPLIED_CONTENT_VERSION_ID, id);
document.setField(LAST_INCOMING_CONTENT_VERSION_ID, Map.of("removeregex", "^(?!"+id+"$).*$"));},
() -> document.setField(LAST_INCOMING_CONTENT_VERSION_ID, CONTENT_OUTDATED_MARKER));
.ifPresent(id -> document.setField(LATEST_APPLIED_CONTENT_VERSION_ID, id));
document.setField(LAST_INCOMING_CONTENT_VERSION_ID, CONTENT_OUTDATED_MARKER);
}
private void addContentProperty(
search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java
+52 -8
View File
@@ -27,6 +27,7 @@
package org.alfresco.solr;
import static java.util.Optional.ofNullable;
import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ANY;
import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.ASSOC_REF;
import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.BOOLEAN;
@@ -45,6 +46,10 @@ import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.PATH;
import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.PERIOD;
import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.QNAME;
import static org.alfresco.service.cmr.dictionary.DataTypeDefinition.TEXT;
import static org.alfresco.solr.SolrInformationServer.REQUEST_HANDLER_GET;
import static org.alfresco.solr.SolrInformationServer.REQUEST_HANDLER_NATIVE;
import static org.alfresco.solr.SolrInformationServer.RESPONSE_DEFAULT_ID;
import static org.alfresco.solr.SolrInformationServer.RESPONSE_DEFAULT_IDS;
import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_DAY_FIELD_SUFFIX;
import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_HOUR_FIELD_SUFFIX;
import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_MINUTE_FIELD_SUFFIX;
@@ -57,6 +62,8 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -76,12 +83,15 @@ import org.alfresco.solr.client.SOLRAPIClient;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.BasicResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.DocList;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -124,6 +134,8 @@ public class SolrInformationServerTest
public void setUp()
{
when(core.getResourceLoader()).thenReturn(resourceLoader);
when(core.getRequestHandler(REQUEST_HANDLER_GET)).thenReturn(handler);
when(core.getRequestHandler(REQUEST_HANDLER_NATIVE)).thenReturn(handler);
when(resourceLoader.getCoreProperties()).thenReturn(new Properties());
infoServer = new SolrInformationServer(adminHandler, core, client)
{
@@ -145,15 +157,14 @@ public class SolrInformationServerTest
SolrDocument state = new SolrDocument();
SimpleOrderedMap<SolrDocument> responseContent = new SimpleOrderedMap<>();
responseContent.add(SolrInformationServer.RESPONSE_DEFAULT_ID, state);
responseContent.add(RESPONSE_DEFAULT_ID, state);
when(response.getValues()).thenReturn(responseContent);
when(core.getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET)).thenReturn(handler);
SolrDocument document = infoServer.getState(core, request, id);
assertEquals(id, request.getParams().get(CommonParams.ID));
verify(core).getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET);
verify(core).getRequestHandler(REQUEST_HANDLER_GET);
verify(response).getValues();
assertSame(state, document);
@@ -313,15 +324,14 @@ public class SolrInformationServerTest
String id = String.valueOf(System.currentTimeMillis());
SimpleOrderedMap<Object> responseContent = new SimpleOrderedMap<>();
responseContent.add(SolrInformationServer.RESPONSE_DEFAULT_ID, null);
responseContent.add(RESPONSE_DEFAULT_ID, null);
when(response.getValues()).thenReturn(responseContent);
when(core.getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET)).thenReturn(handler);
SolrDocument document = infoServer.getState(core, request, id);
assertEquals(id, request.getParams().get(CommonParams.ID));
verify(core).getRequestHandler(SolrInformationServer.REQUEST_HANDLER_GET);
verify(core).getRequestHandler(REQUEST_HANDLER_GET);
verify(response).getValues();
assertNull(document);
@@ -417,5 +427,39 @@ public class SolrInformationServerTest
});
}
}
/** Check that the FTS report is derived from the correct parts of the Solr response. */
@Test
public void testAddContentOutdatedAndUpdatedCounts()
{
// Pretend that there are three documents in total.
NamedList<Object> responseContent = new SimpleOrderedMap<>();
DocList docList = mock(DocList.class);
when(docList.matches()).thenReturn(3);
BasicResultContext basicResultContext = mock(BasicResultContext.class);
when(basicResultContext.getDocList()).thenReturn(docList);
responseContent.add(RESPONSE_DEFAULT_IDS, basicResultContext);
// Set the facet to say one document is outdated.
NamedList<Number> facetQueries = new SimpleOrderedMap<>();
facetQueries.add("OUTDATED", 1);
NamedList<NamedList<Number>> facetCounts = new SimpleOrderedMap<>();
facetCounts.add("facet_queries", facetQueries);
responseContent.add("facet_counts", facetCounts);
// Set up the request handler to return the fake response.
doAnswer(invocation -> {
SolrQueryResponse solrQueryResponse = invocation.getArgument(1);
solrQueryResponse.setAllValues(responseContent);
return null;
}).when(handler).handleRequest(any(SolrQueryRequest.class), any(SolrQueryResponse.class));
// Call the method under test.
NamedList<Object> report = new NamedList<>();
infoServer.addContentOutdatedAndUpdatedCounts(report);
// Check the report.
assertEquals("Expected two content nodes to be in sync.", report.get("Node count whose content is in sync"), 2L);
assertEquals("Expected one content node to need an update.", report.get("Node count whose content needs to be updated"), 1L);
}
}