From d98bf928c39f6a04c81a0f8224a7e18b574393a4 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Jan 2020 16:10:31 +0000 Subject: [PATCH 1/8] SEARCH-1949 Add config option to disable cascade tracking. --- .../src/main/resources/solr/instance/conf/shared.properties | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties index e78b8ee32..63bc6bfe2 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties @@ -30,4 +30,7 @@ alfresco.cross.locale.property.1={http://www.alfresco.org/model/content/1.0}lock # alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content # alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext -alfresco.model.tracker.cron=0/10 * * * * ? * \ No newline at end of file +alfresco.model.tracker.cron=0/10 * * * * ? * + +# Whether path queries are enabled. +alfresco.cascade.tracker.enabled=true From 6268b22a2f10b07f062d8d7dc49c135864c35658 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Jan 2020 16:23:14 +0000 Subject: [PATCH 2/8] SEARCH-1949 Rename some integration tests as we can still have cascading without the tracker. --- ...rackerIntegrationTest.java => CascadingIntegrationTest.java} | 2 +- .../solr/tracker/{CascadeTrackerIT.java => CascadingIT.java} | 2 +- ...stributedCascadeTrackerIT.java => DistributedCascadeIT.java} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/{CascadingTrackerIntegrationTest.java => CascadingIntegrationTest.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{CascadeTrackerIT.java => CascadingIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedCascadeTrackerIT.java => DistributedCascadeIT.java} (98%) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java similarity index 99% rename from e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java rename to e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java index 3d66f1922..a61e2dc63 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java @@ -31,7 +31,7 @@ import static org.testng.Assert.assertTrue; * @author Alessandro Benedetti * @author Meenal Bhave */ -public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest +public class CascadingIntegrationTest extends AbstractE2EFunctionalTest { @Autowired protected DataContent dataContent; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java index 00c34cfa4..7d1e145a9 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java @@ -52,7 +52,7 @@ import java.util.List; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class CascadeTrackerIT extends AbstractAlfrescoSolrIT +public class CascadingIT extends AbstractAlfrescoSolrIT { private static long MAX_WAIT_TIME = 80000; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeIT.java index 6e5fbe39f..3d1415bb4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeIT.java @@ -55,7 +55,7 @@ import java.util.Properties; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT +public class DistributedCascadeIT extends AbstractAlfrescoDistributedIT { private Node parentFolder; private NodeMetaData parentFolderMetadata; From 6ceabbaaac524ab05b15145b781fc5884975b658 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 09:25:12 +0000 Subject: [PATCH 3/8] SEARCH-1949 Add helper method to check if cascade tracking is enabled. --- .../alfresco/solr/AlfrescoSolrDataModel.java | 2 +- .../alfresco/solr/SolrInformationServer.java | 165 ++++++++++-------- 2 files changed, 94 insertions(+), 73 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java index c28d7ae8b..50b6d9563 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java @@ -523,7 +523,7 @@ public class AlfrescoSolrDataModel implements QueryConstants } catch (IOException e) { - log.info("Failed to read shared properties fat " + propertiesFile.getAbsolutePath()); + log.info("Failed to read shared properties at " + propertiesFile.getAbsolutePath()); } return props; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 182067271..fe94569ff 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -205,10 +205,7 @@ public class SolrInformationServer implements InformationServer public static final String AND = " AND "; public static final String OR = " OR "; - //public static final String REQUEST_HANDLER_ALFRESCO_FULL_TEXT_SEARCH = "/afts"; private static final String REQUEST_HANDLER_NATIVE = "/native"; - //public static final String REQUEST_HANDLER_ALFRESCO = "/alfresco"; - //public static final String REQUEST_HANDLER_SELECT = "/select"; static final String REQUEST_HANDLER_GET = "/get"; private static final String RESPONSE_DEFAULT_IDS = "response"; static final String RESPONSE_DEFAULT_ID = "doc"; @@ -441,7 +438,7 @@ public class SolrInformationServer implements InformationServer dataModel = AlfrescoSolrDataModel.getInstance(); contentStreamLimit = Integer.parseInt(p.getProperty("alfresco.contentStreamLimit", "10000000")); - + // build base URL - host and port have to come from configuration. props = AlfrescoSolrDataModel.getCommonConfig(); hostName = ConfigUtil.locateProperty(SOLR_HOST, props.getProperty(SOLR_HOST)); @@ -456,6 +453,17 @@ public class SolrInformationServer implements InformationServer return this.adminHandler; } + /** + * Check if cascade tracking is enabled. + * + * @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file). + */ + private boolean cascadeTrackingEnabled() + { + String cascadeTrackerEnabledProp = ofNullable((String) props.get("alfresco.cascade.tracker.enabled")).orElse("true"); + return Boolean.valueOf(cascadeTrackerEnabledProp); + } + @Override public synchronized void initSkippingDescendantDocs() { @@ -507,7 +515,7 @@ public class SolrInformationServer implements InformationServer report.add("Node count with FTSStatus New", newCount); } } - + @Override public void afterInitModels() { @@ -520,7 +528,7 @@ public class SolrInformationServer implements InformationServer String query = FIELD_ACLID + ":" + aclid + AND + FIELD_DOC_TYPE + ":" + DOC_TYPE_ACL; long count = this.getDocListSize(query); aclReport.setIndexedAclDocCount(count); - + // TODO Could add INACLTXID later, but would need acl change set id. return aclReport; } @@ -778,7 +786,7 @@ public class SolrInformationServer implements InformationServer long count = this.getDocListSize(query); nodeReport.setIndexedNodeDocCount(count); } - + @Override public void commit() throws IOException { @@ -891,26 +899,26 @@ public class SolrInformationServer implements InformationServer return searcherOpened; } - + @Override public void deleteByAclChangeSetId(Long aclChangeSetId) throws IOException { deleteById(FIELD_INACLTXID, aclChangeSetId); } - + @Override public void deleteByAclId(Long aclId) throws IOException { isIdIndexCache.clear(); deleteById(FIELD_ACLID, aclId); } - + @Override public void deleteByNodeId(Long nodeId) throws IOException { deleteById(FIELD_DBID, nodeId); } - + @Override public void deleteByTransactionId(Long transactionId) throws IOException { @@ -923,7 +931,7 @@ public class SolrInformationServer implements InformationServer { return this.dataModel.getAlfrescoModels(); } - + @SuppressWarnings({ "unchecked", "rawtypes" }) @Override public Iterable> getCoreStats() throws IOException @@ -1047,7 +1055,7 @@ public class SolrInformationServer implements InformationServer return coreSummary; } - + @Override public DictionaryComponent getDictionaryService(String alternativeDictionary) { @@ -1371,7 +1379,10 @@ public class SolrInformationServer implements InformationServer public void dirtyTransaction(long txnId) { this.cleanContentCache.remove(txnId); - this.cleanCascadeCache.remove(txnId); + if (cascadeTrackingEnabled()) + { + this.cleanCascadeCache.remove(txnId); + } } @Override @@ -1460,9 +1471,9 @@ public class SolrInformationServer implements InformationServer long start = System.nanoTime(); if ((node.getStatus() == SolrApiNodeStatus.DELETED) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) - || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)) + || (node.getStatus() == SolrApiNodeStatus.UNKNOWN) + || cascadeTrackingEnabled() && ((node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) + || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED))) { // fix up any secondary paths NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); @@ -1470,8 +1481,8 @@ public class SolrInformationServer implements InformationServer nmdp.setToNodeId(node.getId()); List nodeMetaDatas; if ((node.getStatus() == SolrApiNodeStatus.DELETED) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) + || cascadeTrackingEnabled() && ((node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) + || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED))) { // Fake the empty node metadata for this parent deleted node NodeMetaData nodeMetaData = createDeletedNodeMetaData(node); @@ -1481,7 +1492,7 @@ public class SolrInformationServer implements InformationServer { nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE); } - + NodeMetaData nodeMetaData; if (!nodeMetaDatas.isEmpty()) { @@ -1499,7 +1510,7 @@ public class SolrInformationServer implements InformationServer finally { unlock(nodeMetaData.getId()); - } + } } } // else, the node has moved on to a later transaction, and it will be indexed later @@ -1508,10 +1519,9 @@ public class SolrInformationServer implements InformationServer deleteNode(processor, request, node); } - - if ((node.getStatus() == SolrApiNodeStatus.UPDATED) - || (node.getStatus() == SolrApiNodeStatus.UNKNOWN) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) + if (node.getStatus() == SolrApiNodeStatus.UPDATED + || node.getStatus() == SolrApiNodeStatus.UNKNOWN + || (cascadeTrackingEnabled() && node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) { long nodeId = node.getId(); @@ -1796,8 +1806,13 @@ public class SolrInformationServer implements InformationServer EnumMap> nodeStatusToNodeIds = new EnumMap<>(SolrApiNodeStatus.class); categorizeNodes(nodes, nodeIdsToNodes, nodeStatusToNodeIds); List deletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.DELETED)); - List shardDeletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_DELETED)); - List shardUpdatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_UPDATED)); + List shardDeletedNodeIds = Collections.emptyList(); + List shardUpdatedNodeIds = Collections.emptyList(); + if (cascadeTrackingEnabled()) + { + shardDeletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_DELETED)); + shardUpdatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_UPDATED)); + } List unknownNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UNKNOWN)); List updatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UPDATED)); @@ -1881,7 +1896,7 @@ public class SolrInformationServer implements InformationServer continue; } - if (nodeIdsToNodes.get(nodeMetaData.getId()).getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) + if (cascadeTrackingEnabled() && nodeIdsToNodes.get(nodeMetaData.getId()).getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) { if (nodeMetaData.getProperties().get(ContentModel.PROP_CASCADE_TX) != null) { @@ -1982,7 +1997,7 @@ public class SolrInformationServer implements InformationServer { StringPropertyValue latProp = ((StringPropertyValue)nodeMetaData.getProperties().get(ContentModel.PROP_LATITUDE)); StringPropertyValue lonProp = ((StringPropertyValue)nodeMetaData.getProperties().get(ContentModel.PROP_LONGITUDE)); - + if((latProp != null) && (lonProp != null)) { String lat = latProp.getValue(); @@ -2020,14 +2035,14 @@ public class SolrInformationServer implements InformationServer } doc.addField(FIELD_ISNODE, "T"); // FIELD_FTSSTATUS is set when adding content properties to indicate whether or not the cache is clean. - + doc.addField(FIELD_TENANT, AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain())); updatePathRelatedFields(nodeMetaData, doc); updateNamePathRelatedFields(nodeMetaData, doc); updateAncestorRelatedFields(nodeMetaData, doc); doc.addField(FIELD_PARENT_ASSOC_CRC, nodeMetaData.getParentAssocsCrc()); - + if (nodeMetaData.getOwner() != null) { doc.addField(FIELD_OWNER, nodeMetaData.getOwner()); @@ -2113,7 +2128,7 @@ public class SolrInformationServer implements InformationServer } } - static void addPropertiesToDoc(Map properties, boolean isContentIndexedForNode, + static void addPropertiesToDoc(Map properties, boolean isContentIndexedForNode, SolrInputDocument newDoc, SolrInputDocument cachedDoc, boolean transformContentFlag) { for (Entry property : properties.entrySet()) @@ -2121,7 +2136,7 @@ public class SolrInformationServer implements InformationServer QName propertyQName = property.getKey(); newDoc.addField(FIELD_PROPERTIES, propertyQName.toString()); newDoc.addField(FIELD_PROPERTIES, propertyQName.getPrefixString()); - + PropertyValue value = property.getValue(); if(value != null) { @@ -2195,22 +2210,22 @@ public class SolrInformationServer implements InformationServer } } } - + private void deleteErrorNode(UpdateRequestProcessor processor, SolrQueryRequest request, Node node) throws IOException { - + String errorDocId = PREFIX_ERROR + node.getId(); - + // Try finding the node before performing removal operation DocSet docSet = request.getSearcher().getDocSet(new TermQuery(new Term(FIELD_SOLR4_ID, errorDocId))); - + if (docSet.size() > 0) { DeleteUpdateCommand delErrorDocCmd = new DeleteUpdateCommand(request); delErrorDocCmd.setId(errorDocId); processor.processDelete(delErrorDocCmd); } - + } private void deleteNode(UpdateRequestProcessor processor, SolrQueryRequest request, Node node) throws IOException @@ -2221,20 +2236,20 @@ public class SolrInformationServer implements InformationServer // MNT-13767 fix, remove by node DBID. deleteNode(processor, request, node.getId()); } - + private void deleteNode(UpdateRequestProcessor processor, SolrQueryRequest request, long dbid) throws IOException { - + // Try finding the node before performing removal operation DocSet docSet = request.getSearcher().getDocSet(LongPoint.newExactQuery(FIELD_DBID, dbid)); - + if (docSet.size() > 0) { DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request); delDocCmd.setQuery(FIELD_DBID + ":" + dbid); processor.processDelete(delDocCmd); } - + } private boolean isContentIndexedForNode(Map properties) @@ -2296,7 +2311,7 @@ public class SolrInformationServer implements InformationServer return fieldName; } - private void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName, + private void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName, AlfrescoSolrDataModel.ContentFieldType type, GetTextContentResponse textContentResponse) { IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getIndexedFieldForContentPropertyMetadata( @@ -2322,7 +2337,7 @@ public class SolrInformationServer implements InformationServer } } - private static void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName, + private static void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName, ContentPropertyValue contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType type) { IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getIndexedFieldForContentPropertyMetadata( @@ -2352,8 +2367,8 @@ public class SolrInformationServer implements InformationServer } } } - - private static void addContentPropertyToDocUsingCache(SolrInputDocument newDoc, SolrInputDocument cachedDoc, + + private static void addContentPropertyToDocUsingCache(SolrInputDocument newDoc, SolrInputDocument cachedDoc, QName propertyQName, ContentPropertyValue contentPropertyValue, boolean transformContentFlag) { addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType.DOCID); @@ -2361,14 +2376,14 @@ public class SolrInformationServer implements InformationServer addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType.LOCALE); addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType.MIMETYPE); addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType.ENCODING); - + if (!transformContentFlag) { // Marks it as Clean so we do not get the actual content markFTSStatus(newDoc, FTSStatus.Clean); return; } - + if (cachedDoc != null) { ofNullable(cachedDoc.getField("MINHASH")) @@ -2386,7 +2401,7 @@ public class SolrInformationServer implements InformationServer addFieldIfNotSet(newDoc, field); } - String transformationStatusFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, + String transformationStatusFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_STATUS); if (transformationStatusFieldName != null){ newDoc.addField(transformationStatusFieldName, cachedDoc.getFieldValue(transformationStatusFieldName)); @@ -2405,19 +2420,19 @@ public class SolrInformationServer implements InformationServer // Gets the new content docid and compares to that of the cachedDoc to mark the content as clean/dirty String fldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, AlfrescoSolrDataModel.ContentFieldType.DOCID); - + if(newDoc.getFieldValue(FIELD_FTSSTATUS) == null) { newDoc.addField(FIELD_FTSSTATUS, cachedDoc.getFieldValue(FIELD_FTSSTATUS)); } - + if(cachedDoc.getFieldValue(fldName) != null) { long cachedDocContentDocid = Long.parseLong(String.valueOf(cachedDoc.getFieldValue(fldName))); long currentContentDocid = contentPropertyValue.getId(); // If we have used out of date content we mark it as dirty // Otherwise we leave it alone - it could already be marked as dirty/New and require an update - + if (cachedDocContentDocid != currentContentDocid) { // The cached content is out of date @@ -2429,7 +2444,7 @@ public class SolrInformationServer implements InformationServer markFTSStatus(newDoc, FTSStatus.Dirty); } } - else + else { // There is not a SolrInputDocument in the solrContentStore, so no content is added now to the new solr doc markFTSStatus(newDoc, FTSStatus.New); @@ -2468,7 +2483,7 @@ public class SolrInformationServer implements InformationServer doc.removeField(FIELD_FTSSTATUS); doc.addField(FIELD_FTSSTATUS, status.toString()); } - + private void addContentToDoc(SolrInputDocument doc, long dbId) throws AuthenticationException, IOException { Collection fieldNames = doc.deepCopy().getFieldNames(); @@ -2484,22 +2499,22 @@ public class SolrInformationServer implements InformationServer // Could update multi content but it is broken .... } } - + private void addContentPropertyToDocUsingAlfrescoRepository(SolrInputDocument doc, QName propertyQName, long dbId, String locale) throws AuthenticationException, IOException { long start = System.nanoTime(); - + // Expensive call to be done with ContentTracker GetTextContentResponse response = repositoryClient.getTextContent(dbId, propertyQName, null); - + addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_STATUS, response); addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_EXCEPTION, response); addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_TIME, response); - + InputStream ris = response.getContent(); if (Objects.equals(response.getContentEncoding(), "gzip")) { @@ -2545,7 +2560,7 @@ public class SolrInformationServer implements InformationServer long end = System.nanoTime(); this.getTrackerStats().addDocTransformationTime(end - start); - + StringBuilder builder = new StringBuilder(textContent.length() + 16); builder.append("\u0000").append(locale).append("\u0000"); builder.append(textContent); @@ -2567,7 +2582,7 @@ public class SolrInformationServer implements InformationServer } private static void addMLTextPropertyToDoc(SolrInputDocument doc, FieldInstance field, MLTextPropertyValue mlTextPropertyValue) - { + { if(field.isLocalised()) { StringBuilder sort = new StringBuilder(128); @@ -2575,20 +2590,20 @@ public class SolrInformationServer implements InformationServer { final String propValue = mlTextPropertyValue.getValue(locale); LOGGER.debug("ML {} in {} of {}", field.getField(), locale, propValue); - + if((locale == null) || (propValue == null)) { continue; - } - + } + StringBuilder builder = new StringBuilder(propValue.length() + 16); builder.append("\u0000").append(locale.toString()).append("\u0000").append(propValue); - + if(!field.isSort()) { doc.addField(field.getField(), builder.toString()); } - + if (sort.length() > 0) { sort.append("\u0000"); @@ -2629,7 +2644,10 @@ public class SolrInformationServer implements InformationServer input.addField(FIELD_INTXID, txn.getId()); input.addField(FIELD_TXCOMMITTIME, txn.getCommitTimeMs()); input.addField(FIELD_DOC_TYPE, DOC_TYPE_TX); - input.addField(FIELD_CASCADE_FLAG, 0); + if (cascadeTrackingEnabled()) + { + input.addField(FIELD_CASCADE_FLAG, 0); + } cmd.solrDoc = input; processor.processAdd(cmd); } @@ -2670,8 +2688,11 @@ public class SolrInformationServer implements InformationServer input.addField(FIELD_S_TXID, info.getId()); input.addField(FIELD_S_TXCOMMITTIME, info.getCommitTimeMs()); - //Set the cascade flag to 1. This means cascading updates have not been done yet. - input.addField(FIELD_CASCADE_FLAG, 1); + if (cascadeTrackingEnabled()) + { + //Set the cascade flag to 1. This means cascading updates have not been done yet. + input.addField(FIELD_CASCADE_FLAG, 1); + } cmd.solrDoc = input; processor.processAdd(cmd); @@ -2898,7 +2919,7 @@ public class SolrInformationServer implements InformationServer { activeTrackerThreadsLock.writeLock().unlock(); } - + } @Override @@ -2925,7 +2946,7 @@ public class SolrInformationServer implements InformationServer SolrIndexSearcher solrIndexSearcher = refCounted.get(); NumericDocValues dbidDocValues = solrIndexSearcher.getSlowAtomicReader().getNumericDocValues(QueryConstants.FIELD_DBID); - + List batch = new ArrayList<>(200); DocList docList = cloud.getDocList(nativeRequestHandler, request, query.startsWith("{") ? query : "{!afts}"+query); for (DocIterator it = docList.iterator(); it.hasNext(); /**/) @@ -2940,7 +2961,7 @@ public class SolrInformationServer implements InformationServer node.setTxnId(Long.MAX_VALUE); batch.add(node); - + if(batch.size() >= 200) { indexNodes(batch, true, true); @@ -3438,7 +3459,7 @@ public class SolrInformationServer implements InformationServer SolrQueryRequest request, UpdateRequestProcessor processor, LinkedHashSet stack) throws AuthenticationException, IOException, JSONException { - + // skipDescendantDocsForSpecificAspects is initialised on a synchronised method, so access must be also synchronised synchronized (this) { From 6b1c8c4a1cb932d9b820b7023be2e474be4000aa Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 09:39:47 +0000 Subject: [PATCH 4/8] SEARCH-1949 Remove unused cascade parameter. --- .../org/alfresco/solr/InformationServer.java | 2 +- .../alfresco/solr/SolrInformationServer.java | 42 +++++++++++++++---- .../solr/tracker/MetadataTracker.java | 2 +- .../solr/tracker/MetadataTrackerTest.java | 2 +- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java index ec64e11b6..2e54bdb46 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java @@ -88,7 +88,7 @@ public interface InformationServer extends InformationServerCollectionProvider void indexNode(Node node, boolean overwrite) throws IOException, AuthenticationException, JSONException; - void indexNodes(List nodes, boolean overwrite, boolean cascade) throws IOException, AuthenticationException, JSONException; + void indexNodes(List nodes, boolean overwrite) throws IOException, AuthenticationException, JSONException; void cascadeNodes(List nodes, boolean overwrite) throws IOException, AuthenticationException, JSONException; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index fe94569ff..67dd1b0ea 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -19,6 +19,7 @@ package org.alfresco.solr; import static java.util.Optional.ofNullable; + import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLID; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLTXCOMMITTIME; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLTXID; @@ -76,17 +77,32 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.text.DecimalFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import com.carrotsearch.hppc.IntArrayList; - import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.cursors.LongCursor; + import org.alfresco.httpclient.AuthenticationException; import org.alfresco.model.ContentModel; import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; @@ -142,7 +158,19 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; -import org.apache.lucene.search.*; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.LegacyNumericRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldCollector; import org.apache.lucene.util.BytesRefBuilder; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -167,8 +195,6 @@ import org.apache.solr.search.DelegatingCollector; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.DocSet; -import org.apache.solr.search.QueryCommand; -import org.apache.solr.search.QueryResult; import org.apache.solr.search.QueryWrapperFilter; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.update.AddUpdateCommand; @@ -1795,7 +1821,7 @@ public class SolrInformationServer implements InformationServer @Override - public void indexNodes(List nodes, boolean overwrite, boolean cascade) throws IOException, JSONException + public void indexNodes(List nodes, boolean overwrite) throws IOException, JSONException { UpdateRequestProcessor processor = null; try (SolrQueryRequest request = newSolrQueryRequest()) @@ -2964,13 +2990,13 @@ public class SolrInformationServer implements InformationServer if(batch.size() >= 200) { - indexNodes(batch, true, true); + indexNodes(batch, true); batch.clear(); } } if(batch.size() > 0) { - indexNodes(batch, true, true); + indexNodes(batch, true); batch.clear(); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 4821278b4..0501108bd 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -957,7 +957,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker List filteredNodes = filterNodes(nodes); if(filteredNodes.size() > 0) { - this.infoServer.indexNodes(filteredNodes, true, false); + this.infoServer.indexNodes(filteredNodes, true); } } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java index 5c68f5974..7d2c4c1ce 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java @@ -120,7 +120,7 @@ public class MetadataTrackerTest this.metadataTracker.doTrack(); InOrder inOrder = inOrder(srv); - inOrder.verify(srv).indexNodes(nodes, true, false); + inOrder.verify(srv).indexNodes(nodes, true); inOrder.verify(srv).indexTransaction(tx, true); inOrder.verify(srv).commit(); } From f23c082f1cbeecdb478e75db18282a24c9851ce9 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 13:26:38 +0000 Subject: [PATCH 5/8] SEARCH-1949 Refactor tracker registration. --- .../alfresco/solr/SolrInformationServer.java | 6 ++- .../solr/lifecycle/SolrCoreLoadListener.java | 40 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 67dd1b0ea..e5cd5f018 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -255,6 +255,8 @@ public class SolrInformationServer implements InformationServer */ private static final int BATCH_FACET_TXS = 4096; private static final String FINGERPRINT_FIELD = "MINHASH"; + /** Shared property to determine if the cascade tracking is enabled. */ + public static final String CASCADE_TRACKER_ENABLED = "alfresco.cascade.tracker.enabled"; private final AlfrescoCoreAdminHandler adminHandler; private final SolrCore core; @@ -484,9 +486,9 @@ public class SolrInformationServer implements InformationServer * * @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file). */ - private boolean cascadeTrackingEnabled() + public boolean cascadeTrackingEnabled() { - String cascadeTrackerEnabledProp = ofNullable((String) props.get("alfresco.cascade.tracker.enabled")).orElse("true"); + String cascadeTrackerEnabledProp = ofNullable((String) props.get(CASCADE_TRACKER_ENABLED)).orElse("true"); return Boolean.valueOf(cascadeTrackerEnabledProp); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index fc7597a0f..2a7c2b2ce 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -21,6 +21,15 @@ package org.alfresco.solr.lifecycle; import static java.util.Arrays.asList; import static java.util.Optional.ofNullable; +import static org.alfresco.solr.SolrInformationServer.CASCADE_TRACKER_ENABLED; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.function.Function; +import java.util.function.Predicate; + import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.AlfrescoSolrDataModel; @@ -54,13 +63,6 @@ import org.apache.solr.search.SolrIndexSearcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Properties; -import java.util.function.Function; -import java.util.function.Predicate; - /** * Listeners for *FIRST SEARCHER* events in order to prepare and register the SolrContentStore and the Tracking Subsystem. * @@ -257,19 +259,27 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener trackerRegistry, scheduler); - CascadeTracker cascadeTracker = - registerAndSchedule( - new CascadeTracker(props, repositoryClient, core.getName(), srv), - core, - props, - trackerRegistry, - scheduler); + List trackers = new ArrayList<>(); + + String cascadeTrackerEnabledProp = ofNullable((String) props.get(CASCADE_TRACKER_ENABLED)).orElse("true"); + if (Boolean.valueOf(cascadeTrackerEnabledProp)) + { + CascadeTracker cascadeTracker = + registerAndSchedule( + new CascadeTracker(props, repositoryClient, core.getName(), srv), + core, + props, + trackerRegistry, + scheduler); + trackers.add(cascadeTracker); + } //The CommitTracker will acquire these locks in order //The ContentTracker will likely have the longest runs so put it first to ensure the MetadataTracker is not paused while //waiting for the ContentTracker to release it's lock. //The aclTracker will likely have the shortest runs so put it last. - return asList(cascadeTracker, contentTracker, metadataTracker, aclTracker); + trackers.addAll(asList(contentTracker, metadataTracker, aclTracker)); + return trackers; } /** From 944b67f0cde668b9a031778a13479289abe15cf5 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 14:28:56 +0000 Subject: [PATCH 6/8] SEARCH-1949 Update other trackers. --- .../alfresco/solr/tracker/CommitTracker.java | 22 +++++++++++++------ .../solr/tracker/MetadataTracker.java | 7 ++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java index 375c21ad8..6f7332035 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java @@ -19,7 +19,11 @@ package org.alfresco.solr.tracker; +import static java.util.Optional.empty; +import static java.util.Optional.ofNullable; + import java.util.List; +import java.util.Optional; import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; @@ -41,7 +45,8 @@ public class CommitTracker extends AbstractTracker private MetadataTracker metadataTracker; private AclTracker aclTracker; private ContentTracker contentTracker; - private CascadeTracker cascadeTracker; + /** The cascade tracker. Note that this may be empty if cascade tracking is disabled. */ + private Optional cascadeTracker = empty(); private AtomicInteger rollbackCount = new AtomicInteger(0); protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class); @@ -71,7 +76,7 @@ public class CommitTracker extends AbstractTracker } else if(tracker instanceof ContentTracker) { this.contentTracker = (ContentTracker)tracker; } else if(tracker instanceof CascadeTracker) { - this.cascadeTracker = (CascadeTracker)tracker; + this.cascadeTracker = ofNullable((CascadeTracker) tracker); } } @@ -182,8 +187,11 @@ public class CommitTracker extends AbstractTracker contentTracker.getWriteLock().acquire(); assert(contentTracker.getWriteLock().availablePermits() == 0); - cascadeTracker.getWriteLock().acquire(); - assert(cascadeTracker.getWriteLock().availablePermits() == 0); + if (cascadeTracker.isPresent()) + { + cascadeTracker.get().getWriteLock().acquire(); + assert (cascadeTracker.get().getWriteLock().availablePermits() == 0); + } infoSrv.rollback(); } @@ -206,12 +214,12 @@ public class CommitTracker extends AbstractTracker contentTracker.invalidateState(); //Reset cascadeTracker - cascadeTracker.setRollback(false); - cascadeTracker.invalidateState(); + cascadeTracker.ifPresent(c -> c.setRollback(false)); + cascadeTracker.ifPresent(c -> invalidateState()); //Release the locks contentTracker.getWriteLock().release(); - cascadeTracker.getWriteLock().release(); + cascadeTracker.ifPresent(c -> c.getWriteLock().release()); rollbackCount.incrementAndGet(); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 0501108bd..3df245e4a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -32,6 +32,7 @@ import org.alfresco.repo.index.shard.ShardState; import org.alfresco.solr.BoundedDeque; import org.alfresco.solr.InformationServer; import org.alfresco.solr.NodeReport; +import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.TrackerState; import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.client.GetNodesParameters; @@ -83,6 +84,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * {@link org.alfresco.solr.client.SOLRAPIClient#GET_TX_INTERVAL_COMMIT_TIME} */ private boolean txIntervalCommitTimeServiceAvailable = false; + /** Whether the cascade tracking is enabled. */ + private boolean cascadeTrackerEnabled = true; public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -107,6 +110,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); + cascadeTrackerEnabled = ((SolrInformationServer) informationServer).cascadeTrackingEnabled(); // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2 @@ -977,9 +981,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { filteredList.add(node); } - else + else if (cascadeTrackerEnabled) { - if(node.getStatus() == SolrApiNodeStatus.UPDATED) { Node doCascade = new Node(); From 12cf62abb84e9c3ad84368465ae937eca3e7e7a6 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 14:56:12 +0000 Subject: [PATCH 7/8] SEARCH-1949 Unit test for SolrCoreLoadListener. --- .../lifecycle/SolrCoreLoadListenerTest.java | 65 ++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadListenerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadListenerTest.java index e1ec88140..233bfa393 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadListenerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadListenerTest.java @@ -18,6 +18,30 @@ */ package org.alfresco.solr.lifecycle; +import static java.util.Arrays.asList; + +import static org.alfresco.solr.SolrInformationServer.CASCADE_TRACKER_ENABLED; +import static org.alfresco.solr.tracker.Tracker.Type.ACL; +import static org.alfresco.solr.tracker.Tracker.Type.CASCADE; +import static org.alfresco.solr.tracker.Tracker.Type.CONTENT; +import static org.alfresco.solr.tracker.Tracker.Type.METADATA; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.tracker.AclTracker; @@ -26,6 +50,7 @@ import org.alfresco.solr.tracker.ContentTracker; import org.alfresco.solr.tracker.MetadataTracker; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; +import org.alfresco.solr.tracker.Tracker.Type; import org.alfresco.solr.tracker.TrackerRegistry; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; @@ -36,20 +61,6 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import org.xml.sax.InputSource; -import java.util.List; -import java.util.Properties; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - /** * Unit tests for the {@link SolrCoreLoadListener}. * @@ -83,6 +94,8 @@ public class SolrCoreLoadListenerTest @Before public void setUp() { + initMocks(this); + listener = new SolrCoreLoadListener(core); when(core.getName()).thenReturn(coreName); @@ -104,7 +117,29 @@ public class SolrCoreLoadListenerTest verify(scheduler).schedule(any(MetadataTracker.class), eq(coreName), same(coreProperties)); verify(scheduler).schedule(any(CascadeTracker.class), eq(coreName), same(coreProperties)); - assertEquals(4, coreTrackers.size()); + Set trackerTypes = coreTrackers.stream().map(Tracker::getType).collect(Collectors.toSet()); + assertEquals("Unexpected trackers found.", Set.of(ACL, CONTENT, METADATA, CASCADE), trackerTypes); + } + + @Test + public void testDisabledCascadeTracking() + { + coreProperties.put(CASCADE_TRACKER_ENABLED, "false"); + + List coreTrackers = listener.createAndScheduleCoreTrackers(core, registry, coreProperties, scheduler, api, informationServer); + + verify(registry).register(eq(coreName), any(AclTracker.class)); + verify(registry).register(eq(coreName), any(ContentTracker.class)); + verify(registry).register(eq(coreName), any(MetadataTracker.class)); + verify(registry, never()).register(eq(coreName), any(CascadeTracker.class)); + + verify(scheduler).schedule(any(AclTracker.class), eq(coreName), same(coreProperties)); + verify(scheduler).schedule(any(ContentTracker.class), eq(coreName), same(coreProperties)); + verify(scheduler).schedule(any(MetadataTracker.class), eq(coreName), same(coreProperties)); + verify(scheduler, never()).schedule(any(CascadeTracker.class), eq(coreName), same(coreProperties)); + + Set trackerTypes = coreTrackers.stream().map(Tracker::getType).collect(Collectors.toSet()); + assertEquals("Unexpected trackers found.", Set.of(ACL, CONTENT, METADATA), trackerTypes); } @Test From f2966a73ab29f19a9d49d55ca301486c248470f2 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 15:54:36 +0000 Subject: [PATCH 8/8] SEARCH-1949 Add cascade enabled check to interface. --- .../src/main/java/org/alfresco/solr/InformationServer.java | 7 +++++++ .../main/java/org/alfresco/solr/SolrInformationServer.java | 7 ++----- .../java/org/alfresco/solr/tracker/MetadataTracker.java | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java index 2e54bdb46..e1b24d82b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java @@ -181,4 +181,11 @@ public interface InformationServer extends InformationServerCollectionProvider String getBaseUrl(); void flushContentStore() throws IOException; + + /** + * Check if cascade tracking is enabled. + * + * @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file). + */ + boolean cascadeTrackingEnabled(); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index e5cd5f018..1008789cd 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -481,11 +481,8 @@ public class SolrInformationServer implements InformationServer return this.adminHandler; } - /** - * Check if cascade tracking is enabled. - * - * @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file). - */ + /** {@inheritDoc} */ + @Override public boolean cascadeTrackingEnabled() { String cascadeTrackerEnabledProp = ofNullable((String) props.get(CASCADE_TRACKER_ENABLED)).orElse("true"); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 3df245e4a..3a7773dd9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -110,7 +110,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); - cascadeTrackerEnabled = ((SolrInformationServer) informationServer).cascadeTrackingEnabled(); + cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled(); // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2