From 24edfa695cabcb058b3d071482eb982bf4a80ba7 Mon Sep 17 00:00:00 2001 From: Eva Vasques Date: Tue, 10 May 2022 10:58:23 +0100 Subject: [PATCH] MNT-23022 - add configurable property alfresco.stats.facetLimit so the facet query limit can be changed from the default 100 limit. This impacts the number of items that are retrieved for index checks. Also validate and warn if not all items from the requested stats action or report were processed (cherry picked from commit f81403f0584cdbda496abba1491dbf3133f56798) --- .../alfresco/solr/SolrInformationServer.java | 28 +++++++++++++++++++ .../noRerank/conf/solrcore.properties | 3 ++ .../templates/rerank/conf/solrcore.properties | 4 +++ .../collection1/conf/solrcore.properties | 4 +++ .../master/conf/solrcore.properties | 4 +++ .../test-files/slave/conf/solrcore.properties | 4 +++ 6 files changed, 47 insertions(+) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 0f9711e68..a574cbb6e 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -149,6 +149,7 @@ import org.alfresco.solr.adapters.ISimpleOrderedMap; import org.alfresco.solr.adapters.SolrOpenBitSetAdapter; import org.alfresco.solr.adapters.SolrSimpleOrderedMap; import org.alfresco.solr.client.AclChangeSet; +import org.alfresco.solr.client.AclChangeSets; import org.alfresco.solr.client.AclReaders; import org.alfresco.solr.client.AlfrescoModel; import org.alfresco.solr.client.ContentPropertyValue; @@ -426,6 +427,7 @@ public class SolrInformationServer implements InformationServer private final long holeRetention; private final boolean fingerprintHasBeenEnabledOnThisInstance; private final int contentStreamLimit; + private final int statsFacetLimit; private long cleanContentLastPurged; @@ -632,6 +634,8 @@ public class SolrInformationServer implements InformationServer getPathsInNodeBatches = Boolean.parseBoolean(coreConfiguration.getProperty("alfresco.metadata.getPathsInNodeBatches", "true")); + statsFacetLimit = Integer.parseInt(coreConfiguration.getProperty("alfresco.stats.facetLimit", "100")); + props = AlfrescoSolrDataModel.getCommonConfig(); hostName = ConfigUtil.locateProperty(SOLR_HOST, props.getProperty(SOLR_HOST)); @@ -3687,6 +3691,7 @@ public class SolrInformationServer implements InformationServer IOpenBitSet idsInIndex = this.getOpenBitSetInstance(); long batchStartId = minId; long batchEndId = Math.min(batchStartId + BATCH_FACET_TXS, maxId); + long lastIdEvaluated = 0; // Continues as long as the batch does not pass the maximum while (batchStartId <= maxId) @@ -3698,6 +3703,7 @@ public class SolrInformationServer implements InformationServer for (Map.Entry idCount : idCounts) { long idInIndex = Long.parseLong(idCount.getKey()); + lastIdEvaluated = idInIndex; // Only looks at facet values that fit the query if (batchStartId <= idInIndex && idInIndex <= batchEndId) @@ -3737,6 +3743,27 @@ public class SolrInformationServer implements InformationServer batchEndId = Math.min(batchStartId + BATCH_FACET_TXS, maxId); } + // Verify we processed all items from request params, if not, send a warning + if (lastIdEvaluated != 0 && lastIdEvaluated < batchEndId) + { + try + { + AclChangeSets changesets = repositoryClient.getAclChangeSets(null, lastIdEvaluated, null, lastIdEvaluated + 1, + 1); + Long changeSetCommitTimeMs = changesets.getAclChangeSets().size() > 0 + ? changesets.getAclChangeSets().get(0).getCommitTimeMs() + : 0L; + + LOGGER.warning("Not all items processed. Last acl changeset (id {} ) with commit time evaluated: {}", + lastIdEvaluated, changeSetCommitTimeMs); + } + catch (JSONException | AuthenticationException | IOException e) + { + LOGGER.warning("Not all items processed. Last acl changeset evaluated: {}", lastIdEvaluated); + } + + } + reporter.reportUniqueIdsInIndex(idsInIndex.cardinality()); } } @@ -3762,6 +3789,7 @@ public class SolrInformationServer implements InformationServer .set(CommonParams.ROWS, 0) .set(FacetParams.FACET, true) .set(FacetParams.FACET_FIELD, field) + .set(FacetParams.FACET_LIMIT, statsFacetLimit) .set(FacetParams.FACET_MINCOUNT, minCount); SolrQueryResponse response = cloud.getResponse(nativeRequestHandler, request, params); diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties index 23b1f8408..a47898e85 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties @@ -202,6 +202,9 @@ solr.request.content.compress=false # solr.initial.transaction.range=0-2000 +# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports +# and fix) +alfresco.stats.facetLimit=100 # # Limit the maximum text size of transformed content sent to the index - in bytes diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index fbcd4369b..f512cb9cd 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -234,6 +234,10 @@ solr.request.content.compress=false # solr.initial.transaction.range=0-2000 +# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports +# and fix) +alfresco.stats.facetLimit=100 + # Backup # The path where the backup will be created. diff --git a/search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrcore.properties b/search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrcore.properties index bedcbe554..57017c56f 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrcore.properties +++ b/search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrcore.properties @@ -125,6 +125,10 @@ alfresco.newSearcherInterval=2000 alfresco.doPermissionChecks=true +# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports +# and fix) +alfresco.stats.facetLimit=100 + # # Metadata pulling control diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrcore.properties b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrcore.properties index bedcbe554..57017c56f 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrcore.properties +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrcore.properties @@ -125,6 +125,10 @@ alfresco.newSearcherInterval=2000 alfresco.doPermissionChecks=true +# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports +# and fix) +alfresco.stats.facetLimit=100 + # # Metadata pulling control diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrcore.properties b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrcore.properties index a46ace7f4..9c3a5c9fe 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrcore.properties +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrcore.properties @@ -125,6 +125,10 @@ alfresco.newSearcherInterval=2000 alfresco.doPermissionChecks=true +# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports +# and fix) +alfresco.stats.facetLimit=100 + # # Metadata pulling control