Merge pull request #1428 from Alfresco/fix/MNT-23022_aclchangeset_hole_fix_20X

MNT-23022 aclchangeset hole fix

search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java

@@ -149,6 +149,7 @@ import org.alfresco.solr.adapters.ISimpleOrderedMap;
 import org.alfresco.solr.adapters.SolrOpenBitSetAdapter;
 import org.alfresco.solr.adapters.SolrSimpleOrderedMap;
 import org.alfresco.solr.client.AclChangeSet;
+import org.alfresco.solr.client.AclChangeSets;
 import org.alfresco.solr.client.AclReaders;
 import org.alfresco.solr.client.AlfrescoModel;
 import org.alfresco.solr.client.ContentPropertyValue;
@@ -426,6 +427,7 @@ public class SolrInformationServer implements InformationServer
     private final long holeRetention;
     private final boolean fingerprintHasBeenEnabledOnThisInstance;
     private final int contentStreamLimit;
+    private final int statsFacetLimit;
 
     private long cleanContentLastPurged;
     
@@ -632,6 +634,8 @@ public class SolrInformationServer implements InformationServer
         
         getPathsInNodeBatches = Boolean.parseBoolean(coreConfiguration.getProperty("alfresco.metadata.getPathsInNodeBatches", "true"));
 
+        statsFacetLimit = Integer.parseInt(coreConfiguration.getProperty("alfresco.stats.facetLimit", "100"));
+
         props = AlfrescoSolrDataModel.getCommonConfig();
         hostName = ConfigUtil.locateProperty(SOLR_HOST, props.getProperty(SOLR_HOST));
 
@@ -3687,6 +3691,7 @@ public class SolrInformationServer implements InformationServer
             IOpenBitSet idsInIndex = this.getOpenBitSetInstance();
             long batchStartId = minId;
             long batchEndId = Math.min(batchStartId + BATCH_FACET_TXS, maxId);
+            long idInIndex = 0L;
 
             // Continues as long as the batch does not pass the maximum
             while (batchStartId <= maxId)
@@ -3697,7 +3702,7 @@ public class SolrInformationServer implements InformationServer
                         field, 1); // Min count of 1 ensures that the id returned is in the index
                 for (Map.Entry<String, Integer> idCount : idCounts)
                 {
-                    long idInIndex = Long.parseLong(idCount.getKey());
+                    idInIndex = Long.parseLong(idCount.getKey());
 
                     // Only looks at facet values that fit the query
                     if (batchStartId <= idInIndex && idInIndex <= batchEndId)
@@ -3737,6 +3742,27 @@ public class SolrInformationServer implements InformationServer
                 batchEndId = Math.min(batchStartId + BATCH_FACET_TXS, maxId);
             }
 
+            // Verify we processed all items from request params, if not, send a warning
+            if (idInIndex != 0L && idInIndex < batchEndId)
+            {
+                try
+                {
+                    AclChangeSets changesets = repositoryClient.getAclChangeSets(null, idInIndex, null, idInIndex + 1,
+                            1);
+                    Long changeSetCommitTimeMs = changesets.getAclChangeSets().size() > 0
+                            ? changesets.getAclChangeSets().get(0).getCommitTimeMs()
+                            : 0L;
+
+                    LOGGER.warning("Not all items processed. Last acl changeset (id {} ) with commit time evaluated: {}",
+                            idInIndex, changeSetCommitTimeMs);
+                }
+                catch (JSONException | AuthenticationException | IOException e)
+                {
+                    LOGGER.warning("Not all items processed. Last acl changeset evaluated: {}", idInIndex);
+                }
+
+            }
+
             reporter.reportUniqueIdsInIndex(idsInIndex.cardinality());
         }
     }
@@ -3762,6 +3788,7 @@ public class SolrInformationServer implements InformationServer
                         .set(CommonParams.ROWS, 0)
                         .set(FacetParams.FACET, true)
                         .set(FacetParams.FACET_FIELD, field)
+                        .set(FacetParams.FACET_LIMIT, statsFacetLimit)
                         .set(FacetParams.FACET_MINCOUNT, minCount);
 
         SolrQueryResponse response = cloud.getResponse(nativeRequestHandler, request, params);

search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java

@@ -723,9 +723,7 @@ public class AclTracker extends ActivatableTracker
 
                 this.state = getTrackerState();
                 
-                Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound,
+                Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, state.getLastGoodChangeSetCommitTimeInIndex());
-                        state.getLastChangeSetCommitTimeOnServer() == 0 ? state.getLastGoodChangeSetCommitTimeInIndex()
-                                : state.getLastChangeSetCommitTimeOnServer());
                 aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, timeStep, maxNumberOfAclChangeSets,
                         state.getTimeToStopIndexing());
                 

search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties

@@ -202,6 +202,9 @@ solr.request.content.compress=false
 #
 solr.initial.transaction.range=0-2000
 
+# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports
+# and fix)
+alfresco.stats.facetLimit=100
 
 #
 # Limit the maximum text size of transformed content sent to the index - in bytes

search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties

@@ -234,6 +234,10 @@ solr.request.content.compress=false
 #
 solr.initial.transaction.range=0-2000
 
+# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports
+# and fix)
+alfresco.stats.facetLimit=100
+
 
 # Backup
 # The path where the backup will be created.

search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrcore.properties

@@ -125,6 +125,10 @@ alfresco.newSearcherInterval=2000
 
 alfresco.doPermissionChecks=true
 
+# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports
+# and fix)
+alfresco.stats.facetLimit=100
+
 
 #
 # Metadata pulling control

search-services/alfresco-search/src/test/resources/test-files/master/conf/solrcore.properties

@@ -125,6 +125,10 @@ alfresco.newSearcherInterval=2000
 
 alfresco.doPermissionChecks=true
 
+# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports
+# and fix)
+alfresco.stats.facetLimit=100
+
 
 #
 # Metadata pulling control

search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrcore.properties

@@ -125,6 +125,10 @@ alfresco.newSearcherInterval=2000
 
 alfresco.doPermissionChecks=true
 
+# Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports
+# and fix)
+alfresco.stats.facetLimit=100
+
 
 #
 # Metadata pulling control