[ACS-9167] Clear more caches

repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2021 Alfresco Software Limited
+ * Copyright (C) 2005 - 2025 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -90,6 +90,8 @@ public class DBQueryEngine implements QueryEngine
     protected static final Log logger = LogFactory.getLog(DBQueryEngine.class);
 
     protected static final String SELECT_BY_DYNAMIC_QUERY = "alfresco.metadata.query.select_byDynamicQuery";
+    protected static final String COUNT_BY_DYNAMIC_QUERY = "alfresco.metadata.query.count_byDynamicQuery";
+    protected static final QueryTemplate QUERY_TEMPLATE = new QueryTemplate(SELECT_BY_DYNAMIC_QUERY, COUNT_BY_DYNAMIC_QUERY);
 
     private static final int DEFAULT_MIN_PAGING_BATCH_SIZE = 2500;
 
@@ -252,7 +254,7 @@ public class DBQueryEngine implements QueryEngine
     }
 
     /* (non-Javadoc)
-     * 
+     *
      * @see org.alfresco.repo.search.impl.querymodel.QueryEngine#executeQuery(org.alfresco.repo.search.impl.querymodel.Query, org.alfresco.repo.search.impl.querymodel.QueryOptions, org.alfresco.repo.search.impl.querymodel.FunctionEvaluationContext) */
     @Override
     public QueryEngineResults executeQuery(Query query, QueryOptions options, FunctionEvaluationContext functionContext)
@@ -331,10 +333,10 @@ public class DBQueryEngine implements QueryEngine
         return asQueryEngineResults(resultSet);
     }
 
-    protected String pickQueryTemplate(QueryOptions options, DBQuery dbQuery)
+    protected QueryTemplate pickQueryTemplate(QueryOptions options, DBQuery dbQuery)
     {
         logger.debug("- using standard table for the query");
-        return SELECT_BY_DYNAMIC_QUERY;
+        return QUERY_TEMPLATE;
     }
 
     private ResultSet selectNodesWithPermissions(QueryOptions options, DBQuery dbQuery)
@@ -370,7 +372,8 @@ public class DBQueryEngine implements QueryEngine
         int requiredNodes = computeRequiredNodesCount(options);
 
         logger.debug("- query sent to the database");
-        performTmdqSelect(pickQueryTemplate(options, dbQuery), dbQuery, requiredNodes, new ResultHandler<Node>() {
+        QueryTemplate queryTemplate = pickQueryTemplate(options, dbQuery);
+        performTmdqSelect(queryTemplate, dbQuery, requiredNodes, new ResultHandler<Node>() {
             @Override
             public void handleResult(ResultContext<? extends Node> context)
             {
@@ -425,8 +428,7 @@ public class DBQueryEngine implements QueryEngine
                 }
             }
         });
-
+        int numberFound = countSelectedNodes(queryTemplate, dbQuery);
-        int numberFound = nodes.size();
         nodes.removeAll(Collections.singleton(null));
 
         DBResultSet rs = createResultSet(options, nodes, numberFound);
@@ -437,8 +439,17 @@ public class DBQueryEngine implements QueryEngine
         return frs;
     }
 
-    private void performTmdqSelect(String statement, DBQuery dbQuery, int requiredNodes, ResultHandler<Node> handler)
+    protected int countSelectedNodes(QueryTemplate queryTemplate, DBQuery dbQuery)
     {
+        dbQuery.setLimit(0);
+        dbQuery.setOffset(0);
+        String countQuery = queryTemplate.count();
+        return template.selectOne(countQuery, dbQuery);
+    }
+
+    private void performTmdqSelect(QueryTemplate queryTemplate, DBQuery dbQuery, int requiredNodes, ResultHandler<Node> handler)
+    {
+        String statement = queryTemplate.select();
         if (usePagingQuery)
         {
             performTmdqSelectPaging(statement, dbQuery, requiredNodes, handler);
@@ -457,8 +468,7 @@ public class DBQueryEngine implements QueryEngine
     private void performTmdqSelectPaging(String statement, DBQuery dbQuery, int requiredNodes, ResultHandler<Node> handler)
     {
         int batchStart = 0;
-        int batchSize = requiredNodes * 2;
+        int batchSize = calculateBatchSize(requiredNodes);
-        batchSize = Math.min(Math.max(batchSize, minPagingBatchSize), maxPagingBatchSize);
         DefaultResultContext<Node> resultCtx = new DefaultResultContext<>();
         while (!resultCtx.isStopped())
         {
@@ -485,6 +495,21 @@ public class DBQueryEngine implements QueryEngine
         }
     }
 
+    private int calculateBatchSize(int requiredNodes)
+    {
+        int batchSize;
+        if (requiredNodes > Integer.MAX_VALUE / 2)
+        {
+            // preventing overflow
+            batchSize = Integer.MAX_VALUE;
+        }
+        else
+        {
+            batchSize = requiredNodes * 2;
+        }
+        return Math.min(Math.max(batchSize, minPagingBatchSize), maxPagingBatchSize);
+    }
+
     private DBResultSet createResultSet(QueryOptions options, List<Node> nodes, int numberFound)
     {
         DBResultSet dbResultSet = new DBResultSet(options.getAsSearchParmeters(), nodes, nodeDAO, nodeService, tenantService, Integer.MAX_VALUE);
@@ -524,7 +549,7 @@ public class DBQueryEngine implements QueryEngine
     }
 
     /* (non-Javadoc)
-     * 
+     *
      * @see org.alfresco.repo.search.impl.querymodel.QueryEngine#getQueryModelFactory() */
     @Override
     public QueryModelFactory getQueryModelFactory()
@@ -534,7 +559,7 @@ public class DBQueryEngine implements QueryEngine
 
     /**
      * Injection of nodes cache for clean-up and warm up when required
-     * 
+     *
      * @param cache
      *            The node cache to set
      */
@@ -588,4 +613,7 @@ public class DBQueryEngine implements QueryEngine
             }
         }
     }
+
+    protected record QueryTemplate(String select, String count)
+    {}
 }

repository/src/main/java/org/alfresco/repo/virtual/template/VirtualQueryImpl.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2025 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -203,9 +203,8 @@ public class VirtualQueryImpl implements VirtualQuery
                 start = 0;
             }
         }
-        final int totlaSecond = !hasMore ? (int) result.getNumberFound() : (int) (start + result.getNumberFound() + 1);
+        final int totalSecond = !hasMore ? (int) result.getNumberFound() : (int) (start + result.getNumberFound());
-        final Pair<Integer, Integer> total = new Pair<Integer, Integer>(totalFirst,
+        final Pair<Integer, Integer> total = new Pair<>(totalFirst, totalSecond);
-                totlaSecond);
         return new PagingResults<Reference>() {
 
             @Override

repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-SqlMap.xml

@@ -8,4 +8,10 @@
     <include refid="sql_select_byDynamicQuery"/>   
   </select>
 
-</mapper>
+  <select id="count_byDynamicQuery" parameterType="org.alfresco.repo.search.impl.querymodel.impl.db.DBQuery" resultType="int">
+    SELECT COUNT(DISTINCT nodes.id)
+    FROM (
+      <include refid="sql_select_byDynamicQuery"/>
+    ) nodes
+  </select>
+</mapper>

repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.MySQLInnoDBDialect/metadata-query-SqlMap.xml

@@ -8,4 +8,10 @@
       <include refid="sql_select_byDynamicQuery"/>   
   </select>
 
-</mapper>
+  <select id="count_byDynamicQuery" parameterType="org.alfresco.repo.search.impl.querymodel.impl.db.DBQuery" resultType="int">
+    SELECT COUNT(DISTINCT nodes.id)
+    FROM (
+      <include refid="sql_select_byDynamicQuery"/>
+    ) nodes
+  </select>
+</mapper>

repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2021 Alfresco Software Limited
+ * Copyright (C) 2005 - 2025 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software.
  * If the software was purchased under a paid Alfresco license, the terms of
@@ -80,6 +80,7 @@ import org.alfresco.util.testing.category.NonBuildTests;
 
         // ACS-1907
         org.alfresco.repo.search.impl.querymodel.impl.db.ACS1907Test.class,
+        org.alfresco.repo.search.impl.querymodel.impl.db.ACS9167Test.class,
 
         // REPO-2963 : Tests causing a cascade of failures in AllDBTestsTestSuite on PostgreSQL/MySQL
         // Moved at the bottom of the suite because DbNodeServiceImplTest.testNodeCleanupRegistry() takes a long time on a clean DB.

repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS9167Test.java

@@ -0,0 +1,216 @@
+/*
+ * #%L
+ * Alfresco Repository
+ * %%
+ * Copyright (C) 2005 - 2025 Alfresco Software Limited
+ * %%
+ * This file is part of the Alfresco software.
+ * If the software was purchased under a paid Alfresco license, the terms of
+ * the paid license agreement will prevail.  Otherwise, the software is
+ * provided under the following open source license terms:
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ * #L%
+ */
+package org.alfresco.repo.search.impl.querymodel.impl.db;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.time.Duration;
+import java.util.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.springframework.context.ApplicationContext;
+
+import org.alfresco.model.ContentModel;
+import org.alfresco.repo.cache.TransactionalCache;
+import org.alfresco.repo.security.authentication.AuthenticationComponent;
+import org.alfresco.repo.transaction.RetryingTransactionHelper;
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.cmr.repository.NodeService;
+import org.alfresco.service.cmr.repository.StoreRef;
+import org.alfresco.service.cmr.search.QueryConsistency;
+import org.alfresco.service.cmr.search.ResultSet;
+import org.alfresco.service.cmr.search.SearchParameters;
+import org.alfresco.service.cmr.search.SearchService;
+import org.alfresco.service.namespace.NamespaceService;
+import org.alfresco.service.namespace.QName;
+import org.alfresco.service.transaction.TransactionService;
+import org.alfresco.test_category.OwnJVMTestsCategory;
+import org.alfresco.util.ApplicationContextHelper;
+import org.alfresco.util.testing.category.DBTests;
+
+@Category({OwnJVMTestsCategory.class, DBTests.class})
+@SuppressWarnings({"PMD.JUnitTestsShouldIncludeAssert"})
+public class ACS9167Test
+{
+    private NodeService nodeService;
+    private AuthenticationComponent authenticationComponent;
+    private SearchService pubSearchService;
+    private TransactionService transactionService;
+    private RetryingTransactionHelper txnHelper;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        setupServices();
+        txnHelper = new RetryingTransactionHelper();
+        txnHelper.setTransactionService(transactionService);
+        txnHelper.setReadOnly(false);
+        txnHelper.setMaxRetries(1);
+        authenticationComponent.setSystemUserAsCurrentUser();
+    }
+
+    private void setupServices()
+    {
+        ApplicationContext ctx = ApplicationContextHelper.getApplicationContext();
+        nodeService = (NodeService) ctx.getBean("dbNodeService");
+        authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent");
+        pubSearchService = (SearchService) ctx.getBean("SearchService");
+        transactionService = (TransactionService) ctx.getBean("TransactionService");
+
+        List<TransactionalCache<?, ?>> cachesToClear = new ArrayList<>();
+        cachesToClear.add((TransactionalCache<?, ?>) ctx.getBean("propertyValueCache"));
+        cachesToClear.add((TransactionalCache<?, ?>) ctx.getBean("node.nodesCache"));
+        cachesToClear.add((TransactionalCache<?, ?>) ctx.getBean("node.propertiesCache"));
+        cachesToClear.add((TransactionalCache<?, ?>) ctx.getBean("aclCache"));
+        cachesToClear.add((TransactionalCache<?, ?>) ctx.getBean("aclEntityCache"));
+        cachesToClear.add((TransactionalCache<?, ?>) ctx.getBean("permissionEntityCache"));
+        cachesToClear.add((TransactionalCache<?, ?>) ctx.getBean("nodeOwnerCache"));
+
+        for (TransactionalCache<?, ?> transactionalCache : cachesToClear)
+        {
+            transactionalCache.clear();
+        }
+    }
+
+    @After
+    public void tearDown() throws Exception
+    {
+        authenticationComponent.clearCurrentSecurityContext();
+    }
+
+    @Test
+    public void testPagination()
+    {
+        String searchMarker = UUID.randomUUID().toString();
+        int contentFilesCount = 185;
+        createFolderWithContentNodes(searchMarker, contentFilesCount);
+
+        prepareParametersQueryAndAssertResult(searchMarker, 0, 50, 50, contentFilesCount);
+        prepareParametersQueryAndAssertResult(searchMarker, 50, 50, 50, contentFilesCount);
+        prepareParametersQueryAndAssertResult(searchMarker, 150, 50, 35, contentFilesCount);
+        prepareParametersQueryAndAssertResult(searchMarker, 200, 50, 0, contentFilesCount);
+
+        prepareParametersQueryAndAssertResult(searchMarker, 0, 100, 100, contentFilesCount);
+        prepareParametersQueryAndAssertResult(searchMarker, 100, 100, 85, contentFilesCount);
+        prepareParametersQueryAndAssertResult(searchMarker, 200, 100, 0, contentFilesCount);
+
+        prepareParametersQueryAndAssertResult(searchMarker, 0, 200, contentFilesCount, contentFilesCount);
+    }
+
+    @Test
+    public void testLargeFilesCount()
+    {
+        String searchMarker = UUID.randomUUID().toString();
+        int contentFilesCount = 10_000;
+        createFolderWithContentNodes(searchMarker, contentFilesCount);
+
+        prepareParametersQueryAndAssertResult(searchMarker, 0, Integer.MAX_VALUE, contentFilesCount, contentFilesCount);
+    }
+
+    private void createFolderWithContentNodes(String searchMarker, int contentFilesCount)
+    {
+        NodeRef testFolder = txnHelper.doInTransaction(this::createFolderNode, false, false);
+        int batchSize = 1000;
+        int fullBatches = contentFilesCount / batchSize;
+        int remainingItems = contentFilesCount % batchSize;
+
+        for (int i = 0; i < fullBatches; i++)
+        {
+            txnHelper.doInTransaction(() -> {
+                for (int j = 0; j < batchSize; j++)
+                {
+                    createContentNode(searchMarker, testFolder);
+                }
+                return null;
+            }, false, false);
+        }
+
+        if (remainingItems > 0)
+        {
+            txnHelper.doInTransaction(() -> {
+                for (int j = 0; j < remainingItems; j++)
+                {
+                    createContentNode(searchMarker, testFolder);
+                }
+                return null;
+            }, false, false);
+        }
+    }
+
+    private void prepareParametersQueryAndAssertResult(String searchMarker, int parameterSkipCount, int parameterMaxItems, int expectedLength, int expectedNumberFound)
+    {
+        txnHelper.doInTransaction(() -> {
+            // given
+            SearchParameters sp = new SearchParameters();
+            sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE);
+            sp.setLanguage(SearchService.LANGUAGE_FTS_ALFRESCO);
+            sp.setQueryConsistency(QueryConsistency.TRANSACTIONAL);
+            sp.setQuery("(+TYPE:'cm:content') and !ASPECT:'cm:checkedOut' and !TYPE:'fm:forum' and !TYPE:'fm:topic' and !TYPE:'cm:systemfolder' and !TYPE:'fm:post' and !TYPE:'fm:forums' and =cm:description:'" + searchMarker + "'");
+            sp.setSkipCount(parameterSkipCount);
+            sp.setMaxItems(parameterMaxItems);
+            sp.setMaxPermissionChecks(Integer.MAX_VALUE);
+            sp.setMaxPermissionCheckTimeMillis(Duration.ofMinutes(10).toMillis());
+            // when
+            ResultSet resultSet = pubSearchService.query(sp);
+            // then
+            assertEquals(expectedLength, resultSet.length());
+            assertEquals(expectedNumberFound, resultSet.getNumberFound());
+            return null;
+        }, false, false);
+    }
+
+    private NodeRef createFolderNode()
+    {
+        NodeRef rootNodeRef = nodeService.getRootNode(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE);
+        Map<QName, Serializable> testFolderProps = new HashMap<>();
+        String folderName = "folder" + UUID.randomUUID();
+        testFolderProps.put(ContentModel.PROP_NAME, folderName);
+        return nodeService.createNode(
+                rootNodeRef,
+                ContentModel.ASSOC_CHILDREN,
+                QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI, folderName),
+                ContentModel.TYPE_FOLDER,
+                testFolderProps).getChildRef();
+    }
+
+    private void createContentNode(String searchMarker, NodeRef testFolder)
+    {
+        Map<QName, Serializable> testContentProps = new HashMap<>();
+        String fileName = "content" + UUID.randomUUID();
+        testContentProps.put(ContentModel.PROP_NAME, fileName);
+        testContentProps.put(ContentModel.PROP_DESCRIPTION, searchMarker);
+        nodeService.createNode(
+                testFolder,
+                ContentModel.ASSOC_CONTAINS,
+                QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI, fileName),
+                ContentModel.TYPE_CONTENT,
+                testContentProps);
+    }
+}

repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngineTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2021 Alfresco Software Limited
+ * Copyright (C) 2005 - 2025 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -60,7 +60,8 @@ import org.alfresco.util.Pair;
 
 public class DBQueryEngineTest
 {
-    private static final String SQL_TEMPLATE_PATH = "alfresco.metadata.query.select_byDynamicQuery";
+    private static final String SQL_SELECT_TEMPLATE_PATH = "alfresco.metadata.query.select_byDynamicQuery";
+    private static final String SQL_COUNT_TEMPLATE_PATH = "alfresco.metadata.query.count_byDynamicQuery";
 
     private DBQueryEngine engine;
     private SqlSessionTemplate template;
@@ -94,6 +95,7 @@ public class DBQueryEngineTest
     public void shouldGetAFilteringResultSetFromAcceleratedNodeSelection()
     {
         withMaxItems(10);
+        when(template.selectOne(any(), eq(dbQuery))).thenReturn(10);
 
         ResultSet result = engine.acceleratedNodeSelection(options, dbQuery, assessor);
 
@@ -226,7 +228,9 @@ public class DBQueryEngineTest
             }
             return null;
 
-        }).when(template).select(eq(SQL_TEMPLATE_PATH), eq(dbQuery), any());
+        }).when(template).select(eq(SQL_SELECT_TEMPLATE_PATH), eq(dbQuery), any());
+
+        when(template.selectOne(eq(SQL_COUNT_TEMPLATE_PATH), eq(dbQuery))).thenReturn(nodes.size());
     }
 
     private QueryOptions createQueryOptions()