diff --git a/data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java b/data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java index e543104e5f..b39cbd572e 100644 --- a/data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java +++ b/data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java @@ -2,7 +2,7 @@ * #%L * Alfresco Data model classes * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited + * Copyright (C) 2005 - 2021 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of diff --git a/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java b/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java index fd98d9bc77..abb7c40efe 100644 --- a/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java +++ b/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.search.impl.querymodel.impl.db; import java.util.ArrayList; @@ -74,6 +74,10 @@ public class DBQuery extends BaseQuery implements DBQueryBuilderComponent Set selectorGroup; + private int limit = 0; + + private int offset = 0; + /** * @param source Source * @param constraint Constraint @@ -133,6 +137,22 @@ public class DBQuery extends BaseQuery implements DBQueryBuilderComponent this.sinceTxId = sinceTxId; } + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + public List getJoins() { HashMap singleJoins = new HashMap(); diff --git a/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java b/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java index 612f958a65..96ca208c73 100644 --- a/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java +++ b/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java @@ -76,8 +76,10 @@ import org.alfresco.service.namespace.QName; import org.alfresco.util.Pair; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ibatis.executor.result.DefaultResultContext; import org.apache.ibatis.session.ResultContext; import org.apache.ibatis.session.ResultHandler; +import org.apache.ibatis.session.RowBounds; import org.mybatis.spring.SqlSessionTemplate; /** @@ -89,7 +91,11 @@ public class DBQueryEngine implements QueryEngine protected static final Log logger = LogFactory.getLog(DBQueryEngine.class); protected static final String SELECT_BY_DYNAMIC_QUERY = "alfresco.metadata.query.select_byDynamicQuery"; - + + private static final int DEFAULT_MIN_PAGING_BATCH_SIZE = 2500; + + private static final int DEFAULT_MAX_PAGING_BATCH_SIZE = 10000; + protected SqlSessionTemplate template; protected QNameDAO qnameDAO; @@ -114,6 +120,12 @@ public class DBQueryEngine implements QueryEngine private boolean maxPermissionCheckEnabled; + private boolean usePagingQuery = false; + + private int minPagingBatchSize = DEFAULT_MIN_PAGING_BATCH_SIZE; + + private int maxPagingBatchSize = DEFAULT_MAX_PAGING_BATCH_SIZE; + protected EntityLookupCache nodesCache; private List> stores; @@ -149,7 +161,31 @@ public class DBQueryEngine implements QueryEngine { this.permissionService = permissionService; } - + + public boolean isUsePagingQuery() { + return usePagingQuery; + } + + public void setUsePagingQuery(boolean usePagingQuery) { + this.usePagingQuery = usePagingQuery; + } + + public int getMinPagingBatchSize() { + return minPagingBatchSize; + } + + public void setMinPagingBatchSize(int minPagingBatchSize) { + this.minPagingBatchSize = minPagingBatchSize; + } + + public int getMaxPagingBatchSize() { + return maxPagingBatchSize; + } + + public void setMaxPagingBatchSize(int maxPagingBatchSize) { + this.maxPagingBatchSize = maxPagingBatchSize; + } + public void setMetadataIndexCheck2(OptionalPatchApplicationCheckBootstrapBean metadataIndexCheck2) { this.metadataIndexCheck2 = metadataIndexCheck2; @@ -331,7 +367,7 @@ public class DBQueryEngine implements QueryEngine int requiredNodes = computeRequiredNodesCount(options); logger.debug("- query sent to the database"); - template.select(pickQueryTemplate(options, dbQuery), dbQuery, new ResultHandler() + performTmdqSelect(pickQueryTemplate(options, dbQuery), dbQuery, requiredNodes, new ResultHandler() { @Override public void handleResult(ResultContext context) @@ -399,6 +435,54 @@ public class DBQueryEngine implements QueryEngine return frs; } + private void performTmdqSelect(String statement, DBQuery dbQuery, int requiredNodes, ResultHandler handler) + { + if (usePagingQuery) + { + performTmdqSelectPaging(statement, dbQuery, requiredNodes, handler); + } + else + { + performTmdqSelectStreaming(statement, dbQuery, handler); + } + } + + private void performTmdqSelectStreaming(String statement, DBQuery dbQuery, ResultHandler handler) + { + template.select(statement, dbQuery, handler); + } + + private void performTmdqSelectPaging(String statement, DBQuery dbQuery, int requiredNodes, ResultHandler handler) + { + int batchStart = 0; + int batchSize = requiredNodes * 2; + batchSize = Math.min(Math.max(batchSize, minPagingBatchSize), maxPagingBatchSize); + DefaultResultContext resultCtx = new DefaultResultContext<>(); + while (!resultCtx.isStopped()) + { + dbQuery.setOffset(batchStart); + dbQuery.setLimit(batchSize); + List batch = template.selectList(statement, dbQuery); + for (Node node : batch) + { + resultCtx.nextResultObject(node); + handler.handleResult(resultCtx); + if (resultCtx.isStopped()) + { + break; + } + } + if (batch.size() < batchSize) + { + resultCtx.stop(); + } + else + { + batchStart += batchSize; + } + } + } + private DBResultSet createResultSet(QueryOptions options, List nodes, int numberFound) { DBResultSet dbResultSet = new DBResultSet(options.getAsSearchParmeters(), nodes, nodeDAO, nodeService, tenantService, Integer.MAX_VALUE); diff --git a/repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml b/repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml index 8230226816..ecc97b3451 100644 --- a/repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml +++ b/repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml @@ -177,7 +177,8 @@ - - + + limit #{offset}, #{limit} + \ No newline at end of file diff --git a/repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml b/repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml index 7410e27f2b..932c4a06ef 100644 --- a/repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml +++ b/repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml @@ -105,8 +105,35 @@ - - + + + + search.dbQueryEngineImpl.#bean.dialect# + + + org.alfresco.repo.search.impl.querymodel.QueryEngine + + + org.alfresco.repo.domain.dialect.Dialect + + + + + + + + + + + + + + + + + diff --git a/repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java b/repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java index ec7aa1105a..1737976f0e 100644 --- a/repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java +++ b/repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java @@ -2,7 +2,7 @@ * #%L * Alfresco Repository * %% - * Copyright (C) 2005 - 2017 Alfresco Software Limited + * Copyright (C) 2005 - 2021 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -80,7 +80,7 @@ import org.junit.runners.Suite; // ACS-1907 org.alfresco.repo.search.impl.querymodel.impl.db.ACS1907Test.class, - // REPO-2963 : Tests causing a cascade of failures in AllDBTestsTestSuite on PostgreSQL/MySQL + // REPO-2963 : Tests causing a cascade of failures in AllDBTestsTestSuite on PostgreSQL/MySQL // Moved at the bottom of the suite because DbNodeServiceImplTest.testNodeCleanupRegistry() takes a long time on a clean DB. org.alfresco.repo.node.db.DbNodeServiceImplTest.class, diff --git a/repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS1907Test.java b/repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS1907Test.java index 65e4741147..50232978e9 100644 --- a/repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS1907Test.java +++ b/repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS1907Test.java @@ -28,6 +28,7 @@ package org.alfresco.repo.search.impl.querymodel.impl.db; import junit.framework.TestCase; import org.alfresco.model.ContentModel; import org.alfresco.repo.cache.TransactionalCache; +import org.alfresco.repo.management.subsystems.SwitchableApplicationContextFactory; import org.alfresco.repo.security.authentication.AuthenticationComponent; import org.alfresco.repo.security.authentication.AuthenticationUtil; import org.alfresco.repo.security.authentication.MutableAuthenticationDao; @@ -36,21 +37,23 @@ import org.alfresco.repo.transaction.RetryingTransactionHelper; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.NodeService; import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.service.cmr.search.ResultSet; -import org.alfresco.service.cmr.search.ResultSetRow; -import org.alfresco.service.cmr.search.SearchParameters; -import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.cmr.search.*; import org.alfresco.service.cmr.security.MutableAuthenticationService; import org.alfresco.service.cmr.security.PermissionService; import org.alfresco.service.namespace.QName; import org.alfresco.service.transaction.TransactionService; +import org.alfresco.test_category.OwnJVMTestsCategory; import org.alfresco.util.ApplicationContextHelper; +import org.alfresco.util.testing.category.DBTests; +import org.junit.experimental.categories.Category; import org.springframework.context.ApplicationContext; import java.io.Serializable; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +@Category({OwnJVMTestsCategory.class, DBTests.class}) public class ACS1907Test extends TestCase { @@ -64,6 +67,7 @@ public class ACS1907Test extends TestCase private PermissionService pubPermissionService; private TransactionService transactionService; private RetryingTransactionHelper txnHelper; + private DBQueryEngine queryEngine; private TransactionalCache aclCache; private TransactionalCache aclEntityCache; @@ -101,7 +105,16 @@ public class ACS1907Test extends TestCase aclCache = (TransactionalCache) ctx.getBean("aclCache"); aclEntityCache = (TransactionalCache) ctx.getBean("aclEntityCache"); permissionEntityCache = (TransactionalCache) ctx.getBean("permissionEntityCache"); - txnHelper = transactionService.getRetryingTransactionHelper(); + SwitchableApplicationContextFactory searchContextFactory = (SwitchableApplicationContextFactory) ctx.getBean("Search"); + ApplicationContext searchCtx = searchContextFactory.getApplicationContext(); + queryEngine = (DBQueryEngine) searchCtx.getBean("search.dbQueryEngineImpl"); + txnHelper = new RetryingTransactionHelper(); + txnHelper.setTransactionService(transactionService); + txnHelper.setReadOnly(false); + txnHelper.setMaxRetries(1); + txnHelper.setMinRetryWaitMs(1); + txnHelper.setMaxRetryWaitMs(10); + txnHelper.setRetryWaitIncrementMs(1); } private void setupTestUser(String userName) @@ -127,7 +140,7 @@ public class ACS1907Test extends TestCase private void setupTestContent() { - for(int f = 0; f < 100; f++) + for(int f = 0; f < 5; f++) { final int ff = f; txnHelper.doInTransaction(new RetryingTransactionHelper.RetryingTransactionCallback() { @@ -142,7 +155,7 @@ public class ACS1907Test extends TestCase ContentModel.TYPE_FOLDER, testFolderProps ).getChildRef(); - for(int c = 0; c < 1000; c++) + for(int c = 0; c < 5; c++) { Map testContentProps = new HashMap<>(); testContentProps.put(ContentModel.PROP_NAME, "content"+c); @@ -179,12 +192,15 @@ public class ACS1907Test extends TestCase public Object doWork() throws Exception { SearchParameters sp = new SearchParameters(); sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); - sp.setLanguage(SearchService.LANGUAGE_LUCENE); + sp.setLanguage(SearchService.LANGUAGE_FTS_ALFRESCO); + sp.setQueryConsistency(QueryConsistency.TRANSACTIONAL); sp.setQuery("TYPE:\"cm:content\""); ResultSet rs = pubSearchService.query(sp); + int cnt = 0; for (ResultSetRow row : rs) { assertNotNull(row.getValue(ContentModel.PROP_NAME)); + cnt++; } return null; } @@ -194,4 +210,51 @@ public class ACS1907Test extends TestCase }, false, false); } + public void testPaging() + { + HashSet resultPageSize2 = queryNodes(2); + HashSet resultPageSize5 = queryNodes(5); + HashSet resultPageSize10 = queryNodes(10); + HashSet resultPageSizeAll = queryNodes(10000); + // all result sets must be equal, independent of page size used to retrieve them + assertTrue(resultPageSize2.size() >= 25); + assertTrue(resultPageSize5.size() >= 25); + assertTrue(resultPageSize10.size() >= 25); + assertTrue(resultPageSizeAll.size() >= 25); + assertTrue(resultPageSize2.containsAll(resultPageSize5)); + assertTrue(resultPageSize2.containsAll(resultPageSize10)); + assertTrue(resultPageSize2.containsAll(resultPageSizeAll)); + assertTrue(resultPageSize5.containsAll(resultPageSize2)); + assertTrue(resultPageSize5.containsAll(resultPageSize10)); + assertTrue(resultPageSize5.containsAll(resultPageSizeAll)); + assertTrue(resultPageSize10.containsAll(resultPageSize2)); + assertTrue(resultPageSize10.containsAll(resultPageSize5)); + assertTrue(resultPageSize10.containsAll(resultPageSizeAll)); + assertTrue(resultPageSizeAll.containsAll(resultPageSize2)); + assertTrue(resultPageSizeAll.containsAll(resultPageSize5)); + assertTrue(resultPageSizeAll.containsAll(resultPageSize10)); + // reset + queryEngine.setMinPagingBatchSize(2500); + queryEngine.setMaxPagingBatchSize(10000); + } + + HashSet queryNodes(int pageSize) + { + queryEngine.setMinPagingBatchSize(pageSize); + queryEngine.setMaxPagingBatchSize(pageSize); + HashSet result = new HashSet<>(); + SearchParameters sp = new SearchParameters(); + sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); + sp.setLanguage(SearchService.LANGUAGE_FTS_ALFRESCO); + sp.setQueryConsistency(QueryConsistency.TRANSACTIONAL); + sp.setQuery("TYPE:\"cm:content\""); + ResultSet rs = pubSearchService.query(sp); + int cnt = 0; + for (ResultSetRow row : rs) + { + result.add(row.getNodeRef()); + } + return result; + } + }