inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-31 17:39:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

"ACS-1907 TMDQ against MySql throws SQLException in certain situations (#679)"

Browse Source 
This reverts commit 83097d452b.
This commit is contained in:
alandavis
2021-08-26 14:25:26 +01:00
parent 1766ce6e31
commit bb58c4a2d9
7 changed files with 238 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java
View File

@@ -2,7 +2,7 @@
* #%L
* Alfresco Data model classes
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* Copyright (C) 2005 - 2021 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of

20
repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java
View File

@@ -74,6 +74,10 @@ public class DBQuery extends BaseQuery implements DBQueryBuilderComponent
Set<String> selectorGroup;
private int limit = 0;
private int offset = 0;
/**
* @param source Source
* @param constraint Constraint
@@ -133,6 +137,22 @@ public class DBQuery extends BaseQuery implements DBQueryBuilderComponent
this.sinceTxId = sinceTxId;
}
public int getLimit() {
return limit;
}
public void setLimit(int limit) {
this.limit = limit;
}
public int getOffset() {
return offset;
}
public void setOffset(int offset) {
this.offset = offset;
}
public List<DBQueryBuilderJoinCommand> getJoins()
{
HashMap<QName, DBQueryBuilderJoinCommand> singleJoins = new HashMap<QName, DBQueryBuilderJoinCommand>();

86
repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java
View File

@@ -76,8 +76,10 @@ import org.alfresco.service.namespace.QName;
import org.alfresco.util.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ibatis.executor.result.DefaultResultContext;
import org.apache.ibatis.session.ResultContext;
import org.apache.ibatis.session.ResultHandler;
import org.apache.ibatis.session.RowBounds;
import org.mybatis.spring.SqlSessionTemplate;
/**
@@ -90,6 +92,10 @@ public class DBQueryEngine implements QueryEngine
protected static final String SELECT_BY_DYNAMIC_QUERY = "alfresco.metadata.query.select_byDynamicQuery";
private static final int DEFAULT_MIN_PAGING_BATCH_SIZE = 2500;
private static final int DEFAULT_MAX_PAGING_BATCH_SIZE = 10000;
protected SqlSessionTemplate template;
protected QNameDAO qnameDAO;
@@ -114,6 +120,12 @@ public class DBQueryEngine implements QueryEngine
private boolean maxPermissionCheckEnabled;
private boolean usePagingQuery = false;
private int minPagingBatchSize = DEFAULT_MIN_PAGING_BATCH_SIZE;
private int maxPagingBatchSize = DEFAULT_MAX_PAGING_BATCH_SIZE;
protected EntityLookupCache<Long, Node, NodeRef> nodesCache;
private List<Pair<Long, StoreRef>> stores;
@@ -150,6 +162,30 @@ public class DBQueryEngine implements QueryEngine
this.permissionService = permissionService;
}
public boolean isUsePagingQuery() {
return usePagingQuery;
}
public void setUsePagingQuery(boolean usePagingQuery) {
this.usePagingQuery = usePagingQuery;
}
public int getMinPagingBatchSize() {
return minPagingBatchSize;
}
public void setMinPagingBatchSize(int minPagingBatchSize) {
this.minPagingBatchSize = minPagingBatchSize;
}
public int getMaxPagingBatchSize() {
return maxPagingBatchSize;
}
public void setMaxPagingBatchSize(int maxPagingBatchSize) {
this.maxPagingBatchSize = maxPagingBatchSize;
}
public void setMetadataIndexCheck2(OptionalPatchApplicationCheckBootstrapBean metadataIndexCheck2)
{
this.metadataIndexCheck2 = metadataIndexCheck2;
@@ -331,7 +367,7 @@ public class DBQueryEngine implements QueryEngine
int requiredNodes = computeRequiredNodesCount(options);
logger.debug("- query sent to the database");
template.select(pickQueryTemplate(options, dbQuery), dbQuery, new ResultHandler<Node>()
performTmdqSelect(pickQueryTemplate(options, dbQuery), dbQuery, requiredNodes, new ResultHandler<Node>()
{
@Override
public void handleResult(ResultContext<? extends Node> context)
@@ -399,6 +435,54 @@ public class DBQueryEngine implements QueryEngine
return frs;
}
private void performTmdqSelect(String statement, DBQuery dbQuery, int requiredNodes, ResultHandler<Node> handler)
{
if (usePagingQuery)
{
performTmdqSelectPaging(statement, dbQuery, requiredNodes, handler);
}
else
{
performTmdqSelectStreaming(statement, dbQuery, handler);
}
}
private void performTmdqSelectStreaming(String statement, DBQuery dbQuery, ResultHandler<Node> handler)
{
template.select(statement, dbQuery, handler);
}
private void performTmdqSelectPaging(String statement, DBQuery dbQuery, int requiredNodes, ResultHandler<Node> handler)
{
int batchStart = 0;
int batchSize = requiredNodes * 2;
batchSize = Math.min(Math.max(batchSize, minPagingBatchSize), maxPagingBatchSize);
DefaultResultContext<Node> resultCtx = new DefaultResultContext<>();
while (!resultCtx.isStopped())
{
dbQuery.setOffset(batchStart);
dbQuery.setLimit(batchSize);
List<Node> batch = template.selectList(statement, dbQuery);
for (Node node : batch)
{
resultCtx.nextResultObject(node);
handler.handleResult(resultCtx);
if (resultCtx.isStopped())
{
break;
}
}
if (batch.size() < batchSize)
{
resultCtx.stop();
}
else
{
batchStart += batchSize;
}
}
}
private DBResultSet createResultSet(QueryOptions options, List<Node> nodes, int numberFound)
{
DBResultSet dbResultSet = new DBResultSet(options.getAsSearchParmeters(), nodes, nodeDAO, nodeService, tenantService, Integer.MAX_VALUE);

1
repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml
View File

@@ -178,6 +178,7 @@
</foreach>
</if>
</if>
<if test="limit != 0">limit #{offset}, #{limit}</if>
</sql>
</mapper>

29
repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml
View File

@@ -106,7 +106,34 @@
</property>
</bean>
<bean id="search.dbQueryEngineImpl" class="org.alfresco.repo.search.impl.querymodel.impl.db.DBQueryEngine" >
<bean id="search.dbQueryEngineImpl" class="org.alfresco.util.bean.HierarchicalBeanLoader">
<property name="targetBeanName">
<value>search.dbQueryEngineImpl.#bean.dialect#</value>
</property>
<property name="targetClass">
<value>org.alfresco.repo.search.impl.querymodel.QueryEngine</value>
</property>
<property name="dialectBaseClass">
<value>org.alfresco.repo.domain.dialect.Dialect</value>
</property>
<property name="dialectClass">
<bean class="org.springframework.beans.factory.config.PropertyPathFactoryBean">
<property name="targetBeanName" value="dialect" />
<property name="propertyPath" value="class.name" />
</bean>
</property>
</bean>
<bean id="search.dbQueryEngineImpl.org.alfresco.repo.domain.dialect.Dialect"
parent="search.baseDbQueryEngineImpl">
<property name="usePagingQuery" value="false"/>
</bean>
<bean id="search.dbQueryEngineImpl.org.alfresco.repo.domain.dialect.MySQLInnoDBDialect"
parent="search.baseDbQueryEngineImpl">
<property name="usePagingQuery" value="true"/>
</bean>
<bean id="search.baseDbQueryEngineImpl" class="org.alfresco.repo.search.impl.querymodel.impl.db.DBQueryEngine" abstract="true">
<property name="permissionService" ref="permissionService"/>
<property name="dictionaryService" ref="dictionaryService" />
<property name="namespaceService" ref="namespaceService" />

2
repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java
View File

@@ -2,7 +2,7 @@
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2017 Alfresco Software Limited
* Copyright (C) 2005 - 2021 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of

79
repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS1907Test.java
View File

@@ -28,6 +28,7 @@ package org.alfresco.repo.search.impl.querymodel.impl.db;
import junit.framework.TestCase;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.cache.TransactionalCache;
import org.alfresco.repo.management.subsystems.SwitchableApplicationContextFactory;
import org.alfresco.repo.security.authentication.AuthenticationComponent;
import org.alfresco.repo.security.authentication.AuthenticationUtil;
import org.alfresco.repo.security.authentication.MutableAuthenticationDao;
@@ -36,21 +37,23 @@ import org.alfresco.repo.transaction.RetryingTransactionHelper;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.service.cmr.search.SearchService;
import org.alfresco.service.cmr.search.*;
import org.alfresco.service.cmr.security.MutableAuthenticationService;
import org.alfresco.service.cmr.security.PermissionService;
import org.alfresco.service.namespace.QName;
import org.alfresco.service.transaction.TransactionService;
import org.alfresco.test_category.OwnJVMTestsCategory;
import org.alfresco.util.ApplicationContextHelper;
import org.alfresco.util.testing.category.DBTests;
import org.junit.experimental.categories.Category;
import org.springframework.context.ApplicationContext;
import java.io.Serializable;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@Category({OwnJVMTestsCategory.class, DBTests.class})
public class ACS1907Test extends TestCase
{
@@ -64,6 +67,7 @@ public class ACS1907Test extends TestCase
private PermissionService pubPermissionService;
private TransactionService transactionService;
private RetryingTransactionHelper txnHelper;
private DBQueryEngine queryEngine;
private TransactionalCache<Serializable, AccessControlList> aclCache;
private TransactionalCache<Serializable, Object> aclEntityCache;
@@ -101,7 +105,16 @@ public class ACS1907Test extends TestCase
aclCache = (TransactionalCache) ctx.getBean("aclCache");
aclEntityCache = (TransactionalCache) ctx.getBean("aclEntityCache");
permissionEntityCache = (TransactionalCache) ctx.getBean("permissionEntityCache");
txnHelper = transactionService.getRetryingTransactionHelper();
SwitchableApplicationContextFactory searchContextFactory = (SwitchableApplicationContextFactory) ctx.getBean("Search");
ApplicationContext searchCtx = searchContextFactory.getApplicationContext();
queryEngine = (DBQueryEngine) searchCtx.getBean("search.dbQueryEngineImpl");
txnHelper = new RetryingTransactionHelper();
txnHelper.setTransactionService(transactionService);
txnHelper.setReadOnly(false);
txnHelper.setMaxRetries(1);
txnHelper.setMinRetryWaitMs(1);
txnHelper.setMaxRetryWaitMs(10);
txnHelper.setRetryWaitIncrementMs(1);
}
private void setupTestUser(String userName)
@@ -127,7 +140,7 @@ public class ACS1907Test extends TestCase
private void setupTestContent()
{
for(int f = 0; f < 100; f++)
for(int f = 0; f < 5; f++)
{
final int ff = f;
txnHelper.doInTransaction(new RetryingTransactionHelper.RetryingTransactionCallback<Object>() {
@@ -142,7 +155,7 @@ public class ACS1907Test extends TestCase
ContentModel.TYPE_FOLDER,
testFolderProps
).getChildRef();
for(int c = 0; c < 1000; c++)
for(int c = 0; c < 5; c++)
{
Map<QName, Serializable> testContentProps = new HashMap<>();
testContentProps.put(ContentModel.PROP_NAME, "content"+c);
@@ -179,12 +192,15 @@ public class ACS1907Test extends TestCase
public Object doWork() throws Exception {
SearchParameters sp = new SearchParameters();
sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE);
sp.setLanguage(SearchService.LANGUAGE_LUCENE);
sp.setLanguage(SearchService.LANGUAGE_FTS_ALFRESCO);
sp.setQueryConsistency(QueryConsistency.TRANSACTIONAL);
sp.setQuery("TYPE:\"cm:content\"");
ResultSet rs = pubSearchService.query(sp);
int cnt = 0;
for (ResultSetRow row : rs)
{
assertNotNull(row.getValue(ContentModel.PROP_NAME));
cnt++;
}
return null;
}
@@ -194,4 +210,51 @@ public class ACS1907Test extends TestCase
}, false, false);
}
public void testPaging()
{
HashSet<NodeRef> resultPageSize2 = queryNodes(2);
HashSet<NodeRef> resultPageSize5 = queryNodes(5);
HashSet<NodeRef> resultPageSize10 = queryNodes(10);
HashSet<NodeRef> resultPageSizeAll = queryNodes(10000);
// all result sets must be equal, independent of page size used to retrieve them
assertTrue(resultPageSize2.size() >= 25);
assertTrue(resultPageSize5.size() >= 25);
assertTrue(resultPageSize10.size() >= 25);
assertTrue(resultPageSizeAll.size() >= 25);
assertTrue(resultPageSize2.containsAll(resultPageSize5));
assertTrue(resultPageSize2.containsAll(resultPageSize10));
assertTrue(resultPageSize2.containsAll(resultPageSizeAll));
assertTrue(resultPageSize5.containsAll(resultPageSize2));
assertTrue(resultPageSize5.containsAll(resultPageSize10));
assertTrue(resultPageSize5.containsAll(resultPageSizeAll));
assertTrue(resultPageSize10.containsAll(resultPageSize2));
assertTrue(resultPageSize10.containsAll(resultPageSize5));
assertTrue(resultPageSize10.containsAll(resultPageSizeAll));
assertTrue(resultPageSizeAll.containsAll(resultPageSize2));
assertTrue(resultPageSizeAll.containsAll(resultPageSize5));
assertTrue(resultPageSizeAll.containsAll(resultPageSize10));
// reset
queryEngine.setMinPagingBatchSize(2500);
queryEngine.setMaxPagingBatchSize(10000);
}
HashSet<NodeRef> queryNodes(int pageSize)
{
queryEngine.setMinPagingBatchSize(pageSize);
queryEngine.setMaxPagingBatchSize(pageSize);
HashSet<NodeRef> result = new HashSet<>();
SearchParameters sp = new SearchParameters();
sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE);
sp.setLanguage(SearchService.LANGUAGE_FTS_ALFRESCO);
sp.setQueryConsistency(QueryConsistency.TRANSACTIONAL);
sp.setQuery("TYPE:\"cm:content\"");
ResultSet rs = pubSearchService.query(sp);
int cnt = 0;
for (ResultSetRow row : rs)
{
result.add(row.getNodeRef());
}
return result;
}
}