Fix for Bug ALF-19985 Metadata query performance - remove store join, make node id based -> better ordering and large result set performance (x2-x3)

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@55166 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Andrew Hind
2013-09-10 10:05:15 +00:00
parent 186b7f3431
commit f89f2abff4
7 changed files with 60 additions and 26 deletions

View File

@@ -224,6 +224,7 @@ Inbound settings from iBatis
<mapper resource="alfresco/ibatis/#resource.dialect#/subscriptions-common-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/tenants-common-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/metadata-query-common-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/metadata-query-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/query-usages-common-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/query-auditable-common-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/query-authorities-common-SqlMap.xml"/>

View File

@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="alfresco.metadata.query">
<select id="select_byDynamicQuery" fetchSize="200" parameterType="org.alfresco.repo.search.impl.querymodel.impl.db.DBQuery" resultMap="alfresco.node.result_NodeRef">
<include refid="sql_select_byDynamicQuery"/>
</select>
</mapper>

View File

@@ -4,15 +4,11 @@
<mapper namespace="alfresco.metadata.query">
<select id="select_byDynamicQuery" fetchSize="200" parameterType="org.alfresco.repo.search.impl.querymodel.impl.db.DBQuery" resultMap="alfresco.node.result_NodeRef">
<sql id="sql_select_byDynamicQuery">
select
node.id as id,
store.protocol as protocol,
store.identifier as identifier,
node.uuid as uuid
node.id as id
from
alf_node node
join alf_store store on (store.id = node.store_id)
<foreach item="item" index="index" collection="joins">
<choose>
<when test="item.type == 'PARENT'">
@@ -38,7 +34,7 @@
</foreach>
where
node.type_qname_id &lt;&gt; #{sysDeletedType}
AND store.id = #{storeId}
AND node.store_id = #{storeId}
<if test="hasPredicate">
AND
<foreach item="item" index="index" collection="predicateParts">
@@ -118,6 +114,6 @@
</choose>
</foreach>
</if>
</select>
</sql>
</mapper>

View File

@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="alfresco.metadata.query">
<select id="select_byDynamicQuery" fetchSize="-2147483648" parameterType="org.alfresco.repo.search.impl.querymodel.impl.db.DBQuery" resultMap="alfresco.node.result_NodeRef">
<include refid="sql_select_byDynamicQuery"/>
</select>
</mapper>

View File

@@ -27,7 +27,6 @@ import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.ResultSetRow;
import org.alfresco.util.Pair;
import org.apache.lucene.search.HitCollector;
/**
* Common result set implementation.

View File

@@ -160,13 +160,13 @@ public class DBQueryEngine implements QueryEngine
}
dbQuery.prepare(namespaceService, dictionaryService, qnameDAO, nodeDAO, selectorGroup, null, functionContext);
List<Node> nodes = (List<Node>)template.selectList(SELECT_BY_DYNAMIC_QUERY, dbQuery);
LinkedHashSet<NodeRef> set = new LinkedHashSet<NodeRef>(nodes.size());
LinkedHashSet<Long> set = new LinkedHashSet<Long>(nodes.size());
for(Node node : nodes)
{
set.add(node.getNodeRef());
set.add(node.getId());
}
List<NodeRef> nodeRefs = new ArrayList<NodeRef>(set);
ResultSet rs = new DBResultSet(options.getAsSearchParmeters(), nodeRefs, nodeDAO, nodeService, Integer.MAX_VALUE);
List<Long> nodeIds = new ArrayList<Long>(set);
ResultSet rs = new DBResultSet(options.getAsSearchParmeters(), nodeIds, nodeDAO, nodeService, Integer.MAX_VALUE);
ResultSet paged = new PagingLuceneResultSet(rs, options.getAsSearchParmeters(), nodeService);
answer.put(key, paged);

View File

@@ -41,7 +41,9 @@ import org.alfresco.service.cmr.search.SearchParameters;
*/
public class DBResultSet extends AbstractResultSet
{
private List<NodeRef> nodeRefs;
private List<Long> dbids;
private NodeRef[] nodeRefs;
private NodeDAO nodeDao;
@@ -51,12 +53,13 @@ public class DBResultSet extends AbstractResultSet
private BitSet prefetch;
public DBResultSet(SearchParameters searchParameters, List<NodeRef> nodeRefs, NodeDAO nodeDao, NodeService nodeService, int maximumResultsFromUnlimitedQuery)
public DBResultSet(SearchParameters searchParameters, List<Long> dbids, NodeDAO nodeDao, NodeService nodeService, int maximumResultsFromUnlimitedQuery)
{
this.nodeDao = nodeDao;
this.nodeRefs = nodeRefs;
this.dbids = dbids;
this.nodeService = nodeService;
this.prefetch = new BitSet(nodeRefs.size());
this.prefetch = new BitSet(dbids.size());
nodeRefs= new NodeRef[(dbids.size())];
final LimitBy limitBy;
int maxResults = -1;
@@ -81,7 +84,7 @@ public class DBResultSet extends AbstractResultSet
}
this.resultSetMetaData = new SimpleResultSetMetaData(
maxResults > 0 && nodeRefs.size() < maxResults ? LimitBy.UNLIMITED : limitBy,
maxResults > 0 && dbids.size() < maxResults ? LimitBy.UNLIMITED : limitBy,
PermissionEvaluationMode.EAGER, searchParameters);
}
@@ -91,7 +94,7 @@ public class DBResultSet extends AbstractResultSet
@Override
public int length()
{
return nodeRefs.size();
return dbids.size();
}
/* (non-Javadoc)
@@ -100,7 +103,7 @@ public class DBResultSet extends AbstractResultSet
@Override
public long getNumberFound()
{
return nodeRefs.size();
return dbids.size();
}
/* (non-Javadoc)
@@ -110,7 +113,7 @@ public class DBResultSet extends AbstractResultSet
public NodeRef getNodeRef(int n)
{
prefetch(n);
return nodeRefs.get(n);
return nodeRefs[n];
}
/* (non-Javadoc)
@@ -185,8 +188,16 @@ public class DBResultSet extends AbstractResultSet
}
// Start at 'n' and process the the next bulk set
int bulkFetchSize = getBulkFetchSize();
List<NodeRef> fetchList = new ArrayList<NodeRef>(bulkFetchSize);
int totalHits = nodeRefs.size();
if(bulkFetchSize < 1)
{
nodeRefs[n] = nodeDao.getNodePair(dbids.get(n)).getSecond();
return;
}
List<Long> fetchList = new ArrayList<Long>(bulkFetchSize);
BitSet done = new BitSet(bulkFetchSize);
int totalHits = dbids.size();
for (int i = 0; i < bulkFetchSize; i++)
{
int next = n + i;
@@ -203,12 +214,17 @@ public class DBResultSet extends AbstractResultSet
// We store the node and mark it as prefetched
prefetch.set(next);
fetchList.add(nodeRefs.get(next));
fetchList.add(dbids.get(next));
done.set(next);
}
// Now bulk fetch
if (fetchList.size() > 1)
{
nodeDao.cacheNodes(fetchList);
nodeDao.cacheNodesById(fetchList);
for (int i = done.nextSetBit(0); i >= 0; i = done.nextSetBit(i+1))
{
nodeRefs[n+i] = nodeDao.getNodePair(fetchList.get(i)).getSecond();
}
}
}