REPO-5376 Query Accelerator Remove all temporary code (#308)

* REPO-5376 Remove all temporary code

* Remove DBStats, SingleTaskRestartableWatch
* Remove propertiesCache and aspectsCache from DBQueryEngine as they were marked as temporary

* Remove further temporary code

Co-authored-by: Nana Insaidoo <insaidoo.nana@yahoo.it>
This commit is contained in:
Alan Davis
2021-02-23 16:54:58 +00:00
committed by GitHub
parent 36937eaad7
commit e88aab47f7
11 changed files with 25 additions and 404 deletions

View File

@@ -25,12 +25,6 @@
*/
package org.alfresco.rest.api.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.alfresco.repo.search.impl.querymodel.impl.db.DBStats;
import org.alfresco.repo.search.impl.querymodel.impl.db.SingleTaskRestartableWatch;
import org.alfresco.rest.api.model.Node;
import org.alfresco.rest.api.search.context.SearchRequestContext;
import org.alfresco.rest.api.search.impl.ResultMapper;
@@ -51,14 +45,14 @@ import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.service.cmr.search.SearchService;
import org.alfresco.util.ParameterCheck;
import org.alfresco.util.PropertyCheck;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.extensions.webscripts.AbstractWebScript;
import org.springframework.extensions.webscripts.WebScriptRequest;
import org.springframework.extensions.webscripts.WebScriptResponse;
import org.springframework.util.StopWatch;
import org.springframework.util.StopWatch.TaskInfo;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* An implementation of the {{baseUrl}}/{{networkId}}/public/search/versions/1/search endpoint
@@ -68,15 +62,12 @@ import org.springframework.util.StopWatch.TaskInfo;
public class SearchApiWebscript extends AbstractWebScript implements RecognizedParamsExtractor, RequestReader, ResponseWriter,
InitializingBean
{
protected static final Log logger = LogFactory.getLog(SearchApiWebscript.class);
private ServiceRegistry serviceRegistry;
private SearchService searchService;
private SearchMapper searchMapper;
private ResultMapper resultMapper;
protected ApiAssistant assistant;
protected ResourceWebScriptHelper helper;
private boolean statsEnabled;
@Override
public void afterPropertiesSet()
@@ -91,7 +82,6 @@ public class SearchApiWebscript extends AbstractWebScript implements RecognizedP
@Override
public void execute(WebScriptRequest webScriptRequest, WebScriptResponse webScriptResponse) throws IOException
{
StopWatch apiStopWatch = new StopWatch();
try {
//Turn JSON into a Java object respresentation
SearchQuery searchQuery = extractJsonContent(webScriptRequest, assistant.getJsonHelper(), SearchQuery.class);
@@ -106,43 +96,12 @@ public class SearchApiWebscript extends AbstractWebScript implements RecognizedP
SearchParameters searchParams = searchMapper.toSearchParameters(params, searchQuery, searchRequestContext);
//Call searchService
apiStopWatch.start("nodes");
ResultSet results = searchService.query(searchParams);
apiStopWatch.stop();
//Turn solr results into JSON
apiStopWatch.start("props");
CollectionWithPagingInfo<Node> resultJson = resultMapper.toCollectionWithPagingInfo(params, searchRequestContext, searchQuery, results);
//Post-process the request and pass in params, eg. params.getFilter()
Object toRender = helper.processAdditionsToTheResponse(null, null, null, params, resultJson);
apiStopWatch.stop();
// store execution stats in a special header if enabled
if (statsEnabled)
{
// store execution time in a special header
StringBuilder sb = new StringBuilder();
sb.append("api={");
sb.append("tot=").append(apiStopWatch.getTotalTimeMillis()).append("ms,");
addStopWatchStats(sb, apiStopWatch);
sb.append("}; ");
sb.append("db={");
addStopWatchStats(sb, DBStats.queryStopWatch());
sb.append("}; ");
sb.append("query={");
addStopWatchStats(sb, DBStats.handlerStopWatch());
sb.append(",");
addStopWatchStats(sb, DBStats.aclReadStopWatch());
sb.append(",");
addStopWatchStats(sb, DBStats.aclOwnerStopWatch());
sb.append("}");
webScriptResponse.addHeader("X-Response-Stats", sb.toString());
}
//Write response
setResponse(webScriptResponse, DEFAULT_SUCCESS);
@@ -153,44 +112,6 @@ public class SearchApiWebscript extends AbstractWebScript implements RecognizedP
}
}
private void addStopWatchStats(StringBuilder sb, StopWatch watch)
{
boolean first = true;
for (TaskInfo task : watch.getTaskInfo())
{
if (first)
{
first = false;
}
else
{
sb.append(",");
}
sb.append(task.getTaskName())
.append("=")
.append(task.getTimeMillis())
.append("ms");
int pc = Math.round(100 * task.getTimeNanos() / watch.getTotalTimeNanos());
sb.append("(")
.append(pc).append("%")
.append(")");
}
}
private void addStopWatchStats(StringBuilder sb, SingleTaskRestartableWatch watch)
{
long decimillis = (watch.getTotalTimeMicros()+5)/100;
double millis = decimillis/10.0;
sb.append(watch.getName())
.append("=")
.append(millis)
.append("ms");
}
/**
* Gets the Params object, parameters come from the SearchQuery json not the request
* @param webScriptRequest
@@ -243,10 +164,4 @@ public class SearchApiWebscript extends AbstractWebScript implements RecognizedP
{
this.helper = helper;
}
// receiving as a string because of known issue: https://jira.spring.io/browse/SPR-9989
public void setStatsEnabled(String enabled) {
this.statsEnabled = Boolean.valueOf(enabled);
logger.info("API stats header: " + (this.statsEnabled ? "enabled" : "disabled"));
}
}

View File

@@ -23,7 +23,4 @@
# See issue REPO-2575 for details.
alfresco.restApi.basicAuthScheme=false
# REPO-4388 allow CORS headers in transaction response
webscripts.transaction.preserveHeadersPattern=Access-Control-.*
# REPO-5371 enable stats header in API response (only search atm)
webscripts.stats.enabled=false
webscripts.transaction.preserveHeadersPattern=Access-Control-.*

View File

@@ -1024,7 +1024,6 @@
<property name="helper" ref="webscriptHelper" />
<property name="resultMapper" ref="searchapiResultMapper" />
<property name="searchMapper" ref="searchapiSearchMapper" />
<property name="statsEnabled" value="${webscripts.stats.enabled}" />
</bean>
<bean id="webscript.org.alfresco.api.SearchSQLApiWebscript.post"

View File

@@ -427,10 +427,6 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
NodeEntity node = new NodeEntity();
node.setId(id);
if (logger.isDebugEnabled())
{
logger.debug("+ Read node with id: "+id);
}
return template.selectOne(SELECT_NODE_BY_ID, node);
}
@@ -454,10 +450,6 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
}
node.setUuid(uuid);
if (logger.isDebugEnabled())
{
logger.debug("+ Read node with uuid: "+uuid);
}
return template.selectOne(SELECT_NODE_BY_NODEREF, node);
}

View File

@@ -26,8 +26,6 @@
package org.alfresco.repo.search.impl.querymodel.impl.db;
import static org.alfresco.repo.domain.node.AbstractNodeDAOImpl.CACHE_REGION_NODES;
import static org.alfresco.repo.search.impl.querymodel.impl.db.DBStats.handlerStopWatch;
import static org.alfresco.repo.search.impl.querymodel.impl.db.DBStats.resetStopwatches;
import java.io.Serializable;
import java.util.ArrayList;
@@ -35,7 +33,6 @@ import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -49,7 +46,6 @@ import org.alfresco.repo.cache.lookup.EntityLookupCache;
import org.alfresco.repo.cache.lookup.EntityLookupCache.EntityLookupCallbackDAOAdaptor;
import org.alfresco.repo.domain.node.Node;
import org.alfresco.repo.domain.node.NodeDAO;
import org.alfresco.repo.domain.node.NodeVersionKey;
import org.alfresco.repo.domain.permissions.AclCrudDAO;
import org.alfresco.repo.domain.permissions.Authority;
import org.alfresco.repo.domain.qname.QNameDAO;
@@ -81,7 +77,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.ibatis.session.ResultContext;
import org.apache.ibatis.session.ResultHandler;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.util.StopWatch;
/**
* @author Andy
@@ -115,12 +110,8 @@ public class DBQueryEngine implements QueryEngine
private long maxPermissionCheckTimeMillis;
private SimpleCache<NodeVersionKey, Map<QName, Serializable>> propertiesCache;
protected EntityLookupCache<Long, Node, NodeRef> nodesCache;
private SimpleCache<NodeVersionKey, Set<QName>> aspectsCache;
AclCrudDAO aclCrudDAO;
public void setAclCrudDAO(AclCrudDAO aclCrudDAO)
@@ -217,8 +208,7 @@ public class DBQueryEngine implements QueryEngine
public QueryEngineResults executeQuery(Query query, QueryOptions options, FunctionEvaluationContext functionContext)
{
logger.debug("Query request received");
resetStopwatches();
Set<String> selectorGroup = null;
if (query.getSource() != null)
{
@@ -275,29 +265,11 @@ public class DBQueryEngine implements QueryEngine
logger.debug("- query is being prepared");
dbQuery.prepare(namespaceService, dictionaryService, qnameDAO, nodeDAO, tenantService, selectorGroup,
null, functionContext, metadataIndexCheck2.getPatchApplied());
ResultSet resultSet;
// TEMPORARY - this first branch of the if statement simply allows us to easily clear the caches for now; it will be removed afterwards
if (cleanCacheRequest(options))
{
nodesCache.clear();
propertiesCache.clear();
aspectsCache.clear();
logger.info("Nodes cache cleared");
resultSet = new DBResultSet(options.getAsSearchParmeters(), Collections.emptyList(), nodeDAO, nodeService,
tenantService, Integer.MAX_VALUE);
}
else if (forceOldPermissionResolution(options))
{
resultSet = selectNodesStandard(options, dbQuery);
logger.debug("Selected " +resultSet.length()+ " nodes with standard permission resolution");
}
else
{
resultSet = selectNodesWithPermissions(options, dbQuery);
logger.debug("Selected " +resultSet.length()+ " nodes with accelerated permission resolution");
}
resultSet = selectNodesWithPermissions(options, dbQuery);
logger.debug("Selected " + resultSet.length() + " nodes with accelerated permission resolution");
return asQueryEngineResults(resultSet);
}
@@ -306,14 +278,7 @@ public class DBQueryEngine implements QueryEngine
logger.debug("- using standard table for the query");
return SELECT_BY_DYNAMIC_QUERY;
}
private ResultSet selectNodesStandard(QueryOptions options, DBQuery dbQuery)
{
List<Node> nodes = removeDuplicates(template.selectList(pickQueryTemplate(options, dbQuery), dbQuery));
DBResultSet rs = new DBResultSet(options.getAsSearchParmeters(), nodes, nodeDAO, nodeService, tenantService, Integer.MAX_VALUE);
return new PagingLuceneResultSet(rs, options.getAsSearchParmeters(), nodeService);
}
private ResultSet selectNodesWithPermissions(QueryOptions options, DBQuery dbQuery)
{
Authority authority = aclCrudDAO.getAuthority(AuthenticationUtil.getRunAsUser());
@@ -340,37 +305,21 @@ public class DBQueryEngine implements QueryEngine
FilteringResultSet acceleratedNodeSelection(QueryOptions options, DBQuery dbQuery, NodePermissionAssessor permissionAssessor)
{
StopWatch sw = DBStats.queryStopWatch();
List<Node> nodes = new ArrayList<>();
int requiredNodes = computeRequiredNodesCount(options);
logger.debug("- query sent to the database");
sw.start("ttfr");
template.select(pickQueryTemplate(options, dbQuery), dbQuery, new ResultHandler<Node>()
{
@Override
public void handleResult(ResultContext<? extends Node> context)
{
handlerStopWatch().start();
try
{
doHandleResult(permissionAssessor, sw, nodes, requiredNodes, context);
}
finally
{
handlerStopWatch().stop();
}
doHandleResult(permissionAssessor, nodes, requiredNodes, context);
}
private void doHandleResult(NodePermissionAssessor permissionAssessor, StopWatch sw, List<Node> nodes,
int requiredNodes, ResultContext<? extends Node> context)
private void doHandleResult(NodePermissionAssessor permissionAssessor, List<Node> nodes,
int requiredNodes, ResultContext<? extends Node> context)
{
if (permissionAssessor.isFirstRecord())
{
sw.stop();
sw.start("ttlr");
}
if (nodes.size() >= requiredNodes)
{
context.stop();
@@ -403,7 +352,6 @@ public class DBQueryEngine implements QueryEngine
}
});
sw.stop();
int numberFound = nodes.size();
nodes.removeAll(Collections.singleton(null));
@@ -455,23 +403,6 @@ public class DBQueryEngine implements QueryEngine
return new QueryEngineResults(answer);
}
private List<Node> removeDuplicates(List<Node> nodes)
{
LinkedHashSet<Node> uniqueNodes = new LinkedHashSet<>(nodes.size());
List<Long> checkedNodeIds = new ArrayList<>(nodes.size());
for (Node node : nodes)
{
if (!checkedNodeIds.contains(node.getId()))
{
checkedNodeIds.add(node.getId());
uniqueNodes.add(node);
}
}
return new ArrayList<Node>(uniqueNodes);
}
/*
* (non-Javadoc)
* @see org.alfresco.repo.search.impl.querymodel.QueryEngine#getQueryModelFactory()
@@ -481,28 +412,7 @@ public class DBQueryEngine implements QueryEngine
{
return new DBQueryModelFactory();
}
private boolean cleanCacheRequest(QueryOptions options)
{
return "xxx".equals(getLocaleLanguage(options));
}
char getMagicCharFromLocale(QueryOptions options, int index)
{
String lang = getLocaleLanguage(options);
return lang.length() > index ? lang.charAt(index) : ' ';
}
private boolean forceOldPermissionResolution(QueryOptions options)
{
return getMagicCharFromLocale(options, 2) == 's';
}
private String getLocaleLanguage(QueryOptions options)
{
return options.getLocales().size() == 1 ? options.getLocales().get(0).getLanguage() : "";
}
/**
* Injection of nodes cache for clean-up and warm up when required
* @param cache The node cache to set
@@ -540,20 +450,4 @@ public class DBQueryEngine implements QueryEngine
return value.getNodeRef();
}
}
/*
* TEMPORARY - Injection of nodes cache for clean-up when required
*/
public void setPropertiesCache(SimpleCache<NodeVersionKey, Map<QName, Serializable>> propertiesCache)
{
this.propertiesCache = propertiesCache;
}
/*
* TEMPORARY - Injection of nodes cache for clean-up when required
*/
public void setAspectsCache(SimpleCache<NodeVersionKey, Set<QName>> aspectsCache)
{
this.aspectsCache = aspectsCache;
}
}

View File

@@ -1,66 +0,0 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2021 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.repo.search.impl.querymodel.impl.db;
import org.springframework.util.StopWatch;
public final class DBStats
{
public static final ThreadLocal<StopWatch> QUERY_STOPWATCH = new ThreadLocal<StopWatch>();
public static final ThreadLocal<SingleTaskRestartableWatch> ACL_READ_STOPWATCH = new ThreadLocal<SingleTaskRestartableWatch>();
public static final ThreadLocal<SingleTaskRestartableWatch> ACL_OWNER_STOPWATCH = new ThreadLocal<SingleTaskRestartableWatch>();
public static final ThreadLocal<SingleTaskRestartableWatch> HANDLER_STOPWATCH = new ThreadLocal<SingleTaskRestartableWatch>();
private DBStats() {}
public static void resetStopwatches()
{
QUERY_STOPWATCH.set(new StopWatch());
HANDLER_STOPWATCH.set(new SingleTaskRestartableWatch("tot"));
ACL_READ_STOPWATCH.set(new SingleTaskRestartableWatch("acl"));
ACL_OWNER_STOPWATCH.set(new SingleTaskRestartableWatch("own"));
}
public static StopWatch queryStopWatch()
{
return QUERY_STOPWATCH.get();
}
public static SingleTaskRestartableWatch aclReadStopWatch()
{
return ACL_READ_STOPWATCH.get();
}
public static SingleTaskRestartableWatch aclOwnerStopWatch()
{
return ACL_OWNER_STOPWATCH.get();
}
public static SingleTaskRestartableWatch handlerStopWatch()
{
return HANDLER_STOPWATCH.get();
}
}

View File

@@ -25,9 +25,6 @@
*/
package org.alfresco.repo.search.impl.querymodel.impl.db;
import static org.alfresco.repo.search.impl.querymodel.impl.db.DBStats.aclOwnerStopWatch;
import static org.alfresco.repo.search.impl.querymodel.impl.db.DBStats.aclReadStopWatch;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
@@ -98,21 +95,13 @@ public class NodePermissionAssessor
protected boolean isOwnerReading(Node node, Authority authority)
{
aclOwnerStopWatch().start();
try
if (authority == null)
{
if (authority == null)
{
return false;
}
String owner = getOwner(node);
return EqualsHelper.nullSafeEquals(authority.getAuthority(), owner);
}
finally
{
aclOwnerStopWatch().stop();
return false;
}
String owner = getOwner(node);
return EqualsHelper.nullSafeEquals(authority.getAuthority(), owner);
}
private String getOwner(Node node)
@@ -176,21 +165,13 @@ public class NodePermissionAssessor
protected boolean canRead(Long aclId)
{
aclReadStopWatch().start();
try
Boolean res = aclReadCache.get(aclId);
if (res == null)
{
Boolean res = aclReadCache.get(aclId);
if (res == null)
{
res = canCurrentUserRead(aclId);
aclReadCache.put(aclId, res);
}
return res;
}
finally
{
aclReadStopWatch().stop();
res = canCurrentUserRead(aclId);
aclReadCache.put(aclId, res);
}
return res;
}
protected boolean canCurrentUserRead(Long aclId)

View File

@@ -1,76 +0,0 @@
/*-
* #%L
* Alfresco Remote API
* %%
* Copyright (C) 2005 - 2021 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.repo.search.impl.querymodel.impl.db;
import java.util.concurrent.TimeUnit;
import javax.annotation.concurrent.NotThreadSafe;
@NotThreadSafe
public class SingleTaskRestartableWatch
{
private final String name;
private long startTimeNanos;
private long totalTimeNanos;
public SingleTaskRestartableWatch(String name)
{
this.name = name;
}
public String getName()
{
return name;
}
public void start()
{
startTimeNanos = System.nanoTime();
}
public void stop()
{
long elapsedNanos = System.nanoTime() - startTimeNanos;
totalTimeNanos += elapsedNanos;
}
public long getTotalTimeNanos()
{
return totalTimeNanos;
}
public long getTotalTimeMicros()
{
return TimeUnit.NANOSECONDS.toMicros(totalTimeNanos);
}
public long getTotalTimeMillis()
{
return TimeUnit.NANOSECONDS.toMillis(totalTimeNanos);
}
}

View File

@@ -116,8 +116,6 @@
<property name="nodeDAO" ref="nodeDAO"/>
<property name="tenantService" ref="tenantService"/>
<property name="nodesCache" ref="node.nodesCache"/>
<property name="aspectsCache" ref="node.aspectsCache"/>
<property name="propertiesCache" ref="node.propertiesCache"/>
<property name="aclCrudDAO" ref="aclCrudDAO"/>
<property name="metadataIndexCheck2">
<ref bean="metadataQueryIndexesCheck2" />

View File

@@ -26,7 +26,6 @@
package org.alfresco.repo.search.impl.querymodel.impl.db;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
@@ -37,21 +36,16 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.alfresco.repo.cache.SimpleCache;
import org.alfresco.repo.cache.lookup.EntityLookupCache;
import org.alfresco.repo.domain.node.Node;
import org.alfresco.repo.domain.node.NodeVersionKey;
import org.alfresco.repo.domain.node.StoreEntity;
import org.alfresco.repo.search.impl.querymodel.QueryOptions;
import org.alfresco.repo.security.permissions.impl.acegi.FilteringResultSet;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.service.namespace.QName;
import org.apache.ibatis.executor.result.DefaultResultContext;
import org.apache.ibatis.session.ResultContext;
import org.apache.ibatis.session.ResultHandler;
@@ -69,7 +63,6 @@ public class DBQueryEngineTest
private DBQuery dbQuery;
private ResultContext<Node> resultContext;
private QueryOptions options;
private SimpleCache<NodeVersionKey, Map<QName, Serializable>> propertiesCache;
@SuppressWarnings("unchecked")
@Before
@@ -84,12 +77,7 @@ public class DBQueryEngineTest
template = mock(SqlSessionTemplate.class);
engine.setSqlSessionTemplate(template);
propertiesCache = mock(SimpleCache.class);
engine.setPropertiesCache(propertiesCache);
engine.nodesCache = mock(EntityLookupCache.class);
DBStats.resetStopwatches();
}
@Test

View File

@@ -54,7 +54,6 @@ public class NodePermissionAssessorPermissionsTest
{
AuthenticationUtil.clearCurrentSecurityContext();
permissionService = mock(PermissionService.class);
DBStats.resetStopwatches();
}
@Test