ACS-1601 Node cleanup job improvements for Postgres

This commit is contained in:
Nithin Nambiar
2022-03-11 16:58:22 +00:00
committed by GitHub
parent b2dd06eef8
commit 8cf9cd3ed5
31 changed files with 953 additions and 191 deletions

View File

@@ -128,8 +128,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -21,7 +21,7 @@ package org.alfresco.util.transaction;
import org.alfresco.error.AlfrescoRuntimeException;
/**
* Exception wraps {@link java.util.NoSuchElementException} from {@link org.apache.commons.dbcp.BasicDataSource}
* Exception wraps {@link java.util.NoSuchElementException} from {@link org.apache.commons.dbcp2.BasicDataSource}
*
* @author alex.mukha
* @since 4.1.9

View File

@@ -65,7 +65,7 @@
<dependency.bouncycastle.version>1.70</dependency.bouncycastle.version>
<dependency.mockito-core.version>3.11.2</dependency.mockito-core.version>
<dependency.org-json.version>20211205</dependency.org-json.version>
<dependency.commons-dbcp.version>1.4-DBCP330</dependency.commons-dbcp.version>
<dependency.commons-dbcp.version>2.9.0</dependency.commons-dbcp.version>
<dependency.commons-io.version>2.11.0</dependency.commons-io.version>
<dependency.gson.version>2.8.5</dependency.gson.version>
<dependency.httpclient.version>4.5.13</dependency.httpclient.version>
@@ -755,8 +755,8 @@
<version>${dependency.mockito-core.version}</version>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
<version>${dependency.commons-dbcp.version}</version>
</dependency>
<dependency>

View File

@@ -77,8 +77,8 @@
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>

View File

@@ -51,7 +51,7 @@ import org.alfresco.service.cmr.workflow.WorkflowAdminService;
import org.alfresco.service.transaction.TransactionService;
import org.alfresco.traitextender.SpringExtensionBundle;
import org.alfresco.util.PropertyCheck;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.InitializingBean;
@@ -89,7 +89,7 @@ import javax.sql.DataSource;
* </li>
* <li><b>db:</b> Database configuration
* <ul>
* <li>maxConnections: int - The maximum number of active connections. {@link BasicDataSource#getMaxActive()}</li>
* <li>maxConnections: int - The maximum number of active connections. {@link BasicDataSource#getMaxTotal()}</li>
* </ul>
* </li>
* <li><b>authentication</b>: Authentication configuration.
@@ -326,7 +326,7 @@ public class ConfigurationDataCollector extends HBBaseDataCollector implements I
if (dataSource instanceof BasicDataSource)
{
Map<String, Object> db = new HashMap<>();
db.put("maxConnections", ((BasicDataSource) dataSource).getMaxActive());
db.put("maxConnections", ((BasicDataSource) dataSource).getMaxTotal());
configurationValues.put("db", db);
}

View File

@@ -31,7 +31,7 @@ import org.alfresco.heartbeat.datasender.HBData;
import org.alfresco.heartbeat.jobs.HeartBeatJobScheduler;
import org.alfresco.repo.descriptor.DescriptorDAO;
import org.alfresco.util.PropertyCheck;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.InitializingBean;

View File

@@ -35,6 +35,7 @@ import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;

View File

@@ -29,6 +29,7 @@ import java.io.Serializable;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -944,4 +945,49 @@ public interface NodeDAO extends NodeBulkLoader
*/
public Long getNextTxCommitTime(Long fromCommitTime);
/**
*
* @param maxCommitTime
* @return Iterator over node ids
*/
default public Iterator<Long> selectDeletedNodesByCommitTime(long maxCommitTime)
{
throw new UnsupportedOperationException("Not Implemented");
}
/**
* Purge the nodes marked as deleted
* @param minAge
* @param deleteBatchSize
* @return the count of nodes deleted in each batch
*/
default public List<String> purgeDeletedNodes(long minAge, int deleteBatchSize)
{
throw new UnsupportedOperationException("This operation is not supported");
}
/**
*
* @param maxCommitTime
* @return Iterator over transaction ids
*/
default public Iterator<Long> selectUnusedTransactionsByCommitTime(long maxCommitTime)
{
throw new UnsupportedOperationException("Not Implemented");
}
/**
* Purge the transactions of purged nodes
* @param minAge
* @param deleteBatchSize
* @return the count of transactions deleted in each batch
*/
default public List<String> purgeEmptyTransactions(long minAge, int deleteBatchSize)
{
throw new UnsupportedOperationException("This operation is not supported");
}
}

View File

@@ -25,16 +25,6 @@
*/
package org.alfresco.repo.domain.node.ibatis;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.ibatis.IdsEntity;
import org.alfresco.model.ContentModel;
@@ -65,6 +55,7 @@ import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.namespace.QName;
import org.alfresco.util.Pair;
import org.apache.ibatis.cursor.Cursor;
import org.apache.ibatis.executor.result.DefaultResultContext;
import org.apache.ibatis.session.ResultContext;
import org.apache.ibatis.session.ResultHandler;
@@ -72,6 +63,17 @@ import org.apache.ibatis.session.RowBounds;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.util.Assert;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
/**
* iBatis-specific extension of the Node abstract DAO
@@ -166,6 +168,12 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
private static final String SELECT_TXN_MIN_TX_ID_IN_NODE_IDRANGE = "alfresco.node.select_TxnMinTxIdInNodeIdRange";
private static final String SELECT_TXN_MAX_TX_ID_IN_NODE_IDRANGE = "alfresco.node.select_TxnMaxTxIdInNodeIdRange";
private static final String SELECT_TXN_NEXT_TXN_COMMIT_TIME = "select_TxnNextTxnCommitTime";
private static final String SELECT_NODES_DELETED_BY_TXN_COMMIT_TIME = "alfresco.node.select.select_Deleted_NodesByTxnCommitTime";
private static final String DELETE_NODES_BY_ID = "alfresco.node.delete_NodesById";
private static final String DELETE_NODE_PROPS_BY_NODE_ID = "alfresco.node.delete_NodePropsByNodeId";
private static final String SELECT_TXNS_UNUSED_BY_TXN_COMMIT_TIME = "alfresco.node.select.select_Txns_UnusedByTxnCommitTime";
private static final String DELETE_TXNS_UNUSED_BY_ID = "alfresco.node.delete_Txns_UnusedById";
protected QNameDAO qnameDAO;
protected DictionaryService dictionaryService;
@@ -1795,6 +1803,136 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
return template.selectOne(SELECT_TXN_NEXT_TXN_COMMIT_TIME, fromCommitTimeEntity);
}
public Iterator<Long> selectDeletedNodesByCommitTime(long maxCommitTime)
{
// Get the deleted nodes
Pair<Long, QName> deletedTypePair = qnameDAO.getQName(ContentModel.TYPE_DELETED);
if (deletedTypePair == null)
{
// Nothing to do
return null;
}
TransactionQueryEntity transactionQueryEntity = new TransactionQueryEntity();
transactionQueryEntity.setMaxCommitTime(maxCommitTime);
transactionQueryEntity.setTypeQNameId(deletedTypePair.getFirst());
Cursor<Long> cursor = template.selectCursor(SELECT_NODES_DELETED_BY_TXN_COMMIT_TIME, transactionQueryEntity);
return cursor.iterator();
}
public Iterator<Long> selectUnusedTransactionsByCommitTime(long maxCommitTime)
{
TransactionQueryEntity maxCommitTimeEntity = new TransactionQueryEntity();
maxCommitTimeEntity.setMaxCommitTime(maxCommitTime);
Cursor<Long> cursor = template.selectCursor(SELECT_TXNS_UNUSED_BY_TXN_COMMIT_TIME, maxCommitTimeEntity);
return cursor.iterator();
}
@Override
public List<String> purgeDeletedNodes(long minAge, int deleteBatchSize)
{
final long maxCommitTime = System.currentTimeMillis() - minAge;
Iterator<Long> nodeIdIterator = this.selectDeletedNodesByCommitTime(maxCommitTime);
ArrayList<Long> nodeIdList = new ArrayList<>();
List<String> deleteResult = new ArrayList<>();
if (isDebugEnabled)
{
logger.debug("nodes selected for deletion, deleteBatchSize:" + deleteBatchSize);
}
while (nodeIdIterator != null && nodeIdIterator.hasNext())
{
if (deleteBatchSize == nodeIdList.size())
{
int count = deleteSelectedNodesAndProperties(nodeIdList);
if (isDebugEnabled)
{
logger.debug("nodes deleted:" + count);
}
deleteResult.add("Purged old nodes: " + count);
nodeIdList.clear();
}
else
{
nodeIdList.add(nodeIdIterator.next());
}
}
if (nodeIdList.size() > 0)
{
int count = deleteSelectedNodesAndProperties(nodeIdList);
if (isDebugEnabled)
{
logger.debug("remaining nodes deleted:" + count);
}
deleteResult.add("Purged old nodes: " + count);
nodeIdList.clear();
}
return deleteResult;
}
public List<String> purgeEmptyTransactions(long minAge, int deleteBatchSize)
{
final long maxCommitTime = System.currentTimeMillis() - minAge;
Iterator<Long> transactionIdIterator = this.selectUnusedTransactionsByCommitTime(maxCommitTime);
ArrayList<Long> transactionIdList = new ArrayList<>();
List<String> deleteResult = new ArrayList<>();
if (isDebugEnabled)
{
logger.debug("transactions selected for deletion, deleteBatchSize:" + deleteBatchSize);
}
while (transactionIdIterator.hasNext())
{
if (deleteBatchSize == transactionIdList.size())
{
int count = deleteSelectedTransactions(transactionIdList);
deleteResult.add("Purged old transactions: " + count);
if (isDebugEnabled)
{
logger.debug("transactions deleted:" + count);
}
transactionIdList.clear();
}
else
{
transactionIdList.add(transactionIdIterator.next());
}
}
if (transactionIdList.size() > 0)
{
int count = deleteSelectedTransactions(transactionIdList);
deleteResult.add("Purged old transactions: " + count);
if (isDebugEnabled)
{
logger.debug("final batch of transactions deleted:" + count);
}
transactionIdList.clear();
}
return deleteResult;
}
private int deleteSelectedNodesAndProperties(List<Long> nodeIdList)
{
int cnt = template.delete(DELETE_NODE_PROPS_BY_NODE_ID, nodeIdList);
if (isDebugEnabled)
{
logger.debug("nodes props deleted:" + cnt);
}
// Finally, remove the nodes
cnt = template.delete(DELETE_NODES_BY_ID, nodeIdList);
if (isDebugEnabled)
{
logger.debug("nodes deleted:" + cnt);
}
return cnt;
}
private int deleteSelectedTransactions(List<Long> transactionIdList)
{
return template.delete(DELETE_TXNS_UNUSED_BY_ID, transactionIdList);
}
/*
* DAO OVERRIDES

View File

@@ -50,6 +50,13 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
// of the chunk (in ms). Default is a couple of hours.
private int purgeSize = 7200000; // ms
//to determine if we need a time based window deletion of nodes or in fixed size batches.
private String algorithm;
private int deleteBatchSize;
private static final String NODE_TABLE_CLEANER_ALG_V2 = "V2";
/**
* Default constructor
*/
@@ -67,15 +74,57 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
{
return Collections.singletonList("Minimum purge age is negative; purge disabled");
}
List<String> purgedNodes, purgedTxns;
List<String> purgedNodes = purgeOldDeletedNodes(minPurgeAgeMs);
List<String> purgedTxns = purgeOldEmptyTransactions(minPurgeAgeMs);
if (NODE_TABLE_CLEANER_ALG_V2.equals(algorithm))
{
refreshLock();
if (logger.isDebugEnabled())
{
logger.debug("DeletedNodeCleanupWorker using batch deletion: About to execute the clean up nodes ");
List<String> allResults = new ArrayList<String>(100);
}
purgedNodes = purgeOldDeletedNodesV2(minPurgeAgeMs);
if (logger.isDebugEnabled())
{
logger.debug(purgedNodes);
}
refreshLock();
if (logger.isDebugEnabled())
{
logger.debug("DeletedNodeCleanupWorker: About to execute the clean up txns ");
}
purgedTxns = purgeOldEmptyTransactionsV2(minPurgeAgeMs);
}
else
{
if (logger.isDebugEnabled())
{
logger.debug("DeletedNodeCleanupWorker: About to start purgeOldDeletedNodes ");
}
purgedNodes = purgeOldDeletedNodes(minPurgeAgeMs);
logger.debug(purgedNodes);
if (logger.isDebugEnabled())
{
logger.debug("DeletedNodeCleanupWorker: About to start purgeOldEmptyTransactions ");
}
purgedTxns = purgeOldEmptyTransactions(minPurgeAgeMs);
}
if (logger.isDebugEnabled())
{
logger.debug(purgedTxns);
}
List<String> allResults = new ArrayList<>(100);
allResults.addAll(purgedNodes);
allResults.addAll(purgedTxns);
// Done
return allResults;
}
@@ -110,6 +159,16 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
this.purgeSize = purgeSize;
}
public void setAlgorithm(String algorithm)
{
this.algorithm = algorithm;
}
public void setDeleteBatchSize(int deleteBatchSize)
{
this.deleteBatchSize = deleteBatchSize;
}
/**
* Cleans up deleted nodes that are older than the given minimum age.
*
@@ -122,10 +181,12 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
final long maxCommitTime = System.currentTimeMillis() - minAge;
long fromCommitTime = fromCustomCommitTime;
if (fromCommitTime <= 0L)
{
fromCommitTime = nodeDAO.getMinTxnCommitTimeForDeletedNodes().longValue();
}
if ( fromCommitTime == 0L )
{
String msg = "There are no old nodes to purge.";
@@ -134,7 +195,10 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
}
long loopPurgeSize = purgeSize;
Long purgeCount = new Long(0);
if(logger.isDebugEnabled())
{
logger.debug("DeletedNodeCleanupWorker: purgeOldDeletedNodes started ");
}
while (true)
{
// Ensure we keep the lock
@@ -153,9 +217,9 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
try
{
DeleteNodesByTransactionsCallback purgeNodesCallback = new DeleteNodesByTransactionsCallback(nodeDAO, fromCommitTime, toCommitTime);
purgeCount = txnHelper.doInTransaction(purgeNodesCallback, false, true);
Long purgeCount = txnHelper.doInTransaction(purgeNodesCallback, false, true);
if (purgeCount.longValue() > 0)
if (purgeCount > 0)
{
String msg =
"Purged old nodes: \n" +
@@ -221,6 +285,7 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
}
}
logger.debug("DeletedNodeCleanupWorker: purgeOldDeletedNodes finished ");
// Done
return results;
}
@@ -245,6 +310,10 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
{
fromCommitTime = nodeDAO.getMinUnusedTxnCommitTime().longValue();
}
if(logger.isDebugEnabled())
{
logger.debug("DeletedNodeCleanupWorker: purgeOldEmptyTransactions started ");
}
// delete unused transactions in batches of size 'purgeTxnBlockSize'
while (true)
{
@@ -298,15 +367,47 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
}
fromCommitTime += purgeSize;
if(fromCommitTime >= maxCommitTime)
if (fromCommitTime >= maxCommitTime)
{
break;
}
}
logger.debug("DeletedNodeCleanupWorker: purgeOldEmptyTransactions finished ");
// Done
return results;
}
private List<String> purgeOldDeletedNodesV2(long minAge)
{
refreshLock();
final List<String> returnList = new ArrayList<>();
RetryingTransactionHelper txnHelper = transactionService.getRetryingTransactionHelper();
RetryingTransactionCallback<Void> callback = () -> {
returnList.addAll(nodeDAO.purgeDeletedNodes(minAge, deleteBatchSize));
return null;
};
txnHelper.doInTransaction(callback, false, true);
return returnList;
}
private List<String> purgeOldEmptyTransactionsV2(long minAge)
{
refreshLock();
final List<String> returnList = new ArrayList<>();
RetryingTransactionHelper txnHelper = transactionService.getRetryingTransactionHelper();
RetryingTransactionCallback<Void> callback = () -> {
returnList.addAll(nodeDAO.purgeEmptyTransactions(minAge, deleteBatchSize));
return null;
};
txnHelper.doInTransaction(callback, false, true);
return returnList;
}
private static abstract class DeleteByTransactionsCallback implements RetryingTransactionCallback<Long>
{
protected NodeDAO nodeDAO;
@@ -356,4 +457,5 @@ public class DeletedNodeCleanupWorker extends AbstractNodeCleanupWorker
return count;
}
}
}

View File

@@ -27,7 +27,7 @@ package org.alfresco.repo.tenant;
import java.sql.SQLException;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp2.BasicDataSource;
/**
* Experimental
@@ -41,7 +41,7 @@ public class TenantBasicDataSource extends BasicDataSource
{
// tenant-specific
this.setUrl(tenantUrl);
this.setMaxActive(tenantMaxActive == -1 ? bds.getMaxActive() : tenantMaxActive);
this.setMaxTotal(tenantMaxActive == -1 ? bds.getMaxTotal() : tenantMaxActive);
// defaults/overrides - see also 'baseDefaultDataSource' (core-services-context.xml + repository.properties)
@@ -54,7 +54,7 @@ public class TenantBasicDataSource extends BasicDataSource
this.setMaxIdle(bds.getMaxIdle());
this.setDefaultAutoCommit(bds.getDefaultAutoCommit());
this.setDefaultTransactionIsolation(bds.getDefaultTransactionIsolation());
this.setMaxWait(bds.getMaxWait());
this.setMaxWaitMillis(bds.getMaxWaitMillis());
this.setValidationQuery(bds.getValidationQuery());
this.setTimeBetweenEvictionRunsMillis(bds.getTimeBetweenEvictionRunsMillis());
this.setMinEvictableIdleTimeMillis(bds.getMinEvictableIdleTimeMillis());
@@ -62,7 +62,7 @@ public class TenantBasicDataSource extends BasicDataSource
this.setTestOnBorrow(bds.getTestOnBorrow());
this.setTestOnReturn(bds.getTestOnReturn());
this.setTestWhileIdle(bds.getTestWhileIdle());
this.setRemoveAbandoned(bds.getRemoveAbandoned());
this.setRemoveAbandonedOnBorrow(bds.getRemoveAbandonedOnBorrow());
this.setRemoveAbandonedTimeout(bds.getRemoveAbandonedTimeout());
this.setPoolPreparedStatements(bds.isPoolPreparedStatements());
this.setMaxOpenPreparedStatements(bds.getMaxOpenPreparedStatements());

View File

@@ -32,7 +32,7 @@ import java.util.Map;
import javax.sql.DataSource;
import org.alfresco.repo.security.authentication.AuthenticationUtil;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp2.BasicDataSource;
import org.springframework.extensions.surf.util.ParameterCheck;
import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource;

View File

@@ -30,7 +30,7 @@
<property name="initialSize">
<value>0</value>
</property>
<property name="maxActive">
<property name="maxTotal">
<value>1</value>
</property>
<property name="maxIdle">

View File

@@ -156,7 +156,7 @@
<bean id="defaultDataSource" parent="baseDefaultDataSource" />
<!-- Datasource bean -->
<bean id="baseDefaultDataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close" abstract="true">
<bean id="baseDefaultDataSource" class="org.apache.commons.dbcp2.BasicDataSource" destroy-method="close" abstract="true">
<property name="driverClassName">
<value>${db.driver}</value>
</property>
@@ -172,7 +172,7 @@
<property name="initialSize" >
<value>${db.pool.initial}</value>
</property>
<property name="maxActive" >
<property name="maxTotal" >
<value>${db.pool.max}</value>
</property>
<property name="minIdle" >
@@ -187,7 +187,7 @@
<property name="defaultTransactionIsolation" >
<value>${db.txn.isolation}</value>
</property>
<property name="maxWait" >
<property name="maxWaitMillis" >
<value>${db.pool.wait.max}</value>
</property>
<property name="validationQuery" >
@@ -211,7 +211,7 @@
<property name="testWhileIdle" >
<value>${db.pool.evict.validate}</value>
</property>
<property name="removeAbandoned" >
<property name="removeAbandonedOnBorrow" >
<value>${db.pool.abandoned.detect}</value>
</property>
<property name="removeAbandonedTimeout" >

View File

@@ -203,6 +203,7 @@ Inbound settings from iBatis
<mapper resource="alfresco/ibatis/#resource.dialect#/content-insert-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/node-common-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/node-select-children-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/node-select-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/node-update-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/node-delete-SqlMap.xml"/>
<mapper resource="alfresco/ibatis/#resource.dialect#/node-insert-SqlMap.xml"/>

View File

@@ -1506,4 +1506,31 @@
commit_time_ms > #{minCommitTime}
</select>
<delete id="delete_NodesById" parameterType="list">
delete from alf_node
where
id IN
<foreach item="item" index="index" collection="list" open="(" separator="," close=")">
#{item}
</foreach>
</delete>
<delete id="delete_NodePropsByNodeId" parameterType="list">
delete from alf_node_properties
where
node_id IN
<foreach item="item" index="index" collection="list" open="(" separator="," close=")">
#{item}
</foreach>
</delete>
<delete id="delete_Txns_UnusedById" parameterType="list">
delete from alf_transaction
where
id in
<foreach item="item" index="index" collection="list" open="(" separator="," close=")">
#{item}
</foreach>
</delete>
</mapper>

View File

@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="alfresco.node.select">
<select id="select_Deleted_NodesByTxnCommitTime" parameterType="TransactionQuery" fetchSize="100000" resultType="java.lang.Long">
select
node.id
from
alf_node node
join alf_transaction txn on (node.transaction_id = txn.id)
where
node.type_qname_id = #{typeQNameId}
<![CDATA[and commit_time_ms < #{maxCommitTime}]]>
</select>
<select id="select_Txns_UnusedByTxnCommitTime" parameterType="TransactionQuery" fetchSize="100000" resultType="java.lang.Long">
select
id
from alf_transaction
where not exists
(
select 1
from
alf_node node
where
node.transaction_id = alf_transaction.id
)
<![CDATA[and commit_time_ms <= #{maxCommitTime}]]>
</select>
</mapper>

View File

@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="alfresco.node.select">
<select id="select_Deleted_NodesByTxnCommitTime" parameterType="TransactionQuery" fetchSize="-2147483648" resultType="java.lang.Long">
select
node.id
from
alf_node node
join alf_transaction txn on (node.transaction_id = txn.id)
where
node.type_qname_id = #{typeQNameId}
<![CDATA[and commit_time_ms < #{maxCommitTime}]]>
</select>
<select id="select_Txns_UnusedByTxnCommitTime" parameterType="TransactionQuery" fetchSize="-2147483648" resultType="java.lang.Long">
select
id
from alf_transaction
where not exists
(
select 1
from
alf_node node
where
node.transaction_id = alf_transaction.id
)
<![CDATA[and commit_time_ms <= #{maxCommitTime}]]>
</select>
</mapper>

View File

@@ -238,6 +238,12 @@
<property name="purgeSize">
<value>${index.tracking.purgeSize}</value>
</property>
<property name="algorithm">
<value>${system.node_table_cleaner.algorithm}</value>
</property>
<property name="deleteBatchSize">
<value>${system.node_cleanup.delete_batchSize}</value>
</property>
</bean>
<!-- String length adjustment -->

View File

@@ -1246,6 +1246,11 @@ system.delete_not_exists.read_only=false
system.delete_not_exists.timeout_seconds=-1
system.prop_table_cleaner.algorithm=V2
# --Node cleanup batch - default settings
system.node_cleanup.delete_batchSize=1000
system.node_table_cleaner.algorithm=V1
# Configure the system-wide (ACS) settings for direct access urls.
#
# For Direct Access URLs to be usable on the service-layer, the feature must be enabled both system-wide and on the

View File

@@ -87,7 +87,8 @@ import org.junit.runners.Suite;
org.alfresco.repo.node.cleanup.TransactionCleanupTest.class,
org.alfresco.repo.security.person.GetPeopleCannedQueryTest.class,
org.alfresco.repo.domain.schema.script.DeleteNotExistsExecutorTest.class
org.alfresco.repo.domain.schema.script.DeleteNotExistsExecutorTest.class,
org.alfresco.repo.node.cleanup.DeletedNodeBatchCleanupTest.class
})
public class AllDBTestsTestSuite
{

View File

@@ -84,6 +84,7 @@ import org.junit.runners.Suite;
org.alfresco.repo.node.archive.ArchiveAndRestoreTest.class,
org.alfresco.repo.node.db.DbNodeServiceImplTest.class,
org.alfresco.repo.node.cleanup.TransactionCleanupTest.class,
org.alfresco.repo.node.cleanup.DeletedNodeBatchCleanupTest.class,
org.alfresco.repo.node.db.DbNodeServiceImplPropagationTest.class,
})
public class AppContext03TestSuite

View File

@@ -57,7 +57,7 @@ import org.alfresco.service.cmr.workflow.WorkflowAdminService;
import org.alfresco.service.descriptor.Descriptor;
import org.alfresco.service.transaction.TransactionService;
import org.alfresco.traitextender.SpringExtensionBundle;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp2.BasicDataSource;
import org.junit.Before;
import org.junit.Test;

View File

@@ -32,7 +32,7 @@ import org.alfresco.heartbeat.jobs.HeartBeatJobScheduler;
import org.alfresco.repo.descriptor.DescriptorDAO;
import org.alfresco.service.cmr.repository.HBDataCollectorService;
import org.alfresco.service.descriptor.Descriptor;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp2.BasicDataSource;
import org.junit.Before;
import org.junit.Test;

View File

@@ -0,0 +1,366 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.repo.node.cleanup;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Stream.of;
import javax.transaction.UserTransaction;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.cache.SimpleCache;
import org.alfresco.repo.domain.node.NodeDAO;
import org.alfresco.repo.domain.node.Transaction;
import org.alfresco.repo.domain.node.ibatis.NodeDAOImpl;
import org.alfresco.repo.node.db.DeletedNodeCleanupWorker;
import org.alfresco.repo.transaction.AlfrescoTransactionSupport;
import org.alfresco.repo.transaction.RetryingTransactionHelper;
import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.search.SearchService;
import org.alfresco.service.cmr.security.AuthenticationService;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.service.transaction.TransactionService;
import org.alfresco.test_category.OwnJVMTestsCategory;
import org.alfresco.util.BaseSpringTest;
import org.alfresco.util.testing.category.DBTests;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.extensions.webscripts.GUID;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.annotation.DirtiesContext.ClassMode;
@Category({ OwnJVMTestsCategory.class, DBTests.class })
@DirtiesContext(classMode = ClassMode.BEFORE_EACH_TEST_METHOD)
public class DeletedNodeBatchCleanupTest extends BaseSpringTest
{
@Autowired
private AuthenticationService authenticationService;
@Autowired
private NodeDAO nodeDAO;
@Autowired
@Qualifier("node.nodesSharedCache")
private SimpleCache<Serializable, Serializable> nodesCache;
@Autowired
private DeletedNodeCleanupWorker worker;
@Autowired
private NamespaceService namespaceService;
@Autowired
private TransactionService transactionService;
@Autowired
private NodeService nodeService;
@Autowired
private SearchService searchService;
private RetryingTransactionHelper helper;
private List<NodeRef> testNodes;
@Before
public void before()
{
helper = transactionService.getRetryingTransactionHelper();
authenticationService.authenticate("admin", "admin".toCharArray());
resetWorkerConfig();
// create 5 test nodes
final NodeRef companyHome = getCompanyHome();
testNodes = IntStream.range(0, 5)
.mapToObj(i -> helper.doInTransaction(createNodeCallback(companyHome), false, true))
.collect(toList());
// clean up pre-existing data
helper.doInTransaction(() -> worker.doClean(), false, true);
}
private void resetWorkerConfig()
{
worker.setMinPurgeAgeDays(0);
worker.setAlgorithm("V2");
worker.setDeleteBatchSize(20);
}
private NodeRef getCompanyHome()
{
StoreRef storeRef = new StoreRef(StoreRef.PROTOCOL_WORKSPACE, "SpacesStore");
NodeRef storeRoot = nodeService.getRootNode(storeRef);
List<NodeRef> nodeRefs = searchService.selectNodes(storeRoot, "/app:company_home", null, namespaceService,
false);
return nodeRefs.get(0);
}
private RetryingTransactionCallback<NodeRef> createNodeCallback(NodeRef companyHome)
{
return () -> nodeService.createNode(
companyHome, ContentModel.ASSOC_CONTAINS, QName.createQName("test", GUID.generate()),
ContentModel.TYPE_CONTENT).getChildRef();
}
private void deleteNodes(NodeRef nodeRef, NodeRef... additionalNodeRefs)
{
Stream.concat(of(nodeRef), of(additionalNodeRefs))
.forEach(this::deleteNode);
}
private void deleteNode(NodeRef nodeRef)
{
helper.doInTransaction(new DeleteNode(nodeRef), false, true);
}
@Test
public void testPurgeNodesDeleted()
{
final NodeRef nodeRef4 = getNode(4);
final NodeRef nodeRef5 = getNode(5);
// delete nodes 4 and 5
deleteNodes(nodeRef4, nodeRef5);
// double-check that node 4 and 5 are present in deleted form
nodesCache.clear();
assertTrue("Node 4 is deleted but not purged", nodeDAO.getNodeRefStatus(nodeRef4).isDeleted());
assertTrue("Node 5 is deleted but not purged", nodeDAO.getNodeRefStatus(nodeRef5).isDeleted());
worker.doClean();
// verify that node 4 and 5 were purged
nodesCache.clear();
assertNull("Node 4 was not cleaned up", nodeDAO.getNodeRefStatus(nodeRef4));
assertNull("Node 5 was not cleaned up", nodeDAO.getNodeRefStatus(nodeRef5));
}
@Test
public void testNodesDeletedNotPurgedWhenNotAfterPurgeAge()
{
final NodeRef nodeRef1 = getNode(1);
final NodeRef nodeRef2 = getNode(2);
// delete nodes 1 and 2
deleteNodes(nodeRef1, nodeRef2);
// double-check that node 1 and 2 are present in deleted form
nodesCache.clear();
assertTrue("Node 1 is deleted but not purged", nodeDAO.getNodeRefStatus(nodeRef1).isDeleted());
assertTrue("Node 2 is deleted but not purged", nodeDAO.getNodeRefStatus(nodeRef2).isDeleted());
// run the worker
worker.setMinPurgeAgeDays(1);
worker.doClean();
// verify that node 1 and 2 were not purged
nodesCache.clear();
assertNotNull("Node 1 was cleaned up", nodeDAO.getNodeRefStatus(nodeRef1));
assertNotNull("Node 2 was cleaned up", nodeDAO.getNodeRefStatus(nodeRef2));
}
@Test
public void testPurgeUnusedTransactions() throws Exception
{
// Execute transactions that update a number of nodes. For nodeRef1, all but the last txn will be unused.
final long start = System.currentTimeMillis();
final Long minTxnId = nodeDAO.getMinTxnId();
final Map<NodeRef, List<String>> txnIds = createTransactions();
final List<String> txnIds1 = txnIds.get(getNode(1));
final List<String> txnIds2 = txnIds.get(getNode(2));
final List<String> txnIds3 = txnIds.get(getNode(3));
// Double-check that n4 and n5 are present in deleted form
nodesCache.clear();
UserTransaction txn = transactionService.getUserTransaction(true);
txn.begin();
try
{
assertTrue("Node 4 is deleted but not purged", nodeDAO.getNodeRefStatus(getNode(4)).isDeleted());
assertTrue("Node 5 is deleted but not purged", nodeDAO.getNodeRefStatus(getNode(5)).isDeleted());
}
finally
{
txn.rollback();
}
// run the transaction cleaner
worker.doClean();
// Get transactions committed after the test started
RetryingTransactionHelper.RetryingTransactionCallback<List<Transaction>> getTxnsCallback = () -> ((NodeDAOImpl) nodeDAO).selectTxns(
start, Long.MAX_VALUE, Integer.MAX_VALUE, null, null, true);
List<Transaction> txns = transactionService.getRetryingTransactionHelper()
.doInTransaction(getTxnsCallback, true, false);
List<String> expectedUnusedTxnIds = new ArrayList<>(10);
expectedUnusedTxnIds.addAll(txnIds1.subList(0, txnIds1.size() - 1));
List<String> expectedUsedTxnIds = new ArrayList<>(5);
expectedUsedTxnIds.add(txnIds1.get(txnIds1.size() - 1));
expectedUsedTxnIds.addAll(txnIds2);
expectedUsedTxnIds.addAll(txnIds3);
// 4 and 5 should not be in the list because they are deletes
// check that the correct transactions have been purged i.e. all except the last one to update the node
// i.e. in this case, all but the last one in txnIds1
List<String> unusedTxnsNotPurged = expectedUnusedTxnIds.stream()
.filter(txnId -> containsTransaction(txns, txnId))
.collect(toList());
if (!unusedTxnsNotPurged.isEmpty())
{
fail("Unused transaction(s) were not purged: " + unusedTxnsNotPurged);
}
long numFoundUnusedTxnIds = expectedUnusedTxnIds.stream()
.filter(txnId -> !containsTransaction(txns, txnId))
.count();
assertEquals(9, numFoundUnusedTxnIds);
// check that the correct transactions remain i.e. all those in txnIds2, txnIds3, txnIds4 and txnIds5
long numFoundUsedTxnIds = expectedUsedTxnIds.stream()
.filter(txnId -> containsTransaction(txns, txnId))
.count();
assertEquals(3, numFoundUsedTxnIds);
// Get transactions committed after the test started
RetryingTransactionHelper.RetryingTransactionCallback<List<Long>> getTxnsUnusedCallback = () -> nodeDAO.getTxnsUnused(
minTxnId, Long.MAX_VALUE, Integer.MAX_VALUE);
List<Long> txnsUnused = transactionService.getRetryingTransactionHelper()
.doInTransaction(getTxnsUnusedCallback, true, false);
assertEquals(0, txnsUnused.size());
// Double-check that n4 and n5 were removed as well
nodesCache.clear();
assertNull("Node 4 was not cleaned up", nodeDAO.getNodeRefStatus(getNode(4)));
assertNull("Node 5 was not cleaned up", nodeDAO.getNodeRefStatus(getNode(5)));
}
private boolean containsTransaction(List<Transaction> txns, String txnId)
{
return txns.stream()
.map(Transaction::getChangeTxnId)
.filter(changeTxnId -> changeTxnId.equals(txnId))
.map(match -> true)
.findFirst()
.orElse(false);
}
private Map<NodeRef, List<String>> createTransactions()
{
Map<NodeRef, List<String>> txnIds = new HashMap<>();
UpdateNode updateNode1 = new UpdateNode(getNode(1));
UpdateNode updateNode2 = new UpdateNode(getNode(2));
UpdateNode updateNode3 = new UpdateNode(getNode(3));
DeleteNode deleteNode4 = new DeleteNode(getNode(4));
DeleteNode deleteNode5 = new DeleteNode(getNode(5));
List<String> txnIds1 = new ArrayList<>();
List<String> txnIds2 = new ArrayList<>();
List<String> txnIds3 = new ArrayList<>();
List<String> txnIds4 = new ArrayList<>();
List<String> txnIds5 = new ArrayList<>();
txnIds.put(getNode(1), txnIds1);
txnIds.put(getNode(2), txnIds2);
txnIds.put(getNode(3), txnIds3);
txnIds.put(getNode(4), txnIds4);
txnIds.put(getNode(5), txnIds5);
for (int i = 0; i < 10; i++)
{
String txnId1 = helper.doInTransaction(updateNode1, false, true);
txnIds1.add(txnId1);
if (i == 0)
{
String txnId2 = helper.doInTransaction(updateNode2, false, true);
txnIds2.add(txnId2);
}
if (i == 1)
{
String txnId3 = helper.doInTransaction(updateNode3, false, true);
txnIds3.add(txnId3);
}
}
String txnId4 = helper.doInTransaction(deleteNode4, false, true);
txnIds4.add(txnId4);
String txnId5 = helper.doInTransaction(deleteNode5, false, true);
txnIds5.add(txnId5);
return txnIds;
}
private class UpdateNode implements RetryingTransactionHelper.RetryingTransactionCallback<String>
{
private final NodeRef nodeRef;
UpdateNode(NodeRef nodeRef)
{
this.nodeRef = nodeRef;
}
@Override
public String execute() throws Throwable
{
nodeService.setProperty(nodeRef, ContentModel.PROP_NAME, GUID.generate());
return AlfrescoTransactionSupport.getTransactionId();
}
}
private class DeleteNode implements RetryingTransactionHelper.RetryingTransactionCallback<String>
{
private final NodeRef nodeRef;
DeleteNode(NodeRef nodeRef)
{
this.nodeRef = nodeRef;
}
@Override
public String execute() throws Throwable
{
nodeService.addAspect(nodeRef, ContentModel.ASPECT_TEMPORARY, null);
nodeService.deleteNode(nodeRef);
return AlfrescoTransactionSupport.getTransactionId();
}
}
private NodeRef getNode(int i)
{
return testNodes.get(i - 1);
}
}

View File

@@ -110,6 +110,8 @@ public class TransactionCleanupTest
this.nodesCache = (SimpleCache<Serializable, Serializable>) ctx.getBean("node.nodesSharedCache");
this.worker = (DeletedNodeCleanupWorker)ctx.getBean("nodeCleanup.deletedNodeCleanup");
this.worker.setMinPurgeAgeDays(0);
this.worker.setAlgorithm("V1");
this.helper = transactionService.getRetryingTransactionHelper();
authenticationService.authenticate("admin", "admin".toCharArray());

View File

@@ -37,7 +37,7 @@
</bean>
<!-- dummy -->
<bean id="defaultDataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
<bean id="defaultDataSource" class="org.apache.commons.dbcp2.BasicDataSource" destroy-method="close">
</bean>
<bean id="dataSource" class="org.alfresco.config.JndiObjectFactoryBean">

View File

@@ -4,7 +4,7 @@
<import resource="classpath:alfresco/extension/dev-context.xml" />
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource"
<bean id="dataSource" class="org.apache.commons.dbcp2.BasicDataSource"
destroy-method="close">
<property name="driverClassName">
<value>${db.driver}</value>
@@ -21,7 +21,7 @@
<property name="initialSize">
<value>${db.pool.initial}</value>
</property>
<property name="maxActive">
<property name="maxTotal">
<value>${db.pool.max}</value>
</property>
<property name="minIdle">
@@ -36,7 +36,7 @@
<property name="defaultTransactionIsolation">
<value>${db.txn.isolation}</value>
</property>
<property name="maxWait">
<property name="maxWaitMillis">
<value>${db.pool.wait.max}</value>
</property>
<property name="validationQuery">
@@ -57,7 +57,7 @@
<property name="testWhileIdle">
<value>${db.pool.evict.validate}</value>
</property>
<property name="removeAbandoned">
<property name="removeAbandonedOnBorrow">
<value>${db.pool.abandoned.detect}</value>
</property>
<property name="removeAbandonedTimeout">

View File

@@ -6,7 +6,7 @@
<import resource="classpath:test/alfresco/test-context.xml" />
<!-- Datasource bean -->
<bean id="testDataSource" class="org.apache.commons.dbcp.BasicDataSource"
<bean id="testDataSource" class="org.apache.commons.dbcp2.BasicDataSource"
destroy-method="close">
<property name="driverClassName">
<value>${db.driver}</value>
@@ -23,7 +23,7 @@
<property name="initialSize">
<value>${db.pool.initial}</value>
</property>
<property name="maxActive">
<property name="maxTotal">
<value>${db.pool.max}</value>
</property>
<property name="minIdle">
@@ -38,7 +38,7 @@
<property name="defaultTransactionIsolation">
<value>${db.txn.isolation}</value>
</property>
<property name="maxWait">
<property name="maxWaitMillis">
<value>${db.pool.wait.max}</value>
</property>
<property name="validationQuery">
@@ -59,7 +59,7 @@
<property name="testWhileIdle">
<value>${db.pool.evict.validate}</value>
</property>
<property name="removeAbandoned">
<property name="removeAbandonedOnBorrow">
<value>${db.pool.abandoned.detect}</value>
</property>
<property name="removeAbandonedTimeout">