inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-31 17:39:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

MNT-23953 - PropTablesCleaner v3 Skip IDs (#2181)

Browse Source 
* Configure v3 to skip ids per table. Added unit test
This commit is contained in:
evasques
2023-09-06 17:13:36 +01:00
committed by GitHub
parent c43dc66e2e
commit ddd0fac97e
6 changed files with 84 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsExecutor.java
View File

@@ -108,7 +108,7 @@ public class DeleteNotExistsExecutor implements StatementExecutor
// --DELETE_NOT_EXISTS primaryTable.key,secondaryTable1.key1,... batch.size.property
String[] args = sql.split("[ \\t]+(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)");
if (args.length == 3 && (args[1].indexOf('.')) != -1)
if (args.length >= 3 && args[1].indexOf('.') != -1)
{
String[] tableColumnArgs = args[1].split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)");
if (tableColumnArgs.length >= 2)
@@ -133,6 +133,10 @@ public class DeleteNotExistsExecutor implements StatementExecutor
String timeoutSecString = globalProperties.getProperty(PROPERTY_TIMEOUT_SECONDS);
timeoutSec = timeoutSecString == null ? -1 : Long.parseLong(timeoutSecString);
// Only implemented in v3. In v2 the skip is not used
String skipToIdString = (args.length == 4) ? globalProperties.getProperty(args[3]) : null;
Long skipToId = skipToIdString == null ? 0L : Long.parseLong(skipToIdString);
// Compute upper limits
Long[] tableUpperLimits = new Long[tableColumnArgs.length];
Pair<String, String>[] tableColumn = new Pair[tableColumnArgs.length];
@@ -159,7 +163,7 @@ public class DeleteNotExistsExecutor implements StatementExecutor
}
}
process(tableColumn, tableUpperLimits, optionalWhereClauses);
process(tableColumn, tableUpperLimits, optionalWhereClauses, skipToId);
}
}
}
@@ -623,4 +627,10 @@ public class DeleteNotExistsExecutor implements StatementExecutor
}
}
}
protected void process(Pair<String, String>[] tableColumn, Long[] tableUpperLimits, String[] optionalWhereClauses,
Long skipToId) throws SQLException
{
process(tableColumn, tableUpperLimits, optionalWhereClauses);
}
}

9
repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3Executor.java
View File

@@ -108,12 +108,19 @@ public class DeleteNotExistsV3Executor extends DeleteNotExistsExecutor
@Override
protected void process(Pair<String, String>[] tableColumn, Long[] tableUpperLimits, String[] optionalWhereClauses)
throws SQLException
{
process(tableColumn, tableUpperLimits, optionalWhereClauses, 0L);
}
@Override
protected void process(Pair<String, String>[] tableColumn, Long[] tableUpperLimits, String[] optionalWhereClauses, Long skipToId)
throws SQLException
{
String primaryTableName = tableColumn[0].getFirst();
String primaryColumnName = tableColumn[0].getSecond();
String primaryWhereClause = optionalWhereClauses[0];
Long primaryId = 0L;
Long primaryId = skipToId;
deletedCount = 0L;
startTime = new Date();

10
repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.MySQLInnoDBDialect/CleanAlfPropTablesV3.sql
View File

@@ -1,9 +1,9 @@
--DELETE_NOT_EXISTS_V3 alf_prop_root.id,alf_audit_app.disabled_paths_id,alf_audit_entry.audit_values_id,alf_prop_unique_ctx.prop1_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_root.id,alf_audit_app.disabled_paths_id,alf_audit_entry.audit_values_id,alf_prop_unique_ctx.prop1_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_root.skipToId
--DELETE_NOT_EXISTS_V3 alf_prop_value.id,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_value.id,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_value.skipToId
--DELETE_NOT_EXISTS_V3 alf_prop_string_value.id,alf_prop_value.long_value."persisted_type in (3, 5, 6)",alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_string_value.id,alf_prop_value.long_value."persisted_type in (3, 5, 6)",alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_string_value.skipToId
--DELETE_NOT_EXISTS_V3 alf_prop_serializable_value.id,alf_prop_value.long_value.persisted_type=4,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_serializable_value.id,alf_prop_value.long_value.persisted_type=4,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_serializable_value.skipToId
--DELETE_NOT_EXISTS_V3 alf_prop_double_value.id,alf_prop_value.long_value.persisted_type=2,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_double_value.id,alf_prop_value.long_value.persisted_type=2,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_double_value.skipToId

10
repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.PostgreSQLDialect/CleanAlfPropTablesV3.sql
View File

@@ -1,9 +1,9 @@
--DELETE_NOT_EXISTS_V3 alf_prop_root.id,alf_audit_app.disabled_paths_id,alf_audit_entry.audit_values_id,alf_prop_unique_ctx.prop1_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_root.id,alf_audit_app.disabled_paths_id,alf_audit_entry.audit_values_id,alf_prop_unique_ctx.prop1_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_root.skipToId
--DELETE_NOT_EXISTS_V3 alf_prop_value.id,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_value.id,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_value.skipToId
--DELETE_NOT_EXISTS_V3 alf_prop_string_value.id,alf_prop_value.long_value."persisted_type in (3, 5, 6)",alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_string_value.id,alf_prop_value.long_value."persisted_type in (3, 5, 6)",alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_string_value.skipToId
--DELETE_NOT_EXISTS_V3 alf_prop_serializable_value.id,alf_prop_value.long_value.persisted_type=4,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_serializable_value.id,alf_prop_value.long_value.persisted_type=4,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_serializable_value.skipToId
--DELETE_NOT_EXISTS_V3 alf_prop_double_value.id,alf_prop_value.long_value.persisted_type=2,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize
--DELETE_NOT_EXISTS_V3 alf_prop_double_value.id,alf_prop_value.long_value.persisted_type=2,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_double_value.skipToId

6
repository/src/main/resources/alfresco/repository.properties
View File

@@ -1254,6 +1254,12 @@ system.prop_table_cleaner.algorithm=V2
system.delete_not_exists.pauseAndRecoverBatchSize=500000
#Duration of the pause in milliseconds (default 10s)
system.delete_not_exists.pauseAndRecoverTime=10000
#Skip ids on job execution
system.delete_not_exists.alf_prop_root.skipToId=0
system.delete_not_exists.alf_prop_value.skipToId=0
system.delete_not_exists.alf_prop_string_value.skipToId=0
system.delete_not_exists.alf_prop_serializable_value.skipToId=0
system.delete_not_exists.alf_prop_double_value.skipToId=0
# --Node cleanup batch - default settings
system.node_cleanup.delete_batchSize=1000

58
repository/src/test/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3ExecutorTest.java
View File

@@ -86,7 +86,7 @@ public class DeleteNotExistsV3ExecutorTest
{
scriptExecutor.executeScriptUrl("scriptexec/${db.script.dialect}/delete-not-exists/test-data1.sql");
String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize";
String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize system.delete_not_exists.temp_tst_tbl_1.skipToId";
int line = 1;
File scriptFile = Mockito.mock(File.class);
Properties properties = Mockito.mock(Properties.class);
@@ -101,8 +101,8 @@ public class DeleteNotExistsV3ExecutorTest
{
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_READ_ONLY)).thenReturn("true");
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_TIMEOUT_SECONDS)).thenReturn("-1");
DeleteNotExistsV3Executor DeleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties);
DeleteNotExistsV3Executor.execute();
DeleteNotExistsV3Executor deleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties);
deleteNotExistsV3Executor.execute();
List<String> res = jdbcTmpl.queryForList(select, String.class);
assertEquals(7, res.size());
@@ -117,8 +117,8 @@ public class DeleteNotExistsV3ExecutorTest
{
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_READ_ONLY)).thenReturn("false");
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_TIMEOUT_SECONDS)).thenReturn("-1");
DeleteNotExistsV3Executor DeleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties);
DeleteNotExistsV3Executor.execute();
DeleteNotExistsV3Executor deleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties);
deleteNotExistsV3Executor.execute();
List<String> res = jdbcTmpl.queryForList(select, String.class);
assertEquals(5, res.size());
@@ -137,7 +137,7 @@ public class DeleteNotExistsV3ExecutorTest
{
scriptExecutor.executeScriptUrl("scriptexec/${db.script.dialect}/delete-not-exists/test-data1.sql");
String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize";
String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize system.delete_not_exists.temp_tst_tbl_1.skipToId";
int line = 1;
File scriptFile = Mockito.mock(File.class);
Properties properties = Mockito.mock(Properties.class);
@@ -150,8 +150,8 @@ public class DeleteNotExistsV3ExecutorTest
{
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_DELETE_BATCH_SIZE)).thenReturn("1");
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_READ_ONLY)).thenReturn("false");
DeleteNotExistsV3Executor DeleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties);
DeleteNotExistsV3Executor.execute();
DeleteNotExistsV3Executor deleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties);
deleteNotExistsV3Executor.execute();
List<String> res = jdbcTmpl.queryForList(select, String.class);
assertEquals(5, res.size());
@@ -170,7 +170,7 @@ public class DeleteNotExistsV3ExecutorTest
{
scriptExecutor.executeScriptUrl("scriptexec/${db.script.dialect}/delete-not-exists/test-data1.sql");
String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize";
String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize system.delete_not_exists.temp_tst_tbl_1.skipToId";
int line = 1;
File scriptFile = Mockito.mock(File.class);
Properties properties = Mockito.mock(Properties.class);
@@ -198,4 +198,42 @@ public class DeleteNotExistsV3ExecutorTest
}
}
}
}
@Test()
public void testSkip() throws Exception
{
scriptExecutor.executeScriptUrl("scriptexec/${db.script.dialect}/delete-not-exists/test-data1.sql");
String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize system.delete_not_exists.temp_tst_tbl_1.skipToId";
int line = 1;
File scriptFile = Mockito.mock(File.class);
Properties properties = Mockito.mock(Properties.class);
String select = "select id from temp_tst_tbl_1 order by id ASC";
try (Connection connection = dataSource.getConnection())
{
connection.setAutoCommit(true);
{
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_BATCH_SIZE)).thenReturn("2");
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_READ_ONLY)).thenReturn("false");
when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_TIMEOUT_SECONDS)).thenReturn("-1");
when(properties.getProperty("system.delete_not_exists.temp_tst_tbl_1.skipToId")).thenReturn("6");
DeleteNotExistsV3Executor deleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties);
deleteNotExistsV3Executor.execute();
List<String> res = jdbcTmpl.queryForList(select, String.class);
assertEquals(7, res.size());
// We are only processing Ids after 6, so all ids < 6 must remain untouched
assertEquals("1", res.get(0));
assertEquals("2", res.get(1));
assertEquals("3", res.get(2));
assertEquals("4", res.get(3));
assertEquals("5", res.get(4));
assertEquals("10", res.get(5));
assertEquals("11", res.get(6));
}
}
}
}