diff --git a/repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsExecutor.java b/repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsExecutor.java index 8c7ee6761b..1d5f837819 100644 --- a/repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsExecutor.java +++ b/repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsExecutor.java @@ -108,7 +108,7 @@ public class DeleteNotExistsExecutor implements StatementExecutor // --DELETE_NOT_EXISTS primaryTable.key,secondaryTable1.key1,... batch.size.property String[] args = sql.split("[ \\t]+(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"); - if (args.length == 3 && (args[1].indexOf('.')) != -1) + if (args.length >= 3 && args[1].indexOf('.') != -1) { String[] tableColumnArgs = args[1].split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"); if (tableColumnArgs.length >= 2) @@ -133,6 +133,10 @@ public class DeleteNotExistsExecutor implements StatementExecutor String timeoutSecString = globalProperties.getProperty(PROPERTY_TIMEOUT_SECONDS); timeoutSec = timeoutSecString == null ? -1 : Long.parseLong(timeoutSecString); + // Only implemented in v3. In v2 the skip is not used + String skipToIdString = (args.length == 4) ? globalProperties.getProperty(args[3]) : null; + Long skipToId = skipToIdString == null ? 0L : Long.parseLong(skipToIdString); + // Compute upper limits Long[] tableUpperLimits = new Long[tableColumnArgs.length]; Pair[] tableColumn = new Pair[tableColumnArgs.length]; @@ -159,7 +163,7 @@ public class DeleteNotExistsExecutor implements StatementExecutor } } - process(tableColumn, tableUpperLimits, optionalWhereClauses); + process(tableColumn, tableUpperLimits, optionalWhereClauses, skipToId); } } } @@ -623,4 +627,10 @@ public class DeleteNotExistsExecutor implements StatementExecutor } } } + + protected void process(Pair[] tableColumn, Long[] tableUpperLimits, String[] optionalWhereClauses, + Long skipToId) throws SQLException + { + process(tableColumn, tableUpperLimits, optionalWhereClauses); + } } diff --git a/repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3Executor.java b/repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3Executor.java index 62d5a4ed2e..3810b4b78b 100644 --- a/repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3Executor.java +++ b/repository/src/main/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3Executor.java @@ -108,12 +108,19 @@ public class DeleteNotExistsV3Executor extends DeleteNotExistsExecutor @Override protected void process(Pair[] tableColumn, Long[] tableUpperLimits, String[] optionalWhereClauses) throws SQLException + { + process(tableColumn, tableUpperLimits, optionalWhereClauses, 0L); + } + + @Override + protected void process(Pair[] tableColumn, Long[] tableUpperLimits, String[] optionalWhereClauses, Long skipToId) + throws SQLException { String primaryTableName = tableColumn[0].getFirst(); String primaryColumnName = tableColumn[0].getSecond(); String primaryWhereClause = optionalWhereClauses[0]; - Long primaryId = 0L; + Long primaryId = skipToId; deletedCount = 0L; startTime = new Date(); diff --git a/repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.MySQLInnoDBDialect/CleanAlfPropTablesV3.sql b/repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.MySQLInnoDBDialect/CleanAlfPropTablesV3.sql index c95d3f0a45..5bb4feff38 100644 --- a/repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.MySQLInnoDBDialect/CleanAlfPropTablesV3.sql +++ b/repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.MySQLInnoDBDialect/CleanAlfPropTablesV3.sql @@ -1,9 +1,9 @@ ---DELETE_NOT_EXISTS_V3 alf_prop_root.id,alf_audit_app.disabled_paths_id,alf_audit_entry.audit_values_id,alf_prop_unique_ctx.prop1_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_root.id,alf_audit_app.disabled_paths_id,alf_audit_entry.audit_values_id,alf_prop_unique_ctx.prop1_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_root.skipToId ---DELETE_NOT_EXISTS_V3 alf_prop_value.id,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_value.id,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_value.skipToId ---DELETE_NOT_EXISTS_V3 alf_prop_string_value.id,alf_prop_value.long_value."persisted_type in (3, 5, 6)",alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_string_value.id,alf_prop_value.long_value."persisted_type in (3, 5, 6)",alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_string_value.skipToId ---DELETE_NOT_EXISTS_V3 alf_prop_serializable_value.id,alf_prop_value.long_value.persisted_type=4,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_serializable_value.id,alf_prop_value.long_value.persisted_type=4,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_serializable_value.skipToId ---DELETE_NOT_EXISTS_V3 alf_prop_double_value.id,alf_prop_value.long_value.persisted_type=2,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_double_value.id,alf_prop_value.long_value.persisted_type=2,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_double_value.skipToId diff --git a/repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.PostgreSQLDialect/CleanAlfPropTablesV3.sql b/repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.PostgreSQLDialect/CleanAlfPropTablesV3.sql index c95d3f0a45..5bb4feff38 100644 --- a/repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.PostgreSQLDialect/CleanAlfPropTablesV3.sql +++ b/repository/src/main/resources/alfresco/dbscripts/utility/org.alfresco.repo.domain.dialect.PostgreSQLDialect/CleanAlfPropTablesV3.sql @@ -1,9 +1,9 @@ ---DELETE_NOT_EXISTS_V3 alf_prop_root.id,alf_audit_app.disabled_paths_id,alf_audit_entry.audit_values_id,alf_prop_unique_ctx.prop1_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_root.id,alf_audit_app.disabled_paths_id,alf_audit_entry.audit_values_id,alf_prop_unique_ctx.prop1_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_root.skipToId ---DELETE_NOT_EXISTS_V3 alf_prop_value.id,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_value.id,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_value.skipToId ---DELETE_NOT_EXISTS_V3 alf_prop_string_value.id,alf_prop_value.long_value."persisted_type in (3, 5, 6)",alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_string_value.id,alf_prop_value.long_value."persisted_type in (3, 5, 6)",alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_string_value.skipToId ---DELETE_NOT_EXISTS_V3 alf_prop_serializable_value.id,alf_prop_value.long_value.persisted_type=4,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_serializable_value.id,alf_prop_value.long_value.persisted_type=4,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_serializable_value.skipToId ---DELETE_NOT_EXISTS_V3 alf_prop_double_value.id,alf_prop_value.long_value.persisted_type=2,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize +--DELETE_NOT_EXISTS_V3 alf_prop_double_value.id,alf_prop_value.long_value.persisted_type=2,alf_audit_app.app_name_id,alf_audit_entry.audit_user_id,alf_prop_link.key_prop_id,alf_prop_link.value_prop_id,alf_prop_unique_ctx.value1_prop_id,alf_prop_unique_ctx.value2_prop_id,alf_prop_unique_ctx.value3_prop_id system.delete_not_exists.batchsize system.delete_not_exists.alf_prop_double_value.skipToId diff --git a/repository/src/main/resources/alfresco/repository.properties b/repository/src/main/resources/alfresco/repository.properties index 78d1558dbc..96fa268f7b 100644 --- a/repository/src/main/resources/alfresco/repository.properties +++ b/repository/src/main/resources/alfresco/repository.properties @@ -1254,6 +1254,12 @@ system.prop_table_cleaner.algorithm=V2 system.delete_not_exists.pauseAndRecoverBatchSize=500000 #Duration of the pause in milliseconds (default 10s) system.delete_not_exists.pauseAndRecoverTime=10000 +#Skip ids on job execution +system.delete_not_exists.alf_prop_root.skipToId=0 +system.delete_not_exists.alf_prop_value.skipToId=0 +system.delete_not_exists.alf_prop_string_value.skipToId=0 +system.delete_not_exists.alf_prop_serializable_value.skipToId=0 +system.delete_not_exists.alf_prop_double_value.skipToId=0 # --Node cleanup batch - default settings system.node_cleanup.delete_batchSize=1000 diff --git a/repository/src/test/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3ExecutorTest.java b/repository/src/test/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3ExecutorTest.java index af6e91e0da..bda4485243 100644 --- a/repository/src/test/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3ExecutorTest.java +++ b/repository/src/test/java/org/alfresco/repo/domain/schema/script/DeleteNotExistsV3ExecutorTest.java @@ -86,7 +86,7 @@ public class DeleteNotExistsV3ExecutorTest { scriptExecutor.executeScriptUrl("scriptexec/${db.script.dialect}/delete-not-exists/test-data1.sql"); - String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize"; + String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize system.delete_not_exists.temp_tst_tbl_1.skipToId"; int line = 1; File scriptFile = Mockito.mock(File.class); Properties properties = Mockito.mock(Properties.class); @@ -101,8 +101,8 @@ public class DeleteNotExistsV3ExecutorTest { when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_READ_ONLY)).thenReturn("true"); when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_TIMEOUT_SECONDS)).thenReturn("-1"); - DeleteNotExistsV3Executor DeleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties); - DeleteNotExistsV3Executor.execute(); + DeleteNotExistsV3Executor deleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties); + deleteNotExistsV3Executor.execute(); List res = jdbcTmpl.queryForList(select, String.class); assertEquals(7, res.size()); @@ -117,8 +117,8 @@ public class DeleteNotExistsV3ExecutorTest { when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_READ_ONLY)).thenReturn("false"); when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_TIMEOUT_SECONDS)).thenReturn("-1"); - DeleteNotExistsV3Executor DeleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties); - DeleteNotExistsV3Executor.execute(); + DeleteNotExistsV3Executor deleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties); + deleteNotExistsV3Executor.execute(); List res = jdbcTmpl.queryForList(select, String.class); assertEquals(5, res.size()); @@ -137,7 +137,7 @@ public class DeleteNotExistsV3ExecutorTest { scriptExecutor.executeScriptUrl("scriptexec/${db.script.dialect}/delete-not-exists/test-data1.sql"); - String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize"; + String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize system.delete_not_exists.temp_tst_tbl_1.skipToId"; int line = 1; File scriptFile = Mockito.mock(File.class); Properties properties = Mockito.mock(Properties.class); @@ -150,8 +150,8 @@ public class DeleteNotExistsV3ExecutorTest { when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_DELETE_BATCH_SIZE)).thenReturn("1"); when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_READ_ONLY)).thenReturn("false"); - DeleteNotExistsV3Executor DeleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties); - DeleteNotExistsV3Executor.execute(); + DeleteNotExistsV3Executor deleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties); + deleteNotExistsV3Executor.execute(); List res = jdbcTmpl.queryForList(select, String.class); assertEquals(5, res.size()); @@ -170,7 +170,7 @@ public class DeleteNotExistsV3ExecutorTest { scriptExecutor.executeScriptUrl("scriptexec/${db.script.dialect}/delete-not-exists/test-data1.sql"); - String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize"; + String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize system.delete_not_exists.temp_tst_tbl_1.skipToId"; int line = 1; File scriptFile = Mockito.mock(File.class); Properties properties = Mockito.mock(Properties.class); @@ -198,4 +198,42 @@ public class DeleteNotExistsV3ExecutorTest } } } -} \ No newline at end of file + + @Test() + public void testSkip() throws Exception + { + scriptExecutor.executeScriptUrl("scriptexec/${db.script.dialect}/delete-not-exists/test-data1.sql"); + + String sql = "--DELETE_NOT_EXISTS_V3 temp_tst_tbl_1.id,temp_tst_tbl_2.tbl_2_id,temp_tst_tbl_3.tbl_3_id,temp_tst_tbl_4.tbl_4_id system.delete_not_exists.batchsize system.delete_not_exists.temp_tst_tbl_1.skipToId"; + int line = 1; + File scriptFile = Mockito.mock(File.class); + Properties properties = Mockito.mock(Properties.class); + + String select = "select id from temp_tst_tbl_1 order by id ASC"; + + try (Connection connection = dataSource.getConnection()) + { + connection.setAutoCommit(true); + { + when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_BATCH_SIZE)).thenReturn("2"); + when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_READ_ONLY)).thenReturn("false"); + when(properties.getProperty(DeleteNotExistsV3Executor.PROPERTY_TIMEOUT_SECONDS)).thenReturn("-1"); + when(properties.getProperty("system.delete_not_exists.temp_tst_tbl_1.skipToId")).thenReturn("6"); + DeleteNotExistsV3Executor deleteNotExistsV3Executor = createDeleteNotExistsV3Executor(dialect, connection, sql, line, scriptFile, properties); + deleteNotExistsV3Executor.execute(); + + List res = jdbcTmpl.queryForList(select, String.class); + assertEquals(7, res.size()); + + // We are only processing Ids after 6, so all ids < 6 must remain untouched + assertEquals("1", res.get(0)); + assertEquals("2", res.get(1)); + assertEquals("3", res.get(2)); + assertEquals("4", res.get(3)); + assertEquals("5", res.get(4)); + assertEquals("10", res.get(5)); + assertEquals("11", res.get(6)); + } + } + } +}