diff --git a/config/alfresco/dbscripts/utility/org.hibernate.dialect.MySQLInnoDBDialect/CleanAlfPropTables.sql b/config/alfresco/dbscripts/utility/org.hibernate.dialect.MySQLInnoDBDialect/CleanAlfPropTables.sql index f59add0c2c..841b00304a 100644 --- a/config/alfresco/dbscripts/utility/org.hibernate.dialect.MySQLInnoDBDialect/CleanAlfPropTables.sql +++ b/config/alfresco/dbscripts/utility/org.hibernate.dialect.MySQLInnoDBDialect/CleanAlfPropTables.sql @@ -68,6 +68,18 @@ create table temp_del_double2 PRIMARY KEY (id) ) ENGINE=MyISAM; +-- Determine the maximum IDs in order to constrain deletion ranges and avoid deleting new data +--ASSIGN:PROP_ROOT_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_root; +--ASSIGN:PROP_VAL_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_value; +--ASSIGN:PROP_STRING_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_string_value; +--ASSIGN:PROP_SERIALIZABLE_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_serializable_value; +--ASSIGN:PROP_DOUBLE_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_double_value; + -- get all active references to alf_prop_root --FOREACH alf_audit_app.id system.upgrade.clean_alf_prop_tables.batchsize insert into temp_prop_root_ref select disabled_paths_id as id from alf_audit_app where id >= ${LOWERBOUND} and id <= ${UPPERBOUND}; @@ -78,7 +90,7 @@ insert into temp_prop_root_ref select prop1_id from alf_prop_unique_ctx where pr -- determine the obsolete entries from alf_prop_root --FOREACH alf_prop_root.id system.upgrade.clean_alf_prop_tables.batchsize -insert into temp_prop_root_obs select alf_prop_root.id from alf_prop_root left join temp_prop_root_ref on temp_prop_root_ref.id = alf_prop_root.id where temp_prop_root_ref.id is null and alf_prop_root.id >= ${LOWERBOUND} and alf_prop_root.id <= ${UPPERBOUND}; +insert into temp_prop_root_obs select alf_prop_root.id from alf_prop_root left join temp_prop_root_ref on temp_prop_root_ref.id = alf_prop_root.id where temp_prop_root_ref.id is null and alf_prop_root.id >= ${LOWERBOUND} and alf_prop_root.id <= ${UPPERBOUND} and alf_prop_root.id <= ${PROP_ROOT_MAX_ID}; -- clear alf_prop_root which cascades DELETE to alf_prop_link --FOREACH temp_prop_root_obs.id system.upgrade.clean_alf_prop_tables.batchsize @@ -102,7 +114,7 @@ insert into temp_prop_val_ref select value3_prop_id from alf_prop_unique_ctx whe -- determine the obsolete entries from alf_prop_value --FOREACH alf_prop_value.id system.upgrade.clean_alf_prop_tables.batchsize -insert into temp_prop_val_obs select apv.id, apv.persisted_type, apv.long_value from alf_prop_value apv left join temp_prop_val_ref on (apv.id = temp_prop_val_ref.id) where temp_prop_val_ref.id is null and apv.id >= ${LOWERBOUND} and apv.id <= ${UPPERBOUND}; +insert into temp_prop_val_obs select apv.id, apv.persisted_type, apv.long_value from alf_prop_value apv left join temp_prop_val_ref on (apv.id = temp_prop_val_ref.id) where temp_prop_val_ref.id is null and apv.id >= ${LOWERBOUND} and apv.id <= ${UPPERBOUND} and apv.id <= ${PROP_VAL_MAX_ID}; -- clear the obsolete entries --FOREACH temp_prop_val_obs.id system.upgrade.clean_alf_prop_tables.batchsize diff --git a/config/alfresco/dbscripts/utility/org.hibernate.dialect.PostgreSQLDialect/CleanAlfPropTables.sql b/config/alfresco/dbscripts/utility/org.hibernate.dialect.PostgreSQLDialect/CleanAlfPropTables.sql index cfd8a5e263..5b6fda3cf6 100644 --- a/config/alfresco/dbscripts/utility/org.hibernate.dialect.PostgreSQLDialect/CleanAlfPropTables.sql +++ b/config/alfresco/dbscripts/utility/org.hibernate.dialect.PostgreSQLDialect/CleanAlfPropTables.sql @@ -13,7 +13,6 @@ -- the rest of the values used in audit can be deleted from alf_prop_string_value, alf_prop_serializable_value, alf_prop_double_value. -- create temp tables ---BEGIN TXN create table temp_prop_root_ref ( id INT8 NOT NULL @@ -69,6 +68,18 @@ create table temp_del_double2 PRIMARY KEY (id) ); +-- Determine the maximum IDs in order to constrain deletion ranges and avoid deleting new data +--ASSIGN:PROP_ROOT_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_root; +--ASSIGN:PROP_VAL_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_value; +--ASSIGN:PROP_STRING_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_string_value; +--ASSIGN:PROP_SERIALIZABLE_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_serializable_value; +--ASSIGN:PROP_DOUBLE_MAX_ID=idmax!-1 +select max(id) as idmax from alf_prop_double_value; + -- get all active references to alf_prop_root --FOREACH alf_audit_app.id system.upgrade.clean_alf_prop_tables.batchsize insert into temp_prop_root_ref select disabled_paths_id as id from alf_audit_app where id >= ${LOWERBOUND} and id <= ${UPPERBOUND}; @@ -79,7 +90,7 @@ insert into temp_prop_root_ref select prop1_id from alf_prop_unique_ctx where pr -- determine the obsolete entries from alf_prop_root --FOREACH alf_prop_root.id system.upgrade.clean_alf_prop_tables.batchsize -insert into temp_prop_root_obs select alf_prop_root.id from alf_prop_root left join temp_prop_root_ref on temp_prop_root_ref.id = alf_prop_root.id where temp_prop_root_ref.id is null and alf_prop_root.id >= ${LOWERBOUND} and alf_prop_root.id <= ${UPPERBOUND}; +insert into temp_prop_root_obs select alf_prop_root.id from alf_prop_root left join temp_prop_root_ref on temp_prop_root_ref.id = alf_prop_root.id where temp_prop_root_ref.id is null and alf_prop_root.id >= ${LOWERBOUND} and alf_prop_root.id <= ${UPPERBOUND} and alf_prop_root.id <= ${PROP_ROOT_MAX_ID}; -- clear alf_prop_root which cascades DELETE to alf_prop_link --FOREACH temp_prop_root_obs.id system.upgrade.clean_alf_prop_tables.batchsize @@ -103,7 +114,7 @@ insert into temp_prop_val_ref select value3_prop_id from alf_prop_unique_ctx whe -- determine the obsolete entries from alf_prop_value --FOREACH alf_prop_value.id system.upgrade.clean_alf_prop_tables.batchsize -insert into temp_prop_val_obs select apv.id, apv.persisted_type, apv.long_value from alf_prop_value apv left join temp_prop_val_ref on (apv.id = temp_prop_val_ref.id) where temp_prop_val_ref.id is null and apv.id >= ${LOWERBOUND} and apv.id <= ${UPPERBOUND}; +insert into temp_prop_val_obs select apv.id, apv.persisted_type, apv.long_value from alf_prop_value apv left join temp_prop_val_ref on (apv.id = temp_prop_val_ref.id) where temp_prop_val_ref.id is null and apv.id >= ${LOWERBOUND} and apv.id <= ${UPPERBOUND} and apv.id <= ${PROP_VAL_MAX_ID}; -- clear the obsolete entries --FOREACH temp_prop_val_obs.id system.upgrade.clean_alf_prop_tables.batchsize @@ -148,5 +159,3 @@ delete from alf_prop_double_value apd using temp_del_double1 tdd where apd.id = -- insert into temp_del_double2 select apd.id from alf_prop_double_value apd left join alf_prop_value apv on apv.long_value = apd.id and apv.persisted_type = 2 where apv.id is null and apd.id >= ${LOWERBOUND} and apd.id <= ${UPPERBOUND}; -- FOREACH temp_del_double2.id system.upgrade.clean_alf_prop_tables.batchsize -- delete from alf_prop_double_value apd using temp_del_double2 tdd where apd.id = tdd.id and tdd.id >= ${LOWERBOUND} and tdd.id <= ${UPPERBOUND}; - ---END TXN diff --git a/source/test-java/org/alfresco/repo/domain/DomainTestSuite.java b/source/test-java/org/alfresco/repo/domain/DomainTestSuite.java index de52a6ed55..0ba469c6a0 100644 --- a/source/test-java/org/alfresco/repo/domain/DomainTestSuite.java +++ b/source/test-java/org/alfresco/repo/domain/DomainTestSuite.java @@ -27,6 +27,7 @@ import org.alfresco.repo.domain.mimetype.MimetypeDAOTest; import org.alfresco.repo.domain.node.NodeDAOTest; import org.alfresco.repo.domain.patch.AppliedPatchDAOTest; import org.alfresco.repo.domain.permissions.AclCrudDAOTest; +import org.alfresco.repo.domain.propval.PropertyValueCleanupTest; import org.alfresco.repo.domain.propval.PropertyValueDAOTest; import org.alfresco.repo.domain.qname.QNameDAOTest; import org.alfresco.repo.domain.query.CannedQueryDAOTest; @@ -52,6 +53,7 @@ import org.junit.runners.Suite; LocaleDAOTest.class, QNameDAOTest.class, PropertyValueDAOTest.class, + PropertyValueCleanupTest.class, AuditDAOTest.class, AppliedPatchDAOTest.class, AclCrudDAOTest.class, diff --git a/source/test-java/org/alfresco/repo/domain/propval/PropertyValueCleanupTest.java b/source/test-java/org/alfresco/repo/domain/propval/PropertyValueCleanupTest.java new file mode 100644 index 0000000000..c5924ffadb --- /dev/null +++ b/source/test-java/org/alfresco/repo/domain/propval/PropertyValueCleanupTest.java @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.repo.domain.propval; + +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.alfresco.repo.transaction.RetryingTransactionHelper; +import org.alfresco.service.cmr.attributes.AttributeService; +import org.alfresco.service.transaction.TransactionService; +import org.alfresco.test_category.OwnJVMTestsCategory; +import org.alfresco.util.ApplicationContextHelper; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.springframework.context.ApplicationContext; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * @see PropertyValueDAO#cleanupUnusedValues() + * + * @author Derek Hulley + * @since 5.1 + */ +@Category(OwnJVMTestsCategory.class) +public class PropertyValueCleanupTest +{ + private ApplicationContext ctx = ApplicationContextHelper.getApplicationContext(); + + private TransactionService transactionService; + private AttributeService attributeService; + private RetryingTransactionHelper txnHelper; + private PropertyValueDAO propertyValueDAO; + + @Before + public void setUp() throws Exception + { + transactionService = (TransactionService) ctx.getBean("TransactionService"); + txnHelper = transactionService.getRetryingTransactionHelper(); + txnHelper.setMaxRetries(0); + + attributeService = (AttributeService) ctx.getBean("AttributeService"); + propertyValueDAO = (PropertyValueDAO) ctx.getBean("propertyValueDAO"); + + // Remove the caches to test all functionality + clearCaches(); + } + + private void clearCaches() + { + ((AbstractPropertyValueDAOImpl)propertyValueDAO).clearCaches(); + } + + @Test + public synchronized void testRapidCreationDuringCleanup() throws Exception + { + // Create and delete some well-known attributes + String toDeleteKey1 = "testRapidCreationDuringCleanup"; + String toDeleteKey2 = UUID.randomUUID().toString(); + byte[] toDeleteProp = new String("Key is " + toDeleteKey2).getBytes("US-ASCII"); + assertNull("Did not expect to find the attribute ", attributeService.getAttribute(toDeleteKey1, toDeleteKey2)); + assertNull("Key2 should be NOT present as a property value", propertyValueDAO.getPropertyStringValue(toDeleteKey2)); + attributeService.createAttribute(toDeleteProp, toDeleteKey1, toDeleteKey2); + assertNotNull("Did not find the attribute ", attributeService.getAttribute(toDeleteKey1, toDeleteKey2)); + // Check that we can get hold of the underlying property + assertNotNull("Key2 should be present as a property value", propertyValueDAO.getPropertyStringValue(toDeleteKey2)); + // Delete the attribute + attributeService.removeAttribute(toDeleteKey1, toDeleteKey2); + // Check that we can STILL get hold of the underlying property + assertNotNull("Key2 should be present as a property value (even if unreferenced)", propertyValueDAO.getPropertyStringValue(toDeleteKey2)); + + // Start threads that throw stuff into the AttributeService + ThreadGroup threadGroup = new ThreadGroup("testRapidCreationDuringCleanup"); + InsertSerializableAttributes[] runnables = new InsertSerializableAttributes[2]; + for (int i = 0; i < runnables.length; i++) + { + // Runnable + runnables[i] = new InsertSerializableAttributes(); + // Put it in a thread + String threadName = "" + i; + Thread thread = new Thread(threadGroup, runnables[i], threadName); + thread.setDaemon(true); // Precautionary + // Start it + thread.start(); + } + + // Wait a bit for data to really get in there + wait(1000L); + + // Now run the cleanup script + propertyValueDAO.cleanupUnusedValues(); + + // Stop the threads + for (int i = 0; i < runnables.length; i++) + { + // Runnable + runnables[i].running.set(false); + } + + // Clear any caches + clearCaches(); + + // The cleanup should have removed the key2 + assertNull("Key2 should be NOT present as a property value (cleanup job)", propertyValueDAO.getPropertyStringValue(toDeleteKey2)); + + // Now check that all the properties written can still be retrieved + for (int i = 0; i < runnables.length; i++) + { + // Runnable + String key1 = runnables[i].key1; + String key2 = runnables[i].key2; + int maxKey3 = runnables[i].counter.get(); + for (int j = 0; j < maxKey3; j++) + { + Integer key3 = j; + // Get the attribute + byte[] propFetched = (byte[]) attributeService.getAttribute(key1, key2, key3); + assertTrue( + "Arrays were not equal for " + key1 + ", " + key2 + ", " + key3, + Arrays.equals(runnables[i].prop, propFetched)); + } + } + } + + /** + * Simple runnable that continuously creates new serializable attributes until stopped. + * Each thread has a unique second key value, a sequential third key and generates serializable (unshared) + * property values. + * + * @author Derek Hulley + */ + private class InsertSerializableAttributes implements Runnable + { + private final String key1; + private final String key2; + private final byte[] prop; + + private AtomicBoolean running = new AtomicBoolean(true); + private AtomicInteger counter = new AtomicInteger(0); + + private InsertSerializableAttributes() throws UnsupportedEncodingException + { + key1 = "PropertyValueCleanupTest"; + key2 = UUID.randomUUID().toString(); + prop = new String("Key is " + key2).getBytes("US-ASCII"); + } + + @Override + public synchronized void run() + { + while (running.get()) + { + Integer key3 = counter.get(); + attributeService.createAttribute(prop, key1, key2, key3); + // Increment the counter + counter.incrementAndGet(); + // Wait a bit so that we don't drown the test + try { wait(10L); } catch (InterruptedException e) {} + } + } + } +} diff --git a/source/test-java/org/alfresco/repo/domain/propval/PropertyValueDAOTest.java b/source/test-java/org/alfresco/repo/domain/propval/PropertyValueDAOTest.java index f61d65708e..6d34efe5f9 100644 --- a/source/test-java/org/alfresco/repo/domain/propval/PropertyValueDAOTest.java +++ b/source/test-java/org/alfresco/repo/domain/propval/PropertyValueDAOTest.java @@ -18,8 +18,6 @@ */ package org.alfresco.repo.domain.propval; -import static org.junit.Assert.*; - import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; @@ -36,20 +34,26 @@ import javax.naming.CompositeName; import org.alfresco.repo.domain.propval.PropertyValueDAO.PropertyFinderCallback; import org.alfresco.repo.transaction.RetryingTransactionHelper; import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; -import org.alfresco.service.ServiceRegistry; import org.alfresco.service.transaction.TransactionService; import org.alfresco.test_category.OwnJVMTestsCategory; import org.alfresco.util.ApplicationContextHelper; import org.alfresco.util.GUID; import org.alfresco.util.Pair; -import org.junit.experimental.categories.Category; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.springframework.context.ApplicationContext; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.extensions.surf.util.ISO8601DateFormat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * @see PropertyValueDAO * @@ -68,8 +72,7 @@ public class PropertyValueDAOTest @Before public void setUp() throws Exception { - ServiceRegistry serviceRegistry = (ServiceRegistry) ctx.getBean(ServiceRegistry.SERVICE_REGISTRY); - transactionService = serviceRegistry.getTransactionService(); + transactionService = (TransactionService) ctx.getBean("TransactionService"); txnHelper = transactionService.getRetryingTransactionHelper(); txnHelper.setMaxRetries(0);