From e0b40d177f5473ee70920e4d3054b54342955dd7 Mon Sep 17 00:00:00 2001 From: Dave Ward Date: Fri, 15 Oct 2010 15:43:21 +0000 Subject: [PATCH] =?UTF-8?q?Merged=20V3.3-BUG-FIX=20to=20HEAD=20=20=20=2023?= =?UTF-8?q?080:=20Fix=20for=20ALF-3815=20-=20Error=20occur=20on=20creating?= =?UTF-8?q?=20user=20(Active=20Directory=20+LDAP=20authentication)=20=20?= =?UTF-8?q?=20=2023084:=20MERGED=20DEV=20to=20V3.3-BUG-FIX=20=20=20=20=20?= =?UTF-8?q?=20=2022839=20:=20=20ALF-4920=20-=20IMAP=20server=20UID=20failu?= =?UTF-8?q?re=20=20=20=2023102:=20Checked=20in=20file=20with=20my=20Hostna?= =?UTF-8?q?me!=20=20=20=2023141:=20Merged=20PATCHES/V3.2.0=20to=20V3.3-BUG?= =?UTF-8?q?-FIX=20=20=20=20=20=20=2022977:=20ALF-5057:=20Don't=20use=20luc?= =?UTF-8?q?ene=20to=20locate=20tag=20nodes=20-=20unreliable=20in=20a=20clu?= =?UTF-8?q?ster=20=20=20=20=20=20=20=20=20=20-=20CategoryService=20extende?= =?UTF-8?q?d=20with=20root=20category=20retrieval=20method=20using=20node?= =?UTF-8?q?=20service=20=20=20=20=20=20=2023043:=20ALF-5057:=20Merged=20V3?= =?UTF-8?q?.2=20to=20PATCHES/V3.2.0=20(partial)=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=2018052:=20Merged=20DEV/REPO-DOCLIB=20to=20V3.2=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=2017653:=20Checkpoint=20Repo=20DocLib=20p?= =?UTF-8?q?rototype=20work=20=20=20=2023142:=20Merged=20PATCHES/V3.2.0=20t?= =?UTF-8?q?o=20V3.3-BUG-FIX=20=20=20=20=20=20=2022981:=20ALF-5141:=20Need?= =?UTF-8?q?=20to=20limit=20webscript=20response=20times=20and=20reject=20t?= =?UTF-8?q?raffic=20at=20high=20load=20=20=20=20=20=20=20=20=20=20-=20serv?= =?UTF-8?q?er.web.transaction.max-duration-ms=20property=20now=20specifies?= =?UTF-8?q?=20a=20maximum=20time=20for=20repository=20webscript=20transact?= =?UTF-8?q?ion=20execution.=20Default=20is=2010=20seconds.=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20-=20transaction=20retrying=20will=20not=20contin?= =?UTF-8?q?ue=20when=20the=20projected=20time=20is=20greater=20than=20this?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20-=20Once=20a=20transaction=20hits?= =?UTF-8?q?=20this=20execution=20time=20the=20number=20of=20concurrently?= =?UTF-8?q?=20executing=20transactions=20at=20the=20time=20it=20was=20star?= =?UTF-8?q?ted=20becomes=20the=20=E2=80=98ceiling=E2=80=99=20for=20the=20n?= =?UTF-8?q?umber=20of=20concurrent=20transactions=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20-=20The=20ceiling=20will=20dynamically=20rise=20and=20fal?= =?UTF-8?q?l,=20based=20on=20transaction=20execution=20times=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20-=20When=20a=20transaction=20is=20started=20?= =?UTF-8?q?=E2=80=98above=E2=80=99=20the=20current=20ceiling=20a=20TooBusy?= =?UTF-8?q?Exception=20is=20thrown,=20which=20is=20mapped=20to=20an=20imme?= =?UTF-8?q?diate=20status=20503=20response=20=20=20=20=20=20=20=20=20=20-?= =?UTF-8?q?=20New=20unit=20test=20added=20for=20this=20=20=20=20=20=20=202?= =?UTF-8?q?3006:=20ALF-5141:=20Reverting=20IndexInfo=20changes=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20-=20'fairness'=20flag=20on=20ReentrantReadWri?= =?UTF-8?q?teLock=20appears=20to=20cause=20deadlock=20on=20JDK=201.5=20in?= =?UTF-8?q?=20IndexInfoTest=20=20=20=20=20=20=20=20=20=20-=20lucene.indexe?= =?UTF-8?q?r.maxMergeWait=20property=20and=20associated=20throttling=20'ba?= =?UTF-8?q?ck=20off'=20behaviour=20abandoned=20as=20it=20has=20the=20risk?= =?UTF-8?q?=20of=20leaving=20indexes=20in=20incomplete=20uncommited=20stat?= =?UTF-8?q?e=20=20=20=20=20=20=20=20=20=20-=20transaction=20limiter=20feat?= =?UTF-8?q?ure=20should=20be=20enough=20to=20avoid=20excessive=20wait=20ti?= =?UTF-8?q?mes=20=20=20=20=20=20=2023011:=20ALF-5141:=20Reintroduce=20fair?= =?UTF-8?q?=20locking=20to=20IndexInfo=20and=20fix=20RetryingTransactionHe?= =?UTF-8?q?lperTest=20=20=20=20=20=20=20=20=20=20-=20Bugs=20surrounding=20?= =?UTF-8?q?ReentrantReadWriteLock=20in=20old=20JVMs=20mean=20that=20it's?= =?UTF-8?q?=20not=20safe=20to=20make=20fair=20locking=20the=20default=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20-=20However,=20it=20would=20be=20use?= =?UTF-8?q?ful=20in=20new=20JVMs=20as=20it=20should=20guarantee=20that=20w?= =?UTF-8?q?e=20don't=20lock=20out=20waiting=20writers=20indefinitely=20und?= =?UTF-8?q?er=20high=20load=20=20=20=20=20=20=20=20=20=20-=20Now=20control?= =?UTF-8?q?led=20by=20lucene.indexer.fairLocking=20property.=20Default=20v?= =?UTF-8?q?alue=20is=20false=20in=20V3.2.0=20but=20true=20in=20V3.3.4=20on?= =?UTF-8?q?wards.=20=20=20=20=20=20=20=20=20=20-=20RetryingTransactionHelp?= =?UTF-8?q?erTest=20now=20uses=20latches=20to=20ensure=20test=20threads=20?= =?UTF-8?q?start=20up=20in=20strict=20sequential=20order=20=20=20=20=20=20?= =?UTF-8?q?=2023014:=20ALF-5141:=20Correct=20error=20that=20could=20allow?= =?UTF-8?q?=20transaction=20ceiling=20to=20be=20lowered=20to=20zero=20=20?= =?UTF-8?q?=20=2023146:=20(RECORD=20ONLY)=20ALF-5028:=20Merged=20HEAD=20to?= =?UTF-8?q?=20V3.3-BUG-FIX=20=20=20=20=20=20=2021471:=20SAIL-240=20(SAIL-2?= =?UTF-8?q?94)=20AuditDAO:=20AuditService=20enhancements=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20-=20Added=20isAuditEnabled=20and=20enableAudit=20fo?= =?UTF-8?q?r=20global=20case=20(system-wide)=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?-=20Some=20neatening=20up=20of=20Audit=20SQL=20(common=20WHERE?= =?UTF-8?q?=20and=20ORDER=20BY=20clauses)=20=20=20=20=20=20=20=20=20=20-?= =?UTF-8?q?=20AuditService=20enforces=20'admin'=20role=20for=20all=20metho?= =?UTF-8?q?ds=20=20=20=20=20=20=2022109:=20ALF-4106:=20Added=20entry=20del?= =?UTF-8?q?etion=20count=20return=20value=20for=20clear()=20=20=20=20=20?= =?UTF-8?q?=20=2022726:=20Coding=20standards=20=20=20=20=20=20=2022857:=20?= =?UTF-8?q?Fix=20typo=20in=20javadoc=20=20=20=20=20=20=2022980:=20Added=20?= =?UTF-8?q?AuditService.clearAudit(List)=20=20=20=20=20=20=2022986:?= =?UTF-8?q?=20ALF-5028=20-=20Tagging=20Service=20Update=20-=20Use=20the=20?= =?UTF-8?q?audit=20service=20as=20a=20persisted=20event=20log,=20so=20that?= =?UTF-8?q?=20tag=20scope=20updates=20can=20occur=20in=20batches=20and=20w?= =?UTF-8?q?ithout=20contention=20issues.=20(Further=20tests=20and=20post-s?= =?UTF-8?q?tartup=20executor=20still=20needed)=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20This=20commit=20enables=20the=20Audit=20Service=20by=20defau?= =?UTF-8?q?lt,=20but=20turns=20off=20all=20the=20audit=20applications=20ex?= =?UTF-8?q?cept=20tagging=20by=20default,=20so=20there=20shouldn't=20be=20?= =?UTF-8?q?any=20noticable=20changes=20=20=20=20=20=20=2022997:=20ALF-5028?= =?UTF-8?q?=20-=20More=20tag=20scope=20updates=20and=20unit=20tests.=20Sho?= =?UTF-8?q?rtly=20after=20the=20system=20is=20started,=20check=20for=20un-?= =?UTF-8?q?applied=20tag=20scope=20updates,=20and=20apply=20them.=20=20=20?= =?UTF-8?q?=20=20=20=2023015:=20ALF-5028:=20Tagging=20test=20mods=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20-=20Join=20onto=20first-level=20threads=20?= =?UTF-8?q?to=20be=20sure=20that=20first=20round=20of=20tagging=20has=20be?= =?UTF-8?q?en=20done=20=20=20=20=20=20=20=20=20=20-=20Double-checks=20for?= =?UTF-8?q?=20transaction=20leaks=20(found=201)=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20-=20Some=20formatting=20(new=20test=20only,=20but=20should?= =?UTF-8?q?=20be=20applied=20to=20file)=20=20=20=2023148:=20Merged=20PATCH?= =?UTF-8?q?ES/V3.2.0=20to=20V3.3-BUG-FIX=20=20=20=20=20=20=2023133:=20ALF-?= =?UTF-8?q?5221:=20Fixed=20file=20handle=20leaks=20in=20TaggingService=20?= =?UTF-8?q?=20=20=2023149:=20Merged=20V3.2=20to=20V3.3-BUG-FIX=20=20=20=20?= =?UTF-8?q?=20=20=2023070:=20Part-fix=20ALF-5134:=20Performance=20of=20Alf?= =?UTF-8?q?resco=20cluster=20less=20than=20performance=20of=20single=20nod?= =?UTF-8?q?e=20=20=20=20=20=20=20=20=20=20-=20Prevent=20cache=20being=20up?= =?UTF-8?q?dated=20even=20when=20there=20are=20no=20changes=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20-=20Prevents=20some=20cache=20invalidation=20mes?= =?UTF-8?q?sages=20during=20read=20operations=20=20=20=20=20=20=2023071:?= =?UTF-8?q?=20ALF-5134:=20Performance=20of=20Alfresco=20cluster=20less=20t?= =?UTF-8?q?han=20performance=20of=20single=20node=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20-=20Removed=20null-equivalence=20check=20in=20Transaction?= =?UTF-8?q?alCache=20=20=20=20=20=20=20=20=20=20-=20Avoids=20cache=20updat?= =?UTF-8?q?e=20messages=20when=20running=20against=20empty=20caches=20=20?= =?UTF-8?q?=20=2023150:=20(RECORD=20ONLY)=20ALF-5235:=20Merged=20HEAD=20to?= =?UTF-8?q?=20V3.3-BUG-FIX=20=20=20=20=20=20=2022695:=20ALF-3800=20"File?= =?UTF-8?q?=20is=20uploaded=20to=20the=20Document=20Library=20when=20its?= =?UTF-8?q?=20size=20more=20than=20user=20quota":=20make=20sure=20the=20ex?= =?UTF-8?q?ception=20is=20thrown=20back=20up=20to=20the=20transaction=20ma?= =?UTF-8?q?chinery=20to=20perform=20a=20rollback=20=20=20=2023156:=20Merge?= =?UTF-8?q?d=20V3.3=20to=20V3.3-BUG-FIX=20=20=20=20=20=20=2022913:=20Add?= =?UTF-8?q?=20jars=20back=20into=20Tomcat=20bundles=20=20=20=20=20=20=2023?= =?UTF-8?q?028:=20Merged=20DEV=20to=20V33:=20=20=20=20=20=20=20=20=20=2023?= =?UTF-8?q?022:=20ALF-4760=20:=20XAM=20post-retention=20cleanup=20job:=20X?= =?UTF-8?q?AMArchiveJob=20=20=20=20=20=20=20=20=20=20=20=20=201.=20Post-re?= =?UTF-8?q?tention=20xam=20cleanup=20job=20was=20implemented=20according?= =?UTF-8?q?=20to=20requirements=20provided=20by=20Derek.=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=202.=20Unit=20tests=20was=20added=20for=20ne?= =?UTF-8?q?w=20functionality.=20=20=20=20=20=20=2023125:=20Merged=20HEAD?= =?UTF-8?q?=20to=20V3.3=20=20=20=20=20=20=20=20=20=2020752:=20BatchProcess?= =?UTF-8?q?or=20is=20fed=20work=20by=20a=20BatchProcessWorkProvider=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=2022297:=20Fixed=20ALF-4676:=20WorkProvid?= =?UTF-8?q?erIterator=20over=20BatchProcessWorkProvider=20does=20not=20fet?= =?UTF-8?q?ch=20all=20results=20=20=20=20=20=20=2023126:=20(RECORD=20ONLY)?= =?UTF-8?q?=20Merged=20BRANCHES/DEV/V3.3-BUG-FIX=20to=20BRANCHES/V3.3:=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=2022883:=20ALF-4800=20-=20AVM=20-=20in?= =?UTF-8?q?termittent=20test=20failure=20(layered=20file=20delete)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@23161 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261 --- config/alfresco/core-services-context.xml | 28 + config/alfresco/domain/transaction.properties | 5 +- config/alfresco/repository.properties | 6 + .../org/alfresco/repo/cache/CacheTest.java | 11 +- .../repo/cache/TransactionalCache.java | 59 +- .../repo/imap/AbstractMimeMessage.java | 4 + .../repo/imap/AlfrescoImapFolder.java | 96 +++- .../repo/imap/ContentModelMessage.java | 24 +- .../alfresco/repo/imap/ImapMessageTest.java | 523 ++++++++++++++++++ ...stractLuceneIndexerAndSearcherFactory.java | 13 +- .../lucene/LuceneCategoryServiceImpl.java | 56 +- .../repo/search/impl/lucene/LuceneConfig.java | 8 + .../search/impl/lucene/index/IndexInfo.java | 7 +- .../security/person/PersonServiceImpl.java | 11 +- .../repo/tagging/TaggingServiceImpl.java | 54 +- .../RetryingTransactionHelper.java | 403 ++++++++------ .../RetryingTransactionHelperTest.java | 178 ++++++ .../repo/transaction/TooBusyException.java | 75 +++ .../service/cmr/search/CategoryService.java | 31 ++ 19 files changed, 1340 insertions(+), 252 deletions(-) create mode 100644 source/java/org/alfresco/repo/imap/ImapMessageTest.java create mode 100644 source/java/org/alfresco/repo/transaction/TooBusyException.java diff --git a/config/alfresco/core-services-context.xml b/config/alfresco/core-services-context.xml index b4c9b25500..a735a97796 100644 --- a/config/alfresco/core-services-context.xml +++ b/config/alfresco/core-services-context.xml @@ -401,6 +401,31 @@ + + + + + + + ${server.transaction.max-retries} + + + ${server.transaction.min-retry-wait-ms} + + + ${server.transaction.max-retry-wait-ms} + + + ${server.transaction.wait-increment-ms} + + + ${server.web.transaction.max-duration-ms} + + + @@ -718,6 +743,9 @@ ${lucene.indexer.maxFieldLength} + + ${lucene.indexer.fairLocking} + ${lucene.write.lock.timeout} diff --git a/config/alfresco/domain/transaction.properties b/config/alfresco/domain/transaction.properties index 0071eb5c2a..82862d0b2e 100644 --- a/config/alfresco/domain/transaction.properties +++ b/config/alfresco/domain/transaction.properties @@ -18,4 +18,7 @@ server.transaction.wait-increment-ms=100 server.setup.transaction.max-retries=40 server.setup.transaction.min-retry-wait-ms=15000 server.setup.transaction.max-retry-wait-ms=15000 -server.setup.transaction.wait-increment-ms=10 \ No newline at end of file +server.setup.transaction.wait-increment-ms=10 + +# Try to limit web transactions to a maximum of 10 seconds +server.web.transaction.max-duration-ms=10000 \ No newline at end of file diff --git a/config/alfresco/repository.properties b/config/alfresco/repository.properties index 7560aa2432..9c3c3d5898 100644 --- a/config/alfresco/repository.properties +++ b/config/alfresco/repository.properties @@ -238,6 +238,12 @@ lucene.indexer.defaultMLSearchAnalysisMode=EXACT_LANGUAGE_AND_ALL # The number of terms from a document that will be indexed # lucene.indexer.maxFieldLength=10000 + +# Should we use a 'fair' locking policy, giving queue-like access behaviour to +# the indexes and avoiding starvation of waiting writers? Set to false on old +# JVMs where this appears to cause deadlock +lucene.indexer.fairLocking=true + # # Index locks (mostly deprecated and will be tidied up with the next lucene upgrade) # diff --git a/source/java/org/alfresco/repo/cache/CacheTest.java b/source/java/org/alfresco/repo/cache/CacheTest.java index 80b2adc76c..76ffbe1308 100644 --- a/source/java/org/alfresco/repo/cache/CacheTest.java +++ b/source/java/org/alfresco/repo/cache/CacheTest.java @@ -27,11 +27,10 @@ import javax.transaction.UserTransaction; import junit.framework.TestCase; import net.sf.ehcache.CacheManager; -import org.alfresco.repo.cache.TransactionalCache.NullValueMarker; import org.alfresco.repo.transaction.AlfrescoTransactionSupport; import org.alfresco.repo.transaction.RetryingTransactionHelper; -import org.alfresco.repo.transaction.TransactionListenerAdapter; import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; +import org.alfresco.repo.transaction.TransactionListenerAdapter; import org.alfresco.service.ServiceRegistry; import org.alfresco.service.transaction.TransactionService; import org.alfresco.util.ApplicationContextHelper; @@ -206,9 +205,9 @@ public class CacheTest extends TestCase // update 3 in the cache transactionalCache.put(UPDATE_TXN_THREE, "XXX"); assertEquals("Item not updated in txn cache", "XXX", transactionalCache.get(UPDATE_TXN_THREE)); - assertFalse("Item was put into backing cache (excl. NullValueMarker)", - backingCache.contains(UPDATE_TXN_THREE) && - !(backingCache.get(UPDATE_TXN_THREE) instanceof NullValueMarker)); + assertFalse( + "Item was put into backing cache (excl. NullValueMarker)", + backingCache.contains(UPDATE_TXN_THREE)); // check that the keys collection is correct Collection transactionalKeys = transactionalCache.getKeys(); @@ -597,7 +596,7 @@ public class CacheTest extends TestCase return null; } }; - executeAndCheck(callback, COMMON_KEY, null); + executeAndCheck(callback, COMMON_KEY, "AAA"); // Relaxed for ALF-5134 } /** *
    diff --git a/source/java/org/alfresco/repo/cache/TransactionalCache.java b/source/java/org/alfresco/repo/cache/TransactionalCache.java index e29f78fe3a..caf24fd27c 100644 --- a/source/java/org/alfresco/repo/cache/TransactionalCache.java +++ b/source/java/org/alfresco/repo/cache/TransactionalCache.java @@ -249,7 +249,7 @@ public class TransactionalCache } /** - * Fetches a value from the shared cache, checking for {@link NullValueMarker null markers}. + * Fetches a value from the shared cache. * * @param key the key * @return Returns the value or null @@ -257,16 +257,7 @@ public class TransactionalCache @SuppressWarnings("unchecked") private V getSharedCacheValue(K key) { - Object valueObj = sharedCache.get(key); - if (valueObj instanceof NullValueMarker) - { - // Someone has already marked this as a null - return null; - } - else - { - return (V) valueObj; - } + return (V) sharedCache.get(key); } /** @@ -413,16 +404,13 @@ public class TransactionalCache CacheBucket bucket = null; if (existingValueObj == null) { - // Insert a 'null' marker into the shared cache - NullValueMarker nullMarker = new NullValueMarker(); - sharedCache.put(key, nullMarker); + // ALF-5134: Performance of Alfresco cluster less than performance of single node + // The 'null' marker that used to be inserted also triggered an update in the afterCommit + // phase; the update triggered cache invalidation in the cluster. Now, the null cannot + // be verified to be the same null - there is no null equivalence + // // The value didn't exist before - bucket = new NewCacheBucket(nullMarker, value); - } - else if (existingValueObj instanceof NullValueMarker) - { - // Record the null as is - bucket = new NewCacheBucket((NullValueMarker)existingValueObj, value); + bucket = new NewCacheBucket(value); } else { @@ -698,17 +686,6 @@ public class TransactionalCache txnData.isClosed = true; } - /** - * Instances of this class are used to mark the shared cache null values for cases where - * new values are going to be inserted into it. - * - * @author Derek Hulley - */ - public static class NullValueMarker implements Serializable - { - private static final long serialVersionUID = -8384777298845693563L; - } - /** * Interface for the transactional cache buckets. These hold the actual values along * with some state and behaviour around writing from the in-transaction caches to the @@ -733,7 +710,6 @@ public class TransactionalCache /** * A bucket class to hold values for the caches.
    - * The cache assumes the presence of a marker object to * * @author Derek Hulley */ @@ -742,11 +718,9 @@ public class TransactionalCache private static final long serialVersionUID = -8536386687213957425L; private final BV value; - private final NullValueMarker nullMarker; - public NewCacheBucket(NullValueMarker nullMarker, BV value) + public NewCacheBucket(BV value) { this.value = value; - this.nullMarker = nullMarker; } public BV getValue() { @@ -757,20 +731,13 @@ public class TransactionalCache Object sharedValue = sharedCache.get(key); if (sharedValue != null) { - if (sharedValue == nullMarker) - { - // The shared cache entry didn't change during the txn and is safe for writing - sharedCache.put(key, value); - } - else - { - // The shared value has moved on since - sharedCache.remove(key); - } + // A value was added during the txn + sharedCache.remove(key); } else { - // The shared cache no longer has a value + // The shared cache value is still null (it might have changed a few times, though) so write + sharedCache.put(key, value); } } } diff --git a/source/java/org/alfresco/repo/imap/AbstractMimeMessage.java b/source/java/org/alfresco/repo/imap/AbstractMimeMessage.java index 8968060689..318e77f9b0 100644 --- a/source/java/org/alfresco/repo/imap/AbstractMimeMessage.java +++ b/source/java/org/alfresco/repo/imap/AbstractMimeMessage.java @@ -281,6 +281,10 @@ public abstract class AbstractMimeMessage extends MimeMessage return model; } + protected void updateMessageID() throws MessagingException + { + setHeader("Message-ID", this.messageFileInfo.getNodeRef().getId()); + } } diff --git a/source/java/org/alfresco/repo/imap/AlfrescoImapFolder.java b/source/java/org/alfresco/repo/imap/AlfrescoImapFolder.java index 9d0067da5b..72a0e46c68 100644 --- a/source/java/org/alfresco/repo/imap/AlfrescoImapFolder.java +++ b/source/java/org/alfresco/repo/imap/AlfrescoImapFolder.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -127,6 +128,35 @@ public class AlfrescoImapFolder extends AbstractImapFolder private Map messages = new TreeMap(); private Map msnCache = new HashMap(); + + private Map messagesCache = Collections.synchronizedMap(new MaxSizeMap(10, CACHE_SIZE)); + + /** + * Map that ejects the last recently used element(s) to keep the size to a + * specified maximum + * + * @param Key + * @param Value + */ + private class MaxSizeMap extends LinkedHashMap + { + private int maxSize; + + public MaxSizeMap(int initialSize, int maxSize) + { + super(initialSize, 0.75f, true); + this.maxSize = maxSize; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) + { + boolean remove = super.size() > this.maxSize; + return remove; + } + } + + private final static int CACHE_SIZE = 20; private static final Flags PERMANENT_FLAGS = new Flags(); @@ -441,10 +471,40 @@ public class AlfrescoImapFolder extends AbstractImapFolder */ @Override protected SimpleStoredMessage getMessageInternal(long uid) throws MessagingException - { + { AbstractMimeMessage mes = (AbstractMimeMessage) messages.get(uid).getMimeMessage(); FileInfo mesInfo = mes.getMessageInfo(); - return createImapMessage(mesInfo, uid, true); + + Date modified = (Date) serviceRegistry.getNodeService().getProperty(mesInfo.getNodeRef(), ContentModel.PROP_MODIFIED); + if(modified != null) + { + CacheItem cached = messagesCache.get(uid); + if (cached != null) + { + if (logger.isDebugEnabled()) + { + logger.debug("retrieved message from cache uid: " + uid); + } + if (cached.getModified().equals(modified)) + { + return cached.getMessage(); + } + } + SimpleStoredMessage message = createImapMessage(mesInfo, uid, true); + messagesCache.put(uid, new CacheItem(modified, message)); + + if (logger.isDebugEnabled()) + { + logger.debug("caching message uid: " + uid + " cacheSize: " + messagesCache.size()); + } + + return message; + } + else + { + SimpleStoredMessage message = createImapMessage(mesInfo, uid, true); + return message; + } } /** @@ -1055,4 +1115,36 @@ public class AlfrescoImapFolder extends AbstractImapFolder FileCopyUtils.copy(part.getInputStream(), os); } + class CacheItem + { + private Date modified; + private SimpleStoredMessage message; + + public CacheItem(Date modified, SimpleStoredMessage message) + { + this.setMessage(message); + this.setModified(modified); + } + + public void setModified(Date modified) + { + this.modified = modified; + } + + public Date getModified() + { + return modified; + } + + public void setMessage(SimpleStoredMessage message) + { + this.message = message; + } + + public SimpleStoredMessage getMessage() + { + return message; + } + } + } diff --git a/source/java/org/alfresco/repo/imap/ContentModelMessage.java b/source/java/org/alfresco/repo/imap/ContentModelMessage.java index ae1a22f396..ff49be33a4 100644 --- a/source/java/org/alfresco/repo/imap/ContentModelMessage.java +++ b/source/java/org/alfresco/repo/imap/ContentModelMessage.java @@ -25,6 +25,7 @@ import java.util.Map; import javax.mail.Address; import javax.mail.MessagingException; import javax.mail.Multipart; +import javax.mail.internet.ContentType; import javax.mail.internet.MimeBodyPart; import javax.mail.internet.MimeMessage; import javax.mail.internet.MimeMultipart; @@ -93,7 +94,7 @@ public class ContentModelMessage extends AbstractMimeMessage */ private Multipart buildContentModelMultipart() throws MessagingException { - MimeMultipart rootMultipart = new MimeMultipart("alternative"); + MimeMultipart rootMultipart = new AlfrescoMimeMultipart("alternative", this.messageFileInfo); // Cite MOB-395: "email agent will be used to select an appropriate template" - we are not able to // detect an email agent so we use a default template for all messages. // See AlfrescoImapConst to see the possible templates to use. @@ -113,4 +114,25 @@ public class ContentModelMessage extends AbstractMimeMessage return result; } + + class AlfrescoMimeMultipart extends MimeMultipart + { + public AlfrescoMimeMultipart(String subtype, FileInfo messageFileInfo) + { + super(); + String boundary = getBoundaryValue(messageFileInfo); + ContentType cType = new ContentType("multipart", subtype, null); + cType.setParameter("boundary", boundary); + contentType = cType.toString(); + } + + public String getBoundaryValue(FileInfo messageFileInfo) + { + StringBuffer s = new StringBuffer(); + s.append("----=_Part_").append(messageFileInfo.getNodeRef().getId()); + return s.toString(); + } + } + + } diff --git a/source/java/org/alfresco/repo/imap/ImapMessageTest.java b/source/java/org/alfresco/repo/imap/ImapMessageTest.java new file mode 100644 index 0000000000..fd2e2a5a40 --- /dev/null +++ b/source/java/org/alfresco/repo/imap/ImapMessageTest.java @@ -0,0 +1,523 @@ +/* + * Copyright (C) 2005-2010 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.repo.imap; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.SequenceInputStream; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import javax.mail.Folder; +import javax.mail.MessagingException; +import javax.mail.Session; +import javax.mail.Store; +import javax.mail.internet.MimeMessage; +import javax.mail.internet.MimeMultipart; +import javax.transaction.UserTransaction; + +import org.alfresco.model.ContentModel; +import org.alfresco.repo.importer.ACPImportPackageHandler; +import org.alfresco.repo.management.subsystems.ChildApplicationContextFactory; +import org.alfresco.repo.node.integrity.IntegrityChecker; +import org.alfresco.service.ServiceRegistry; +import org.alfresco.service.cmr.model.FileFolderService; +import org.alfresco.service.cmr.model.FileFolderUtil; +import org.alfresco.service.cmr.model.FileInfo; +import org.alfresco.service.cmr.repository.ContentWriter; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.cmr.security.MutableAuthenticationService; +import org.alfresco.service.cmr.security.PersonService; +import org.alfresco.service.cmr.view.ImporterService; +import org.alfresco.service.cmr.view.Location; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.transaction.TransactionService; +import org.alfresco.util.ApplicationContextHelper; +import org.alfresco.util.PropertyMap; +import org.alfresco.util.config.RepositoryFolderConfigBean; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.core.io.ClassPathResource; + +import com.sun.mail.iap.ProtocolException; +import com.sun.mail.iap.Response; +import com.sun.mail.imap.IMAPFolder; +import com.sun.mail.imap.protocol.BODY; +import com.sun.mail.imap.protocol.FetchResponse; +import com.sun.mail.imap.protocol.IMAPProtocol; +import com.sun.mail.imap.protocol.UID; + +import junit.framework.TestCase; + +public class ImapMessageTest extends TestCase +{ + private static Log logger = LogFactory.getLog(ImapMessageTest.class); + + // IMAP client settings + private static final String PROTOCOL = "imap"; + private static final String HOST = "localhost"; + private static final int PORT = 143; + + private static final String ADMIN_USER_NAME = "admin"; + private static final String ADMIN_USER_PASSWORD = "admin"; + private static final String IMAP_FOLDER_NAME = "test"; + + private Session session = null; + private Store store = null; + private IMAPFolder folder = null; + + private static ApplicationContext ctx = ApplicationContextHelper.getApplicationContext(); + private TransactionService transactionService; + private NodeService nodeService; + private ImporterService importerService; + private PersonService personService; + private SearchService searchService; + private NamespaceService namespaceService; + private FileFolderService fileFolderService; + private MutableAuthenticationService authenticationService; + + String anotherUserName; + private NodeRef testImapFolderNodeRef; + private NodeRef storeRootNodeRef; + private final String storePath = "workspace://SpacesStore"; + private final String companyHomePathInStore = "/app:company_home"; + + private static final String TEST_FOLDER = "Alfresco IMAP/" + IMAP_FOLDER_NAME + "/___-___folder_a/" + "___-___folder_a_a"; + private static final String TEST_FILE = "/" + NamespaceService.CONTENT_MODEL_PREFIX + ":" + IMAP_FOLDER_NAME + "/" + NamespaceService.CONTENT_MODEL_PREFIX + + ":___-___folder_a/" + NamespaceService.CONTENT_MODEL_PREFIX + ":___-___folder_a_a/" + NamespaceService.CONTENT_MODEL_PREFIX + ":___-___file_a_a"; + + @Override + public void setUp() throws Exception + { + ServiceRegistry serviceRegistry = (ServiceRegistry) ctx.getBean("ServiceRegistry"); + transactionService = serviceRegistry.getTransactionService(); + nodeService = serviceRegistry.getNodeService(); + importerService = serviceRegistry.getImporterService(); + personService = serviceRegistry.getPersonService(); + authenticationService = serviceRegistry.getAuthenticationService(); + searchService = serviceRegistry.getSearchService(); + namespaceService = serviceRegistry.getNamespaceService(); + fileFolderService = serviceRegistry.getFileFolderService(); + + // start the transaction + UserTransaction txn = transactionService.getUserTransaction(); + txn.begin(); + authenticationService.authenticate(ADMIN_USER_NAME, ADMIN_USER_PASSWORD.toCharArray()); + + // downgrade integrity + IntegrityChecker.setWarnInTransaction(); + + anotherUserName = "user" + System.currentTimeMillis(); + + PropertyMap testUser = new PropertyMap(); + testUser.put(ContentModel.PROP_USERNAME, anotherUserName); + testUser.put(ContentModel.PROP_FIRSTNAME, anotherUserName); + testUser.put(ContentModel.PROP_LASTNAME, anotherUserName); + testUser.put(ContentModel.PROP_EMAIL, anotherUserName + "@alfresco.com"); + testUser.put(ContentModel.PROP_JOBTITLE, "jobTitle"); + + personService.createPerson(testUser); + + // create the ACEGI Authentication instance for the new user + authenticationService.createAuthentication(anotherUserName, anotherUserName.toCharArray()); + + StoreRef storeRef = new StoreRef(storePath); + storeRootNodeRef = nodeService.getRootNode(storeRef); + + List nodeRefs = searchService.selectNodes(storeRootNodeRef, companyHomePathInStore, null, namespaceService, false); + NodeRef companyHomeNodeRef = nodeRefs.get(0); + + nodeRefs = searchService.selectNodes(storeRootNodeRef, companyHomePathInStore + "/" + NamespaceService.CONTENT_MODEL_PREFIX + ":" + IMAP_FOLDER_NAME, null, + namespaceService, false); + if (nodeRefs != null && nodeRefs.size() > 0) + { + fileFolderService.delete(nodeRefs.get(0)); + } + + ChildApplicationContextFactory imap = (ChildApplicationContextFactory) ctx.getBean("imap"); + ApplicationContext imapCtx = imap.getApplicationContext(); + ImapServiceImpl imapServiceImpl = (ImapServiceImpl) imapCtx.getBean("imapService"); + + // Creating IMAP test folder for IMAP root + LinkedList folders = new LinkedList(); + folders.add(IMAP_FOLDER_NAME); + FileFolderUtil.makeFolders(fileFolderService, companyHomeNodeRef, folders, ContentModel.TYPE_FOLDER); + + // Setting IMAP root + RepositoryFolderConfigBean imapHome = new RepositoryFolderConfigBean(); + imapHome.setStore(storePath); + imapHome.setRootPath(companyHomePathInStore); + imapHome.setFolderPath(IMAP_FOLDER_NAME); + imapServiceImpl.setImapHome(imapHome); + + // Starting IMAP + imapServiceImpl.startup(); + + nodeRefs = searchService.selectNodes(storeRootNodeRef, companyHomePathInStore + "/" + NamespaceService.CONTENT_MODEL_PREFIX + ":" + IMAP_FOLDER_NAME, null, + namespaceService, false); + testImapFolderNodeRef = nodeRefs.get(0); + + /* + * Importing test folders: Test folder contains: "___-___folder_a" "___-___folder_a" contains: "___-___folder_a_a", "___-___file_a", "Message_485.eml" (this is IMAP + * Message) "___-___folder_a_a" contains: "____-____file_a_a" + */ + importInternal("imap/imapservice_test_folder_a.acp", testImapFolderNodeRef); + + txn.commit(); + + // Init mail client session + Properties props = new Properties(); + props.setProperty("mail.imap.partialfetch", "false"); + this.session = Session.getDefaultInstance(props, null); + + // Get the store + this.store = session.getStore(PROTOCOL); + this.store.connect(HOST, PORT, anotherUserName, anotherUserName); + + // Get folder + folder = (IMAPFolder) store.getFolder(TEST_FOLDER); + folder.open(Folder.READ_ONLY); + + } + + private void importInternal(String acpName, NodeRef space) throws IOException + { + // Importing IMAP test acp + ClassPathResource acpResource = new ClassPathResource(acpName); + ACPImportPackageHandler acpHandler = new ACPImportPackageHandler(acpResource.getFile(), null); + Location importLocation = new Location(space); + importerService.importView(acpHandler, importLocation, null, null); + } + + public void testMessageModifiedBetweenReads() throws Exception + { + // Get test message UID + final Long uid = getMessageUid(folder, 1); + // Get Message size + final int count = getMessageSize(folder, uid); + + // Get first part + BODY body = getMessageBodyPart(folder, uid, 0, count - 100); + + // Modify message. The size of letter describing the node may change + // These changes should be committed because it should be visible from client + NodeRef contentNode = findNode(companyHomePathInStore + TEST_FILE); + UserTransaction txn = transactionService.getUserTransaction(); + txn.begin(); + ContentWriter writer = fileFolderService.getWriter(contentNode); + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < 2000; i++) + { + sb.append("test string"); + } + writer.putContent(sb.toString()); + txn.commit(); + + // Read second message part + BODY bodyRest = getMessageBodyPart(folder, uid, count - 10, 10); + + // Creating and parsing message from 2 parts + MimeMessage message = new MimeMessage(Session.getDefaultInstance(new Properties()), new SequenceInputStream(new BufferedInputStream(body.getByteArrayInputStream()), + new BufferedInputStream(bodyRest.getByteArrayInputStream()))); + + // Reading first part - should be successful + MimeMultipart content = (MimeMultipart) message.getContent(); + assertNotNull(content.getBodyPart(0).getContent()); + + try + { + // Reading second part cause error + content.getBodyPart(1).getContent(); + fail("Should raise an IOException"); + } + catch (IOException e) + { + } + } + + public void testMessageRenamedBetweenReads() throws Exception + { + // Get test message UID + final Long uid = getMessageUid(folder, 1); + // Get Message size + final int count = getMessageSize(folder, uid); + + // Get first part + BODY body = getMessageBodyPart(folder, uid, 0, count - 100); + + // Rename message. The size of letter describing the node will change + // These changes should be committed because it should be visible from client + NodeRef contentNode = findNode(companyHomePathInStore + TEST_FILE); + UserTransaction txn = transactionService.getUserTransaction(); + txn.begin(); + fileFolderService.rename(contentNode, "testtesttesttesttesttesttesttesttesttest"); + txn.commit(); + + // Read second message part + BODY bodyRest = getMessageBodyPart(folder, uid, count - 100, 100); + + // Creating and parsing message from 2 parts + MimeMessage message = new MimeMessage(Session.getDefaultInstance(new Properties()), new SequenceInputStream(new BufferedInputStream(body.getByteArrayInputStream()), + new BufferedInputStream(bodyRest.getByteArrayInputStream()))); + + // Reading first part - should be successful + MimeMultipart content = (MimeMultipart) message.getContent(); + assertNotNull(content.getBodyPart(0).getContent()); + + try + { + // Reading second part cause error + content.getBodyPart(1).getContent(); + fail("Should raise an IOException"); + } + catch (IOException e) + { + } + } + + public void testMessageCache() throws Exception + { + + // Create messages + NodeRef contentNode = findNode(companyHomePathInStore + TEST_FILE); + UserTransaction txn = transactionService.getUserTransaction(); + txn.begin(); + + // Create messages more than cache capacity + for (int i = 0; i < 51; i++) + { + FileInfo fi = fileFolderService.create(nodeService.getParentAssocs(contentNode).get(0).getParentRef(), "test" + i, ContentModel.TYPE_CONTENT); + ContentWriter writer = fileFolderService.getWriter(fi.getNodeRef()); + writer.putContent("test"); + } + + txn.commit(); + + // Reload folder + folder.close(false); + folder = (IMAPFolder) store.getFolder(TEST_FOLDER); + folder.open(Folder.READ_ONLY); + + // Read all messages + for (int i = 1; i < 51; i++) + { + // Get test message UID + final Long uid = getMessageUid(folder, i); + // Get Message size + final int count = getMessageSize(folder, uid); + + // Get first part + BODY body = getMessageBodyPart(folder, uid, 0, count - 100); + // Read second message part + BODY bodyRest = getMessageBodyPart(folder, uid, count - 100, 100); + + // Creating and parsing message from 2 parts + MimeMessage message = new MimeMessage(Session.getDefaultInstance(new Properties()), new SequenceInputStream(new BufferedInputStream(body.getByteArrayInputStream()), + new BufferedInputStream(bodyRest.getByteArrayInputStream()))); + + // Reading first part - should be successful + MimeMultipart content = (MimeMultipart) message.getContent(); + assertNotNull(content.getBodyPart(0).getContent()); + assertNotNull(content.getBodyPart(1).getContent()); + } + } + + + + public void testUnmodifiedMessage() throws Exception + { + // Get test message UID + final Long uid = getMessageUid(folder, 1); + // Get Message size + final int count = getMessageSize(folder, uid); + + // Make multiple message reading + for (int i = 0; i < 100; i++) + { + // Get random offset + int n = (int) ((int) 100 * Math.random()); + + // Get first part + BODY body = getMessageBodyPart(folder, uid, 0, count - n); + // Read second message part + BODY bodyRest = getMessageBodyPart(folder, uid, count - n, n); + + // Creating and parsing message from 2 parts + MimeMessage message = new MimeMessage(Session.getDefaultInstance(new Properties()), new SequenceInputStream(new BufferedInputStream(body.getByteArrayInputStream()), + new BufferedInputStream(bodyRest.getByteArrayInputStream()))); + + MimeMultipart content = (MimeMultipart) message.getContent(); + // Reading first part - should be successful + assertNotNull(content.getBodyPart(0).getContent()); + // Reading second part - should be successful + assertNotNull(content.getBodyPart(1).getContent()); + } + } + + /** + * Returns BODY object containing desired message fragment + * + * @param folder Folder containing the message + * @param uid Message UID + * @param from starting byte + * @param count bytes to read + * @return BODY containing desired message fragment + * @throws MessagingException + */ + private static BODY getMessageBodyPart(IMAPFolder folder, final Long uid, final Integer from, final Integer count) throws MessagingException + { + return (BODY) folder.doCommand(new IMAPFolder.ProtocolCommand() + { + public Object doCommand(IMAPProtocol p) throws ProtocolException + { + Response[] r = p.command("UID FETCH " + uid + " (FLAGS BODY.PEEK[]<" + from + "." + count + ">)", null); + logResponse(r); + Response response = r[r.length - 1]; + + // Grab response + if (!response.isOK()) + { + throw new ProtocolException("Unable to retrieve message part <" + from + "." + count + ">"); + } + + FetchResponse fetchResponse = (FetchResponse) r[0]; + BODY body = (BODY) fetchResponse.getItem(com.sun.mail.imap.protocol.BODY.class); + return body; + } + }); + + } + + /** + * Finds node by its path + * + * @param path + * @return NodeRef + */ + private NodeRef findNode(String path) + { + List nodeRefs = searchService.selectNodes(storeRootNodeRef, path, null, namespaceService, false); + return nodeRefs.size() > 0 ? nodeRefs.get(0) : null; + } + + /** + * Returns the UID of the first message in folder + * + * @param folder Folder containing the message + * @param msn message sequence number + * @return UID of the first message + * @throws MessagingException + */ + private static Long getMessageUid(IMAPFolder folder, final int msn) throws MessagingException + { + return (Long) folder.doCommand(new IMAPFolder.ProtocolCommand() + { + public Object doCommand(IMAPProtocol p) throws ProtocolException + { + Response[] r = p.command("FETCH " + msn + " (UID)", null); + logResponse(r); + Response response = r[r.length - 1]; + + // Grab response + if (!response.isOK()) + { + throw new ProtocolException("Unable to retrieve message UID"); + } + FetchResponse fetchResponse = (FetchResponse) r[0]; + UID uid = (UID) fetchResponse.getItem(UID.class); + return uid.uid; + } + }); + } + + /** + * Returns size of the message + * + * @param folder Folder containing the message + * @param uid Message UID + * @return Returns size of the message + * @throws MessagingException + */ + private static Integer getMessageSize(IMAPFolder folder, final Long uid) throws MessagingException + { + return (Integer) folder.doCommand(new IMAPFolder.ProtocolCommand() + { + public Object doCommand(IMAPProtocol p) throws ProtocolException + { + Response[] r = p.command("UID FETCH " + uid + " (FLAGS BODY.PEEK[])", null); + logResponse(r); + Response response = r[r.length - 1]; + + // Grab response + if (!response.isOK()) + { + throw new ProtocolException("Unable to retrieve message size"); + } + FetchResponse fetchResponse = (FetchResponse) r[0]; + BODY body = (BODY) fetchResponse.getItem(BODY.class); + return body.data.getCount(); + } + }); + } + + /** + * Simple util for logging response + * + * @param r response + */ + private static void logResponse(Response[] r) + { + for (int i = 0; i < r.length; i++) + { + logger.debug(r[i]); + //logger.info(r[i]); + } + } + + @Override + public void tearDown() throws Exception + { + // Deleting created test environment + + UserTransaction txn = transactionService.getUserTransaction(); + txn.begin(); + + List nodeRefs = searchService.selectNodes(storeRootNodeRef, companyHomePathInStore + "/" + NamespaceService.CONTENT_MODEL_PREFIX + ":" + IMAP_FOLDER_NAME, null, + namespaceService, false); + if (nodeRefs != null && nodeRefs.size() > 0) + { + fileFolderService.delete(nodeRefs.get(0)); + } + + authenticationService.deleteAuthentication(anotherUserName); + personService.deletePerson(anotherUserName); + + txn.commit(); + + // Closing client connection + folder.close(false); + store.close(); + } + +} diff --git a/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java b/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java index b7060f6328..bdf9ddddc3 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java @@ -67,7 +67,6 @@ import org.quartz.JobDataMap; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.springframework.beans.BeansException; -import org.springframework.beans.factory.InitializingBean; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.ConfigurableApplicationContext; @@ -181,6 +180,8 @@ public abstract class AbstractLuceneIndexerAndSearcherFactory implements LuceneI private int mergerTargetOverlayCount = 5; private int mergerTargetOverlaysBlockingFactor = 1; + + private boolean fairLocking; private int termIndexInterval = IndexWriter.DEFAULT_TERM_INDEX_INTERVAL; @@ -1748,6 +1749,16 @@ public abstract class AbstractLuceneIndexerAndSearcherFactory implements LuceneI this.mergerTargetOverlaysBlockingFactor = mergerTargetOverlaysBlockingFactor; } + public boolean getFairLocking() + { + return this.fairLocking; + } + + public void setFairLocking(boolean fairLocking) + { + this.fairLocking = fairLocking; + } + public int getTermIndexInterval() { return termIndexInterval; diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java index 1ef8983cfb..613efba10a 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java @@ -18,12 +18,12 @@ */ package org.alfresco.repo.search.impl.lucene; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -249,7 +249,7 @@ public class LuceneCategoryServiceImpl implements CategoryService private Collection resultSetToChildAssocCollection(ResultSet resultSet) { - List collection = new ArrayList(); + List collection = new LinkedList(); if (resultSet != null) { for (ResultSetRow row : resultSet) @@ -264,7 +264,7 @@ public class LuceneCategoryServiceImpl implements CategoryService public Collection getCategories(StoreRef storeRef, QName aspectQName, Depth depth) { - Collection assocs = new ArrayList(); + Collection assocs = new LinkedList(); Set nodeRefs = getClassificationNodes(storeRef, aspectQName); for (NodeRef nodeRef : nodeRefs) { @@ -331,7 +331,7 @@ public class LuceneCategoryServiceImpl implements CategoryService public Collection getRootCategories(StoreRef storeRef, QName aspectName) { - Collection assocs = new ArrayList(); + Collection assocs = new LinkedList(); Set nodeRefs = getClassificationNodes(storeRef, aspectName); for (NodeRef nodeRef : nodeRefs) { @@ -340,7 +340,49 @@ public class LuceneCategoryServiceImpl implements CategoryService return assocs; } + public ChildAssociationRef getCategory(NodeRef parent, QName aspectName, String name) + { + String uri = nodeService.getPrimaryParent(parent).getQName().getNamespaceURI(); + String validLocalName = QName.createValidLocalName(name); + Collection assocs = nodeService.getChildAssocs(parent, ContentModel.ASSOC_SUBCATEGORIES, + QName.createQName(uri, validLocalName), false); + if (assocs.isEmpty()) + { + return null; + } + return assocs.iterator().next(); + } + + public Collection getRootCategories(StoreRef storeRef, QName aspectName, String name, + boolean create) + { + Set nodeRefs = getClassificationNodes(storeRef, aspectName); + if (nodeRefs.isEmpty()) + { + return Collections.emptySet(); + } + Collection assocs = new LinkedList(); + for (NodeRef nodeRef : nodeRefs) + { + ChildAssociationRef category = getCategory(nodeRef, aspectName, name); + if (category != null) + { + assocs.add(category); + } + } + if (create && assocs.isEmpty()) + { + assocs.add(createCategoryInternal(nodeRefs.iterator().next(), name)); + } + return assocs; + } + public NodeRef createCategory(NodeRef parent, String name) + { + return createCategoryInternal(parent, name).getChildRef(); + } + + private ChildAssociationRef createCategoryInternal(NodeRef parent, String name) { if (!nodeService.exists(parent)) { @@ -348,8 +390,8 @@ public class LuceneCategoryServiceImpl implements CategoryService } String uri = nodeService.getPrimaryParent(parent).getQName().getNamespaceURI(); String validLocalName = QName.createValidLocalName(name); - NodeRef newCategory = publicNodeService.createNode(parent, ContentModel.ASSOC_SUBCATEGORIES, QName.createQName(uri, validLocalName), ContentModel.TYPE_CATEGORY).getChildRef(); - publicNodeService.setProperty(newCategory, ContentModel.PROP_NAME, name); + ChildAssociationRef newCategory = publicNodeService.createNode(parent, ContentModel.ASSOC_SUBCATEGORIES, QName.createQName(uri, validLocalName), ContentModel.TYPE_CATEGORY); + publicNodeService.setProperty(newCategory.getChildRef(), ContentModel.PROP_NAME, name); return newCategory; } @@ -412,7 +454,7 @@ public class LuceneCategoryServiceImpl implements CategoryService { LuceneSearcher luceneSearcher = (LuceneSearcher)searchService; List> topTerms = luceneSearcher.getTopTerms(field, count); - List> answer = new ArrayList>(); + List> answer = new LinkedList>(); for (Pair term : topTerms) { Pair toAdd; diff --git a/source/java/org/alfresco/repo/search/impl/lucene/LuceneConfig.java b/source/java/org/alfresco/repo/search/impl/lucene/LuceneConfig.java index 22c6bfaeb7..58d5a1eb8c 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/LuceneConfig.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/LuceneConfig.java @@ -268,4 +268,12 @@ public interface LuceneConfig */ public double getMaxRamInMbForInMemoryIndex(); + /** + * Should we use a 'fair' locking policy, giving queue-like access behaviour to the indexes and avoiding starvation? + * Default is false since fair locking appears to cause deadlock on old JVMs. + * + * @return true if a fair locking policy should be used + */ + public boolean getFairLocking(); + } diff --git a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java index e16438c0c8..16ae2aafc2 100644 --- a/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java +++ b/source/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java @@ -68,7 +68,6 @@ import org.alfresco.service.namespace.NamespaceService; import org.alfresco.util.ApplicationContextHelper; import org.alfresco.util.GUID; import org.alfresco.util.TraceableThreadFactory; -import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.analysis.Analyzer; @@ -94,7 +93,6 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMDirectory; -import org.apache.xml.dtm.ref.sax2dtm.SAX2DTM2.FollowingSiblingIterator; import org.safehaus.uuid.UUID; import org.saxpath.SAXPathException; import org.springframework.context.ApplicationContext; @@ -224,7 +222,7 @@ public class IndexInfo implements IndexMonitor /** * Lock for the index entries */ - private ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + private final ReentrantReadWriteLock readWriteLock; private ReentrantReadWriteLock readOnlyLock = new ReentrantReadWriteLock(); @@ -485,6 +483,7 @@ public class IndexInfo implements IndexMonitor if (config != null) { + this.readWriteLock = new ReentrantReadWriteLock(config.getFairLocking()); this.maxFieldLength = config.getIndexerMaxFieldLength(); this.threadPoolExecutor = config.getThreadPoolExecutor(); IndexInfo.useNIOMemoryMapping = config.getUseNioMemoryMapping(); @@ -518,6 +517,8 @@ public class IndexInfo implements IndexMonitor } else { + this.readWriteLock = new ReentrantReadWriteLock(false); + // need a default thread pool .... TraceableThreadFactory threadFactory = new TraceableThreadFactory(); threadFactory.setThreadDaemon(true); diff --git a/source/java/org/alfresco/repo/security/person/PersonServiceImpl.java b/source/java/org/alfresco/repo/security/person/PersonServiceImpl.java index ad741eb4cb..725c39d879 100644 --- a/source/java/org/alfresco/repo/security/person/PersonServiceImpl.java +++ b/source/java/org/alfresco/repo/security/person/PersonServiceImpl.java @@ -47,9 +47,9 @@ import org.alfresco.repo.security.authentication.AuthenticationUtil; import org.alfresco.repo.security.permissions.PermissionServiceSPI; import org.alfresco.repo.tenant.TenantService; import org.alfresco.repo.transaction.AlfrescoTransactionSupport; -import org.alfresco.repo.transaction.TransactionListenerAdapter; import org.alfresco.repo.transaction.AlfrescoTransactionSupport.TxnReadState; import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; +import org.alfresco.repo.transaction.TransactionListenerAdapter; import org.alfresco.service.cmr.dictionary.DictionaryService; import org.alfresco.service.cmr.repository.ChildAssociationRef; import org.alfresco.service.cmr.repository.NodeRef; @@ -356,6 +356,7 @@ public class PersonServiceImpl extends TransactionListenerAdapter implements Per private NodeRef getPersonOrNull(String searchUserName) { Set allRefs = getFromCache(searchUserName); + boolean addToCache = false; if (allRefs == null) { List childRefs = nodeService.getChildAssocs( @@ -370,6 +371,7 @@ public class PersonServiceImpl extends TransactionListenerAdapter implements Per NodeRef nodeRef = childRef.getChildRef(); allRefs.add(nodeRef); } + addToCache = true; } List refs = new ArrayList(allRefs.size()); @@ -392,8 +394,11 @@ public class PersonServiceImpl extends TransactionListenerAdapter implements Per { returnRef = refs.get(0); - // Don't bother caching unless we get a result that doesn't need duplicate processing - putToCache(searchUserName, allRefs); + if (addToCache) + { + // Don't bother caching unless we get a result that doesn't need duplicate processing + putToCache(searchUserName, allRefs); + } } return returnRef; } diff --git a/source/java/org/alfresco/repo/tagging/TaggingServiceImpl.java b/source/java/org/alfresco/repo/tagging/TaggingServiceImpl.java index 91740f436c..e73b82c717 100644 --- a/source/java/org/alfresco/repo/tagging/TaggingServiceImpl.java +++ b/source/java/org/alfresco/repo/tagging/TaggingServiceImpl.java @@ -448,12 +448,8 @@ public class TaggingServiceImpl implements TaggingService, { // Lower the case of the tag tag = tag.toLowerCase(); - - if (isTag(storeRef, tag) == false) - { - return this.categoryService.createRootCategory(storeRef, ContentModel.ASPECT_TAGGABLE, tag); - } - return null; + + return getTagNodeRef(storeRef, tag, true); } /** @@ -527,12 +523,7 @@ public class TaggingServiceImpl implements TaggingService, String tag = tagName.toLowerCase(); // Get the tag node reference - NodeRef newTagNodeRef = getTagNodeRef(nodeRef.getStoreRef(), tag); - if (newTagNodeRef == null) - { - // Create the new tag - newTagNodeRef = categoryService.createRootCategory(nodeRef.getStoreRef(), ContentModel.ASPECT_TAGGABLE, tag); - } + NodeRef newTagNodeRef = getTagNodeRef(nodeRef.getStoreRef(), tag, true); List tagNodeRefs = new ArrayList(5); if (nodeService.hasAspect(nodeRef, ContentModel.ASPECT_TAGGABLE) == false) @@ -586,16 +577,28 @@ public class TaggingServiceImpl implements TaggingService, * @return NodeRef tag node reference or null not exist */ public NodeRef getTagNodeRef(StoreRef storeRef, String tag) + { + return getTagNodeRef(storeRef, tag, false); + } + + /** + * Gets the node reference for a given tag. + *

    + * Returns null if tag is not present and not created. + * + * @param storeRef store reference + * @param tag tag + * @param create create a node if one doesn't exist? + * @return NodeRef tag node reference or null not exist + */ + private NodeRef getTagNodeRef(StoreRef storeRef, String tag, boolean create) { NodeRef tagNodeRef = null; - String query = "+PATH:\"cm:taggable/cm:" + ISO9075.encode(tag) + "\""; - ResultSet resultSet = this.searchService.query(storeRef, SearchService.LANGUAGE_LUCENE, query); - if (resultSet.length() != 0) + Collection results = this.categoryService.getRootCategories(storeRef, ContentModel.ASPECT_TAGGABLE, tag, create); + if (!results.isEmpty()) { - tagNodeRef = resultSet.getNodeRef(0); + tagNodeRef = results.iterator().next().getChildRef(); } - resultSet.close(); - return tagNodeRef; } @@ -701,12 +704,7 @@ public class TaggingServiceImpl implements TaggingService, tag = tag.toLowerCase(); // Get the tag node reference - NodeRef newTagNodeRef = getTagNodeRef(nodeRef.getStoreRef(), tag); - if (newTagNodeRef == null) - { - // Create the new tag - newTagNodeRef = this.categoryService.createRootCategory(nodeRef.getStoreRef(), ContentModel.ASPECT_TAGGABLE, tag); - } + NodeRef newTagNodeRef = getTagNodeRef(nodeRef.getStoreRef(), tag, true); if (tagNodeRefs.contains(newTagNodeRef) == false) { @@ -953,10 +951,10 @@ public class TaggingServiceImpl implements TaggingService, /*package*/ static List readTagDetails(InputStream is) { List result = new ArrayList(25); - + BufferedReader reader = null; try { - BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); String nextLine = reader.readLine(); while (nextLine != null) { @@ -970,6 +968,10 @@ public class TaggingServiceImpl implements TaggingService, { throw new AlfrescoRuntimeException("Unable to read tag details", exception); } + finally + { + try { reader.close(); } catch (Exception e) {} + } return result; } diff --git a/source/java/org/alfresco/repo/transaction/RetryingTransactionHelper.java b/source/java/org/alfresco/repo/transaction/RetryingTransactionHelper.java index a6205c7e87..4179ca3c6e 100644 --- a/source/java/org/alfresco/repo/transaction/RetryingTransactionHelper.java +++ b/source/java/org/alfresco/repo/transaction/RetryingTransactionHelper.java @@ -121,6 +121,21 @@ public class RetryingTransactionHelper private int maxRetryWaitMs; /** How much to increase the wait time with each retry. */ private int retryWaitIncrementMs; + + /** + * Optional time limit for execution time. When non-zero, retries will not continue when the projected time is + * beyond this time. + */ + private long maxExecutionMs; + + /** The number of concurrently exeucting transactions. Only maintained when maxExecutionMs is set. */ + private int txnCount; + + /** + * A 'ceiling' for the number of concurrent transactions that can execute. Dynamically maintained so that exeuction + * time is within maxExecutionMs. Transactions above this limit will be rejected with a {@link TooBusyException}. + */ + private Integer txnCeiling; /** * Whether the the transactions may only be reads @@ -205,6 +220,11 @@ public class RetryingTransactionHelper this.retryWaitIncrementMs = retryWaitIncrementMs; } + public void setMaxExecutionMs(long maxExecutionMs) + { + this.maxExecutionMs = maxExecutionMs; + } + /** * Set whether this helper only supports read transactions. */ @@ -273,185 +293,256 @@ public class RetryingTransactionHelper { throw new AccessDeniedException(MSG_READ_ONLY); } - // Track the last exception caught, so that we - // can throw it if we run out of retries. - RuntimeException lastException = null; - for (int count = 0; count == 0 || count < maxRetries; count++) + + // If we are time limiting, set ourselves a time limit and maintain the count of concurrent transactions + long startTime = 0, endTime = 0, txnStartTime = 0; + int txnCountWhenStarted = 0; + if (maxExecutionMs > 0) { - UserTransaction txn = null; - try + startTime = System.currentTimeMillis(); + synchronized (this) { - if (requiresNew) + // If this transaction would take us above our ceiling, reject it + if (txnCeiling != null && txnCount >= txnCeiling) { - txn = txnService.getNonPropagatingUserTransaction(readOnly); + throw new TooBusyException("Too busy: " + txnCount + " transactions"); } - else + txnCountWhenStarted = ++txnCount; + } + endTime = startTime + maxExecutionMs; + } + + try + { + // Track the last exception caught, so that we + // can throw it if we run out of retries. + RuntimeException lastException = null; + for (int count = 0; count == 0 || count < maxRetries; count++) + { + // Monitor duration of each retry so that we can project an end time + if (maxExecutionMs > 0) + { + txnStartTime = System.currentTimeMillis(); + } + + UserTransaction txn = null; + try { - TxnReadState readState = AlfrescoTransactionSupport.getTransactionReadState(); - switch (readState) + if (requiresNew) { - case TXN_READ_ONLY: - if (!readOnly) - { - // The current transaction is read-only, but a writable transaction is requested - throw new AlfrescoRuntimeException("Read-Write transaction started within read-only transaction"); - } - // We are in a read-only transaction and this is what we require so continue with it. - break; - case TXN_READ_WRITE: - // We are in a read-write transaction. It cannot be downgraded so just continue with it. - break; - case TXN_NONE: - // There is no current transaction so we need a new one. - txn = txnService.getUserTransaction(readOnly); - break; - default: - throw new RuntimeException("Unknown transaction state: " + readState); - } - } - if (txn != null) - { - txn.begin(); - // Wrap it to protect it - UserTransactionProtectionAdvise advise = new UserTransactionProtectionAdvise(); - ProxyFactory proxyFactory = new ProxyFactory(txn); - proxyFactory.addAdvice(advise); - UserTransaction wrappedTxn = (UserTransaction) proxyFactory.getProxy(); - // Store the UserTransaction for static retrieval. There is no need to unbind it - // because the transaction management will do that for us. - AlfrescoTransactionSupport.bindResource(KEY_ACTIVE_TRANSACTION, wrappedTxn); - } - // Do the work. - R result = cb.execute(); - // Only commit if we 'own' the transaction. - if (txn != null) - { - if (txn.getStatus() == Status.STATUS_MARKED_ROLLBACK) - { - if (logger.isDebugEnabled()) - { - logger.debug("\n" + - "Transaction marked for rollback: \n" + - " Thread: " + Thread.currentThread().getName() + "\n" + - " Txn: " + txn + "\n" + - " Iteration: " + count); - } - // Something caused the transaction to be marked for rollback - // There is no recovery or retrying with this - txn.rollback(); + txn = txnService.getNonPropagatingUserTransaction(readOnly); } else { - // The transaction hasn't been flagged for failure so the commit - // sould still be good. - txn.commit(); - } - } - if (logger.isDebugEnabled()) - { - if (count != 0) - { - logger.debug("\n" + - "Transaction succeeded: \n" + - " Thread: " + Thread.currentThread().getName() + "\n" + - " Txn: " + txn + "\n" + - " Iteration: " + count); - } - } - return result; - } - catch (Throwable e) - { - // Somebody else 'owns' the transaction, so just rethrow. - if (txn == null) - { - RuntimeException ee = AlfrescoRuntimeException.makeRuntimeException( - e, "Exception from transactional callback: " + cb); - throw ee; - } - if (logger.isDebugEnabled()) - { - logger.debug("\n" + - "Transaction commit failed: \n" + - " Thread: " + Thread.currentThread().getName() + "\n" + - " Txn: " + txn + "\n" + - " Iteration: " + count + "\n" + - " Exception follows:", - e); - } - // Rollback if we can. - if (txn != null) - { - try - { - int txnStatus = txn.getStatus(); - // We can only rollback if a transaction was started (NOT NO_TRANSACTION) and - // if that transaction has not been rolled back (NOT ROLLEDBACK). - // If an exception occurs while the transaction is being created (e.g. no database connection) - // then the status will be NO_TRANSACTION. - if (txnStatus != Status.STATUS_NO_TRANSACTION && txnStatus != Status.STATUS_ROLLEDBACK) + TxnReadState readState = AlfrescoTransactionSupport.getTransactionReadState(); + switch (readState) { - txn.rollback(); + case TXN_READ_ONLY: + if (!readOnly) + { + // The current transaction is read-only, but a writable transaction is requested + throw new AlfrescoRuntimeException("Read-Write transaction started within read-only transaction"); + } + // We are in a read-only transaction and this is what we require so continue with it. + break; + case TXN_READ_WRITE: + // We are in a read-write transaction. It cannot be downgraded so just continue with it. + break; + case TXN_NONE: + // There is no current transaction so we need a new one. + txn = txnService.getUserTransaction(readOnly); + break; + default: + throw new RuntimeException("Unknown transaction state: " + readState); } } - catch (Throwable e1) + if (txn != null) { - // A rollback failure should not preclude a retry, but logging of the rollback failure is required - logger.error("Rollback failure. Normal retry behaviour will resume.", e1); + txn.begin(); + // Wrap it to protect it + UserTransactionProtectionAdvise advise = new UserTransactionProtectionAdvise(); + ProxyFactory proxyFactory = new ProxyFactory(txn); + proxyFactory.addAdvice(advise); + UserTransaction wrappedTxn = (UserTransaction) proxyFactory.getProxy(); + // Store the UserTransaction for static retrieval. There is no need to unbind it + // because the transaction management will do that for us. + AlfrescoTransactionSupport.bindResource(KEY_ACTIVE_TRANSACTION, wrappedTxn); } - } - if (e instanceof RollbackException) - { - lastException = (e.getCause() instanceof RuntimeException) ? - (RuntimeException)e.getCause() : new AlfrescoRuntimeException("Exception in Transaction.", e.getCause()); - } - else - { - lastException = (e instanceof RuntimeException) ? - (RuntimeException)e : new AlfrescoRuntimeException("Exception in Transaction.", e); - } - // Check if there is a cause for retrying - Throwable retryCause = extractRetryCause(e); - if (retryCause != null) - { - // Sleep a random amount of time before retrying. - // The sleep interval increases with the number of retries. - int sleepIntervalRandom = (count > 0 && retryWaitIncrementMs > 0) - ? random.nextInt(count * retryWaitIncrementMs) - : minRetryWaitMs; - int sleepInterval = Math.min(maxRetryWaitMs, sleepIntervalRandom); - sleepInterval = Math.max(sleepInterval, minRetryWaitMs); - if (logger.isInfoEnabled() && !logger.isDebugEnabled()) + // Do the work. + R result = cb.execute(); + // Only commit if we 'own' the transaction. + if (txn != null) { - String msg = String.format( - "Retrying %s: count %2d; wait: %1.1fs; msg: \"%s\"; exception: (%s)", - Thread.currentThread().getName(), - count, (double)sleepInterval/1000D, - retryCause.getMessage(), - retryCause.getClass().getName()); - logger.info(msg); + if (txn.getStatus() == Status.STATUS_MARKED_ROLLBACK) + { + if (logger.isDebugEnabled()) + { + logger.debug("\n" + + "Transaction marked for rollback: \n" + + " Thread: " + Thread.currentThread().getName() + "\n" + + " Txn: " + txn + "\n" + + " Iteration: " + count); + } + // Something caused the transaction to be marked for rollback + // There is no recovery or retrying with this + txn.rollback(); + } + else + { + // The transaction hasn't been flagged for failure so the commit + // sould still be good. + txn.commit(); + } } - try + if (logger.isDebugEnabled()) { - Thread.sleep(sleepInterval); + if (count != 0) + { + logger.debug("\n" + + "Transaction succeeded: \n" + + " Thread: " + Thread.currentThread().getName() + "\n" + + " Txn: " + txn + "\n" + + " Iteration: " + count); + } } - catch (InterruptedException ie) - { - // Do nothing. - } - // Try again - continue; + return result; } - else + catch (Throwable e) { - // It was a 'bad' exception. - throw lastException; + // Somebody else 'owns' the transaction, so just rethrow. + if (txn == null) + { + RuntimeException ee = AlfrescoRuntimeException.makeRuntimeException( + e, "Exception from transactional callback: " + cb); + throw ee; + } + if (logger.isDebugEnabled()) + { + logger.debug("\n" + + "Transaction commit failed: \n" + + " Thread: " + Thread.currentThread().getName() + "\n" + + " Txn: " + txn + "\n" + + " Iteration: " + count + "\n" + + " Exception follows:", + e); + } + // Rollback if we can. + if (txn != null) + { + try + { + int txnStatus = txn.getStatus(); + // We can only rollback if a transaction was started (NOT NO_TRANSACTION) and + // if that transaction has not been rolled back (NOT ROLLEDBACK). + // If an exception occurs while the transaction is being created (e.g. no database connection) + // then the status will be NO_TRANSACTION. + if (txnStatus != Status.STATUS_NO_TRANSACTION && txnStatus != Status.STATUS_ROLLEDBACK) + { + txn.rollback(); + } + } + catch (Throwable e1) + { + // A rollback failure should not preclude a retry, but logging of the rollback failure is required + logger.error("Rollback failure. Normal retry behaviour will resume.", e1); + } + } + if (e instanceof RollbackException) + { + lastException = (e.getCause() instanceof RuntimeException) ? + (RuntimeException)e.getCause() : new AlfrescoRuntimeException("Exception in Transaction.", e.getCause()); + } + else + { + lastException = (e instanceof RuntimeException) ? + (RuntimeException)e : new AlfrescoRuntimeException("Exception in Transaction.", e); + } + // Check if there is a cause for retrying + Throwable retryCause = extractRetryCause(e); + if (retryCause != null) + { + // Sleep a random amount of time before retrying. + // The sleep interval increases with the number of retries. + int sleepIntervalRandom = (count > 0 && retryWaitIncrementMs > 0) + ? random.nextInt(count * retryWaitIncrementMs) + : minRetryWaitMs; + int maxRetryWaitMs; + + // If we are time limiting only continue if we have enough time, based on the last duration + if (maxExecutionMs > 0) + { + long txnEndTime = System.currentTimeMillis(); + long projectedEndTime = txnEndTime + (txnEndTime - txnStartTime); + if (projectedEndTime > endTime) + { + // Force the ceiling to be lowered and reject + endTime = 0; + throw new TooBusyException("Too busy to retry", e); + } + // Limit the wait duration to fit into the time we have left + maxRetryWaitMs = Math.min(this.maxRetryWaitMs, (int)(endTime - projectedEndTime)); + } + else + { + maxRetryWaitMs = this.maxRetryWaitMs; + } + int sleepInterval = Math.min(maxRetryWaitMs, sleepIntervalRandom); + sleepInterval = Math.max(sleepInterval, minRetryWaitMs); + if (logger.isInfoEnabled() && !logger.isDebugEnabled()) + { + String msg = String.format( + "Retrying %s: count %2d; wait: %1.1fs; msg: \"%s\"; exception: (%s)", + Thread.currentThread().getName(), + count, (double)sleepInterval/1000D, + retryCause.getMessage(), + retryCause.getClass().getName()); + logger.info(msg); + } + try + { + Thread.sleep(sleepInterval); + } + catch (InterruptedException ie) + { + // Do nothing. + } + // Try again + continue; + } + else + { + // It was a 'bad' exception. + throw lastException; + } } } + // We've worn out our welcome and retried the maximum number of times. + // So, fail. + throw lastException; + } + finally + { + if (maxExecutionMs > 0) + { + synchronized (this) + { + txnCount--; + if(System.currentTimeMillis() > endTime) + { + // Lower the ceiling + if (txnCeiling == null || txnCeiling > txnCountWhenStarted - 1) + { + txnCeiling = Math.max(1, txnCountWhenStarted - 1); + } + } + else if (txnCeiling != null && txnCeiling < txnCountWhenStarted + 1) + { + // Raise the ceiling + txnCeiling = txnCountWhenStarted + 1; + } + } + } } - // We've worn out our welcome and retried the maximum number of times. - // So, fail. - throw lastException; } /** diff --git a/source/java/org/alfresco/repo/transaction/RetryingTransactionHelperTest.java b/source/java/org/alfresco/repo/transaction/RetryingTransactionHelperTest.java index 6f6c2a709b..2343342650 100644 --- a/source/java/org/alfresco/repo/transaction/RetryingTransactionHelperTest.java +++ b/source/java/org/alfresco/repo/transaction/RetryingTransactionHelperTest.java @@ -18,7 +18,12 @@ */ package org.alfresco.repo.transaction; +import java.util.Collections; import java.util.ConcurrentModificationException; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import javax.transaction.Status; import javax.transaction.UserTransaction; @@ -40,6 +45,7 @@ import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; import org.alfresco.service.transaction.TransactionService; import org.alfresco.util.ApplicationContextHelper; +import org.alfresco.util.Pair; import org.apache.commons.lang.mutable.MutableInt; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -543,6 +549,178 @@ public class RetryingTransactionHelperTest extends TestCase assertEquals("Should have been called exactly once", 1, callCount.intValue()); } + public void testTimeLimit() + { + final RetryingTransactionHelper txnHelper = new RetryingTransactionHelper(); + txnHelper.setTransactionService(transactionService); + txnHelper.setMaxExecutionMs(3000); + final List caughtExceptions = Collections.synchronizedList(new LinkedList()); + + // Force ceiling of 2 + runThreads(txnHelper, caughtExceptions, new Pair(2, 1000), new Pair(1, 5000)); + if (caughtExceptions.size() > 0) + { + throw new RuntimeException("Unexpected exception", caughtExceptions.get(0)); + } + + + // Try breaching ceiling + runThreads(txnHelper, caughtExceptions, new Pair(3, 1000)); + assertTrue("Expected exception", caughtExceptions.size() > 0); + assertTrue("Excpected TooBusyException", caughtExceptions.get(0) instanceof TooBusyException); + + // Stay within ceiling, forcing expansion + caughtExceptions.clear(); + runThreads(txnHelper, caughtExceptions, new Pair(1, 1000), new Pair(1, 2000)); + if (caughtExceptions.size() > 0) + { + throw new RuntimeException("Unexpected exception", caughtExceptions.get(0)); + } + + // Test expansion + caughtExceptions.clear(); + runThreads(txnHelper, caughtExceptions, new Pair(3, 1000)); + if (caughtExceptions.size() > 0) + { + throw new RuntimeException("Unexpected exception", caughtExceptions.get(0)); + } + + // Ensure expansion no too fast + caughtExceptions.clear(); + runThreads(txnHelper, caughtExceptions, new Pair(5, 1000)); + assertTrue("Expected exception", caughtExceptions.size() > 0); + assertTrue("Excpected TooBusyException", caughtExceptions.get(0) instanceof TooBusyException); + + // Test contraction + caughtExceptions.clear(); + runThreads(txnHelper, caughtExceptions, new Pair(2, 1000), new Pair(1, 5000)); + if (caughtExceptions.size() > 0) + { + throw new RuntimeException("Unexpected exception", caughtExceptions.get(0)); + } + + // Try breaching new ceiling + runThreads(txnHelper, caughtExceptions, new Pair(3, 1000)); + assertTrue("Expected exception", caughtExceptions.size() > 0); + assertTrue("Excpected TooBusyException", caughtExceptions.get(0) instanceof TooBusyException); + + // Check retry limitation + long startTime = System.currentTimeMillis(); + try + { + txnHelper.doInTransaction(new RetryingTransactionCallback() + { + + public Void execute() throws Throwable + { + Thread.sleep(1000); + throw new ConcurrencyFailureException("Fake concurrency failure"); + } + }); + fail("Expected TooBusyException"); + } + catch (TooBusyException e) + { + assertNotNull("Expected cause", e.getCause()); + assertTrue("Too long", System.currentTimeMillis() < startTime + 5000); + } + } + + private void runThreads(final RetryingTransactionHelper txnHelper, final List caughtExceptions, + Pair... countDurationPairs) + { + int threadCount = 0; + for (Pair pair : countDurationPairs) + { + threadCount += pair.getFirst(); + } + + final CountDownLatch endLatch = new CountDownLatch(threadCount); + + class Callback implements RetryingTransactionCallback + { + private final CountDownLatch startLatch; + private final int duration; + + public Callback(CountDownLatch startLatch, int duration) + { + this.startLatch = startLatch; + this.duration = duration; + } + + public Void execute() throws Throwable + { + long endTime = System.currentTimeMillis() + duration; + + // Signal that we've started + startLatch.countDown(); + + long duration = endTime - System.currentTimeMillis(); + if (duration > 0) + { + Thread.sleep(duration); + } + return null; + } + } + ; + class Work implements Runnable + { + private final Callback callback; + + public Work(Callback callback) + { + this.callback = callback; + } + + public void run() + { + try + { + txnHelper.doInTransaction(callback); + } + catch (Throwable e) + { + caughtExceptions.add(e); + } + endLatch.countDown(); + } + } + ; + + // Fire the threads + int j = 0; + for (Pair pair : countDurationPairs) + { + CountDownLatch startLatch = new CountDownLatch(1); + Runnable work = new Work(new Callback(startLatch, pair.getSecond())); + for (int i = 0; i < pair.getFirst(); i++) + { + Thread thread = new Thread(work); + thread.setName(getName() + "-" + j++); + thread.setDaemon(true); + thread.start(); + try + { + // Wait for the thread to get up and running. We need them starting in sequence + startLatch.await(60, TimeUnit.SECONDS); + } + catch (InterruptedException e) + { + } + } + } + // Wait for the threads to have finished + try + { + endLatch.await(60, TimeUnit.SECONDS); + } + catch (InterruptedException e) + { + } + + } + /** * Helper class to kill the session's DB connection */ diff --git a/source/java/org/alfresco/repo/transaction/TooBusyException.java b/source/java/org/alfresco/repo/transaction/TooBusyException.java new file mode 100644 index 0000000000..3099fda25d --- /dev/null +++ b/source/java/org/alfresco/repo/transaction/TooBusyException.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2005-2010 Alfresco Software Limited. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * As a special exception to the terms and conditions of version 2.0 of + * the GPL, you may redistribute this Program in connection with Free/Libre + * and Open Source Software ("FLOSS") applications as described in Alfresco's + * FLOSS exception. You should have received a copy of the text describing + * the FLOSS exception, and it is also available here: + * http://www.alfresco.com/legal/licensing" + */ +package org.alfresco.repo.transaction; + +import org.alfresco.error.AlfrescoRuntimeException; + +/** + * An exception thrown by {@link RetryingTransactionHelper} when its maxExecutionMs property is set and there isn't + * enough capacity to execute / retry the transaction. + * + * @author dward + */ +public class TooBusyException extends AlfrescoRuntimeException +{ + private static final long serialVersionUID = 1L; + + /** + * @param msgId + */ + public TooBusyException(String msgId) + { + super(msgId); + } + + /** + * @param msgId + * @param msgParams + */ + public TooBusyException(String msgId, Object[] msgParams) + { + super(msgId, msgParams); + } + + /** + * @param msgId + * @param cause + */ + public TooBusyException(String msgId, Throwable cause) + { + super(msgId, cause); + } + + /** + * @param msgId + * @param msgParams + * @param cause + */ + public TooBusyException(String msgId, Object[] msgParams, Throwable cause) + { + super(msgId, msgParams, cause); + } + +} diff --git a/source/java/org/alfresco/service/cmr/search/CategoryService.java b/source/java/org/alfresco/service/cmr/search/CategoryService.java index 0b3f7d8b7c..7489d56565 100644 --- a/source/java/org/alfresco/service/cmr/search/CategoryService.java +++ b/source/java/org/alfresco/service/cmr/search/CategoryService.java @@ -101,7 +101,38 @@ public interface CategoryService */ @Auditable(parameters = {"storeRef", "aspectName"}) public Collection getRootCategories(StoreRef storeRef, QName aspectName); + + /** + * Looks up a category by name under its immediate parent. Index-independent so can be used for cluster-safe + * existence checks. + * + * @param parent + * the parent + * @param aspectName + * the aspect name + * @param name + * the category name + * @return the category child association reference + */ + public ChildAssociationRef getCategory(NodeRef parent, QName aspectName, String name); + /** + * Gets root categories by name, optionally creating one if one does not exist. Index-independent so can be used for + * cluster-safe existence checks. + * + * @param storeRef + * the store ref + * @param aspectName + * the aspect name + * @param name + * the aspect name + * @param create + * should a category node be created if one does not exist? + * @return the root categories + */ + public Collection getRootCategories(StoreRef storeRef, QName aspectName, String name, + boolean create); + /** * Get all the types that represent categories *