Merged HEAD-BUG-FIX (4.3/Cloud) to HEAD (4.3/Cloud)

60122: Merged V4.2-BUG-FIX (4.2.2) to HEAD-BUG-FIX (Cloud/4.3)
      60121: Merged V4.1-BUG-FIX (4.1.8) to V4.2-BUG-FIX (4.2.2)
         60118: Merged DEV (V4.1-BUG-FIX-2013_11_14) to V4.1-BUG-FIX (4.1.8)
            60117: Merged DEV (V4.2.1-2014_01_09) to DEV (V4.1-BUG-FIX-2013_11_14)
               60116: MNT-10399: AutomaticBuilds: Alfresco doesn't shut down because of FullTextSearchIndexerImpl 
                  - Try to spot dead daemon threads holding the lock on subsystem shutdown


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@62267 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/search/impl/lucene/fts/FullTextSearchIndexerImpl.java

@@ -61,6 +61,77 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
 
     private int batchSize = 1000;
     
+    /**
+     * Although not really allowed, locks are being held by daemon threads. This
+     * field holds the thread with the lock, so that other threads that 'wait'
+     * don't wait forever if a daemon thread that has the lock is killed on shutdown.
+     * 
+     * On entry to a synchronized method or block the value is either null or the
+     * current Thread (if a nested call). On exit it is set back to its original value
+     * unless the Thread dies.
+     * 
+     * Prior to calling wait it must be set to null and on return must be set back to
+     * the current thread.
+     * 
+     * If a daemon thread has just been killed, on return from the wait,
+     * it will not be null and the thread holding the lock will not be alive.
+     * 
+     * The alternative to this approach might include:
+     * - Making all threads that use this class daemons (might be simplest)
+     */
+    private static Thread threadHoldingLock;
+
+    // Helper class to keep track of which Thread has the lock, so we can give up if it dies.
+    private abstract class SynchronizedHelper<Result>
+    {
+    	public Result executeNoInterruptedException()
+    	{
+    		try {
+				return execute();
+			} catch (InterruptedException e) {
+				// ignore
+				return null;
+			}
+    	}
+    	
+    	@SuppressWarnings("finally")
+		public Result execute() throws InterruptedException
+    	{
+        	Thread origThreadHoldingLock = threadHoldingLock;
+        	threadHoldingLock = Thread.currentThread();
+        	try
+        	{
+        		return run();
+        	}
+        	catch (ThreadDeath threadDeath)
+        	{
+    			origThreadHoldingLock = null;
+    			throw threadDeath;
+        	}
+        	finally
+        	{
+            	threadHoldingLock = origThreadHoldingLock;
+        	}
+    	}
+    	
+    	public abstract Result run() throws InterruptedException;
+    	
+    	// Does a wait(10000). Returns true if the Thread that had the lock has died.
+    	public boolean waitAndCheckForSubsystemShutdown() throws InterruptedException
+    	{
+    		try
+    		{
+    			threadHoldingLock = null;
+                FullTextSearchIndexerImpl.this.wait(10000);
+                return threadHoldingLock != null && !threadHoldingLock.isAlive();
+    		}
+    		finally
+    		{
+    			threadHoldingLock = Thread.currentThread();
+    		}
+    	}
+    }
+
     /**
      * 
      */
@@ -75,13 +146,20 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
      * 
      * @see org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer#requiresIndex(org.alfresco.repo.ref.StoreRef)
      */
-    public synchronized void requiresIndex(StoreRef storeRef)
+    public synchronized void requiresIndex(final StoreRef storeRef)
+    {
+    	new SynchronizedHelper<Void>()
+    	{
+    		public Void run()
     		{
                 if(s_logger.isDebugEnabled())
                 {
                     s_logger.debug("FTS index request for "+storeRef);
                 }
                 requiresIndex.add(storeRef);
+				return null;
+    		}
+    	}.executeNoInterruptedException();
     }
 
     /*
@@ -89,7 +167,11 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
      * 
      * @see org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer#indexCompleted(org.alfresco.repo.ref.StoreRef, int, java.lang.Exception)
      */
-    public synchronized void indexCompleted(StoreRef storeRef, int remaining, Throwable t)
+    public synchronized void indexCompleted(final StoreRef storeRef, final int remaining, final Throwable t)
+    {
+    	new SynchronizedHelper<Void>()
+    	{
+    		public Void run()
     		{
                 try
                 {
@@ -109,8 +191,11 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
                 }
                 finally
                 {
-            this.notifyAll();
+                	FullTextSearchIndexerImpl.this.notifyAll();
                 }
+				return null;
+    		}
+    	}.executeNoInterruptedException();
     }
 
     /*
@@ -119,6 +204,10 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
      * @see org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer#pause()
      */
     public synchronized void pause() throws InterruptedException
+    {
+    	new SynchronizedHelper<Void>()
+    	{
+    		public Void run() throws InterruptedException
     		{
             	pauseCount++;
                 if(s_logger.isTraceEnabled())
@@ -131,18 +220,25 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
                     {
                         s_logger.trace("Pause: Waiting with count of "+indexing.size()+" id is "+this);
                     }
-            this.wait();
+                    
+                    if (waitAndCheckForSubsystemShutdown())
+                    {
+                    	indexing.clear();
+                    }
                 }
                 pauseCount--;
                 if (pauseCount == 0)
                 {
                     paused = true;
-            this.notifyAll(); // only resumers
+                    FullTextSearchIndexerImpl.this.notifyAll(); // only resumers
                 }
                 if(s_logger.isTraceEnabled())
                 {
                     s_logger.trace("..Remaining "+pauseCount +" paused = "+paused+" id is "+this);
                 }
+				return null;
+    		}
+    	}.execute();
     }
 
     /*
@@ -151,6 +247,10 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
      * @see org.alfresco.repo.search.impl.lucene.fts.FullTextSearchIndexer#resume()
      */
     public synchronized void resume() throws InterruptedException
+    {
+    	new SynchronizedHelper<Void>()
+    	{
+    		public Void run() throws InterruptedException
     		{
                 if (pauseCount == 0)
                 {
@@ -168,14 +268,25 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
                         {
                             s_logger.trace("Resume waiting on "+pauseCount+" id is "+this);
                         }
-                this.wait();
+
+                        if (waitAndCheckForSubsystemShutdown())
+                        {
+                        	break;
+                        }
                     }
                     paused = false;
                 }
+				return null;
+    		}
+    	}.execute();
     }
 
     @SuppressWarnings("unused")
     private synchronized boolean isPaused() throws InterruptedException
+    {
+    	return new SynchronizedHelper<Boolean>()
+    	{
+    		public Boolean run() throws InterruptedException
     		{
                 if (pauseCount == 0)
                 {
@@ -185,11 +296,16 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
                 {
                     while (pauseCount > 0)
                     {
-                this.wait();
+                        if (waitAndCheckForSubsystemShutdown())
+                        {
+                        	break;
+                        }
                     }
                     return paused;
                 }
     		}
+    	}.execute();
+    }
 
     /*
      * (non-Javadoc)
@@ -261,6 +377,10 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
     }
 
     private synchronized StoreRef getNextRef()
+    {
+    	return new SynchronizedHelper<StoreRef>()
+    	{
+    		public StoreRef run()
     		{
                 if (paused || (pauseCount > 0))
                 {
@@ -291,6 +411,8 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
 
                 return nextStoreRef;
     		}
+    	}.executeNoInterruptedException();
+    }
 
     /**
      * @param indexerAndSearcherFactory
@@ -353,6 +475,4 @@ public class FullTextSearchIndexerImpl implements FTSIndexerAware, FullTextSearc
         requiresIndex.clear();
         indexing.clear();
     }
-    
-    
 }