JobLockService implementation

- Provides transaction-aware locking
 - Post-transaction cleanup is automatically done
 - Retrying for lock acquisition is handled internally as well
 - Downgraded the lock concurrency tests for the build machine (maybe 50 threads was too much)
 - Deadlock tests added for the high-level locking


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@13968 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/core-services-context.xml

@@ -1257,4 +1257,30 @@
        </property>
     </bean>
     
+    <!-- Clustered (DB) locking Service -->
+    <bean id="jobLockService" class="org.alfresco.repo.lock.JobLockServiceImpl">
+        <property name="retryingTransactionHelper">
+            <bean class="org.alfresco.repo.transaction.RetryingTransactionHelper">
+                <property name="transactionService">
+                    <ref bean="transactionService"/>
+                </property>
+                <property name="maxRetries">
+                    <value>10</value>
+                </property>
+                <property name="minRetryWaitMs">
+                    <value>10</value>
+                </property>
+                <property name="maxRetryWaitMs">
+                    <value>10</value>
+                </property>
+                <property name="retryWaitIncrementMs">
+                    <value>0</value>
+                </property>
+            </bean>
+        </property>
+        <property name="lockDAO" ref="lockDAO" />
+        <property name="defaultRetryCount"><value>10</value></property>
+        <property name="defaultRetryWait"><value>20</value></property>
+    </bean>
+
 </beans>

config/alfresco/messages/system-messages.properties

@@ -24,4 +24,4 @@ system.openoffice.err.connection_remade=The OpenOffice connection was re-establi
 system.locks.err.failed_to_acquire_lock=Failed to get lock ''{0}'' using token ''{1}''.
 system.locks.err.lock_resource_missing=Failed to manipulate lock ''{0}'' using token ''{1}''.  The lock resource no longer exists.
 system.locks.err.lock_update_count=Failed to update lock ''{0}'' using token ''{1}''.  {2} locks were updated when {3} should have been.
-system.locks.err.excl_lock_exists=Failed to get lock ''{0}'' using token ''{1}''.  An exclusive lock in the hierarchy exists.
+system.locks.err.excl_lock_exists=Failed to get lock ''{0}'' using token ''{1}''.  An exclusive lock exists: {2}

source/java/org/alfresco/repo/domain/locks/AbstractLockDAOImpl.java

@@ -117,7 +117,7 @@ public abstract class AbstractLockDAOImpl implements LockDAO
             {
                 throw new LockAcquisitionException(
                         LockAcquisitionException.ERR_EXCLUSIVE_LOCK_EXISTS,
-                        lockQName, lockToken);
+                        lockQName, lockToken, existingLock);
             }
             existingLocksMap.put(existingLock, existingLock);
         }

source/java/org/alfresco/repo/domain/locks/LockDAOTest.java

@@ -335,7 +335,7 @@ public class LockDAOTest extends TestCase
     public synchronized void testConcurrentLockAquisition() throws Exception
     {
         ReentrantLock threadLock = new ReentrantLock();
-        GetLockThread[] threads = new GetLockThread[50];
+        GetLockThread[] threads = new GetLockThread[5];
         for (int i = 0; i < threads.length; i++)
         {
             threads[i] = new GetLockThread(threadLock);
@@ -346,7 +346,7 @@ public class LockDAOTest extends TestCase
         waitLoop:
         for (int waitLoop = 0; waitLoop < 50; waitLoop++)
         {
-            wait(2000L);
+            wait(1000L);
             for (int i = 0; i < threads.length; i++)
             {
                 if (!threads[i].done)

source/java/org/alfresco/repo/domain/locks/LockEntity.java

@@ -37,6 +37,8 @@ import org.alfresco.util.EqualsHelper;
  */
 public class LockEntity
 {
+    public static final Long CONST_LONG_ZERO = new Long(0L);
+    
     private Long id;
     private Long version;
     private Long sharedResourceId;
@@ -71,6 +73,18 @@ public class LockEntity
         }
     }
     
+    @Override
+    public String toString()
+    {
+        StringBuilder sb = new StringBuilder(512);
+        sb.append("LockEntity")
+          .append("[ ID=").append(id)
+          .append(", sharedResourceId=").append(sharedResourceId)
+          .append(", exclusiveResourceId=").append(exclusiveResourceId)
+          .append("]");
+        return sb.toString();
+    }
+
     /**
      * Determine if the lock is logically exclusive.  A lock is <b>exclusive</b> if the
      * shared lock resource matches the exclusive lock resource.
@@ -111,10 +125,21 @@ public class LockEntity
         this.version = version;
     }
     
+    /**
+     * Increments the version number or resets it if it reaches a large number
+     */
     public void incrementVersion()
+    {
+        long currentVersion = version.longValue();
+        if (currentVersion >= 10E6)
+        {
+            this.version = CONST_LONG_ZERO;
+        }
+        else
         {
             this.version = new Long(version.longValue() + 1L);
         }
+    }
 
     /**
      * @return                  Returns the ID of the shared lock resource

source/java/org/alfresco/repo/domain/locks/ibatis/LockDAOImpl.java

@@ -41,7 +41,6 @@ import org.springframework.orm.ibatis.SqlMapClientTemplate;
  */
 public class LockDAOImpl extends AbstractLockDAOImpl
 {
-    private static final Long CONST_LONG_ZERO = new Long(0L);
     private static final String SELECT_LOCKRESOURCE_BY_QNAME = "select.LockResourceByQName";
     private static final String SELECT_LOCK_BY_ID = "select.LockByID";
     private static final String SELECT_LOCK_BY_KEY = "select.LockByKey";
@@ -73,7 +72,7 @@ public class LockDAOImpl extends AbstractLockDAOImpl
     protected LockResourceEntity createLockResource(Long qnameNamespaceId, String qnameLocalName)
     {
         LockResourceEntity lockResource = new LockResourceEntity();
-        lockResource.setVersion(CONST_LONG_ZERO);
+        lockResource.setVersion(LockEntity.CONST_LONG_ZERO);
         lockResource.setQnameNamespaceId(qnameNamespaceId);
         lockResource.setQnameLocalName(qnameLocalName);
         Long id = (Long) template.insert(INSERT_LOCKRESOURCE, lockResource);
@@ -120,7 +119,7 @@ public class LockDAOImpl extends AbstractLockDAOImpl
             long timeToLive)
     {
         LockEntity lock = new LockEntity();
-        lock.setVersion(CONST_LONG_ZERO);
+        lock.setVersion(LockEntity.CONST_LONG_ZERO);
         lock.setSharedResourceId(sharedResourceId);
         lock.setExclusiveResourceId(exclusiveResourceId);
         lock.setLockToken(lockToken);

source/java/org/alfresco/repo/lock/JobLockService.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2005-2009 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing"
+ */
+package org.alfresco.repo.lock;
+
+import org.alfresco.service.namespace.QName;
+
+/**
+ * Service interface for managing job locks.
+ * <p>
+ * Locks are identified by a fully qualified name ({@link QName}) and follow a hierarchical
+ * naming convention i.e. locks higher up a hierarchy can be shared but will prevent explicit
+ * (exclusive) locks from being taken.  For example:  If exclusive lock <b>a.a.a</b> has been
+ * taken, then <b>a.a</b> and <b>a</b> are all implicitly taken as shared locks.  Exclusive lock
+ * <b>a.a.b</b> can be taken by another process and will share locks <b>a.a</b> and <b>a</b>
+ * with the first process.  It will not be possible for a third process to take a lock on
+ * <b>a.a</b>, however.
+ * <p>
+ * <b><u>LOCK ORDERING</u>:</b><br>
+ * The transactional locks will be applied in strict alphabetical order.  A very basic deadlock
+ * prevention system (at least) must be in place when applying or reapplying locks and be biased
+ * against locks applied non-alphabetically.
+ * 
+ * @author Derek Hulley
+ * @since 3.2
+ */
+public interface JobLockService
+{
+    /**
+     * Take a transactionally-managed lock.  This method can be called repeatedly to both
+     * initially acquire the lock as well as to maintain the lock.  This method should
+     * either be called again before the lock expires or the transaction should end before
+     * the lock expires.
+     * <p>
+     * The following rules apply to taking and releasing locks:<br>
+     *  - Expired locks can be taken by any process<br>
+     *  - Lock expiration does not prevent a lock from being refreshed or released<br>
+     *  - Only locks that were manipulated using another token will cause failure
+     * <p>
+     * The locks are automatically released when the transaction is terminated.
+     * <p>
+     * Any failure to acquire the lock (after retries), refresh the lock or subsequently
+     * release the owned locks will invalidate the transaction and cause rollback.
+     * 
+     * @param lockQName             the name of the lock to acquire
+     * @param timeToLive            the time (in milliseconds) for the lock to remain valid
+     * @throws LockAcquisitionException if the lock could not be acquired
+     * @throws IllegalStateException if a transaction is not active
+     */
+    void getTransacionalLock(QName lockQName, long timeToLive);
+    
+    /**
+     * {@inheritDoc JobLockService#getTransacionalLock(QName, long)}
+     * <p>
+     * If the lock cannot be immediately acquired, the process will wait and retry.  Note
+     * that second and subsequent attempts to get the lock during a transaction cannot
+     * make use of retrying; the lock is actually being refreshed and will therefore never
+     * become valid if it doesn't refresh directly.
+     * 
+     * @param retryWait             the time (in milliseconds) to wait before trying again
+     * @param retryCount            the maximum number of times to attempt the lock acquisition
+     * @throws LockAcquisitionException if the lock could not be acquired
+     * @throws IllegalStateException if a transaction is not active
+     */
+    void getTransacionalLock(QName lockQName, long timeToLive, long retryWait, int retryCount);
+}

source/java/org/alfresco/repo/lock/JobLockServiceImpl.java

@@ -0,0 +1,394 @@
+/*
+ * Copyright (C) 2005-2009 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing"
+ */
+package org.alfresco.repo.lock;
+
+import java.util.TreeSet;
+
+import org.alfresco.repo.domain.locks.LockDAO;
+import org.alfresco.repo.transaction.AlfrescoTransactionSupport;
+import org.alfresco.repo.transaction.RetryingTransactionHelper;
+import org.alfresco.repo.transaction.TransactionListenerAdapter;
+import org.alfresco.repo.transaction.TransactionalResourceHelper;
+import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
+import org.alfresco.service.namespace.QName;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * {@inheritDoc JobLockService}
+ * 
+ * @author Derek Hulley
+ * @since 3.2
+ */
+public class JobLockServiceImpl implements JobLockService
+{
+    private static final String KEY_RESOURCE_LOCKS = "JobLockServiceImpl.Locks";
+    
+    private static Log logger = LogFactory.getLog(JobLockServiceImpl.class);
+    
+    private LockDAO lockDAO;
+    private RetryingTransactionHelper retryingTransactionHelper;
+    private int defaultRetryCount;
+    private long defaultRetryWait;
+    
+    /**
+     * Stateless listener that does post-transaction cleanup.
+     */
+    private final LockTransactionListener txnListener;
+    
+    public JobLockServiceImpl()
+    {
+        defaultRetryWait = 20;
+        defaultRetryCount = 10;
+        txnListener = new LockTransactionListener();
+    }
+
+    /**
+     * Set the lock DAO
+     */
+    public void setLockDAO(LockDAO lockDAO)
+    {
+        this.lockDAO = lockDAO;
+    }
+
+    /**
+     * Set the helper that will handle low-level concurrency conditions i.e. that
+     * enforces optimistic locking and deals with stale state issues.
+     */
+    public void setRetryingTransactionHelper(RetryingTransactionHelper retryingTransactionHelper)
+    {
+        this.retryingTransactionHelper = retryingTransactionHelper;
+    }
+
+    /**
+     * Set the maximum number of attempts to make at getting a lock
+     * @param defaultRetryCount         the number of attempts
+     */
+    public void setDefaultRetryCount(int defaultRetryCount)
+    {
+        this.defaultRetryCount = defaultRetryCount;
+    }
+
+    /**
+     * Set the default time to wait between attempts to acquire a lock
+     * @param defaultRetryWait          the wait time in milliseconds
+     */
+    public void setDefaultRetryWait(long defaultRetryWait)
+    {
+        this.defaultRetryWait = defaultRetryWait;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public void getTransacionalLock(QName lockQName, long timeToLive)
+    {
+        getTransacionalLock(lockQName, timeToLive, defaultRetryWait, defaultRetryCount);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public void getTransacionalLock(QName lockQName, long timeToLive, long retryWait, int retryCount)
+    {
+        // Check that transaction is present
+        final String txnId = AlfrescoTransactionSupport.getTransactionId();
+        if (txnId == null)
+        {
+            throw new IllegalStateException("Locking requires an active transaction");
+        }
+        // Get the set of currently-held locks
+        TreeSet<QName> heldLocks = TransactionalResourceHelper.getTreeSet(KEY_RESOURCE_LOCKS);
+        // We don't want the lock registered as being held if something goes wrong
+        TreeSet<QName> heldLocksTemp = new TreeSet<QName>(heldLocks);
+        boolean added = heldLocksTemp.add(lockQName);
+        if (!added)
+        {
+            // It's a refresh.  Ordering is not important here as we already hold the lock.
+            refreshLock(lockQName, timeToLive);
+        }
+        else
+        {
+            QName lastLock = heldLocksTemp.last();
+            if (lastLock.equals(lockQName))
+            {
+                if (logger.isDebugEnabled())
+                {
+                    logger.debug(
+                            "Attempting to acquire ordered lock: \n" +
+                            "   Lock:     " + lockQName + "\n" +
+                            "   TTL:      " + timeToLive + "\n" +
+                            "   Txn:      " + txnId);
+                }
+                // If it was last in the set, then the order is correct and we use the
+                // full retry behaviour.
+                getLock(lockQName, timeToLive, retryWait, retryCount);
+            }
+            else
+            {
+                if (logger.isDebugEnabled())
+                {
+                    logger.debug(
+                            "Attempting to acquire UNORDERED lock: \n" +
+                            "   Lock:     " + lockQName + "\n" +
+                            "   TTL:      " + timeToLive + "\n" +
+                            "   Txn:      " + txnId);
+                }
+                // The lock request is made out of natural order.
+                // Unordered locks do not get any retry behaviour
+                getLock(lockQName, timeToLive, retryWait, 1);
+            }
+        }
+        // It went in, so add it to the transactionally-stored set
+        heldLocks.add(lockQName);
+        // Done
+    }
+    
+    /**
+     * @throws LockAcquisitionException on failure
+     */
+    private void refreshLock(final QName lockQName, final long timeToLive)
+    {
+        // The lock token is the current transaction ID
+        final String txnId = AlfrescoTransactionSupport.getTransactionId();
+        RetryingTransactionCallback<Object> refreshLockCallback = new RetryingTransactionCallback<Object>()
+        {
+            public Object execute() throws Throwable
+            {
+                lockDAO.releaseLock(lockQName, txnId);
+                return null;
+            }
+        };
+        try
+        {
+            // It must succeed
+            retryingTransactionHelper.doInTransaction(refreshLockCallback, false, true);
+            // Success
+            if (logger.isDebugEnabled())
+            {
+                logger.debug(
+                        "Refreshed Lock: \n" +
+                        "   Lock:     " + lockQName + "\n" +
+                        "   TTL:      " + timeToLive + "\n" +
+                        "   Txn:      " + txnId);
+            }
+        }
+        catch (LockAcquisitionException e)
+        {
+            // Failure
+            if (logger.isDebugEnabled())
+            {
+                logger.debug(
+                        "Lock refresh failed: \n" +
+                        "   Lock:     " + lockQName + "\n" +
+                        "   TTL:      " + timeToLive + "\n" +
+                        "   Txn:      " + txnId + "\n" +
+                        "   Error:    " + e.getMessage());
+            }
+            throw e;
+        }
+    }
+    
+    /**
+     * @throws LockAcquisitionException on failure
+     */
+    private void getLock(final QName lockQName, final long timeToLive, long retryWait, int retryCount)
+    {
+        // The lock token is the current transaction ID
+        final String txnId = AlfrescoTransactionSupport.getTransactionId();
+        RetryingTransactionCallback<Object> getLockCallback = new RetryingTransactionCallback<Object>()
+        {
+            public Object execute() throws Throwable
+            {
+                lockDAO.getLock(lockQName, txnId, timeToLive);
+                return null;
+            }
+        };
+        try
+        {
+            int iterations = doWithRetry(getLockCallback, retryWait, retryCount);
+            // Bind in a listener
+            AlfrescoTransactionSupport.bindListener(txnListener);
+            // Success
+            if (logger.isDebugEnabled())
+            {
+                logger.debug(
+                        "Acquired Lock: \n" +
+                        "   Lock:     " + lockQName + "\n" +
+                        "   TTL:      " + timeToLive + "\n" +
+                        "   Txn:      " + txnId + "\n" +
+                        "   Attempts: " + iterations);
+            }
+        }
+        catch (LockAcquisitionException e)
+        {
+            // Failure
+            if (logger.isDebugEnabled())
+            {
+                logger.debug(
+                        "Lock acquisition failed: \n" +
+                        "   Lock:     " + lockQName + "\n" +
+                        "   TTL:      " + timeToLive + "\n" +
+                        "   Txn:      " + txnId + "\n" +
+                        "   Error:    " + e.getMessage());
+            }
+            throw e;
+        }
+    }
+    
+    /**
+     * Does the high-level retrying around the callback
+     */
+    private int doWithRetry(RetryingTransactionCallback<? extends Object> callback, long retryWait, int retryCount)
+    {
+        int iteration = 0;
+        LockAcquisitionException lastException = null;
+        while (iteration++ < retryCount)
+        {
+            try
+            {
+                retryingTransactionHelper.doInTransaction(callback, false, true);
+                // Success.  Clear the exception indicator! 
+                lastException = null;
+                break;
+            }
+            catch (LockAcquisitionException e)
+            {
+                if (logger.isDebugEnabled())
+                {
+                    logger.debug("Lock attempt " + iteration + " of " + retryCount + " failed: " + e.getMessage());
+                }
+                lastException = e;
+                if (iteration >= retryCount)
+                {
+                    // Avoid an unnecessary wait if this is the last attempt
+                    break;
+                }
+            }
+            // Before running again, do a wait
+            synchronized(callback)
+            {
+                try { callback.wait(retryWait); } catch (InterruptedException e) {}
+            }
+        }
+        if (lastException == null)
+        {
+            // Success
+            return iteration;
+        }
+        else
+        {
+            // Failure
+            throw lastException;
+        }
+    }
+    
+    /**
+     * Handles the transction synchronization activity, ensuring locks are rolled back as
+     * required.
+     * 
+     * @author Derek Hulley
+     * @since 3.2
+     */
+    private class LockTransactionListener extends TransactionListenerAdapter
+    {
+        /**
+         * Release any open locks with extreme prejudice i.e. the commit will fail if the
+         * locks cannot be released.  The locks are released in a single transaction -
+         * ordering is therefore not important.  Should this fail, the post-commit phase
+         * will do a final cleanup with individual locks.
+         */
+        @Override
+        public void beforeCommit(boolean readOnly)
+        {
+            final String txnId = AlfrescoTransactionSupport.getTransactionId();
+            final TreeSet<QName> heldLocks = TransactionalResourceHelper.getTreeSet(KEY_RESOURCE_LOCKS);
+            // Shortcut if there are no locks
+            if (heldLocks.size() == 0)
+            {
+                return;
+            }
+            // Clean up the locks
+            RetryingTransactionCallback<Object> releaseCallback = new RetryingTransactionCallback<Object>()
+            {
+                public Object execute() throws Throwable
+                {
+                    // Any one of the them could fail
+                    for (QName lockQName : heldLocks)
+                    {
+                        lockDAO.releaseLock(lockQName, txnId);
+                    }
+                    return null;
+                }
+            };
+            retryingTransactionHelper.doInTransaction(releaseCallback, false, true);
+            // So they were all successful
+            heldLocks.clear();
+        }
+
+        /**
+         * This will be called if something went wrong.  It might have been the lock releases, but
+         * it could be anything else as well.  Each remaining lock is released with warnings where
+         * it fails.
+         */
+        @Override
+        public void afterRollback()
+        {
+            final String txnId = AlfrescoTransactionSupport.getTransactionId();
+            final TreeSet<QName> heldLocks = TransactionalResourceHelper.getTreeSet(KEY_RESOURCE_LOCKS);
+            // Shortcut if there are no locks
+            if (heldLocks.size() == 0)
+            {
+                return;
+            }
+            // Clean up any remaining locks
+            for (final QName lockQName : heldLocks)
+            {
+                RetryingTransactionCallback<Object> releaseCallback = new RetryingTransactionCallback<Object>()
+                {
+                    public Object execute() throws Throwable
+                    {
+                        lockDAO.releaseLock(lockQName, txnId);
+                        return null;
+                    }
+                };
+                try
+                {
+                    retryingTransactionHelper.doInTransaction(releaseCallback, false, true);
+                }
+                catch (Throwable e)
+                {
+                    // There is no point propagating this, so just log a warning and
+                    // hope that it expires soon enough
+                    logger.warn(
+                            "Failed to release a lock in 'afterRollback':\n" +
+                            "   Lock Name:  " + lockQName + "\n" +
+                            "   Lock Token: " + txnId,
+                            e);
+                }
+            }
+        }
+    }
+}

source/java/org/alfresco/repo/lock/JobLockServiceTest.java

@@ -0,0 +1,287 @@
+/*
+ * Copyright (C) 2005-2009 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing"
+ */
+package org.alfresco.repo.lock;
+
+import junit.framework.TestCase;
+
+import org.alfresco.repo.domain.locks.LockDAO;
+import org.alfresco.repo.transaction.RetryingTransactionHelper;
+import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
+import org.alfresco.service.ServiceRegistry;
+import org.alfresco.service.namespace.QName;
+import org.alfresco.service.transaction.TransactionService;
+import org.alfresco.util.ApplicationContextHelper;
+import org.springframework.context.ApplicationContext;
+
+/**
+ * Tests the high-level capabilities provided by the service implementation.  The DAO tests
+ * stress the underlying database work, so we only need to deal with deadlock resolution, etc.
+ * 
+ * @see JobLockService      the service being tested
+ * @see LockDAO             the DAO being indirectly tested
+ * 
+ * @author Derek Hulley
+ * @since 3.2
+ */
+public class JobLockServiceTest extends TestCase
+{
+    public static final String NAMESPACE = "http://www.alfresco.org/test/JobLockServiceTest";
+    
+    private ApplicationContext ctx = ApplicationContextHelper.getApplicationContext();
+
+    private TransactionService transactionService;
+    private RetryingTransactionHelper txnHelper;
+    private JobLockService jobLockService;
+    // Lock names for the tests
+    private QName lockA;
+    private QName lockAA;
+    private QName lockAAA;
+    private QName lockAAB;
+    private QName lockAAC;
+    private QName lockAB;
+    private QName lockABA;
+    private QName lockABB;
+    private QName lockABC;
+    
+    @Override
+    public void setUp() throws Exception
+    {
+        ServiceRegistry serviceRegistry = (ServiceRegistry) ctx.getBean(ServiceRegistry.SERVICE_REGISTRY);
+        transactionService = serviceRegistry.getTransactionService();
+        txnHelper = transactionService.getRetryingTransactionHelper();
+        
+        jobLockService = (JobLockService) ctx.getBean("jobLockService");
+        
+        // Get the test name
+        String testName = getName();
+        // Build lock names for the test
+        lockA = QName.createQName(NAMESPACE, "a-" + testName);
+        lockAA = QName.createQName(NAMESPACE, "a-" + testName + ".a-" + testName);
+        lockAAA = QName.createQName(NAMESPACE, "a-" + testName + ".a-" + testName + ".a-" + testName);
+        lockAAB = QName.createQName(NAMESPACE, "a-" + testName + ".a-" + testName + ".b-" + testName);
+        lockAAC = QName.createQName(NAMESPACE, "a-" + testName + ".a-" + testName + ".c-" + testName);
+        lockAB = QName.createQName(NAMESPACE, "a-" + testName + ".b-" + testName);
+        lockABA = QName.createQName(NAMESPACE, "a-" + testName + ".b-" + testName + ".a-" + testName);
+        lockABB = QName.createQName(NAMESPACE, "a-" + testName + ".b-" + testName + ".b-" + testName);
+        lockABC = QName.createQName(NAMESPACE, "a-" + testName + ".b-" + testName + ".c-" + testName);
+    }
+    
+    public void testSetUp()
+    {
+        assertNotNull(jobLockService);
+    }
+    
+    public void testEnforceTxn()
+    {
+        try
+        {
+            jobLockService.getTransacionalLock(lockAAA, 50L);
+            fail("Service did not enforce the presence of a transaction");
+        }
+        catch (IllegalStateException e)
+        {
+            // Expected
+        }
+    }
+    
+    /**
+     * Checks that the lock can be aquired by a read-only transaction i.e. that locking is
+     * independent of the outer transaction.
+     */
+    public void testLockInReadOnly() throws Exception
+    {
+        RetryingTransactionCallback<Object> lockCallback = new RetryingTransactionCallback<Object>()
+        {
+            public Object execute() throws Throwable
+            {
+                jobLockService.getTransacionalLock(lockAAA, 500);
+                return null;
+            }
+        };
+        txnHelper.doInTransaction(lockCallback, true, true);
+    }
+    
+    /**
+     * Checks that locks are released on commit
+     */
+    public void testLockReleaseOnCommit() throws Exception
+    {
+        RetryingTransactionCallback<Object> lockCallback = new RetryingTransactionCallback<Object>()
+        {
+            public Object execute() throws Throwable
+            {
+                jobLockService.getTransacionalLock(lockAAA, 5000);
+                return null;
+            }
+        };
+        txnHelper.doInTransaction(lockCallback, true, true);
+        // The lock should be free now, even though the TTL was high
+        RetryingTransactionCallback<Object> lockCheckCallback = new RetryingTransactionCallback<Object>()
+        {
+            public Object execute() throws Throwable
+            {
+                jobLockService.getTransacionalLock(lockAAA, 50);
+                return null;
+            }
+        };
+        txnHelper.doInTransaction(lockCheckCallback, true, true);
+    }
+    
+    /**
+     * Checks that locks are released on rollback
+     */
+    public void testLockReleaseOnRollback() throws Exception
+    {
+        RetryingTransactionCallback<Object> lockCallback = new RetryingTransactionCallback<Object>()
+        {
+            public Object execute() throws Throwable
+            {
+                jobLockService.getTransacionalLock(lockAAA, 5000);
+                throw new UnsupportedOperationException("ALERT!");
+            }
+        };
+        try
+        {
+            txnHelper.doInTransaction(lockCallback, true, true);
+            fail("Expected transaction failure");
+        }
+        catch (UnsupportedOperationException e)
+        {
+            // Expected
+        }
+        // The lock should be free now, even though the TTL was high
+        RetryingTransactionCallback<Object> lockCheckCallback = new RetryingTransactionCallback<Object>()
+        {
+            public Object execute() throws Throwable
+            {
+                jobLockService.getTransacionalLock(lockAAA, 50);
+                return null;
+            }
+        };
+        txnHelper.doInTransaction(lockCheckCallback, true, true);
+    }
+    
+    /**
+     * Sets up two threads in a deadlock scenario.  Each of the threads has a long wait timeout
+     * for the required locks.  If there were a deadlock, the shorter of the the wait times would
+     * be how long it would take before one of them is thrown out.  Firstly, we check that one
+     * of the threads <i>is</i> thrown out.  Then we check that the thread is thrown out quickly.
+     */
+    public synchronized void testDeadlockPrevention() throws Throwable
+    {
+        DeadlockingThread t1 = new DeadlockingThread(lockAAA, lockAAB);
+        DeadlockingThread t2 = new DeadlockingThread(lockAAB, lockAAA);
+        // Start them
+        t1.start();
+        t2.start();
+        // They can take their first locks (there should be no contention)
+        t1.incrementNextLock();
+        t2.incrementNextLock();
+        // Wait for them to do this
+        try { this.wait(2000L); } catch (InterruptedException e) {}
+        // Advance again
+        t1.incrementNextLock();
+        t2.incrementNextLock();
+        // Wait for them to do this
+        try { this.wait(2000L); } catch (InterruptedException e) {}
+        // Advance again, to end threads
+        t1.incrementNextLock();
+        t2.incrementNextLock();
+        // Wait for them to end (commit/rollback)
+        try { this.wait(2000L); } catch (InterruptedException e) {}
+        
+        if (t1.otherFailure != null)
+        {
+            throw t1.otherFailure;
+        }
+        if (t2.otherFailure != null)
+        {
+            throw t2.otherFailure;
+        }
+        assertNull("T1 should have succeeded as the ordered locker: " + t1.lockFailure, t1.lockFailure);
+        assertNotNull("T2 should have failed as the unordered locker.", t2.lockFailure);
+    }
+    
+    private class DeadlockingThread extends Thread
+    {
+        private final QName[] lockQNames;
+        private volatile int nextLock = -1;
+        private LockAcquisitionException lockFailure;
+        private Throwable otherFailure;
+        
+        private DeadlockingThread(QName ... lockQNames)
+        {
+            super("DeadlockingThread");
+            this.lockQNames = lockQNames;
+            setDaemon(true);
+        }
+        
+        private void incrementNextLock()
+        {
+            nextLock++;
+        }
+        
+        @Override
+        public void run()
+        {
+            RetryingTransactionCallback<Object> runCallback = new RetryingTransactionCallback<Object>()
+            {
+                public synchronized Object execute() throws Throwable
+                {
+                    int currentLock = -1;
+                    // Take the locks in turn, quitting when told to take a lock that's not there
+                    while (currentLock < lockQNames.length - 1)
+                    {
+                        // Check if we have been instructed to take a lock
+                        if (nextLock > currentLock)
+                        {
+                            // Advance and grab the lock
+                            currentLock++;
+                            jobLockService.getTransacionalLock(lockQNames[currentLock], 5000L);
+                        }
+                        else
+                        {
+                            // No advance, so wait a bit more
+                            try { this.wait(20L); } catch (InterruptedException e) {}
+                        }
+                    }
+                    return null;
+                }
+            };
+            try
+            {
+                txnHelper.doInTransaction(runCallback, true);
+            }
+            catch (LockAcquisitionException e)
+            {
+                lockFailure = e;
+            }
+            catch (Throwable e)
+            {
+                otherFailure = e;
+            }
+        }
+    }
+}

source/java/org/alfresco/repo/lock/LockAcquisitionException.java

@@ -64,6 +64,7 @@ public class LockAcquisitionException extends AlfrescoRuntimeException
      * <ul>
      *   <li>1: the qname</li>
      *   <li>2: the lock token</li>
+     *   <li>3: the existing other lock</li>
      * </ul>
      */
     public static final String ERR_EXCLUSIVE_LOCK_EXISTS = "system.locks.err.excl_lock_exists";

source/java/org/alfresco/repo/transaction/TransactionalResourceHelper.java

@@ -30,6 +30,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 /**
  * Helper class that will look up or create transactional resources.
@@ -77,6 +78,24 @@ public abstract class TransactionalResourceHelper
         return set;
     }
 
+    /**
+     * Support method to retrieve or create and bind a <tt>TreeSet</tt> to the current transaction.
+     * 
+     * @param <V>           the set value type
+     * @param resourceKey   the key under which the resource will be stored
+     * @return              Returns an previously-bound <tt>TreeSet</tt> or else a newly-bound <tt>TreeSet</tt>
+     */
+    public static final <V> TreeSet<V> getTreeSet(Object resourceKey)
+    {
+        TreeSet<V> set = AlfrescoTransactionSupport.<TreeSet<V>>getResource(resourceKey);
+        if (set == null)
+        {
+            set = new TreeSet<V>();
+            AlfrescoTransactionSupport.bindResource(resourceKey, set);
+        }
+        return set;
+    }
+
     /**
      * Support method to retrieve or create and bind a <tt>ArrayList</tt> to the current transaction.
      * 

source/java/org/alfresco/service/namespace/QName.java

@@ -40,7 +40,7 @@ import org.alfresco.repo.domain.hibernate.NamespaceEntityImpl;
  * @author David Caruana
  * 
  */
-public final class QName implements QNamePattern, Serializable, Cloneable
+public final class QName implements QNamePattern, Serializable, Cloneable, Comparable<QName>
 {
     private static final long serialVersionUID = 3977016258204348976L;
 
@@ -348,6 +348,21 @@ public final class QName implements QNamePattern, Serializable, Cloneable
                                     .append(localName).toString();
     }
 
+    /**
+     * Uses the {@link #getNamespaceURI() namespace URI} and then the {@link #getLocalName() localname}
+     * to do the comparison i.e. the comparison is alphabetical.
+     */
+    public int compareTo(QName qname)
+    {
+        int namespaceComparison = this.namespaceURI.compareTo(qname.namespaceURI);
+        if (namespaceComparison != 0)
+        {
+            return namespaceComparison;
+        }
+        // Namespaces are the same.  Do comparison on localname
+        return this.localName.compareTo(qname.localName);
+    }
+
 
     /**
      * Render string representation of QName using format: