MOB-710: LDAP queries now performed with RFC 2696 paging and a configurable page size, defaulting to 1000 (the AD default maximum)

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@14648 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/subsystems/Authentication/ldap/ldap-authentication-context.xml

@@ -149,6 +149,16 @@
         <property name="active">
             <value>${ldap.synchronization.active}</value>
         </property>
+
+        <!--
+        If positive, this property indicates that RFC 2696 paged results should be
+        used to split query results into batches of the specified size. This
+        overcomes any size limits imposed by the LDAP server.        
+        -->
+        <property name="queryBatchSize">
+            <value>${ldap.synchronization.queryBatchSize}</value>
+        </property>
+
         <!--
         The query to select all objects that represent the groups to import.
         

config/alfresco/subsystems/Authentication/ldap/ldap-authentication.properties

@@ -49,6 +49,11 @@ ldap.authentication.defaultAdministratorUserNames=
 # authentication, in which case this flag should be set to false.
 ldap.synchronization.active=true
 
+# If positive, this property indicates that RFC 2696 paged results should be
+# used to split query results into batches of the specified size. This
+# overcomes any size limits imposed by the LDAP server.
+ldap.synchronization.queryBatchSize=1000
+
 # The query to select all objects that represent the groups to import.
 ldap.synchronization.groupQuery=(objectclass\=groupOfNames)
 

source/java/org/alfresco/repo/security/authentication/ldap/LDAPInitialDirContextFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005-2007 Alfresco Software Limited.
+ * Copyright (C) 2005-2009 Alfresco Software Limited.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -18,7 +18,7 @@
  * As a special exception to the terms and conditions of version 2.0 of 
  * the GPL, you may redistribute this Program in connection with Free/Libre 
  * and Open Source Software ("FLOSS") applications as described in Alfresco's 
- * FLOSS exception.  You should have recieved a copy of the text describing 
+ * FLOSS exception.  You should have received a copy of the text describing 
  * the FLOSS exception, and it is also available here: 
  * http://www.alfresco.com/legal/licensing"
  */
@@ -26,6 +26,7 @@ package org.alfresco.repo.security.authentication.ldap;
 
 import java.util.Map;
 
+import javax.naming.directory.DirContext;
 import javax.naming.directory.InitialDirContext;
 
 import org.alfresco.repo.security.authentication.AuthenticationException;
@@ -44,6 +45,19 @@ public interface LDAPInitialDirContextFactory
      */
     public void setInitialDirContextEnvironment(Map<String, String> environment);
     
+    /**
+     * Use the environment properties and connect to the LDAP server, optionally configuring RFC 2696 paged results.
+     * Used to obtain read only access to the LDAP server.
+     * 
+     * @param pageSize
+     *            if a positive value, indicates that a LDAP v3 RFC 2696 paged results control should be used. The
+     *            results of a search operation should be returned by the LDAP server in batches of the specified size.
+     * @return the default intial dir context
+     * @throws AuthenticationException
+     *             the authentication exception
+     */
+    public InitialDirContext getDefaultIntialDirContext(int pageSize) throws AuthenticationException;
+    
     /**
      * Use the environment properties and connect to the LDAP server.
      * Used to obtain read only access to the LDAP server.
@@ -53,6 +67,19 @@ public interface LDAPInitialDirContextFactory
      */
     public InitialDirContext getDefaultIntialDirContext() throws AuthenticationException;
     
+    /**
+     * Determines whether there is another page to fetch from the last search to be run in this context. Also prepares
+     * the request controls so that the appropriate cookie will be passed in the next search.
+     * 
+     * @param ctx
+     *            the context
+     * @param pageSize
+     *            if a positive value, indicates that a LDAP v3 RFC 2696 paged results control should be used. The
+     *            results of a search operation should be returned by the LDAP server in batches of the specified size.
+     * @return true, if is ready for next page
+     */
+    public boolean hasNextPage(DirContext ctx, int pageSize);    
+
     /**
      * Augment the connection environment with the identity and credentials and bind to the ldap server.
      * Mainly used to validate a user's credentials during authentication. 

source/java/org/alfresco/repo/security/authentication/ldap/LDAPInitialDirContextFactoryImpl.java

@@ -24,6 +24,7 @@
  */
 package org.alfresco.repo.security.authentication.ldap;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Hashtable;
 import java.util.Map;
@@ -35,7 +36,13 @@ import javax.naming.directory.Attribute;
 import javax.naming.directory.Attributes;
 import javax.naming.directory.BasicAttribute;
 import javax.naming.directory.BasicAttributes;
+import javax.naming.directory.DirContext;
 import javax.naming.directory.InitialDirContext;
+import javax.naming.ldap.Control;
+import javax.naming.ldap.InitialLdapContext;
+import javax.naming.ldap.LdapContext;
+import javax.naming.ldap.PagedResultsControl;
+import javax.naming.ldap.PagedResultsResponseControl;
 
 import org.alfresco.repo.security.authentication.AuthenticationException;
 import org.alfresco.util.ApplicationContextHelper;
@@ -71,19 +78,38 @@ public class LDAPInitialDirContextFactoryImpl implements LDAPInitialDirContextFa
     }
 
     public InitialDirContext getDefaultIntialDirContext() throws AuthenticationException
+    {
+        return getDefaultIntialDirContext(0);
+    }
+
+    public InitialDirContext getDefaultIntialDirContext(int pageSize) throws AuthenticationException
     {
         Hashtable<String, String> env = new Hashtable<String, String>(initialDirContextEnvironment.size());
         env.putAll(initialDirContextEnvironment);
         env.put("javax.security.auth.useSubjectCredsOnly", "false");
-        return buildInitialDirContext(env);
+        return buildInitialDirContext(env, pageSize);
     }
 
-    private InitialDirContext buildInitialDirContext(Hashtable<String, String> env) throws AuthenticationException
+    private InitialDirContext buildInitialDirContext(Hashtable<String, String> env, int pageSize)
+            throws AuthenticationException
     {
         try
+        {
+            // If a page size has been requested, use LDAP v3 paging
+            if (pageSize > 0)
+            {
+                InitialLdapContext ctx = new InitialLdapContext(env, null);
+                ctx.setRequestControls(new Control[]
+                {
+                    new PagedResultsControl(pageSize, Control.CRITICAL)
+                });
+                return ctx;
+            }
+            else
             {
                 return new InitialDirContext(env);
             }
+        }
         catch (javax.naming.AuthenticationException ax)
         {
             throw new AuthenticationException("LDAP authentication failed.", ax);
@@ -92,6 +118,54 @@ public class LDAPInitialDirContextFactoryImpl implements LDAPInitialDirContextFa
         {
             throw new AuthenticationException("Unable to connect to LDAP Server; check LDAP configuration", nx);
         }
+        catch (IOException e)
+        {
+            throw new AuthenticationException("Unable to encode LDAP v3 request controls; check LDAP configuration", e);
+        }
+    }
+
+    public boolean hasNextPage(DirContext ctx, int pageSize)
+    {
+        if (pageSize > 0)
+        {
+            try
+            {
+                LdapContext ldapContext = (LdapContext) ctx;
+                Control[] controls = ldapContext.getResponseControls();
+
+                // Retrieve the paged result cookie if there is one
+                if (controls != null)
+                {
+                    for (Control control : controls)
+                    {
+                        if (control instanceof PagedResultsResponseControl)
+                        {
+                            byte[] cookie = ((PagedResultsResponseControl) control).getCookie();
+                            if (cookie != null)
+                            {
+                                // Prepare for next page
+                                ldapContext.setRequestControls(new Control[]
+                                {
+                                    new PagedResultsControl(pageSize, cookie, Control.CRITICAL)
+                                });
+                                return true;
+                            }
+                        }
+                    }
+                }
+            }
+            catch (NamingException nx)
+            {
+                throw new AuthenticationException("Unable to connect to LDAP Server; check LDAP configuration", nx);
+            }
+            catch (IOException e)
+            {
+                throw new AuthenticationException(
+                        "Unable to encode LDAP v3 request controls; check LDAP configuration", e);
+            }
+
+        }
+        return false;
     }
 
     public InitialDirContext getInitialDirContext(String principal, String credentials) throws AuthenticationException
@@ -121,7 +195,7 @@ public class LDAPInitialDirContextFactoryImpl implements LDAPInitialDirContextFa
         env.put(Context.SECURITY_PRINCIPAL, principal);
         env.put(Context.SECURITY_CREDENTIALS, credentials);
 
-        return buildInitialDirContext(env);
+        return buildInitialDirContext(env, 0);
     }
 
     public static void main(String[] args)

source/java/org/alfresco/repo/security/sync/ldap/LDAPUserRegistry.java

@@ -120,6 +120,12 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
     /** The attribute defaults. */
     private Map<String, String> attributeDefaults;
 
+    /**
+     * The query batch size. If positive, indicates that RFC 2696 paged results should be used to split query results
+     * into batches of the specified size. Overcomes any size limits imposed by the LDAP server.
+     */
+    private int queryBatchSize;
+
     /** Should we error on missing group members? */
     private boolean errorOnMissingMembers;
 
@@ -377,6 +383,18 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
         this.attributeMapping = attributeMapping;
     }
 
+    /**
+     * Sets the query batch size.
+     * 
+     * @param queryBatchSize
+     *            If positive, indicates that RFC 2696 paged results should be used to split query results into batches
+     *            of the specified size. Overcomes any size limits imposed by the LDAP server.
+     */
+    public void setQueryBatchSize(int queryBatchSize)
+    {
+        this.queryBatchSize = queryBatchSize;
+    }
+
     /*
      * (non-Javadoc)
      * @see org.alfresco.repo.management.subsystems.ActivateableBean#isActive()
@@ -426,15 +444,16 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
     public Iterator<NodeDescription> getGroups(Date modifiedSince)
     {
         Map<String, NodeDescription> lookup = new TreeMap<String, NodeDescription>();
-        InitialDirContext ctx = null;
-        try
-        {
-            ctx = this.ldapInitialContextFactory.getDefaultIntialDirContext();
-
         SearchControls userSearchCtls = new SearchControls();
         userSearchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE);
         userSearchCtls.setReturningAttributes(this.groupAttributeNames);
 
+        InitialDirContext ctx = null;
+        try
+        {
+            ctx = this.ldapInitialContextFactory.getDefaultIntialDirContext(this.queryBatchSize);
+            do
+            {
                 NamingEnumeration<SearchResult> searchResults;
 
                 if (modifiedSince == null)
@@ -511,7 +530,7 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
 
                                 // If the user and group search bases are different we may be able to recognise user and
                                 // group DNs without a secondary lookup
-                            if (!this.userSearchBase.equals(this.groupSearchBase))
+                                if (!this.userSearchBase.equalsIgnoreCase(this.groupSearchBase))
                                 {
                                     Attributes nameAttributes = distinguishedName.getRdn(distinguishedName.size() - 1)
                                             .toAttributes();
@@ -545,7 +564,7 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
                                             "objectclass", this.groupIdAttributeName, this.userIdAttributeName
                                         });
                                         String objectclass = (String) childAttributes.get("objectclass").get();
-                                    if (objectclass.equals(this.personType))
+                                        if (objectclass.equalsIgnoreCase(this.personType))
                                         {
                                             nameAttribute = childAttributes.get(this.userIdAttributeName);
                                             if (nameAttribute == null)
@@ -553,8 +572,8 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
                                                 if (this.errorOnMissingUID)
                                                 {
                                                     throw new AlfrescoRuntimeException(
-                                                        "User missing user id attribute DN =" + attribute + "  att = "
+                                                            "User missing user id attribute DN =" + attribute
-                                                                + this.userIdAttributeName);
+                                                                    + "  att = " + this.userIdAttributeName);
                                                 }
                                                 else
                                                 {
@@ -567,7 +586,7 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
                                             childAssocs.add((String) nameAttribute.get());
                                             continue;
                                         }
-                                    else if (objectclass.equals(this.groupType))
+                                        else if (objectclass.equalsIgnoreCase(this.groupType))
                                         {
 
                                             nameAttribute = childAttributes.get(this.groupIdAttributeName);
@@ -596,13 +615,16 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
                                 }
                                 if (this.errorOnMissingMembers)
                                 {
-                                throw new AlfrescoRuntimeException("Failed to resolve distinguished name: " + attribute);
+                                    throw new AlfrescoRuntimeException("Failed to resolve distinguished name: "
+                                            + attribute);
                                 }
                                 LDAPUserRegistry.logger.warn("Failed to resolve distinguished name: " + attribute);
                             }
                         }
                     }
                 }
+            }
+            while (this.ldapInitialContextFactory.hasNextPage(ctx, this.queryBatchSize));
 
             if (LDAPUserRegistry.logger.isDebugEnabled())
             {
@@ -643,6 +665,10 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
         /** The directory context. */
         private InitialDirContext ctx;
 
+        private SearchControls userSearchCtls;
+
+        private Date modifiedSince;
+
         /** The search results. */
         private NamingEnumeration<SearchResult> searchResults;
 
@@ -663,29 +689,18 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
         {
             try
             {
-                this.ctx = LDAPUserRegistry.this.ldapInitialContextFactory.getDefaultIntialDirContext();
+                this.ctx = LDAPUserRegistry.this.ldapInitialContextFactory
+                        .getDefaultIntialDirContext(LDAPUserRegistry.this.queryBatchSize);
 
                 // Authentication has been successful.
                 // Set the current user, they are now authenticated.
 
-                SearchControls userSearchCtls = new SearchControls();
+                this.userSearchCtls = new SearchControls();
-                userSearchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE);
+                this.userSearchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE);
-                userSearchCtls.setCountLimit(Integer.MAX_VALUE);
+                this.userSearchCtls.setReturningAttributes(LDAPUserRegistry.this.userAttributeNames);
-                userSearchCtls.setReturningAttributes(LDAPUserRegistry.this.userAttributeNames);
+
+                this.modifiedSince = modifiedSince;
 
-                if (modifiedSince == null)
-                {
-                    this.searchResults = this.ctx.search(LDAPUserRegistry.this.userSearchBase,
-                            LDAPUserRegistry.this.personQuery, userSearchCtls);
-                }
-                else
-                {
-                    this.searchResults = this.ctx.search(LDAPUserRegistry.this.userSearchBase,
-                            LDAPUserRegistry.this.personDifferentialQuery, new Object[]
-                            {
-                                LDAPUserRegistry.LDAP_GENERALIZED_TIME_FORMAT.format(modifiedSince)
-                            }, userSearchCtls);
-                }
                 this.next = fetchNext();
             }
             catch (NamingException e)
@@ -748,7 +763,11 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
          */
         private NodeDescription fetchNext() throws NamingException
         {
-            while (this.searchResults.hasMoreElements())
+            boolean readyForNextPage;
+            do
+            {
+                readyForNextPage = this.searchResults == null;
+                while (!readyForNextPage && this.searchResults.hasMoreElements())
                 {
                     SearchResult result = this.searchResults.next();
                     Attributes attributes = result.getAttributes();
@@ -758,7 +777,8 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
                         if (LDAPUserRegistry.this.errorOnMissingUID)
                         {
                             throw new AlfrescoRuntimeException(
-                                "User returned by user search does not have mandatory user id attribute " + attributes);
+                                    "User returned by user search does not have mandatory user id attribute "
+                                            + attributes);
                         }
                         else
                         {
@@ -838,6 +858,33 @@ public class LDAPUserRegistry implements UserRegistry, InitializingBean, Activat
                     }
                     return person;
                 }
+
+                // Examine the paged results control response for an indication that another page is available
+                if (!readyForNextPage)
+                {
+                    readyForNextPage = LDAPUserRegistry.this.ldapInitialContextFactory.hasNextPage(this.ctx,
+                            LDAPUserRegistry.this.queryBatchSize);
+                }
+
+                // Fetch the next page if there is one
+                if (readyForNextPage)
+                {
+                    if (this.modifiedSince == null)
+                    {
+                        this.searchResults = this.ctx.search(LDAPUserRegistry.this.userSearchBase,
+                                LDAPUserRegistry.this.personQuery, this.userSearchCtls);
+                    }
+                    else
+                    {
+                        this.searchResults = this.ctx.search(LDAPUserRegistry.this.userSearchBase,
+                                LDAPUserRegistry.this.personDifferentialQuery, new Object[]
+                                {
+                                    LDAPUserRegistry.LDAP_GENERALIZED_TIME_FORMAT.format(this.modifiedSince)
+                                }, this.userSearchCtls);
+                    }
+                }
+            }
+            while (readyForNextPage);
             this.searchResults.close();
             this.searchResults = null;
             this.ctx.close();