Merged V3.2 to HEAD

15200: Fixed ETHREEOH-2480: Cannot upload files with .doc extension owing to exception from ContentMetadataExtracter
   15203: ETHREEOH-2246: Rework patch.authorityMigration to be more performant and report progress
   15254: ETHREEOH-2339: Fixed NullPointerException when editing LDAP-synced user who doesn't have an email address
   15267: Applied patch for ETHREEOH-1448: Exception when using the move up/down button when editting a web form
   15270: Mereg (sic) - Record only 15238
   15285: Merged V3.1 to V3.2
      15281: Merged V2.2 to V3.1
         15280: Absorb all metadata exceptions at all levels
      15282: Merged V2.2 to V3.1
         15273: Merged V3.2 to V2.2
            15200: Fixed ETHREEOH-2480: Cannot upload files with .doc extension owing to exception from ContentMetadataExtracter
-------------------------
Modified: svn:mergeinfo
   Merged /alfresco/BRANCHES/V2.2:r15273,15280
   Merged /alfresco/BRANCHES/V3.1:r15281-15282
   Merged /alfresco/BRANCHES/V3.2:r15200,15203,15254,15267,15270,15285



git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@16854 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/content-services-context.xml

@@ -201,7 +201,6 @@
    <!--  Unsupported experimental extractor commented out -->
    <!-- <bean id="extracter.MP3"           class="org.alfresco.repo.content.metadata.MP3MetadataExtracter"           parent="baseMetadataExtracter" /> -->
    <bean id="extracter.OpenDocument"  class="org.alfresco.repo.content.metadata.OpenDocumentMetadataExtracter"  parent="baseMetadataExtracter" />
-   
    <bean id="extracter.RFC822"        class="org.alfresco.repo.content.metadata.RFC822MetadataExtracter"        parent="baseMetadataExtracter" >
       <property name="supportedDateFormats">
          <list>
@@ -209,21 +208,10 @@
          </list>
       </property>
    </bean>
-   
    <bean id="extracter.OpenOffice" class="org.alfresco.repo.content.metadata.OpenOfficeMetadataExtracter" parent="baseMetadataExtracter">
       <property name="worker">
          <ref bean="extracter.worker.OpenOffice" />
       </property>
-      <property name="supportedMimetypes">
-         <list>
-            <value>application/msword</value>
-            <value>application/vnd.excel</value>
-            <value>application/vnd.powerpoint</value>
-            <value>application/vnd.openxmlformats-officedocument.wordprocessingml.document</value>
-            <value>application/vnd.openxmlformats-officedocument.spreadsheetml.sheet</value>
-            <value>application/vnd.openxmlformats-officedocument.presentationml.presentation</value>
-         </list>
-      </property>
    </bean>
 
    <!-- Content Transformation Regisitry -->

config/alfresco/messages/patch-service.properties

@@ -269,7 +269,9 @@ patch.imapFolders.result.created=The 'Imap Configs' folder was successfully crea
 patch.zonedAuthorities.description=Adds the remodelled cm:authority container to the spaces store
 
 patch.authorityMigration.description=Copies any old authorities from the user store to the spaces store.
-patch.authorityMigration.result=Migrated {0} authorities to the spaces store.
+patch.authorityMigration.progress.authority={0} groups migrated
+patch.authorityMigration.progress.assoc={0} group associations migrated
+patch.authorityMigration.result=Migrated {0} groups and {1} group associations to the spaces store.
 
 patch.authorityDefaultZonesPatch.description=Adds groups and people to the appropriate zones for wcm, share and everything else.
 patch.authorityDefaultZonesPatch.result=Unzoned groups and people added to the default zones.

source/java/org/alfresco/repo/action/executer/ContentMetadataExtracter.java

@@ -25,6 +25,7 @@
 package org.alfresco.repo.action.executer;
 
 import java.io.Serializable;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -179,6 +180,7 @@ public class ContentMetadataExtracter extends ActionExecuterAbstractBase
                         "   Content:   " + reader + "\n" +
                         "   Failure:   " + e.getMessage());
             }
+            modifiedProperties = new HashMap<QName, Serializable>(0);
         }
 
         // If none of the properties where changed, then there is nothing more to do

source/java/org/alfresco/repo/admin/patch/impl/AuthorityMigrationPatch.java

@@ -25,12 +25,21 @@
 package org.alfresco.repo.admin.patch.impl;
 
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
 
 import org.alfresco.i18n.I18NUtil;
 import org.alfresco.model.ContentModel;
 import org.alfresco.repo.admin.patch.AbstractPatch;
+import org.alfresco.repo.admin.patch.PatchExecuter;
 import org.alfresco.repo.importer.ImporterBootstrap;
+import org.alfresco.repo.transaction.RetryingTransactionHelper;
+import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
 import org.alfresco.service.cmr.repository.ChildAssociationRef;
 import org.alfresco.service.cmr.repository.NodeRef;
 import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
@@ -38,6 +47,8 @@ import org.alfresco.service.cmr.security.AuthorityService;
 import org.alfresco.service.cmr.security.AuthorityType;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.service.namespace.RegexQNamePattern;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 /**
  * Migrates authority information previously stored in the user store to the spaces store, using the new structure used
@@ -48,19 +59,31 @@ import org.alfresco.service.namespace.RegexQNamePattern;
 public class AuthorityMigrationPatch extends AbstractPatch
 {
 
+    /** Progress message for authorities. */
+    private static final String MSG_PROGRESS_AUTHORITY = "patch.authorityMigration.progress.authority";
+
+    /** Progress message for associations. */
+    private static final String MSG_PROGRESS_ASSOC = "patch.authorityMigration.progress.assoc";
+
     /** Success message. */
     private static final String MSG_SUCCESS = "patch.authorityMigration.result";
 
-    /** The old authority name property */
+    /** The progress_logger. */
+    private static Log progress_logger = LogFactory.getLog(PatchExecuter.class);
+
+    /** The old authority name property. */
     private static final QName PROP_AUTHORITY_NAME = QName.createQName(ContentModel.USER_MODEL_URI, "authorityName");
 
-    /** The old authority display name property */
+    /** The old authority display name property. */
     private static final QName PROP_AUTHORITY_DISPLAY_NAME = QName.createQName(ContentModel.USER_MODEL_URI,
             "authorityDisplayName");
 
-    /** The old authority members property */
+    /** The old authority members property. */
     private static final QName PROP_MEMBERS = QName.createQName(ContentModel.USER_MODEL_URI, "members");
 
+    /** The number of items to create in a transaction. */
+    private static final int BATCH_SIZE = 10;
+
     /** The authority service. */
     private AuthorityService authorityService;
 
@@ -90,41 +113,51 @@ public class AuthorityMigrationPatch extends AbstractPatch
     }
 
     /**
-     * Recursively migrates the authorities under the given node
+     * Recursively retrieves the authorities under the given node and their associations.
      * 
      * @param parentAuthority
      *            the full name of the parent authority corresponding to the given node, or <code>null</code> if it is
      *            not an authority node.
      * @param nodeRef
      *            the node to find authorities below
-     * @return the number of processed authorities
+     * @param authoritiesToCreate
+     *            the authorities to create
+     * @param childAssocs
+     *            the child associations
      */
-    private int migrateAuthorities(String parentAuthority, NodeRef nodeRef)
+    private void retrieveAuthorities(String parentAuthority, NodeRef nodeRef, Map<String, String> authoritiesToCreate,
+            Map<String, Set<String>> childAssocs)
     {
-        int processedCount = 0;
+        // If we have a parent authority, prepare a list for recording its children
+        Set<String> children = parentAuthority == null ? null : childAssocs.get(parentAuthority);
+
+        // Process all children
         List<ChildAssociationRef> cars = this.nodeService.getChildAssocs(nodeRef);
+
         for (ChildAssociationRef car : cars)
         {
             NodeRef current = car.getChildRef();
+
+            // Record an authority to create
             String authorityName = DefaultTypeConverter.INSTANCE.convert(String.class, this.nodeService.getProperty(
                     current, AuthorityMigrationPatch.PROP_AUTHORITY_NAME));
-            boolean existed = this.authorityService.authorityExists(authorityName);
+            authoritiesToCreate.put(authorityName, DefaultTypeConverter.INSTANCE.convert(String.class, this.nodeService
-            if (!existed)
+                    .getProperty(current, AuthorityMigrationPatch.PROP_AUTHORITY_DISPLAY_NAME)));
+
+            // If we have a parent, remember the child association
+            if (parentAuthority != null)
             {
-                String authorityDisplayName = DefaultTypeConverter.INSTANCE.convert(String.class, this.nodeService
+                if (children == null)
-                        .getProperty(current, AuthorityMigrationPatch.PROP_AUTHORITY_DISPLAY_NAME));
+                {
-                this.authorityService.createAuthority(AuthorityType.getAuthorityType(authorityName),
+                    children = new TreeSet<String>();
-                        this.authorityService.getShortName(authorityName), authorityDisplayName, null);
+                    childAssocs.put(parentAuthority, children);
-                processedCount++;
+                }
-            }
+                children.add(authorityName);
-            if (parentAuthority != null
-                    && (!existed || !this.authorityService.getContainingAuthorities(AuthorityType.GROUP, authorityName,
-                            true).contains(parentAuthority)))
-            {
-                this.authorityService.addAuthority(parentAuthority, authorityName);
             }
 
             // loop over properties
+            Set<String> propChildren = childAssocs.get(authorityName);
+
             Collection<String> members = DefaultTypeConverter.INSTANCE.getCollection(String.class, this.nodeService
                     .getProperty(current, AuthorityMigrationPatch.PROP_MEMBERS));
             if (members != null)
@@ -132,15 +165,119 @@ public class AuthorityMigrationPatch extends AbstractPatch
                 for (String user : members)
                 {
                     // Believe it or not, some old authorities have null members in them!
-                    if (user != null
+                    if (user != null)
-                            && (!existed || !this.authorityService.getContainingAuthorities(AuthorityType.GROUP, user,
-                                    true).contains(authorityName)))
                     {
-                        this.authorityService.addAuthority(authorityName, user);
+                        if (propChildren == null)
+                        {
+                            propChildren = new TreeSet<String>();
+                            childAssocs.put(authorityName, propChildren);
+                        }
+                        propChildren.add(user);
                     }
                 }
             }
-            processedCount += migrateAuthorities(authorityName, current);
+            retrieveAuthorities(authorityName, current, authoritiesToCreate, childAssocs);
+        }
+    }
+
+    /**
+     * Migrates the authorities.
+     * 
+     * @param authoritiesToCreate
+     *            the authorities to create
+     * @return the number of authorities migrated
+     */
+    private int migrateAuthorities(Map<String, String> authoritiesToCreate)
+    {
+        int processedCount = 0;
+        final Iterator<Map.Entry<String, String>> i = authoritiesToCreate.entrySet().iterator();
+        RetryingTransactionHelper retryingTransactionHelper = this.transactionService.getRetryingTransactionHelper();
+
+        // Process batches in separate transactions for maximum performance
+        while (i.hasNext())
+        {
+            processedCount += retryingTransactionHelper.doInTransaction(new RetryingTransactionCallback<Integer>()
+            {
+                public Integer execute() throws Throwable
+                {
+                    int processedCount = 0;
+                    do
+                    {
+                        Map.Entry<String, String> authority = i.next();
+                        String authorityName = authority.getKey();
+                        boolean existed = AuthorityMigrationPatch.this.authorityService.authorityExists(authorityName);
+                        if (existed)
+                        {
+                            i.remove();
+                        }
+                        else
+                        {
+                            AuthorityMigrationPatch.this.authorityService.createAuthority(AuthorityType
+                                    .getAuthorityType(authorityName), AuthorityMigrationPatch.this.authorityService
+                                    .getShortName(authorityName), authority.getValue(), null);
+                            processedCount++;
+                        }
+                    }
+                    while (processedCount < AuthorityMigrationPatch.BATCH_SIZE && i.hasNext());
+                    return processedCount;
+                }
+            }, false, true);
+
+            // Report progress
+            AuthorityMigrationPatch.progress_logger.info(I18NUtil.getMessage(
+                    AuthorityMigrationPatch.MSG_PROGRESS_AUTHORITY, processedCount));
+        }
+        return processedCount;
+    }
+
+    /**
+     * Migrates the group associations.
+     * 
+     * @param authoritiesCreated
+     *            the authorities created
+     * @param childAssocs
+     *            the child associations
+     * @return the number of associations migrated
+     */
+    private int migrateAssocs(final Map<String, String> authoritiesCreated, Map<String, Set<String>> childAssocs)
+    {
+        int processedCount = 0;
+        final Iterator<Map.Entry<String, Set<String>>> j = childAssocs.entrySet().iterator();
+        RetryingTransactionHelper retryingTransactionHelper = this.transactionService.getRetryingTransactionHelper();
+
+        // Process batches in separate transactions for maximum performance
+        while (j.hasNext())
+        {
+            processedCount += retryingTransactionHelper.doInTransaction(new RetryingTransactionCallback<Integer>()
+            {
+                public Integer execute() throws Throwable
+                {
+                    int processedCount = 0;
+                    do
+                    {
+                        Map.Entry<String, Set<String>> childAssoc = j.next();
+                        String parentAuthority = childAssoc.getKey();
+                        Set<String> knownChildren = authoritiesCreated.containsKey(parentAuthority) ? Collections
+                                .<String> emptySet() : AuthorityMigrationPatch.this.authorityService
+                                .getContainedAuthorities(AuthorityType.GROUP, parentAuthority, true);
+                        for (String authorityName : childAssoc.getValue())
+                        {
+                            if (!knownChildren.contains(authorityName))
+                            {
+                                AuthorityMigrationPatch.this.authorityService.addAuthority(parentAuthority,
+                                        authorityName);
+                                processedCount++;
+                            }
+                        }
+                    }
+                    while (processedCount < AuthorityMigrationPatch.BATCH_SIZE && j.hasNext());
+                    return processedCount;
+                }
+            }, false, true);
+
+            // Report progress
+            AuthorityMigrationPatch.progress_logger.info(I18NUtil.getMessage(
+                    AuthorityMigrationPatch.MSG_PROGRESS_ASSOC, processedCount));
         }
         return processedCount;
     }
@@ -148,7 +285,7 @@ public class AuthorityMigrationPatch extends AbstractPatch
     /**
      * Gets the old authority container.
      * 
-     * @return Returns the old authority container or <code>null</code> if not found
+     * @return the old authority container or <code>null</code> if not found
      */
     private NodeRef getAuthorityContainer()
     {
@@ -186,13 +323,18 @@ public class AuthorityMigrationPatch extends AbstractPatch
     @Override
     protected String applyInternal() throws Exception
     {
-        int processedCount = 0;
+        int authorities = 0;
+        int assocs = 0;
         NodeRef authorityContainer = getAuthorityContainer();
         if (authorityContainer != null)
         {
-            processedCount = migrateAuthorities(null, authorityContainer);
+            Map<String, String> authoritiesToCreate = new TreeMap<String, String>();
+            Map<String, Set<String>> childAssocs = new TreeMap<String, Set<String>>();
+            retrieveAuthorities(null, authorityContainer, authoritiesToCreate, childAssocs);
+            authorities = migrateAuthorities(authoritiesToCreate);
+            assocs = migrateAssocs(authoritiesToCreate, childAssocs);
         }
         // build the result message
-        return I18NUtil.getMessage(AuthorityMigrationPatch.MSG_SUCCESS, processedCount);
+        return I18NUtil.getMessage(AuthorityMigrationPatch.MSG_SUCCESS, authorities, assocs);
     }
 }

source/java/org/alfresco/repo/content/MimetypeMap.java

@@ -104,6 +104,10 @@ public class MimetypeMap implements MimetypeService
     public static final String MIMETYPE_OPENOFFICE1_CALC = "application/vnd.sun.xml.calc";
     public static final String MIMETYPE_OPENOFFICE1_DRAW = "application/vnd.sun.xml.draw";
     public static final String MIMETYPE_OPENOFFICE1_IMPRESS = "application/vnd.sun.xml.impress";
+    // Open XML
+    public static final String MIMETYPE_OPENXML_WORDPROCESSING = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+    public static final String MIMETYPE_OPENXML_SPREADSHEET = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+    public static final String MIMETYPE_OPENXML_PRESENTATION = "application/vnd.openxmlformats-officedocument.presentationml.presentation";
     // Star Office
     public static final String MIMETYPE_STAROFFICE5_DRAW = "application/vnd.stardivision.draw";
     public static final String MIMETYPE_STAROFFICE5_CALC = "application/vnd.stardivision.calc";

source/java/org/alfresco/repo/content/metadata/AbstractMappingMetadataExtracter.java

@@ -645,9 +645,22 @@ abstract public class AbstractMappingMetadataExtracter implements MetadataExtrac
         }
         catch (Throwable e)
         {
-            throw new ContentIOException("Metadata extraction failed: \n" +
+            if (logger.isDebugEnabled())
-                    "   reader: " + reader,
+            {
-                    e);
+                logger.debug(
+                        "Metadata extraction failed: \n" +
+                        "   Extracter: " + this + "\n" +
+                        "   Content:   " + reader,
+                        e);
+            }
+            else
+            {
+                logger.warn(
+                        "Metadata extraction failed (turn on DEBUG for full error): \n" +
+                        "   Extracter: " + this + "\n" +
+                        "   Content:   " + reader + "\n" +
+                        "   Failure:   " + e.getMessage());
+            }
         }
         finally
         {
@@ -658,6 +671,11 @@ abstract public class AbstractMappingMetadataExtracter implements MetadataExtrac
                         "   reader: " + reader + "\n" +
                         "   extracter: " + this);
             }
+            // Make sure that we have something to return
+            if (changedProperties == null)
+            {
+                changedProperties = new HashMap<QName, Serializable>(0);
+            }
         }
         
         // Done

source/java/org/alfresco/repo/content/metadata/OpenOfficeMetadataExtracter.java

@@ -47,7 +47,10 @@ public class OpenOfficeMetadataExtracter extends AbstractMappingMetadataExtracte
         MimetypeMap.MIMETYPE_STAROFFICE5_WRITER,
         MimetypeMap.MIMETYPE_STAROFFICE5_IMPRESS,
         MimetypeMap.MIMETYPE_OPENOFFICE1_WRITER,
-        MimetypeMap.MIMETYPE_OPENOFFICE1_IMPRESS
+        MimetypeMap.MIMETYPE_OPENOFFICE1_IMPRESS,
+        MimetypeMap.MIMETYPE_OPENXML_WORDPROCESSING,
+        MimetypeMap.MIMETYPE_OPENXML_SPREADSHEET,
+        MimetypeMap.MIMETYPE_OPENXML_PRESENTATION
     };
 
     private OpenOfficeMetadataWorker worker;