diff --git a/config/alfresco/content-services-context.xml b/config/alfresco/content-services-context.xml index 410294368c..ac8b916846 100644 --- a/config/alfresco/content-services-context.xml +++ b/config/alfresco/content-services-context.xml @@ -201,7 +201,6 @@ - @@ -209,21 +208,10 @@ - - - - application/msword - application/vnd.excel - application/vnd.powerpoint - application/vnd.openxmlformats-officedocument.wordprocessingml.document - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - application/vnd.openxmlformats-officedocument.presentationml.presentation - - diff --git a/config/alfresco/messages/patch-service.properties b/config/alfresco/messages/patch-service.properties index f6594c093b..7ab66be489 100644 --- a/config/alfresco/messages/patch-service.properties +++ b/config/alfresco/messages/patch-service.properties @@ -269,7 +269,9 @@ patch.imapFolders.result.created=The 'Imap Configs' folder was successfully crea patch.zonedAuthorities.description=Adds the remodelled cm:authority container to the spaces store patch.authorityMigration.description=Copies any old authorities from the user store to the spaces store. -patch.authorityMigration.result=Migrated {0} authorities to the spaces store. +patch.authorityMigration.progress.authority={0} groups migrated +patch.authorityMigration.progress.assoc={0} group associations migrated +patch.authorityMigration.result=Migrated {0} groups and {1} group associations to the spaces store. patch.authorityDefaultZonesPatch.description=Adds groups and people to the appropriate zones for wcm, share and everything else. patch.authorityDefaultZonesPatch.result=Unzoned groups and people added to the default zones. diff --git a/source/java/org/alfresco/repo/action/executer/ContentMetadataExtracter.java b/source/java/org/alfresco/repo/action/executer/ContentMetadataExtracter.java index 746631f832..272c697d72 100644 --- a/source/java/org/alfresco/repo/action/executer/ContentMetadataExtracter.java +++ b/source/java/org/alfresco/repo/action/executer/ContentMetadataExtracter.java @@ -25,6 +25,7 @@ package org.alfresco.repo.action.executer; import java.io.Serializable; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -179,6 +180,7 @@ public class ContentMetadataExtracter extends ActionExecuterAbstractBase " Content: " + reader + "\n" + " Failure: " + e.getMessage()); } + modifiedProperties = new HashMap(0); } // If none of the properties where changed, then there is nothing more to do diff --git a/source/java/org/alfresco/repo/admin/patch/impl/AuthorityMigrationPatch.java b/source/java/org/alfresco/repo/admin/patch/impl/AuthorityMigrationPatch.java index 84522c4c32..ea1ba9def6 100644 --- a/source/java/org/alfresco/repo/admin/patch/impl/AuthorityMigrationPatch.java +++ b/source/java/org/alfresco/repo/admin/patch/impl/AuthorityMigrationPatch.java @@ -25,12 +25,21 @@ package org.alfresco.repo.admin.patch.impl; import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; import org.alfresco.i18n.I18NUtil; import org.alfresco.model.ContentModel; import org.alfresco.repo.admin.patch.AbstractPatch; +import org.alfresco.repo.admin.patch.PatchExecuter; import org.alfresco.repo.importer.ImporterBootstrap; +import org.alfresco.repo.transaction.RetryingTransactionHelper; +import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; import org.alfresco.service.cmr.repository.ChildAssociationRef; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; @@ -38,6 +47,8 @@ import org.alfresco.service.cmr.security.AuthorityService; import org.alfresco.service.cmr.security.AuthorityType; import org.alfresco.service.namespace.QName; import org.alfresco.service.namespace.RegexQNamePattern; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; /** * Migrates authority information previously stored in the user store to the spaces store, using the new structure used @@ -48,19 +59,31 @@ import org.alfresco.service.namespace.RegexQNamePattern; public class AuthorityMigrationPatch extends AbstractPatch { + /** Progress message for authorities. */ + private static final String MSG_PROGRESS_AUTHORITY = "patch.authorityMigration.progress.authority"; + + /** Progress message for associations. */ + private static final String MSG_PROGRESS_ASSOC = "patch.authorityMigration.progress.assoc"; + /** Success message. */ private static final String MSG_SUCCESS = "patch.authorityMigration.result"; - /** The old authority name property */ + /** The progress_logger. */ + private static Log progress_logger = LogFactory.getLog(PatchExecuter.class); + + /** The old authority name property. */ private static final QName PROP_AUTHORITY_NAME = QName.createQName(ContentModel.USER_MODEL_URI, "authorityName"); - /** The old authority display name property */ + /** The old authority display name property. */ private static final QName PROP_AUTHORITY_DISPLAY_NAME = QName.createQName(ContentModel.USER_MODEL_URI, "authorityDisplayName"); - /** The old authority members property */ + /** The old authority members property. */ private static final QName PROP_MEMBERS = QName.createQName(ContentModel.USER_MODEL_URI, "members"); + /** The number of items to create in a transaction. */ + private static final int BATCH_SIZE = 10; + /** The authority service. */ private AuthorityService authorityService; @@ -90,41 +113,51 @@ public class AuthorityMigrationPatch extends AbstractPatch } /** - * Recursively migrates the authorities under the given node + * Recursively retrieves the authorities under the given node and their associations. * * @param parentAuthority * the full name of the parent authority corresponding to the given node, or null if it is * not an authority node. * @param nodeRef * the node to find authorities below - * @return the number of processed authorities + * @param authoritiesToCreate + * the authorities to create + * @param childAssocs + * the child associations */ - private int migrateAuthorities(String parentAuthority, NodeRef nodeRef) + private void retrieveAuthorities(String parentAuthority, NodeRef nodeRef, Map authoritiesToCreate, + Map> childAssocs) { - int processedCount = 0; + // If we have a parent authority, prepare a list for recording its children + Set children = parentAuthority == null ? null : childAssocs.get(parentAuthority); + + // Process all children List cars = this.nodeService.getChildAssocs(nodeRef); + for (ChildAssociationRef car : cars) { NodeRef current = car.getChildRef(); + + // Record an authority to create String authorityName = DefaultTypeConverter.INSTANCE.convert(String.class, this.nodeService.getProperty( current, AuthorityMigrationPatch.PROP_AUTHORITY_NAME)); - boolean existed = this.authorityService.authorityExists(authorityName); - if (!existed) + authoritiesToCreate.put(authorityName, DefaultTypeConverter.INSTANCE.convert(String.class, this.nodeService + .getProperty(current, AuthorityMigrationPatch.PROP_AUTHORITY_DISPLAY_NAME))); + + // If we have a parent, remember the child association + if (parentAuthority != null) { - String authorityDisplayName = DefaultTypeConverter.INSTANCE.convert(String.class, this.nodeService - .getProperty(current, AuthorityMigrationPatch.PROP_AUTHORITY_DISPLAY_NAME)); - this.authorityService.createAuthority(AuthorityType.getAuthorityType(authorityName), - this.authorityService.getShortName(authorityName), authorityDisplayName, null); - processedCount++; - } - if (parentAuthority != null - && (!existed || !this.authorityService.getContainingAuthorities(AuthorityType.GROUP, authorityName, - true).contains(parentAuthority))) - { - this.authorityService.addAuthority(parentAuthority, authorityName); + if (children == null) + { + children = new TreeSet(); + childAssocs.put(parentAuthority, children); + } + children.add(authorityName); } // loop over properties + Set propChildren = childAssocs.get(authorityName); + Collection members = DefaultTypeConverter.INSTANCE.getCollection(String.class, this.nodeService .getProperty(current, AuthorityMigrationPatch.PROP_MEMBERS)); if (members != null) @@ -132,15 +165,119 @@ public class AuthorityMigrationPatch extends AbstractPatch for (String user : members) { // Believe it or not, some old authorities have null members in them! - if (user != null - && (!existed || !this.authorityService.getContainingAuthorities(AuthorityType.GROUP, user, - true).contains(authorityName))) + if (user != null) { - this.authorityService.addAuthority(authorityName, user); + if (propChildren == null) + { + propChildren = new TreeSet(); + childAssocs.put(authorityName, propChildren); + } + propChildren.add(user); } } } - processedCount += migrateAuthorities(authorityName, current); + retrieveAuthorities(authorityName, current, authoritiesToCreate, childAssocs); + } + } + + /** + * Migrates the authorities. + * + * @param authoritiesToCreate + * the authorities to create + * @return the number of authorities migrated + */ + private int migrateAuthorities(Map authoritiesToCreate) + { + int processedCount = 0; + final Iterator> i = authoritiesToCreate.entrySet().iterator(); + RetryingTransactionHelper retryingTransactionHelper = this.transactionService.getRetryingTransactionHelper(); + + // Process batches in separate transactions for maximum performance + while (i.hasNext()) + { + processedCount += retryingTransactionHelper.doInTransaction(new RetryingTransactionCallback() + { + public Integer execute() throws Throwable + { + int processedCount = 0; + do + { + Map.Entry authority = i.next(); + String authorityName = authority.getKey(); + boolean existed = AuthorityMigrationPatch.this.authorityService.authorityExists(authorityName); + if (existed) + { + i.remove(); + } + else + { + AuthorityMigrationPatch.this.authorityService.createAuthority(AuthorityType + .getAuthorityType(authorityName), AuthorityMigrationPatch.this.authorityService + .getShortName(authorityName), authority.getValue(), null); + processedCount++; + } + } + while (processedCount < AuthorityMigrationPatch.BATCH_SIZE && i.hasNext()); + return processedCount; + } + }, false, true); + + // Report progress + AuthorityMigrationPatch.progress_logger.info(I18NUtil.getMessage( + AuthorityMigrationPatch.MSG_PROGRESS_AUTHORITY, processedCount)); + } + return processedCount; + } + + /** + * Migrates the group associations. + * + * @param authoritiesCreated + * the authorities created + * @param childAssocs + * the child associations + * @return the number of associations migrated + */ + private int migrateAssocs(final Map authoritiesCreated, Map> childAssocs) + { + int processedCount = 0; + final Iterator>> j = childAssocs.entrySet().iterator(); + RetryingTransactionHelper retryingTransactionHelper = this.transactionService.getRetryingTransactionHelper(); + + // Process batches in separate transactions for maximum performance + while (j.hasNext()) + { + processedCount += retryingTransactionHelper.doInTransaction(new RetryingTransactionCallback() + { + public Integer execute() throws Throwable + { + int processedCount = 0; + do + { + Map.Entry> childAssoc = j.next(); + String parentAuthority = childAssoc.getKey(); + Set knownChildren = authoritiesCreated.containsKey(parentAuthority) ? Collections + . emptySet() : AuthorityMigrationPatch.this.authorityService + .getContainedAuthorities(AuthorityType.GROUP, parentAuthority, true); + for (String authorityName : childAssoc.getValue()) + { + if (!knownChildren.contains(authorityName)) + { + AuthorityMigrationPatch.this.authorityService.addAuthority(parentAuthority, + authorityName); + processedCount++; + } + } + } + while (processedCount < AuthorityMigrationPatch.BATCH_SIZE && j.hasNext()); + return processedCount; + } + }, false, true); + + // Report progress + AuthorityMigrationPatch.progress_logger.info(I18NUtil.getMessage( + AuthorityMigrationPatch.MSG_PROGRESS_ASSOC, processedCount)); } return processedCount; } @@ -148,7 +285,7 @@ public class AuthorityMigrationPatch extends AbstractPatch /** * Gets the old authority container. * - * @return Returns the old authority container or null if not found + * @return the old authority container or null if not found */ private NodeRef getAuthorityContainer() { @@ -186,13 +323,18 @@ public class AuthorityMigrationPatch extends AbstractPatch @Override protected String applyInternal() throws Exception { - int processedCount = 0; + int authorities = 0; + int assocs = 0; NodeRef authorityContainer = getAuthorityContainer(); if (authorityContainer != null) { - processedCount = migrateAuthorities(null, authorityContainer); + Map authoritiesToCreate = new TreeMap(); + Map> childAssocs = new TreeMap>(); + retrieveAuthorities(null, authorityContainer, authoritiesToCreate, childAssocs); + authorities = migrateAuthorities(authoritiesToCreate); + assocs = migrateAssocs(authoritiesToCreate, childAssocs); } // build the result message - return I18NUtil.getMessage(AuthorityMigrationPatch.MSG_SUCCESS, processedCount); + return I18NUtil.getMessage(AuthorityMigrationPatch.MSG_SUCCESS, authorities, assocs); } } \ No newline at end of file diff --git a/source/java/org/alfresco/repo/content/MimetypeMap.java b/source/java/org/alfresco/repo/content/MimetypeMap.java index fe51adbe8d..777a8b3df4 100644 --- a/source/java/org/alfresco/repo/content/MimetypeMap.java +++ b/source/java/org/alfresco/repo/content/MimetypeMap.java @@ -104,6 +104,10 @@ public class MimetypeMap implements MimetypeService public static final String MIMETYPE_OPENOFFICE1_CALC = "application/vnd.sun.xml.calc"; public static final String MIMETYPE_OPENOFFICE1_DRAW = "application/vnd.sun.xml.draw"; public static final String MIMETYPE_OPENOFFICE1_IMPRESS = "application/vnd.sun.xml.impress"; + // Open XML + public static final String MIMETYPE_OPENXML_WORDPROCESSING = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + public static final String MIMETYPE_OPENXML_SPREADSHEET = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + public static final String MIMETYPE_OPENXML_PRESENTATION = "application/vnd.openxmlformats-officedocument.presentationml.presentation"; // Star Office public static final String MIMETYPE_STAROFFICE5_DRAW = "application/vnd.stardivision.draw"; public static final String MIMETYPE_STAROFFICE5_CALC = "application/vnd.stardivision.calc"; diff --git a/source/java/org/alfresco/repo/content/metadata/AbstractMappingMetadataExtracter.java b/source/java/org/alfresco/repo/content/metadata/AbstractMappingMetadataExtracter.java index 180802399b..22efcf1f43 100644 --- a/source/java/org/alfresco/repo/content/metadata/AbstractMappingMetadataExtracter.java +++ b/source/java/org/alfresco/repo/content/metadata/AbstractMappingMetadataExtracter.java @@ -645,9 +645,22 @@ abstract public class AbstractMappingMetadataExtracter implements MetadataExtrac } catch (Throwable e) { - throw new ContentIOException("Metadata extraction failed: \n" + - " reader: " + reader, - e); + if (logger.isDebugEnabled()) + { + logger.debug( + "Metadata extraction failed: \n" + + " Extracter: " + this + "\n" + + " Content: " + reader, + e); + } + else + { + logger.warn( + "Metadata extraction failed (turn on DEBUG for full error): \n" + + " Extracter: " + this + "\n" + + " Content: " + reader + "\n" + + " Failure: " + e.getMessage()); + } } finally { @@ -658,6 +671,11 @@ abstract public class AbstractMappingMetadataExtracter implements MetadataExtrac " reader: " + reader + "\n" + " extracter: " + this); } + // Make sure that we have something to return + if (changedProperties == null) + { + changedProperties = new HashMap(0); + } } // Done diff --git a/source/java/org/alfresco/repo/content/metadata/OpenOfficeMetadataExtracter.java b/source/java/org/alfresco/repo/content/metadata/OpenOfficeMetadataExtracter.java index 800f4049be..90491ee82c 100644 --- a/source/java/org/alfresco/repo/content/metadata/OpenOfficeMetadataExtracter.java +++ b/source/java/org/alfresco/repo/content/metadata/OpenOfficeMetadataExtracter.java @@ -47,7 +47,10 @@ public class OpenOfficeMetadataExtracter extends AbstractMappingMetadataExtracte MimetypeMap.MIMETYPE_STAROFFICE5_WRITER, MimetypeMap.MIMETYPE_STAROFFICE5_IMPRESS, MimetypeMap.MIMETYPE_OPENOFFICE1_WRITER, - MimetypeMap.MIMETYPE_OPENOFFICE1_IMPRESS + MimetypeMap.MIMETYPE_OPENOFFICE1_IMPRESS, + MimetypeMap.MIMETYPE_OPENXML_WORDPROCESSING, + MimetypeMap.MIMETYPE_OPENXML_SPREADSHEET, + MimetypeMap.MIMETYPE_OPENXML_PRESENTATION }; private OpenOfficeMetadataWorker worker;