Merged HEAD-BUG-FIX (5.1/Cloud) to HEAD (5.0/Cloud)

86233: Merged V4.2-BUG-FIX (4.2.4) to HEAD-BUG-FIX (5.0/Cloud)
      86211: Merged V4.1-BUG-FIX (4.1.10) to V4.2-BUG-FIX (4.2.4)
         86138: MNT-12400: Merged DEV to V4.1-BUG-FIX (4.1.10)
            84644: MNT-12400: Bulk File System Import Tool logs incorrectly failed files
               - In case of IntegrityException show "Failed on batch commit" message instead of incorrect id entry. Show cm:name for nodes for which constraint check failed.
            84647: MNT-12400: Bulk File System Import Tool logs incorrectly failed files
               - Skip import for content with invalid metadata files.


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@94530 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/bulk-import-context.xml

@@ -47,6 +47,7 @@
     <property name="importStatus"    ref="bfsiStatus" />
     <property name="importFilters"   ref="bfsiImportFilters" />
     <property name="nameChecker"   ref="nameChecker" />
+    <property name="dictionaryService"   ref="dictionaryService" />
   </bean>
   
   <bean id="abstractNodeImporterFactory" class="org.alfresco.repo.bulkimport.impl.AbstractNodeImporterFactory" abstract="true">

source/java/org/alfresco/repo/batch/BatchProcessor.java

@@ -37,6 +37,7 @@ import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
 import org.alfresco.error.AlfrescoRuntimeException;
+import org.alfresco.repo.node.integrity.IntegrityException;
 import org.alfresco.repo.transaction.AlfrescoTransactionSupport;
 import org.alfresco.repo.transaction.RetryingTransactionHelper;
 import org.alfresco.repo.transaction.TransactionListenerAdapter;
@@ -773,12 +774,13 @@ public class BatchProcessor<T> implements BatchMonitor
                 if (this.splitTxns)
                 {
                     this.txnLastError = t;
-                    this.txnLastErrorEntryId = this.txnEntryId;
+                    this.txnLastErrorEntryId = (t instanceof IntegrityException) ? "unknown" : this.txnEntryId;
                     this.txnErrors++;
                     if (BatchProcessor.this.logger.isWarnEnabled())
                     {
-                        BatchProcessor.this.logger.warn(getProcessName() + ": Failed to process entry \""
-                                + BatchProcessor.this.currentEntryId + "\".", t);
+                        String message = (t instanceof IntegrityException) ? ": Failed on batch commit." : ": Failed to process entry \""
+                                + this.txnEntryId + "\".";
+                        BatchProcessor.this.logger.warn(getProcessName() + message, t);
                     }
                 }
                 // Otherwise, we have a retryable exception that we should propagate

source/java/org/alfresco/repo/bulkimport/impl/DirectoryAnalyserImpl.java

@@ -27,13 +27,16 @@ package org.alfresco.repo.bulkimport.impl;
 
 import java.io.File;
 import java.io.FileFilter;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.alfresco.error.AlfrescoRuntimeException;
 import org.alfresco.repo.bulkimport.AnalysedDirectory;
 import org.alfresco.repo.bulkimport.DirectoryAnalyser;
 import org.alfresco.repo.bulkimport.ImportFilter;
@@ -41,7 +44,12 @@ import org.alfresco.repo.bulkimport.ImportableItem;
 import org.alfresco.repo.bulkimport.ImportableItem.FileType;
 import org.alfresco.repo.bulkimport.MetadataLoader;
 import org.alfresco.repo.dictionary.constraint.NameChecker;
+import org.alfresco.service.cmr.dictionary.Constraint;
+import org.alfresco.service.cmr.dictionary.ConstraintDefinition;
 import org.alfresco.service.cmr.dictionary.ConstraintException;
+import org.alfresco.service.cmr.dictionary.DictionaryService;
+import org.alfresco.service.cmr.dictionary.PropertyDefinition;
+import org.alfresco.service.namespace.QName;
 import org.alfresco.util.ISO8601DateFormat;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -64,6 +72,7 @@ public class DirectoryAnalyserImpl implements DirectoryAnalyser
     private BulkImportStatusImpl importStatus;
     private List<ImportFilter> importFilters;
     private NameChecker nameChecker;
+    private DictionaryService dictionaryService;
 
     
     public DirectoryAnalyserImpl(MetadataLoader metadataLoader, BulkImportStatusImpl importStatus, List<ImportFilter> importFilters,
@@ -79,6 +88,11 @@ public class DirectoryAnalyserImpl implements DirectoryAnalyser
     {
     }
     
+    public void setDictionaryService(DictionaryService dictionaryService)
+    {
+        this.dictionaryService = dictionaryService;
+    }
+    
     public void setNameChecker(NameChecker nameChecker)
     {
         this.nameChecker = nameChecker;
@@ -230,7 +244,7 @@ public class DirectoryAnalyserImpl implements DirectoryAnalyser
         {
             ImportableItem importableItem = iter.next();
 
-            if (!importableItem.isValid())
+            if (!importableItem.isValid() || !isMetadataValid(importableItem))
             {
                 iter.remove();
             }
@@ -255,6 +269,51 @@ public class DirectoryAnalyserImpl implements DirectoryAnalyser
         return result;
     }
 
+    private boolean isMetadataValid(ImportableItem importableItem)
+    {
+        if (!importableItem.getHeadRevision().metadataFileExists())
+        {
+            return true;
+        }
+        
+        if (metadataLoader != null)
+        {
+            MetadataLoader.Metadata result = new MetadataLoader.Metadata();
+            metadataLoader.loadMetadata(importableItem.getHeadRevision(), result);
+            
+            Map<QName, Serializable> metadataProperties = result.getProperties();
+            for (QName propertyName : metadataProperties.keySet())
+            {
+                PropertyDefinition propDef = dictionaryService.getProperty(propertyName);
+                if (propDef != null)
+                {
+                    for (ConstraintDefinition constraintDef : propDef.getConstraints())
+                    {
+                        Constraint constraint = constraintDef.getConstraint();
+                        if (constraint != null)
+                        {
+                            try
+                            {
+                                constraint.evaluate(metadataProperties.get(propertyName));
+                            }
+                            catch (ConstraintException e)
+                            {
+                                if (log.isWarnEnabled())
+                                {
+                                    log.warn("Skipping file '" + FileUtils.getFileName(importableItem.getHeadRevision().getContentFile())
+                                       +"' with invalid metadata: '" + FileUtils.getFileName(importableItem.getHeadRevision().getMetadataFile()) + "'.", e);
+                                }
+                                return false;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        
+        return true;
+    }
+
     private boolean isVersionFile(File file)
     {
         Matcher matcher = VERSION_SUFFIX_PATTERN.matcher(file.getName());

source/java/org/alfresco/repo/node/integrity/PropertiesIntegrityEvent.java

@@ -153,9 +153,11 @@ public class PropertiesIntegrityEvent extends AbstractIntegrityEvent
             // check that enforced, mandatoryproperties are set
             if (propertyDef.isMandatory() && propertyDef.isMandatoryEnforced() && !nodeProperties.containsKey(propertyQName))
             {
+                String nameProp = (String) nodeService.getProperty(nodeRef, ContentModel.PROP_NAME);
                 IntegrityRecord result = new IntegrityRecord(
                         "Mandatory property not set: \n" +
                         "   Node: " + nodeRef + "\n" +
+                        (nameProp != null ? "   Name: " + nameProp + "\n" : "") +
                         "   Type: " + typeQName + "\n" +
                         "   Property: " + propertyQName);
                 eventResults.add(result);
@@ -168,9 +170,11 @@ public class PropertiesIntegrityEvent extends AbstractIntegrityEvent
             {
                 if (propertyValue != null && !(propertyValue instanceof SealedObject))
                 {
+                    String nameProp = (String) nodeService.getProperty(nodeRef, ContentModel.PROP_NAME);
                     IntegrityRecord result = new IntegrityRecord(
                             "Property must be encrypted: \n" +
                             "   Node: " + nodeRef + "\n" +
+                            (nameProp != null ? "   Name: " + nameProp + "\n" : "") +
                             "   Type: " + typeQName + "\n" +
                             "   Property: " + propertyQName);
                     eventResults.add(result);
@@ -188,9 +192,11 @@ public class PropertiesIntegrityEvent extends AbstractIntegrityEvent
                 }
                 catch (ConstraintException e)
                 {
+                    String nameProp = (String) nodeService.getProperty(nodeRef, ContentModel.PROP_NAME);
                     IntegrityRecord result = new IntegrityRecord(
                             "Invalid property value: \n" +
                             "   Node: " + nodeRef + "\n" +
+                            (nameProp != null ? "   Name: " + nameProp + "\n" : "") +
                             "   Type: " + typeQName + "\n" +
                             "   Property: " + propertyQName + "\n" +
                             "   Constraint: " + e.getMessage());