Fix/mnt 23087 export of records failing (#1333)

* MNT-23087 - Split the export list into several smaller list for better performance

repository/src/main/java/org/alfresco/repo/exporter/ExporterComponent.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software.
  * If the software was purchased under a paid Alfresco license, the terms of
@@ -39,6 +39,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.Arrays;
 
 import org.alfresco.model.ContentModel;
 import org.alfresco.repo.node.MLPropertyInterceptor;
@@ -77,6 +78,7 @@ import org.alfresco.service.descriptor.DescriptorService;
 import org.alfresco.service.namespace.NamespaceService;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.service.namespace.RegexQNamePattern;
+import org.apache.commons.lang3.math.NumberUtils;
 import org.dom4j.io.OutputFormat;
 import org.dom4j.io.XMLWriter;
 import org.springframework.extensions.surf.util.ParameterCheck;
@@ -100,6 +102,8 @@ public class ExporterComponent
     private AuthenticationService authenticationService;
     private PermissionService permissionService;
 
+    private String exportChunkSize;
+    
 
     /** Indent Size */
     private int indentSize = 2;
@@ -179,6 +183,14 @@ public class ExporterComponent
         this.exportSecondaryNodes = exportSecondaryNodes;
     }
 
+    /**
+     * @param exportChunkSize  the exportChunkSize
+     */
+    public void setExportChunkSize(String exportChunkSize)
+    {
+        this.exportChunkSize = exportChunkSize;
+    }
+    
     /* (non-Javadoc)
      * @see org.alfresco.service.cmr.view.ExporterService#exportView(java.io.OutputStream, org.alfresco.service.cmr.view.ExporterCrawlerParameters, org.alfresco.service.cmr.view.Exporter)
      */
@@ -943,26 +955,21 @@ public class ExporterComponent
             try
             {
                 // Current strategy is to determine if node is a child of the root exported node
-                for (NodeRef exportRoot : context.getExportList())
+                if (context.getExportMap() != null)
                 {
-                    if (nodeRef.equals(exportRoot) && parameters.isCrawlSelf() == true)
+                    for (NodeRef[] listNodeRef : context.getExportMap().values())
                     {
-                        // node to export is the root export node (and root is to be exported)
+                        for (NodeRef exportRoot : listNodeRef)
-                        isWithin = true;
+                        {
+                            isWithin = checkIsWithin(nodeRef, exportRoot, parameters);
+                        }
+                    }
                 }
                 else
                 {
-                        // locate export root in primary parent path of node
+                    for (NodeRef exportRoot : context.getExportList())
-                        Path nodePath = nodeService.getPath(nodeRef);
-                        for (int i = nodePath.size() - 1; i >= 0; i--)
                     {
-                            Path.ChildAssocElement pathElement = (Path.ChildAssocElement) nodePath.get(i);
+                        isWithin = checkIsWithin(nodeRef, exportRoot, parameters);
-                            if (pathElement.getRef().getChildRef().equals(exportRoot))
-                            {
-                                isWithin = true;
-                                break;
-                            }
-                        }
                     }
                 }
             }
@@ -979,6 +986,28 @@ public class ExporterComponent
         }
     }
 
+    private boolean checkIsWithin(NodeRef nodeRef, NodeRef exportRoot, ExporterCrawlerParameters parameters){
+        if (nodeRef.equals(exportRoot) && parameters.isCrawlSelf() == true)
+        {
+            // node to export is the root export node (and root is to be exported)
+            return true;
+        }
+        else
+        {
+            // locate export root in primary parent path of node
+            Path nodePath = nodeService.getPath(nodeRef);
+            for (int i = nodePath.size() - 1; i >= 0; i--)
+            {
+                Path.ChildAssocElement pathElement = (Path.ChildAssocElement) nodePath.get(i);
+                if (pathElement.getRef().getChildRef().equals(exportRoot))
+                {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
 
     /**
      * Exporter Context
@@ -986,7 +1015,9 @@ public class ExporterComponent
     private class ExporterContextImpl implements ExporterContext
     {
         private NodeRef[] exportList;
+        private Map<Integer,NodeRef[]> exportListMap;
         private NodeRef[] parentList;
+        private Map<Integer,NodeRef[]> parentListMap;
         private String exportedBy;
         private Date exportedDate;
         private String exporterVersion;
@@ -995,6 +1026,8 @@ public class ExporterComponent
         private Map<Integer, Set<NodeRef>> nodesWithAssociations = new HashMap<Integer, Set<NodeRef>>();
         
         private int index;
+        private int indexSubList;
+        private int chunkSize;
 
 
         /**
@@ -1005,6 +1038,16 @@ public class ExporterComponent
         public ExporterContextImpl(ExporterCrawlerParameters parameters)
         {
             index = 0;
+            indexSubList = 0;
+
+
+            if(!NumberUtils.isParsable(exportChunkSize)){
+                chunkSize = 10;
+            }
+            else
+            {
+                chunkSize = Integer.parseInt(exportChunkSize);
+            }
 
             // get current user performing export
             String currentUserName = authenticationService.getCurrentUserName();
@@ -1022,24 +1065,80 @@ public class ExporterComponent
                 NodeRef exportOf = getNodeRef(parameters.getExportFrom());
                 exportList[0] = exportOf;
             }
+            if(exportList.length > chunkSize)
+            {
+                exportListMap = splitArray(exportList);
+
+                parentListMap = new HashMap<>();
+                for(Map.Entry<Integer, NodeRef[]> exportEntrySet : exportListMap.entrySet())
+                {
+                    parentList= new NodeRef[exportEntrySet.getValue().length];
+                    for (int i = 0; i < exportEntrySet.getValue().length; i++)
+                    {
+                        parentList[i] = getParent(exportEntrySet.getValue()[i], parameters.isCrawlSelf());
+                    }
+                    parentListMap.put(exportEntrySet.getKey(), parentList);
+                }
+            }
+            else{
                 parentList = new NodeRef[exportList.length];
                 for (int i = 0; i < exportList.length; i++)
                 {
                     parentList[i] = getParent(exportList[i], parameters.isCrawlSelf());
                 }
+            }
 
             // get exporter version
             exporterVersion = descriptorService.getServerDescriptor().getVersion();
         }
 
+        public Map<Integer, NodeRef[]> splitArray(NodeRef[] arrayToSplit){
+            if(chunkSize <= 0){
+                return null;
+            }
+            int rest = arrayToSplit.length % chunkSize;
+            int chunks = arrayToSplit.length / chunkSize + (rest > 0 ? 1 : 0);
+            Map<Integer, NodeRef[]> arrays = new HashMap<>() ;
+            for(Integer i = 0; i < (rest > 0 ? chunks - 1 : chunks); i++){
+                arrays.put(i, Arrays.copyOfRange(arrayToSplit, i * chunkSize, i * chunkSize + chunkSize));
+            }
+            if(rest > 0){
+                arrays.put(chunks - 1, Arrays.copyOfRange(arrayToSplit, (chunks - 1) * chunkSize, (chunks - 1) * chunkSize + rest));
+            }
+            return arrays;
+        }
+        
         public boolean canRetrieve()
         {
+            if(exportListMap != null)
+            {
+                if (exportListMap.containsKey(indexSubList))
+                {
+                    return index < exportListMap.get(indexSubList).length;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+            else {
                 return index < exportList.length;
             }
+        }
         
         public int setNextValue()
         {
-            return ++index;
+            if(exportListMap != null && (index == exportListMap.get(indexSubList).length-1)){
+                resetContext();
+                if(indexSubList <= exportListMap.size())
+                {
+                    ++indexSubList;
+                }
+            }
+            else{
+                ++index;
+            }
+            return index;
         }
         
         public void resetContext()
@@ -1078,8 +1177,14 @@ public class ExporterComponent
         {
             if (canRetrieve())
             {
+                if(exportListMap!=null)
+                {
+                    return exportListMap.get(indexSubList)[index];
+                }
+                else {
                     return exportList[index];
                 }
+            }
             return null;
         }
         
@@ -1091,8 +1196,14 @@ public class ExporterComponent
         {
             if (canRetrieve())
             {
+                if(parentListMap!=null)
+                {
+                    return parentListMap.get(indexSubList)[index];
+                }
+                else {
                     return parentList[index];
                 }
+            }
             return null;
         }
         
@@ -1105,6 +1216,11 @@ public class ExporterComponent
             return exportList;
         }
 
+        public Map<Integer, NodeRef[]> getExportMap()
+        {
+            return exportListMap;
+        }
+
         /*
          * (non-Javadoc)
          * @see org.alfresco.service.cmr.view.ExporterContext#getExportParentList()

repository/src/main/java/org/alfresco/service/cmr/view/ExporterContext.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software.
  * If the software was purchased under a paid Alfresco license, the terms of
@@ -26,6 +26,7 @@
 package org.alfresco.service.cmr.view;
 
 import java.util.Date;
+import java.util.Map;
 
 import org.alfresco.service.cmr.repository.NodeRef;
 
@@ -74,6 +75,8 @@ public interface ExporterContext
      */
     public NodeRef[] getExportList();
 
+    public Map<Integer, NodeRef[]> getExportMap();
+    
     /**
      * Gets list of parents for exporting nodes
      * 

repository/src/main/java/org/alfresco/service/cmr/view/ExporterService.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software.
  * If the software was purchased under a paid Alfresco license, the terms of
@@ -76,4 +76,5 @@ public interface ExporterService
     @Auditable(parameters = {"exporter", "parameters", "progress"})
     public void exportView(Exporter exporter, ExporterCrawlerParameters parameters, Exporter progress);
 
+    public void setExportChunkSize(String exportChunkSize);
 }

repository/src/main/resources/alfresco/import-export-context.xml

@@ -130,6 +130,9 @@
         <property name="permissionService">
             <ref bean="PermissionService" />
         </property>
+        <property name="exportChunkSize">
+            <value>${rm.export.chunk.size}</value>
+        </property>
     </bean>
 
    <bean id="repositoryExporterComponent" class="org.alfresco.repo.exporter.RepositoryExporterComponent">

repository/src/test/java/org/alfresco/repo/exporter/ExporterComponentTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -33,7 +33,10 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.io.BufferedReader;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Enumeration;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -41,6 +44,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 import org.alfresco.model.ContentModel;
 import org.alfresco.repo.importer.ACPImportPackageHandler;
@@ -50,12 +55,7 @@ import org.alfresco.repo.security.permissions.PermissionServiceSPI;
 import org.alfresco.service.ServiceRegistry;
 import org.alfresco.service.cmr.model.FileFolderService;
 import org.alfresco.service.cmr.model.FileInfo;
-import org.alfresco.service.cmr.repository.ChildAssociationRef;
+import org.alfresco.service.cmr.repository.*;
-import org.alfresco.service.cmr.repository.ContentData;
-import org.alfresco.service.cmr.repository.MLText;
-import org.alfresco.service.cmr.repository.NodeRef;
-import org.alfresco.service.cmr.repository.NodeService;
-import org.alfresco.service.cmr.repository.StoreRef;
 import org.alfresco.service.cmr.search.CategoryService;
 import org.alfresco.service.cmr.security.AccessPermission;
 import org.alfresco.service.cmr.security.AccessStatus;
@@ -68,6 +68,7 @@ import org.alfresco.service.cmr.view.ExporterService;
 import org.alfresco.service.cmr.view.ImportPackageHandler;
 import org.alfresco.service.cmr.view.ImporterService;
 import org.alfresco.service.cmr.view.Location;
+
 import org.alfresco.service.namespace.QName;
 import org.alfresco.service.transaction.TransactionService;
 import org.alfresco.test_category.OwnJVMTestsCategory;
@@ -82,6 +83,7 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.springframework.extensions.surf.util.I18NUtil;
+import org.springframework.extensions.surf.util.InputStreamContent;
 import org.springframework.transaction.annotation.Transactional;
 
 @Category({OwnJVMTestsCategory.class, LuceneTests.class})
@@ -95,6 +97,7 @@ public class ExporterComponentTest extends BaseSpringTest
     private FileFolderService fileFolderService;
     private CategoryService categoryService;
     private TransactionService transactionService;
+    private ContentService contentService;
     private StoreRef storeRef;
     private AuthenticationComponent authenticationComponent;
     private PermissionServiceSPI permissionService;
@@ -112,6 +115,7 @@ public class ExporterComponentTest extends BaseSpringTest
         categoryService = (CategoryService) applicationContext.getBean("categoryService");     
         transactionService = (TransactionService) applicationContext.getBean("transactionService");
         permissionService = (PermissionServiceSPI) applicationContext.getBean("permissionService");
+        contentService = (ContentService) applicationContext.getBean("contentService");
 
         this.authenticationService = (MutableAuthenticationService) applicationContext.getBean("AuthenticationService");
         this.authenticationComponent = (AuthenticationComponent) applicationContext.getBean("authenticationComponent");
@@ -151,8 +155,6 @@ public class ExporterComponentTest extends BaseSpringTest
         OutputStream output = new FileOutputStream(tempFile);
         ExporterCrawlerParameters parameters = new ExporterCrawlerParameters();
         parameters.setExportFrom(location);
-//        parameters.setExcludeAspects(new QName[] { ContentModel.ASPECT_AUDITABLE });
-//        parameters.setExcludeChildAssocs(new QName[] { ContentModel.ASSOC_CONTAINS });
 
         File acpFile = TempFileProvider.createTempFile("alf", ACPExportPackageHandler.ACP_EXTENSION);
         File dataFile = new File("test");
@@ -162,6 +164,81 @@ public class ExporterComponentTest extends BaseSpringTest
         acpHandler.setExportAsFolders(true);
         exporterService.exportView(acpHandler, parameters, testProgress);
         output.close();
+
+
+    }
+
+    @Test
+    public void testExportWithChunkedList()
+            throws Exception
+    {
+        TestProgress testProgress = new TestProgress();
+        Location location = new Location(storeRef);
+
+        String testFile = "_testFile";
+        int numberOfNodesToExport = 20;
+        // now export
+        location.setPath("/system");
+        File tempFile = TempFileProvider.createTempFile("xmlexporttest", ".xml");
+        OutputStream output = new FileOutputStream(tempFile);
+        ExporterCrawlerParameters parameters = new ExporterCrawlerParameters();
+        parameters.setExportFrom(location);
+
+        File acpFile = TempFileProvider.createTempFile("alf", ACPExportPackageHandler.ACP_EXTENSION);
+        File dataFile = new File("test");
+        File contentDir = new File("test");
+        ACPExportPackageHandler acpHandler = new ACPExportPackageHandler(new FileOutputStream(acpFile), dataFile, contentDir, null);
+        acpHandler.setNodeService(nodeService);
+        acpHandler.setExportAsFolders(true);
+        NodeRef nodeRef = (location == null) ? null : location.getNodeRef();
+        if (nodeRef == null)
+        {
+            // If a specific node has not been provided, default to the root
+            nodeRef = nodeService.getRootNode(location.getStoreRef());
+        }
+        NodeRef[] childRefs = new NodeRef[numberOfNodesToExport];
+
+        for (int i = 0; i < numberOfNodesToExport; i++)
+        {
+            Map<QName, Serializable> props = new HashMap<QName, Serializable>();
+            props.put(ContentModel.PROP_NAME, this.getClass() + testFile + i);
+            childRefs[i] = nodeService.createNode(nodeRef, ContentModel.ASSOC_CONTAINS, ContentModel.ASSOC_CONTAINS, ContentModel.TYPE_CONTENT, props).getChildRef();
+        }
+        parameters.getExportFrom().setNodeRefs(childRefs);
+        parameters.setCrawlSelf(true);
+        exporterService.setExportChunkSize("3");
+        exporterService.exportView(acpHandler, parameters, testProgress);
+        output.close();
+        ZipFile zipFile = new ZipFile(acpFile.getAbsolutePath());
+
+        Enumeration<? extends ZipEntry> entries = zipFile.entries();
+        int numberOfExportedNodes = 0;
+        while(entries.hasMoreElements()){
+            ZipEntry entry = entries.nextElement();
+            InputStream stream = zipFile.getInputStream(entry);
+            try (BufferedReader br = new BufferedReader(new InputStreamReader(
+                    stream, StandardCharsets.UTF_8));) {
+
+                String line;
+
+                while ((line = br.readLine()) != null) {
+
+                    if(line.contains(testFile)){
+                        numberOfExportedNodes++;
+                    }
+                }
+            }
+            stream.close();
+        }
+        zipFile.close();
+
+        assertEquals(numberOfNodesToExport, numberOfExportedNodes);
+
+        parameters.getExportFrom().setNodeRefs(null);
+        for (int i = 0; i < numberOfNodesToExport; i++)
+        {
+            nodeService.deleteNode(childRefs[i]);
+        }
     }
 
     /**