diff --git a/repository/src/main/java/org/alfresco/repo/exporter/ExporterComponent.java b/repository/src/main/java/org/alfresco/repo/exporter/ExporterComponent.java index 352826b815..ebf1fcc99f 100644 --- a/repository/src/main/java/org/alfresco/repo/exporter/ExporterComponent.java +++ b/repository/src/main/java/org/alfresco/repo/exporter/ExporterComponent.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.exporter; import java.io.IOException; @@ -39,6 +39,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.Arrays; import org.alfresco.model.ContentModel; import org.alfresco.repo.node.MLPropertyInterceptor; @@ -77,6 +78,7 @@ import org.alfresco.service.descriptor.DescriptorService; import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; import org.alfresco.service.namespace.RegexQNamePattern; +import org.apache.commons.lang3.math.NumberUtils; import org.dom4j.io.OutputFormat; import org.dom4j.io.XMLWriter; import org.springframework.extensions.surf.util.ParameterCheck; @@ -99,6 +101,8 @@ public class ExporterComponent private DescriptorService descriptorService; private AuthenticationService authenticationService; private PermissionService permissionService; + + private String exportChunkSize; /** Indent Size */ @@ -178,6 +182,14 @@ public class ExporterComponent { this.exportSecondaryNodes = exportSecondaryNodes; } + + /** + * @param exportChunkSize the exportChunkSize + */ + public void setExportChunkSize(String exportChunkSize) + { + this.exportChunkSize = exportChunkSize; + } /* (non-Javadoc) * @see org.alfresco.service.cmr.view.ExporterService#exportView(java.io.OutputStream, org.alfresco.service.cmr.view.ExporterCrawlerParameters, org.alfresco.service.cmr.view.Exporter) @@ -943,28 +955,23 @@ public class ExporterComponent try { // Current strategy is to determine if node is a child of the root exported node - for (NodeRef exportRoot : context.getExportList()) + if (context.getExportMap() != null) { - if (nodeRef.equals(exportRoot) && parameters.isCrawlSelf() == true) + for (NodeRef[] listNodeRef : context.getExportMap().values()) { - // node to export is the root export node (and root is to be exported) - isWithin = true; - } - else - { - // locate export root in primary parent path of node - Path nodePath = nodeService.getPath(nodeRef); - for (int i = nodePath.size() - 1; i >= 0; i--) + for (NodeRef exportRoot : listNodeRef) { - Path.ChildAssocElement pathElement = (Path.ChildAssocElement) nodePath.get(i); - if (pathElement.getRef().getChildRef().equals(exportRoot)) - { - isWithin = true; - break; - } + isWithin = checkIsWithin(nodeRef, exportRoot, parameters); } } } + else + { + for (NodeRef exportRoot : context.getExportList()) + { + isWithin = checkIsWithin(nodeRef, exportRoot, parameters); + } + } } catch (AccessDeniedException accessErr) { @@ -979,6 +986,28 @@ public class ExporterComponent } } + private boolean checkIsWithin(NodeRef nodeRef, NodeRef exportRoot, ExporterCrawlerParameters parameters){ + if (nodeRef.equals(exportRoot) && parameters.isCrawlSelf() == true) + { + // node to export is the root export node (and root is to be exported) + return true; + } + else + { + // locate export root in primary parent path of node + Path nodePath = nodeService.getPath(nodeRef); + for (int i = nodePath.size() - 1; i >= 0; i--) + { + Path.ChildAssocElement pathElement = (Path.ChildAssocElement) nodePath.get(i); + if (pathElement.getRef().getChildRef().equals(exportRoot)) + { + return true; + } + } + } + return false; + } + /** * Exporter Context @@ -986,7 +1015,9 @@ public class ExporterComponent private class ExporterContextImpl implements ExporterContext { private NodeRef[] exportList; + private Map exportListMap; private NodeRef[] parentList; + private Map parentListMap; private String exportedBy; private Date exportedDate; private String exporterVersion; @@ -995,8 +1026,10 @@ public class ExporterComponent private Map> nodesWithAssociations = new HashMap>(); private int index; - - + private int indexSubList; + private int chunkSize; + + /** * Construct * @@ -1005,7 +1038,17 @@ public class ExporterComponent public ExporterContextImpl(ExporterCrawlerParameters parameters) { index = 0; - + indexSubList = 0; + + + if(!NumberUtils.isParsable(exportChunkSize)){ + chunkSize = 10; + } + else + { + chunkSize = Integer.parseInt(exportChunkSize); + } + // get current user performing export String currentUserName = authenticationService.getCurrentUserName(); exportedBy = (currentUserName == null) ? "unknown" : currentUserName; @@ -1022,24 +1065,80 @@ public class ExporterComponent NodeRef exportOf = getNodeRef(parameters.getExportFrom()); exportList[0] = exportOf; } - parentList = new NodeRef[exportList.length]; - for (int i = 0; i < exportList.length; i++) + if(exportList.length > chunkSize) { - parentList[i] = getParent(exportList[i], parameters.isCrawlSelf()); + exportListMap = splitArray(exportList); + + parentListMap = new HashMap<>(); + for(Map.Entry exportEntrySet : exportListMap.entrySet()) + { + parentList= new NodeRef[exportEntrySet.getValue().length]; + for (int i = 0; i < exportEntrySet.getValue().length; i++) + { + parentList[i] = getParent(exportEntrySet.getValue()[i], parameters.isCrawlSelf()); + } + parentListMap.put(exportEntrySet.getKey(), parentList); + } } - + else{ + parentList = new NodeRef[exportList.length]; + for (int i = 0; i < exportList.length; i++) + { + parentList[i] = getParent(exportList[i], parameters.isCrawlSelf()); + } + } + // get exporter version exporterVersion = descriptorService.getServerDescriptor().getVersion(); } + + public Map splitArray(NodeRef[] arrayToSplit){ + if(chunkSize <= 0){ + return null; + } + int rest = arrayToSplit.length % chunkSize; + int chunks = arrayToSplit.length / chunkSize + (rest > 0 ? 1 : 0); + Map arrays = new HashMap<>() ; + for(Integer i = 0; i < (rest > 0 ? chunks - 1 : chunks); i++){ + arrays.put(i, Arrays.copyOfRange(arrayToSplit, i * chunkSize, i * chunkSize + chunkSize)); + } + if(rest > 0){ + arrays.put(chunks - 1, Arrays.copyOfRange(arrayToSplit, (chunks - 1) * chunkSize, (chunks - 1) * chunkSize + rest)); + } + return arrays; + } public boolean canRetrieve() { - return index < exportList.length; + if(exportListMap != null) + { + if (exportListMap.containsKey(indexSubList)) + { + return index < exportListMap.get(indexSubList).length; + } + else + { + return false; + } + } + else { + return index < exportList.length; + } } public int setNextValue() { - return ++index; + if(exportListMap != null && (index == exportListMap.get(indexSubList).length-1)){ + resetContext(); + if(indexSubList <= exportListMap.size()) + { + ++indexSubList; + } + } + else{ + ++index; + } + return index; } public void resetContext() @@ -1078,7 +1177,13 @@ public class ExporterComponent { if (canRetrieve()) { - return exportList[index]; + if(exportListMap!=null) + { + return exportListMap.get(indexSubList)[index]; + } + else { + return exportList[index]; + } } return null; } @@ -1091,7 +1196,13 @@ public class ExporterComponent { if (canRetrieve()) { - return parentList[index]; + if(parentListMap!=null) + { + return parentListMap.get(indexSubList)[index]; + } + else { + return parentList[index]; + } } return null; } @@ -1105,6 +1216,11 @@ public class ExporterComponent return exportList; } + public Map getExportMap() + { + return exportListMap; + } + /* * (non-Javadoc) * @see org.alfresco.service.cmr.view.ExporterContext#getExportParentList() diff --git a/repository/src/main/java/org/alfresco/service/cmr/view/ExporterContext.java b/repository/src/main/java/org/alfresco/service/cmr/view/ExporterContext.java index b3162b0519..d6b5dc1c93 100644 --- a/repository/src/main/java/org/alfresco/service/cmr/view/ExporterContext.java +++ b/repository/src/main/java/org/alfresco/service/cmr/view/ExporterContext.java @@ -1,31 +1,32 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.service.cmr.view; import java.util.Date; +import java.util.Map; import org.alfresco.service.cmr.repository.NodeRef; @@ -73,6 +74,8 @@ public interface ExporterContext * @return NodeRef[] */ public NodeRef[] getExportList(); + + public Map getExportMap(); /** * Gets list of parents for exporting nodes diff --git a/repository/src/main/java/org/alfresco/service/cmr/view/ExporterService.java b/repository/src/main/java/org/alfresco/service/cmr/view/ExporterService.java index 934653bd51..47e5fe3320 100644 --- a/repository/src/main/java/org/alfresco/service/cmr/view/ExporterService.java +++ b/repository/src/main/java/org/alfresco/service/cmr/view/ExporterService.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.service.cmr.view; import java.io.OutputStream; @@ -75,5 +75,6 @@ public interface ExporterService */ @Auditable(parameters = {"exporter", "parameters", "progress"}) public void exportView(Exporter exporter, ExporterCrawlerParameters parameters, Exporter progress); - + + public void setExportChunkSize(String exportChunkSize); } diff --git a/repository/src/main/resources/alfresco/import-export-context.xml b/repository/src/main/resources/alfresco/import-export-context.xml index d986471533..5e66eb6ec4 100644 --- a/repository/src/main/resources/alfresco/import-export-context.xml +++ b/repository/src/main/resources/alfresco/import-export-context.xml @@ -130,6 +130,9 @@ + + ${rm.export.chunk.size} + diff --git a/repository/src/test/java/org/alfresco/repo/exporter/ExporterComponentTest.java b/repository/src/test/java/org/alfresco/repo/exporter/ExporterComponentTest.java index d3d37a8ee3..c3aadc0643 100644 --- a/repository/src/test/java/org/alfresco/repo/exporter/ExporterComponentTest.java +++ b/repository/src/test/java/org/alfresco/repo/exporter/ExporterComponentTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Repository * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -33,7 +33,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Serializable; +import java.io.BufferedReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Enumeration; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -41,6 +44,8 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import org.alfresco.model.ContentModel; import org.alfresco.repo.importer.ACPImportPackageHandler; @@ -50,12 +55,7 @@ import org.alfresco.repo.security.permissions.PermissionServiceSPI; import org.alfresco.service.ServiceRegistry; import org.alfresco.service.cmr.model.FileFolderService; import org.alfresco.service.cmr.model.FileInfo; -import org.alfresco.service.cmr.repository.ChildAssociationRef; -import org.alfresco.service.cmr.repository.ContentData; -import org.alfresco.service.cmr.repository.MLText; -import org.alfresco.service.cmr.repository.NodeRef; -import org.alfresco.service.cmr.repository.NodeService; -import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.cmr.repository.*; import org.alfresco.service.cmr.search.CategoryService; import org.alfresco.service.cmr.security.AccessPermission; import org.alfresco.service.cmr.security.AccessStatus; @@ -68,6 +68,7 @@ import org.alfresco.service.cmr.view.ExporterService; import org.alfresco.service.cmr.view.ImportPackageHandler; import org.alfresco.service.cmr.view.ImporterService; import org.alfresco.service.cmr.view.Location; + import org.alfresco.service.namespace.QName; import org.alfresco.service.transaction.TransactionService; import org.alfresco.test_category.OwnJVMTestsCategory; @@ -82,6 +83,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.springframework.extensions.surf.util.I18NUtil; +import org.springframework.extensions.surf.util.InputStreamContent; import org.springframework.transaction.annotation.Transactional; @Category({OwnJVMTestsCategory.class, LuceneTests.class}) @@ -95,6 +97,7 @@ public class ExporterComponentTest extends BaseSpringTest private FileFolderService fileFolderService; private CategoryService categoryService; private TransactionService transactionService; + private ContentService contentService; private StoreRef storeRef; private AuthenticationComponent authenticationComponent; private PermissionServiceSPI permissionService; @@ -112,6 +115,7 @@ public class ExporterComponentTest extends BaseSpringTest categoryService = (CategoryService) applicationContext.getBean("categoryService"); transactionService = (TransactionService) applicationContext.getBean("transactionService"); permissionService = (PermissionServiceSPI) applicationContext.getBean("permissionService"); + contentService = (ContentService) applicationContext.getBean("contentService"); this.authenticationService = (MutableAuthenticationService) applicationContext.getBean("AuthenticationService"); this.authenticationComponent = (AuthenticationComponent) applicationContext.getBean("authenticationComponent"); @@ -151,9 +155,7 @@ public class ExporterComponentTest extends BaseSpringTest OutputStream output = new FileOutputStream(tempFile); ExporterCrawlerParameters parameters = new ExporterCrawlerParameters(); parameters.setExportFrom(location); -// parameters.setExcludeAspects(new QName[] { ContentModel.ASPECT_AUDITABLE }); -// parameters.setExcludeChildAssocs(new QName[] { ContentModel.ASSOC_CONTAINS }); - + File acpFile = TempFileProvider.createTempFile("alf", ACPExportPackageHandler.ACP_EXTENSION); File dataFile = new File("test"); File contentDir = new File("test"); @@ -162,6 +164,81 @@ public class ExporterComponentTest extends BaseSpringTest acpHandler.setExportAsFolders(true); exporterService.exportView(acpHandler, parameters, testProgress); output.close(); + + + } + + @Test + public void testExportWithChunkedList() + throws Exception + { + TestProgress testProgress = new TestProgress(); + Location location = new Location(storeRef); + + String testFile = "_testFile"; + int numberOfNodesToExport = 20; + // now export + location.setPath("/system"); + File tempFile = TempFileProvider.createTempFile("xmlexporttest", ".xml"); + OutputStream output = new FileOutputStream(tempFile); + ExporterCrawlerParameters parameters = new ExporterCrawlerParameters(); + parameters.setExportFrom(location); + + File acpFile = TempFileProvider.createTempFile("alf", ACPExportPackageHandler.ACP_EXTENSION); + File dataFile = new File("test"); + File contentDir = new File("test"); + ACPExportPackageHandler acpHandler = new ACPExportPackageHandler(new FileOutputStream(acpFile), dataFile, contentDir, null); + acpHandler.setNodeService(nodeService); + acpHandler.setExportAsFolders(true); + NodeRef nodeRef = (location == null) ? null : location.getNodeRef(); + if (nodeRef == null) + { + // If a specific node has not been provided, default to the root + nodeRef = nodeService.getRootNode(location.getStoreRef()); + } + NodeRef[] childRefs = new NodeRef[numberOfNodesToExport]; + + for (int i = 0; i < numberOfNodesToExport; i++) + { + Map props = new HashMap(); + props.put(ContentModel.PROP_NAME, this.getClass() + testFile + i); + childRefs[i] = nodeService.createNode(nodeRef, ContentModel.ASSOC_CONTAINS, ContentModel.ASSOC_CONTAINS, ContentModel.TYPE_CONTENT, props).getChildRef(); + } + parameters.getExportFrom().setNodeRefs(childRefs); + parameters.setCrawlSelf(true); + exporterService.setExportChunkSize("3"); + exporterService.exportView(acpHandler, parameters, testProgress); + output.close(); + ZipFile zipFile = new ZipFile(acpFile.getAbsolutePath()); + + Enumeration entries = zipFile.entries(); + int numberOfExportedNodes = 0; + while(entries.hasMoreElements()){ + ZipEntry entry = entries.nextElement(); + InputStream stream = zipFile.getInputStream(entry); + try (BufferedReader br = new BufferedReader(new InputStreamReader( + stream, StandardCharsets.UTF_8));) { + + String line; + + while ((line = br.readLine()) != null) { + + if(line.contains(testFile)){ + numberOfExportedNodes++; + } + } + } + stream.close(); + } + zipFile.close(); + + assertEquals(numberOfNodesToExport, numberOfExportedNodes); + + parameters.getExportFrom().setNodeRefs(null); + for (int i = 0; i < numberOfNodesToExport; i++) + { + nodeService.deleteNode(childRefs[i]); + } } /**