inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-07-24 17:32:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix/mnt 23087 export of records failing (#1333)

Browse Source 
* MNT-23087 - Split the export list into several smaller list for better performance
This commit is contained in:
Antonio Felix
2022-08-25 15:36:50 +01:00
committed by GitHub
parent 6e1d5c81e2
commit 9c98f7b0fb
5 changed files with 313 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
repository/src/main/java/org/alfresco/repo/exporter/ExporterComponent.java
View File

@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.repo.exporter;
import java.io.IOException;
@@ -39,6 +39,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.Arrays;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.node.MLPropertyInterceptor;
@@ -77,6 +78,7 @@ import org.alfresco.service.descriptor.DescriptorService;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.service.namespace.RegexQNamePattern;
import org.apache.commons.lang3.math.NumberUtils;
import org.dom4j.io.OutputFormat;
import org.dom4j.io.XMLWriter;
import org.springframework.extensions.surf.util.ParameterCheck;
@@ -99,6 +101,8 @@ public class ExporterComponent
private DescriptorService descriptorService;
private AuthenticationService authenticationService;
private PermissionService permissionService;
private String exportChunkSize;
/** Indent Size */
@@ -178,6 +182,14 @@ public class ExporterComponent
{
this.exportSecondaryNodes = exportSecondaryNodes;
}
/**
* @param exportChunkSize the exportChunkSize
*/
public void setExportChunkSize(String exportChunkSize)
{
this.exportChunkSize = exportChunkSize;
}
/* (non-Javadoc)
* @see org.alfresco.service.cmr.view.ExporterService#exportView(java.io.OutputStream, org.alfresco.service.cmr.view.ExporterCrawlerParameters, org.alfresco.service.cmr.view.Exporter)
@@ -943,28 +955,23 @@ public class ExporterComponent
try
{
// Current strategy is to determine if node is a child of the root exported node
for (NodeRef exportRoot : context.getExportList())
if (context.getExportMap() != null)
{
if (nodeRef.equals(exportRoot) && parameters.isCrawlSelf() == true)
for (NodeRef[] listNodeRef : context.getExportMap().values())
{
// node to export is the root export node (and root is to be exported)
isWithin = true;
}
else
{
// locate export root in primary parent path of node
Path nodePath = nodeService.getPath(nodeRef);
for (int i = nodePath.size() - 1; i >= 0; i--)
for (NodeRef exportRoot : listNodeRef)
{
Path.ChildAssocElement pathElement = (Path.ChildAssocElement) nodePath.get(i);
if (pathElement.getRef().getChildRef().equals(exportRoot))
{
isWithin = true;
break;
}
isWithin = checkIsWithin(nodeRef, exportRoot, parameters);
}
}
}
else
{
for (NodeRef exportRoot : context.getExportList())
{
isWithin = checkIsWithin(nodeRef, exportRoot, parameters);
}
}
}
catch (AccessDeniedException accessErr)
{
@@ -979,6 +986,28 @@ public class ExporterComponent
}
}
private boolean checkIsWithin(NodeRef nodeRef, NodeRef exportRoot, ExporterCrawlerParameters parameters){
if (nodeRef.equals(exportRoot) && parameters.isCrawlSelf() == true)
{
// node to export is the root export node (and root is to be exported)
return true;
}
else
{
// locate export root in primary parent path of node
Path nodePath = nodeService.getPath(nodeRef);
for (int i = nodePath.size() - 1; i >= 0; i--)
{
Path.ChildAssocElement pathElement = (Path.ChildAssocElement) nodePath.get(i);
if (pathElement.getRef().getChildRef().equals(exportRoot))
{
return true;
}
}
}
return false;
}
/**
* Exporter Context
@@ -986,7 +1015,9 @@ public class ExporterComponent
private class ExporterContextImpl implements ExporterContext
{
private NodeRef[] exportList;
private Map<Integer,NodeRef[]> exportListMap;
private NodeRef[] parentList;
private Map<Integer,NodeRef[]> parentListMap;
private String exportedBy;
private Date exportedDate;
private String exporterVersion;
@@ -995,8 +1026,10 @@ public class ExporterComponent
private Map<Integer, Set<NodeRef>> nodesWithAssociations = new HashMap<Integer, Set<NodeRef>>();
private int index;
private int indexSubList;
private int chunkSize;
/**
* Construct
*
@@ -1005,7 +1038,17 @@ public class ExporterComponent
public ExporterContextImpl(ExporterCrawlerParameters parameters)
{
index = 0;
indexSubList = 0;
if(!NumberUtils.isParsable(exportChunkSize)){
chunkSize = 10;
}
else
{
chunkSize = Integer.parseInt(exportChunkSize);
}
// get current user performing export
String currentUserName = authenticationService.getCurrentUserName();
exportedBy = (currentUserName == null) ? "unknown" : currentUserName;
@@ -1022,24 +1065,80 @@ public class ExporterComponent
NodeRef exportOf = getNodeRef(parameters.getExportFrom());
exportList[0] = exportOf;
}
parentList = new NodeRef[exportList.length];
for (int i = 0; i < exportList.length; i++)
if(exportList.length > chunkSize)
{
parentList[i] = getParent(exportList[i], parameters.isCrawlSelf());
exportListMap = splitArray(exportList);
parentListMap = new HashMap<>();
for(Map.Entry<Integer, NodeRef[]> exportEntrySet : exportListMap.entrySet())
{
parentList= new NodeRef[exportEntrySet.getValue().length];
for (int i = 0; i < exportEntrySet.getValue().length; i++)
{
parentList[i] = getParent(exportEntrySet.getValue()[i], parameters.isCrawlSelf());
}
parentListMap.put(exportEntrySet.getKey(), parentList);
}
}
else{
parentList = new NodeRef[exportList.length];
for (int i = 0; i < exportList.length; i++)
{
parentList[i] = getParent(exportList[i], parameters.isCrawlSelf());
}
}
// get exporter version
exporterVersion = descriptorService.getServerDescriptor().getVersion();
}
public Map<Integer, NodeRef[]> splitArray(NodeRef[] arrayToSplit){
if(chunkSize <= 0){
return null;
}
int rest = arrayToSplit.length % chunkSize;
int chunks = arrayToSplit.length / chunkSize + (rest > 0 ? 1 : 0);
Map<Integer, NodeRef[]> arrays = new HashMap<>() ;
for(Integer i = 0; i < (rest > 0 ? chunks - 1 : chunks); i++){
arrays.put(i, Arrays.copyOfRange(arrayToSplit, i * chunkSize, i * chunkSize + chunkSize));
}
if(rest > 0){
arrays.put(chunks - 1, Arrays.copyOfRange(arrayToSplit, (chunks - 1) * chunkSize, (chunks - 1) * chunkSize + rest));
}
return arrays;
}
public boolean canRetrieve()
{
return index < exportList.length;
if(exportListMap != null)
{
if (exportListMap.containsKey(indexSubList))
{
return index < exportListMap.get(indexSubList).length;
}
else
{
return false;
}
}
else {
return index < exportList.length;
}
}
public int setNextValue()
{
return ++index;
if(exportListMap != null && (index == exportListMap.get(indexSubList).length-1)){
resetContext();
if(indexSubList <= exportListMap.size())
{
++indexSubList;
}
}
else{
++index;
}
return index;
}
public void resetContext()
@@ -1078,7 +1177,13 @@ public class ExporterComponent
{
if (canRetrieve())
{
return exportList[index];
if(exportListMap!=null)
{
return exportListMap.get(indexSubList)[index];
}
else {
return exportList[index];
}
}
return null;
}
@@ -1091,7 +1196,13 @@ public class ExporterComponent
{
if (canRetrieve())
{
return parentList[index];
if(parentListMap!=null)
{
return parentListMap.get(indexSubList)[index];
}
else {
return parentList[index];
}
}
return null;
}
@@ -1105,6 +1216,11 @@ public class ExporterComponent
return exportList;
}
public Map<Integer, NodeRef[]> getExportMap()
{
return exportListMap;
}
/*
* (non-Javadoc)
* @see org.alfresco.service.cmr.view.ExporterContext#getExportParentList()

53
repository/src/main/java/org/alfresco/service/cmr/view/ExporterContext.java
View File

@@ -1,31 +1,32 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.service.cmr.view;
import java.util.Date;
import java.util.Map;
import org.alfresco.service.cmr.repository.NodeRef;
@@ -73,6 +74,8 @@ public interface ExporterContext
* @return NodeRef[]
*/
public NodeRef[] getExportList();
public Map<Integer, NodeRef[]> getExportMap();
/**
* Gets list of parents for exporting nodes

53
repository/src/main/java/org/alfresco/service/cmr/view/ExporterService.java
View File

@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.service.cmr.view;
import java.io.OutputStream;
@@ -75,5 +75,6 @@ public interface ExporterService
*/
@Auditable(parameters = {"exporter", "parameters", "progress"})
public void exportView(Exporter exporter, ExporterCrawlerParameters parameters, Exporter progress);
public void setExportChunkSize(String exportChunkSize);
}

3
repository/src/main/resources/alfresco/import-export-context.xml
View File

@@ -130,6 +130,9 @@
<property name="permissionService">
<ref bean="PermissionService" />
</property>
<property name="exportChunkSize">
<value>${rm.export.chunk.size}</value>
</property>
</bean>
<bean id="repositoryExporterComponent" class="org.alfresco.repo.exporter.RepositoryExporterComponent">

97
repository/src/test/java/org/alfresco/repo/exporter/ExporterComponentTest.java
View File

@@ -2,7 +2,7 @@
* #%L
* Alfresco Repository
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
@@ -33,7 +33,10 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Serializable;
import java.io.BufferedReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -41,6 +44,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.importer.ACPImportPackageHandler;
@@ -50,12 +55,7 @@ import org.alfresco.repo.security.permissions.PermissionServiceSPI;
import org.alfresco.service.ServiceRegistry;
import org.alfresco.service.cmr.model.FileFolderService;
import org.alfresco.service.cmr.model.FileInfo;
import org.alfresco.service.cmr.repository.ChildAssociationRef;
import org.alfresco.service.cmr.repository.ContentData;
import org.alfresco.service.cmr.repository.MLText;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.repository.*;
import org.alfresco.service.cmr.search.CategoryService;
import org.alfresco.service.cmr.security.AccessPermission;
import org.alfresco.service.cmr.security.AccessStatus;
@@ -68,6 +68,7 @@ import org.alfresco.service.cmr.view.ExporterService;
import org.alfresco.service.cmr.view.ImportPackageHandler;
import org.alfresco.service.cmr.view.ImporterService;
import org.alfresco.service.cmr.view.Location;
import org.alfresco.service.namespace.QName;
import org.alfresco.service.transaction.TransactionService;
import org.alfresco.test_category.OwnJVMTestsCategory;
@@ -82,6 +83,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.springframework.extensions.surf.util.I18NUtil;
import org.springframework.extensions.surf.util.InputStreamContent;
import org.springframework.transaction.annotation.Transactional;
@Category({OwnJVMTestsCategory.class, LuceneTests.class})
@@ -95,6 +97,7 @@ public class ExporterComponentTest extends BaseSpringTest
private FileFolderService fileFolderService;
private CategoryService categoryService;
private TransactionService transactionService;
private ContentService contentService;
private StoreRef storeRef;
private AuthenticationComponent authenticationComponent;
private PermissionServiceSPI permissionService;
@@ -112,6 +115,7 @@ public class ExporterComponentTest extends BaseSpringTest
categoryService = (CategoryService) applicationContext.getBean("categoryService");
transactionService = (TransactionService) applicationContext.getBean("transactionService");
permissionService = (PermissionServiceSPI) applicationContext.getBean("permissionService");
contentService = (ContentService) applicationContext.getBean("contentService");
this.authenticationService = (MutableAuthenticationService) applicationContext.getBean("AuthenticationService");
this.authenticationComponent = (AuthenticationComponent) applicationContext.getBean("authenticationComponent");
@@ -151,9 +155,7 @@ public class ExporterComponentTest extends BaseSpringTest
OutputStream output = new FileOutputStream(tempFile);
ExporterCrawlerParameters parameters = new ExporterCrawlerParameters();
parameters.setExportFrom(location);
// parameters.setExcludeAspects(new QName[] { ContentModel.ASPECT_AUDITABLE });
// parameters.setExcludeChildAssocs(new QName[] { ContentModel.ASSOC_CONTAINS });
File acpFile = TempFileProvider.createTempFile("alf", ACPExportPackageHandler.ACP_EXTENSION);
File dataFile = new File("test");
File contentDir = new File("test");
@@ -162,6 +164,81 @@ public class ExporterComponentTest extends BaseSpringTest
acpHandler.setExportAsFolders(true);
exporterService.exportView(acpHandler, parameters, testProgress);
output.close();
}
@Test
public void testExportWithChunkedList()
throws Exception
{
TestProgress testProgress = new TestProgress();
Location location = new Location(storeRef);
String testFile = "_testFile";
int numberOfNodesToExport = 20;
// now export
location.setPath("/system");
File tempFile = TempFileProvider.createTempFile("xmlexporttest", ".xml");
OutputStream output = new FileOutputStream(tempFile);
ExporterCrawlerParameters parameters = new ExporterCrawlerParameters();
parameters.setExportFrom(location);
File acpFile = TempFileProvider.createTempFile("alf", ACPExportPackageHandler.ACP_EXTENSION);
File dataFile = new File("test");
File contentDir = new File("test");
ACPExportPackageHandler acpHandler = new ACPExportPackageHandler(new FileOutputStream(acpFile), dataFile, contentDir, null);
acpHandler.setNodeService(nodeService);
acpHandler.setExportAsFolders(true);
NodeRef nodeRef = (location == null) ? null : location.getNodeRef();
if (nodeRef == null)
{
// If a specific node has not been provided, default to the root
nodeRef = nodeService.getRootNode(location.getStoreRef());
}
NodeRef[] childRefs = new NodeRef[numberOfNodesToExport];
for (int i = 0; i < numberOfNodesToExport; i++)
{
Map<QName, Serializable> props = new HashMap<QName, Serializable>();
props.put(ContentModel.PROP_NAME, this.getClass() + testFile + i);
childRefs[i] = nodeService.createNode(nodeRef, ContentModel.ASSOC_CONTAINS, ContentModel.ASSOC_CONTAINS, ContentModel.TYPE_CONTENT, props).getChildRef();
}
parameters.getExportFrom().setNodeRefs(childRefs);
parameters.setCrawlSelf(true);
exporterService.setExportChunkSize("3");
exporterService.exportView(acpHandler, parameters, testProgress);
output.close();
ZipFile zipFile = new ZipFile(acpFile.getAbsolutePath());
Enumeration<? extends ZipEntry> entries = zipFile.entries();
int numberOfExportedNodes = 0;
while(entries.hasMoreElements()){
ZipEntry entry = entries.nextElement();
InputStream stream = zipFile.getInputStream(entry);
try (BufferedReader br = new BufferedReader(new InputStreamReader(
stream, StandardCharsets.UTF_8));) {
String line;
while ((line = br.readLine()) != null) {
if(line.contains(testFile)){
numberOfExportedNodes++;
}
}
}
stream.close();
}
zipFile.close();
assertEquals(numberOfNodesToExport, numberOfExportedNodes);
parameters.getExportFrom().setNodeRefs(null);
for (int i = 0; i < numberOfNodesToExport; i++)
{
nodeService.deleteNode(childRefs[i]);
}
}
/**