From 7e5f8fe6fccecd93ef1075c9c2058209ebb70bc8 Mon Sep 17 00:00:00 2001 From: Andrei Zapodeanu <42929831+andrei-zapodeanu-alfresco@users.noreply.github.com> Date: Mon, 15 Oct 2018 11:40:44 +0300 Subject: [PATCH] REPO-3718: MNT-19833: Download via v1 REST API loaded in memory (#87) (cherry picked from commit f3c3b3876f1fd26b78411dbda708770e4af72427) Cherry-picked from f3c3b38 master to 6.0.N --- .../repo/web/scripts/BufferedResponse.java | 70 +++++---- .../repo/web/scripts/RepositoryContainer.java | 142 +++++++++--------- .../framework/webscripts/ApiWebScript.java | 2 +- .../org/alfresco/AppContext02TestSuite.java | 1 + .../org/alfresco/rest/api/tests/ApiTest.java | 3 +- .../rest/api/tests/BufferedResponseTest.java | 116 ++++++++++++++ 6 files changed, 228 insertions(+), 106 deletions(-) create mode 100644 src/test/java/org/alfresco/rest/api/tests/BufferedResponseTest.java diff --git a/src/main/java/org/alfresco/repo/web/scripts/BufferedResponse.java b/src/main/java/org/alfresco/repo/web/scripts/BufferedResponse.java index 298b2e1fa2..01e9147d04 100644 --- a/src/main/java/org/alfresco/repo/web/scripts/BufferedResponse.java +++ b/src/main/java/org/alfresco/repo/web/scripts/BufferedResponse.java @@ -1,36 +1,37 @@ -/* - * #%L - * Alfresco Remote API - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Remote API + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.web.scripts; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.Writer; import org.alfresco.error.AlfrescoRuntimeException; +import org.apache.chemistry.opencmis.commons.server.TempStoreOutputStream; +import org.apache.chemistry.opencmis.server.shared.TempStoreOutputStreamFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.extensions.surf.util.StringBuilderWriter; @@ -38,6 +39,7 @@ import org.springframework.extensions.webscripts.Cache; import org.springframework.extensions.webscripts.Runtime; import org.springframework.extensions.webscripts.WebScriptResponse; import org.springframework.extensions.webscripts.WrappingWebScriptResponse; +import org.springframework.util.FileCopyUtils; /** * Transactional Buffered Response @@ -47,9 +49,10 @@ public class BufferedResponse implements WrappingWebScriptResponse // Logger protected static final Log logger = LogFactory.getLog(BufferedResponse.class); + private TempStoreOutputStreamFactory streamFactory; private WebScriptResponse res; private int bufferSize; - private ByteArrayOutputStream outputStream = null; + private TempStoreOutputStream outputStream = null; private StringBuilderWriter outputWriter = null; @@ -59,10 +62,11 @@ public class BufferedResponse implements WrappingWebScriptResponse * @param res WebScriptResponse * @param bufferSize int */ - public BufferedResponse(WebScriptResponse res, int bufferSize) + public BufferedResponse(WebScriptResponse res, int bufferSize, TempStoreOutputStreamFactory streamFactory) { this.res = res; this.bufferSize = bufferSize; + this.streamFactory = streamFactory; } /* @@ -129,7 +133,7 @@ public class BufferedResponse implements WrappingWebScriptResponse { throw new AlfrescoRuntimeException("Already buffering output writer"); } - this.outputStream = new ByteArrayOutputStream(bufferSize); + outputStream = streamFactory.newOutputStream(); } return outputStream; } @@ -168,7 +172,7 @@ public class BufferedResponse implements WrappingWebScriptResponse { if (outputStream != null) { - outputStream.reset(); + outputStream = null; } else if (outputWriter != null) { @@ -231,7 +235,7 @@ public class BufferedResponse implements WrappingWebScriptResponse { if (logger.isDebugEnabled() && outputStream != null) { - logger.debug("Writing Transactional response: size=" + outputStream.size()); + logger.debug("Writing Transactional response: size=" + outputStream.getLength()); } if (outputWriter != null) @@ -242,10 +246,10 @@ public class BufferedResponse implements WrappingWebScriptResponse else if (outputStream != null) { if (logger.isDebugEnabled()) - logger.debug("Writing Transactional response: size=" + outputStream.size()); + logger.debug("Writing Transactional response: size=" + outputStream.getLength()); outputStream.flush(); - outputStream.writeTo(res.getOutputStream()); + FileCopyUtils.copy(outputStream.getInputStream(), res.getOutputStream()); } } catch (IOException e) diff --git a/src/main/java/org/alfresco/repo/web/scripts/RepositoryContainer.java b/src/main/java/org/alfresco/repo/web/scripts/RepositoryContainer.java index 776d49b848..0e4fb8c752 100644 --- a/src/main/java/org/alfresco/repo/web/scripts/RepositoryContainer.java +++ b/src/main/java/org/alfresco/repo/web/scripts/RepositoryContainer.java @@ -1,75 +1,75 @@ -/* - * #%L - * Alfresco Remote API - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Remote API + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.web.scripts; -import java.io.File; -import java.io.IOException; -import java.net.SocketException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.servlet.http.HttpServletResponse; -import javax.transaction.Status; -import javax.transaction.UserTransaction; - -import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.error.ExceptionStackUtil; -import org.alfresco.repo.model.Repository; -import org.alfresco.repo.security.authentication.AuthenticationUtil; -import org.alfresco.repo.security.authentication.AuthenticationUtil.RunAsWork; -import org.alfresco.repo.transaction.AlfrescoTransactionSupport; -import org.alfresco.repo.transaction.AlfrescoTransactionSupport.TxnReadState; -import org.alfresco.repo.transaction.RetryingTransactionHelper; -import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; -import org.alfresco.repo.transaction.TooBusyException; -import org.alfresco.repo.web.scripts.bean.LoginPost; -import org.alfresco.service.cmr.repository.NodeRef; -import org.alfresco.service.cmr.repository.TemplateService; -import org.alfresco.service.cmr.security.AuthorityService; -import org.alfresco.service.descriptor.DescriptorService; -import org.alfresco.service.transaction.TransactionService; -import org.alfresco.util.TempFileProvider; -import org.apache.chemistry.opencmis.server.shared.TempStoreOutputStreamFactory; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationEvent; -import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.extensions.webscripts.AbstractRuntimeContainer; -import org.springframework.extensions.webscripts.Authenticator; -import org.springframework.extensions.webscripts.Description; -import org.springframework.extensions.webscripts.Description.RequiredAuthentication; -import org.springframework.extensions.webscripts.Description.RequiredTransaction; -import org.springframework.extensions.webscripts.Description.RequiredTransactionParameters; -import org.springframework.extensions.webscripts.Description.TransactionCapability; -import org.springframework.extensions.webscripts.ServerModel; -import org.springframework.extensions.webscripts.WebScript; -import org.springframework.extensions.webscripts.WebScriptException; -import org.springframework.extensions.webscripts.WebScriptRequest; +import java.io.File; +import java.io.IOException; +import java.net.SocketException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletResponse; +import javax.transaction.Status; +import javax.transaction.UserTransaction; + +import org.alfresco.error.AlfrescoRuntimeException; +import org.alfresco.error.ExceptionStackUtil; +import org.alfresco.repo.model.Repository; +import org.alfresco.repo.security.authentication.AuthenticationUtil; +import org.alfresco.repo.security.authentication.AuthenticationUtil.RunAsWork; +import org.alfresco.repo.transaction.AlfrescoTransactionSupport; +import org.alfresco.repo.transaction.AlfrescoTransactionSupport.TxnReadState; +import org.alfresco.repo.transaction.RetryingTransactionHelper; +import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; +import org.alfresco.repo.transaction.TooBusyException; +import org.alfresco.repo.web.scripts.bean.LoginPost; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.TemplateService; +import org.alfresco.service.cmr.security.AuthorityService; +import org.alfresco.service.descriptor.DescriptorService; +import org.alfresco.service.transaction.TransactionService; +import org.alfresco.util.TempFileProvider; +import org.apache.chemistry.opencmis.server.shared.TempStoreOutputStreamFactory; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.extensions.webscripts.AbstractRuntimeContainer; +import org.springframework.extensions.webscripts.Authenticator; +import org.springframework.extensions.webscripts.Description; +import org.springframework.extensions.webscripts.Description.RequiredAuthentication; +import org.springframework.extensions.webscripts.Description.RequiredTransaction; +import org.springframework.extensions.webscripts.Description.RequiredTransactionParameters; +import org.springframework.extensions.webscripts.Description.TransactionCapability; +import org.springframework.extensions.webscripts.ServerModel; +import org.springframework.extensions.webscripts.WebScript; +import org.springframework.extensions.webscripts.WebScriptException; +import org.springframework.extensions.webscripts.WebScriptRequest; import org.springframework.extensions.webscripts.WebScriptResponse; @@ -480,7 +480,7 @@ public class RepositoryContainer extends AbstractRuntimeContainer // create buffered request and response that allow transaction retrying bufferedReq = new BufferedRequest(scriptReq, streamFactory); - bufferedRes = new BufferedResponse(scriptRes, trxParams.getBufferSize()); + bufferedRes = new BufferedResponse(scriptRes, trxParams.getBufferSize(), streamFactory); } else { diff --git a/src/main/java/org/alfresco/rest/framework/webscripts/ApiWebScript.java b/src/main/java/org/alfresco/rest/framework/webscripts/ApiWebScript.java index 126056e3fe..cbad461112 100644 --- a/src/main/java/org/alfresco/rest/framework/webscripts/ApiWebScript.java +++ b/src/main/java/org/alfresco/rest/framework/webscripts/ApiWebScript.java @@ -138,7 +138,7 @@ public abstract class ApiWebScript extends AbstractWebScript protected BufferedResponse getResponse(final WebScriptResponse resp) { // create buffered request and response that allow transaction retrying - final BufferedResponse bufferedRes = new BufferedResponse(resp, memoryThreshold); + final BufferedResponse bufferedRes = new BufferedResponse(resp, memoryThreshold, streamFactory); return bufferedRes; } diff --git a/src/test/java/org/alfresco/AppContext02TestSuite.java b/src/test/java/org/alfresco/AppContext02TestSuite.java index da4e6866e2..5f7ddeb101 100644 --- a/src/test/java/org/alfresco/AppContext02TestSuite.java +++ b/src/test/java/org/alfresco/AppContext02TestSuite.java @@ -68,6 +68,7 @@ import org.junit.runners.Suite; org.alfresco.rest.api.tests.TestUserPreferences.class, org.alfresco.rest.api.tests.WherePredicateApiTest.class, org.alfresco.rest.api.tests.TestRemovePermissions.class, + org.alfresco.rest.api.tests.BufferedResponseTest.class, org.alfresco.rest.workflow.api.tests.DeploymentWorkflowApiTest.class, org.alfresco.rest.workflow.api.tests.ProcessDefinitionWorkflowApiTest.class, }) diff --git a/src/test/java/org/alfresco/rest/api/tests/ApiTest.java b/src/test/java/org/alfresco/rest/api/tests/ApiTest.java index a9a30c5020..64cce3b353 100644 --- a/src/test/java/org/alfresco/rest/api/tests/ApiTest.java +++ b/src/test/java/org/alfresco/rest/api/tests/ApiTest.java @@ -77,7 +77,8 @@ import org.junit.runners.Suite; TestPublicApi128.class, TestPublicApiCaching.class, TestDownloads.class, - AuditAppTest.class + AuditAppTest.class, + BufferedResponseTest.class }) public class ApiTest { diff --git a/src/test/java/org/alfresco/rest/api/tests/BufferedResponseTest.java b/src/test/java/org/alfresco/rest/api/tests/BufferedResponseTest.java new file mode 100644 index 0000000000..bbcf2a9772 --- /dev/null +++ b/src/test/java/org/alfresco/rest/api/tests/BufferedResponseTest.java @@ -0,0 +1,116 @@ + +/* + * #%L + * Alfresco Remote API + * %% + * Copyright (C) 2005 - 2018 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.rest.api.tests; + +import org.alfresco.repo.web.scripts.BufferedResponse; +import org.alfresco.util.TempFileProvider; +import org.apache.chemistry.opencmis.server.shared.TempStoreOutputStreamFactory; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.stream.Stream; + +/** + * Test that BufferedResponse uses a temp file instead of buffering the entire output stream in memory + * + * @author Andrei Zapodeanu + * @author azapodeanu + */ +public class BufferedResponseTest +{ + private static final String TEMP_FOLDER_PATH = TempFileProvider.getTempDir().getAbsolutePath(); + + private static final String TEMP_DIRECTORY_NAME = "testLargeFile"; + private static final String LARGE_FILE_NAME = "largeFile.tmp"; + private static final String FILE_PREFIX = "opencmis"; + + private static final Integer LARGE_FILE_SIZE_BYTES = 5 * 1024 * 1024; + private static final Integer MEMORY_THRESHOLD = 4 * 1024 * 1024; + private static final Integer MAX_CONTENT_SIZE = 1024 * 1024 * 1024; + + @Before + public void createSourceFile() throws IOException + { + createRandomFileInDirectory(TEMP_FOLDER_PATH, LARGE_FILE_NAME, LARGE_FILE_SIZE_BYTES); + } + + @After + public void tearDown() throws Exception + { + File largeFileSource = new File(TEMP_FOLDER_PATH, LARGE_FILE_NAME); + largeFileSource.delete(); + } + + /** + * Test that the output stream creates a temp file to cache its content when file size was bigger than its memory threshold ( 5 > 4 MB ) + * MNT-19833 + */ + @Test + public void testOutputStream() throws IOException + { + File bufferTempDirectory = TempFileProvider.getTempDir(TEMP_DIRECTORY_NAME); + TempStoreOutputStreamFactory streamFactory = TempStoreOutputStreamFactory.newInstance(bufferTempDirectory, MEMORY_THRESHOLD, MAX_CONTENT_SIZE,false); + BufferedResponse response = new BufferedResponse(null, 0, streamFactory); + + long countBefore = countFilesInDirectoryWithPrefix(bufferTempDirectory, FILE_PREFIX ); + copyFileToOutputStream(response); + long countAfter = countFilesInDirectoryWithPrefix(bufferTempDirectory, FILE_PREFIX); + + Assert.assertEquals(countBefore + 1, countAfter); + + } + + private void copyFileToOutputStream(BufferedResponse response) throws IOException + { + File largeFileSource = new File(TEMP_FOLDER_PATH, LARGE_FILE_NAME); + OutputStream testOutputStream = response.getOutputStream(); + Files.copy(largeFileSource.toPath(), testOutputStream); + } + + private void createRandomFileInDirectory(String path, String fileName, int size) throws IOException + { + String fullPath = new File(path, fileName).getPath(); + RandomAccessFile file = new RandomAccessFile(fullPath,"rw"); + file.setLength(size); + file.close(); + } + + private long countFilesInDirectoryWithPrefix(File directory, String filePrefix) throws IOException + { + Stream fileStream = Arrays.stream(directory.listFiles()); + return fileStream.filter( f -> f.getName().startsWith(filePrefix)).count(); + } +} +