Merged HEAD-BUG-FIX (5.1/Cloud) to HEAD (5.1/Cloud)

99377: BENCH-369: BM-0004: API and internals for Alfresco server
    - Move commons Math3 to 'core' project for general reuse
    - Clone NormalDistributionHelper class from Benchmark projects to Alfresco 'core'
    - API added: http://localhost:8080/alfresco/s/api/model/filefolder/load
      JSON:
      {
      "folderPath":"/Sites/t2/documentLibrary",
      "fileCount":"1",
      "minFileSize":"1024",
      "maxFileSize":"2048",
      "maxUniqueDocuments":"10000"
      }
    - Above JSON will create 1 file in the 't2' site document library with spoofed plain text
    - Change away from deprecated API for TransactionListenerAdapter
    - Fix imports and neatness
    - Improve FileNotFoundException details
    - Disable timestamp propagation on the parent folder to reduce CPU overhead
    - Document changes relating to the addition of cm:description properties
    - Add options to control generation of MLText cm:description fields
      - descriptionCount: number of cm:description translations to include
      - descriptionSize:  size in bytes of each cm:description translation
    - Use released 'alfresco-text-gen' V1.1
    - Use fixed text-gen component to prevent ArrayIndexOutOfBOunds
    - Tighten up error message when errors occur on reading content strings
    - Fix random seed generation bug


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@99503 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

config/alfresco/templates/webscripts/org/alfresco/repository/model/filefolder/load.post.desc.xml

@@ -0,0 +1,25 @@
+<webscript>
+  <shortname>Load files into a folder</shortname>
+  <description>  <![CDATA[
+  Create text documents in a folder.
+  <br />
+  The following properties may be set.
+  
+  <dl>
+  <dt>folderPath</dt><dd>path of the folder to use (default:---)</dd>
+  <dt>fileCount</dt><dd>the number of files to create (default:100)</dd>
+  <dt>filesPerTxn</dt><dd>the number of files committed at a time (default:100)</dd>
+  <dt>minFileSize</dt><dd>the smallest file size (standard deviation lower limit) (default:80K)</dd>
+  <dt>maxFileSize</dt><dd>the largest file size (standard deviation upper limit) (default:120K)</dd>
+  <dt>maxUniqueDocuments</dt><dd>total number of random unique text combinations from which to choose (default:Long.MAX_VALUE)</dd>
+  <dt>forceBinaryStorage</dt><dd>force the text to be written to permanent storage, otherwise it is spoofed (default:false)</dd>
+  <dt>descriptionCount</dt><dd>the number of cm:description properties to create (default:1)</dd>
+  <dt>descriptionSize</dt><dd>the size (bytes) of each cm:description property (default:128)</dd>
+  </dl>
+  ]]></description>
+  <url>/api/model/filefolder/load</url>
+  <format default="json" />
+  <authentication>user</authentication>
+  <transaction>required</transaction>
+  <lifecycle>draft_public_api</lifecycle>
+</webscript>

config/alfresco/web-scripts-application-context.xml

@@ -1886,6 +1886,9 @@
         <property name="transactionHelper" ref="retryingTransactionHelper" />
     </bean>
 
+    <!-- FileFolderLoader API -->
+    <bean id="webscript.org.alfresco.repository.model.filefolder.load.post" class="org.alfresco.repo.web.scripts.model.filefolder.FileFolderLoaderPost" parent="webscript"/>
+    
     <!-- Node browser web script -->
     <bean id="webscript.org.alfresco.slingshot.node-browser.node-browser.get" parent="webscript"
    	class="org.alfresco.slingshot.web.scripts.NodeBrowserScript">

source/java/org/alfresco/repo/web/scripts/model/filefolder/FileFolderLoaderPost.java

@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2005-2015 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.repo.web.scripts.model.filefolder;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.alfresco.repo.model.filefolder.FileFolderLoader;
+import org.alfresco.service.cmr.model.FileNotFoundException;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.extensions.webscripts.AbstractWebScript;
+import org.springframework.extensions.webscripts.Status;
+import org.springframework.extensions.webscripts.WebScriptException;
+import org.springframework.extensions.webscripts.WebScriptRequest;
+import org.springframework.extensions.webscripts.WebScriptResponse;
+
+/**
+ * Link to {@link FileFolderLoader}
+ */
+public class FileFolderLoaderPost extends AbstractWebScript implements ApplicationContextAware
+{
+    public static final String KEY_FOLDER_PATH = "folderPath";
+    public static final String KEY_FILE_COUNT = "fileCount";
+    public static final String KEY_FILES_PER_TXN = "filesPerTxn";
+    public static final String KEY_MIN_FILE_SIZE = "minFileSize";
+    public static final String KEY_MAX_FILE_SIZE = "maxFileSize";
+    public static final String KEY_MAX_UNIQUE_DOCUMENTS = "maxUniqueDocuments";
+    public static final String KEY_FORCE_BINARY_STORAGE = "forceBinaryStorage";
+    public static final String KEY_DESCRIPTION_COUNT = "descriptionCount";
+    public static final String KEY_DESCRIPTION_SIZE = "descriptionSize";
+    public static final String KEY_COUNT = "count";
+    
+    public static final int DEFAULT_FILE_COUNT = 100;
+    public static final int DEFAULT_FILES_PER_TXN = 100;
+    public static final long DEFAULT_MIN_FILE_SIZE = 80*1024L;
+    public static final long DEFAULT_MAX_FILE_SIZE = 120*1024L;
+    public static final long DEFAULT_MAX_UNIQUE_DOCUMENTS = Long.MAX_VALUE;
+    public static final int DEFAULT_DESCRIPTION_COUNT = 1;
+    public static final long DEFAULT_DESCRIPTION_SIZE = 128L;
+    public static final boolean DEFAULT_FORCE_BINARY_STORAGE = false;
+    
+    private ApplicationContext applicationContext;
+    
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException
+    {
+        this.applicationContext = applicationContext;
+    }
+    
+    public void execute(WebScriptRequest req, WebScriptResponse res) throws IOException
+    {
+        FileFolderLoader loader = (FileFolderLoader) applicationContext.getBean("fileFolderLoader");
+        
+        int count = 0;
+        String folderPath = "";
+        try
+        {
+            JSONObject json = new JSONObject(new JSONTokener(req.getContent().getContent()));
+            folderPath = json.getString(KEY_FOLDER_PATH);
+            if (folderPath == null)
+            {
+                throw new WebScriptException(Status.STATUS_BAD_REQUEST, KEY_FOLDER_PATH + " not supplied.");
+            }
+            int fileCount = 100;
+            if (json.has(KEY_FILE_COUNT))
+            {
+                fileCount = json.getInt(KEY_FILE_COUNT);
+            }
+            int filesPerTxn = DEFAULT_FILES_PER_TXN;
+            if (json.has(KEY_FILES_PER_TXN))
+            {
+                filesPerTxn = json.getInt(KEY_FILES_PER_TXN);
+            }
+            long minFileSize = DEFAULT_MIN_FILE_SIZE;
+            if (json.has(KEY_MIN_FILE_SIZE))
+            {
+                minFileSize = json.getInt(KEY_MIN_FILE_SIZE);
+            }
+            long maxFileSize = DEFAULT_MAX_FILE_SIZE;
+            if (json.has(KEY_MAX_FILE_SIZE))
+            {
+                maxFileSize = json.getInt(KEY_MAX_FILE_SIZE);
+            }
+            long maxUniqueDocuments = DEFAULT_MAX_UNIQUE_DOCUMENTS;
+            if (json.has(KEY_MAX_UNIQUE_DOCUMENTS))
+            {
+                maxUniqueDocuments = json.getInt(KEY_MAX_UNIQUE_DOCUMENTS);
+            }
+            boolean forceBinaryStorage = DEFAULT_FORCE_BINARY_STORAGE;
+            if (json.has(KEY_FORCE_BINARY_STORAGE))
+            {
+                forceBinaryStorage = json.getBoolean(KEY_FORCE_BINARY_STORAGE);
+            }
+            int descriptionCount = DEFAULT_DESCRIPTION_COUNT;
+            if (json.has(KEY_DESCRIPTION_COUNT))
+            {
+                descriptionCount = json.getInt(KEY_DESCRIPTION_COUNT);
+            }
+            long descriptionSize = DEFAULT_DESCRIPTION_SIZE;
+            if (json.has(KEY_DESCRIPTION_SIZE))
+            {
+                descriptionSize = json.getLong(KEY_DESCRIPTION_SIZE);
+            }
+            
+            // Perform the load
+            count = loader.createFiles(
+                    folderPath,
+                    fileCount, filesPerTxn,
+                    minFileSize, maxFileSize,
+                    maxUniqueDocuments,
+                    forceBinaryStorage,
+                    descriptionCount, descriptionSize);
+        }
+        catch (FileNotFoundException e)
+        {
+            throw new WebScriptException(Status.STATUS_NOT_FOUND, "Folder not found: ", folderPath);
+        }
+        catch (IOException iox)
+        {
+            throw new WebScriptException(Status.STATUS_BAD_REQUEST, "Could not read content from req.", iox);
+        }
+        catch (JSONException je)
+        {
+            throw new WebScriptException(Status.STATUS_BAD_REQUEST, "Could not parse JSON from req.", je);
+        }
+        // Write the response
+        OutputStream os = res.getOutputStream();
+        try
+        {
+            JSONObject json = new JSONObject();
+            json.put(KEY_COUNT, count);
+            os.write(json.toString().getBytes("UTF-8"));
+        }
+        catch (JSONException e)
+        {
+            throw new WebScriptException(Status.STATUS_INTERNAL_SERVER_ERROR, "Failed to write JSON", e);
+        }
+        finally
+        {
+            os.close();
+        }
+    }
+}

source/test-java/org/alfresco/repo/model/filefolder/RemoteFileFolderLoaderTest.java

@@ -0,0 +1,252 @@
+/*
+ * Copyright (C) 2005-2015 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.repo.model.filefolder;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+
+import org.alfresco.model.ContentModel;
+import org.alfresco.repo.content.filestore.FileContentStore;
+import org.alfresco.repo.content.filestore.SpoofedTextContentReader;
+import org.alfresco.repo.model.Repository;
+import org.alfresco.repo.security.authentication.AuthenticationUtil;
+import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback;
+import org.alfresco.repo.web.scripts.BaseWebScriptTest;
+import org.alfresco.repo.web.scripts.model.filefolder.FileFolderLoaderPost;
+import org.alfresco.service.cmr.model.FileFolderService;
+import org.alfresco.service.cmr.model.FileInfo;
+import org.alfresco.service.cmr.repository.ContentReader;
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.cmr.repository.NodeService;
+import org.alfresco.service.transaction.TransactionService;
+import org.json.simple.JSONObject;
+import org.springframework.extensions.webscripts.Status;
+import org.springframework.extensions.webscripts.TestWebScriptServer.PostRequest;
+import org.springframework.extensions.webscripts.TestWebScriptServer.Response;
+
+/**
+ * Remote FileFolderLoader testing
+ * 
+ * @author Derek Hulley
+ * @since 5.1
+ */
+public class RemoteFileFolderLoaderTest  extends BaseWebScriptTest
+{
+    public static final String URL = "/api/model/filefolder/load";
+
+    private Repository repositoryHelper;
+    private NodeService nodeService;
+    private TransactionService transactionService;
+    private FileFolderService fileFolderService;
+    private String sharedHomePath;
+    private NodeRef loadHomeNodeRef;
+    private String loadHomePath;
+    
+    @Override
+    protected void setUp() throws Exception
+    {
+        super.setUp();
+        
+        this.repositoryHelper = (Repository)getServer().getApplicationContext().getBean("repositoryHelper");
+        this.nodeService = (NodeService)getServer().getApplicationContext().getBean("nodeService");
+        this.transactionService = (TransactionService) getServer().getApplicationContext().getBean("TransactionService");
+        this.fileFolderService = (FileFolderService) getServer().getApplicationContext().getBean("FileFolderService");
+
+        // Get the path of the shared folder home
+        final NodeRef companyHomeNodeRef = repositoryHelper.getCompanyHome();
+        final NodeRef sharedHomeNodeRef = repositoryHelper.getSharedHome();
+        RetryingTransactionCallback<NodeRef> createFolderWork = new RetryingTransactionCallback<NodeRef>()
+        {
+            @Override
+            public NodeRef execute() throws Throwable
+            {
+                List<FileInfo> sharedHomeFileInfos = fileFolderService.getNamePath(companyHomeNodeRef, sharedHomeNodeRef);
+                sharedHomePath = "/" + sharedHomeFileInfos.get(0).getName();
+
+                String folderName = UUID.randomUUID().toString();
+                // Create a folder
+                FileInfo folderInfo = fileFolderService.create(sharedHomeNodeRef, folderName, ContentModel.TYPE_FOLDER);
+                loadHomePath = sharedHomePath + "/" + folderName;
+                // Done
+                return folderInfo.getNodeRef();
+            }
+        };
+        AuthenticationUtil.pushAuthentication();            // Will be cleared later
+        AuthenticationUtil.setAdminUserAsFullyAuthenticatedUser();
+
+        loadHomeNodeRef = transactionService.getRetryingTransactionHelper().doInTransaction(createFolderWork);
+    }
+    
+    @Override
+    protected void tearDown() throws Exception
+    {
+        RetryingTransactionCallback<Void> deleteFolderWork = new RetryingTransactionCallback<Void>()
+        {
+            @Override
+            public Void execute() throws Throwable
+            {
+                fileFolderService.delete(loadHomeNodeRef);
+                // Done
+                return null;
+            }
+        };
+        try
+        {
+            transactionService.getRetryingTransactionHelper().doInTransaction(deleteFolderWork);
+        }
+        finally
+        {
+            AuthenticationUtil.popAuthentication();
+        }
+    }
+    
+    /**
+     * Load with no folder path
+     */
+    public void testLoad_noFolderPath() throws Exception
+    {
+        JSONObject body = new JSONObject();
+        
+        sendRequest(
+                new PostRequest(URL,  body.toString(), "application/json"),
+                Status.STATUS_BAD_REQUEST,
+                "bmarley");
+    }
+    
+    /**
+     * Load with defaults
+     */
+    @SuppressWarnings("unchecked")
+    public void testLoad_default_default() throws Exception
+    {
+        JSONObject body = new JSONObject();
+        body.put(FileFolderLoaderPost.KEY_FOLDER_PATH, loadHomePath);
+        
+        Response response = sendRequest(
+                new PostRequest(URL,  body.toString(), "application/json"),
+                Status.STATUS_OK,
+                "bmarley");
+        assertEquals("{\"count\":100}", response.getContentAsString());
+        
+        // Check file(s)
+        assertEquals(100, nodeService.countChildAssocs(loadHomeNodeRef, true));
+    }
+    
+    /**
+     * Load 15 files with default sizes
+     */
+    @SuppressWarnings("unchecked")
+    public void testLoad_15_default() throws Exception
+    {
+        JSONObject body = new JSONObject();
+        body.put(FileFolderLoaderPost.KEY_FOLDER_PATH, loadHomePath);
+        body.put(FileFolderLoaderPost.KEY_FILE_COUNT, 15);
+        body.put(FileFolderLoaderPost.KEY_FILES_PER_TXN, 10);
+        
+        Response response = null;
+        try
+        {
+            AuthenticationUtil.pushAuthentication();
+            AuthenticationUtil.setFullyAuthenticatedUser("hhoudini");
+            response = sendRequest(
+                    new PostRequest(URL,  body.toString(), "application/json"),
+                    Status.STATUS_OK,
+                    "hhoudini");
+        }
+        finally
+        {
+            AuthenticationUtil.popAuthentication();
+        }
+        assertEquals("{\"count\":15}", response.getContentAsString());
+        
+        // Check file(s)
+        assertEquals(15, nodeService.countChildAssocs(loadHomeNodeRef, true));
+        // Size should be default
+        List<FileInfo> fileInfos = fileFolderService.list(loadHomeNodeRef);
+        for (FileInfo fileInfo : fileInfos)
+        {
+            NodeRef fileNodeRef = fileInfo.getNodeRef();
+            ContentReader reader = fileFolderService.getReader(fileNodeRef);
+            // Expect spoofing by default
+            assertTrue(reader.getContentUrl().startsWith(FileContentStore.SPOOF_PROTOCOL));
+            assertTrue(
+                    "Default file size not correct: " + reader,
+                    FileFolderLoaderPost.DEFAULT_MIN_FILE_SIZE < reader.getSize() &&
+                        reader.getSize() < FileFolderLoaderPost.DEFAULT_MAX_FILE_SIZE);
+            // Check creator
+            assertEquals("hhoudini", nodeService.getProperty(fileNodeRef, ContentModel.PROP_CREATOR));
+            // We also expect the default language description to be present
+            String description = (String) nodeService.getProperty(fileNodeRef, ContentModel.PROP_DESCRIPTION);
+            assertNotNull("No description", description);
+            assertEquals("Description length incorrect: ", 128L, description.getBytes("UTF-8").length);
+        }
+    }
+        
+    /**
+     * Load 15 files; 1K size; 1 document sample; force binary storage
+     */
+    @SuppressWarnings("unchecked")
+    public void testLoad_15_16bytes() throws Exception
+    {
+        JSONObject body = new JSONObject();
+        body.put(FileFolderLoaderPost.KEY_FOLDER_PATH, loadHomePath);
+        body.put(FileFolderLoaderPost.KEY_MIN_FILE_SIZE, 16L);
+        body.put(FileFolderLoaderPost.KEY_MAX_FILE_SIZE, 16L);
+        body.put(FileFolderLoaderPost.KEY_MAX_UNIQUE_DOCUMENTS, 1L);
+        body.put(FileFolderLoaderPost.KEY_FORCE_BINARY_STORAGE, Boolean.TRUE);
+        
+        Response response = null;
+        try
+        {
+            AuthenticationUtil.pushAuthentication();
+            AuthenticationUtil.setFullyAuthenticatedUser("maggi");
+            response = sendRequest(
+                    new PostRequest(URL,  body.toString(), "application/json"),
+                    Status.STATUS_OK,
+                    "maggi");
+        }
+        finally
+        {
+            AuthenticationUtil.popAuthentication();
+        }
+        assertEquals("{\"count\":100}", response.getContentAsString());
+        
+        // Check file(s)
+        assertEquals(100, nodeService.countChildAssocs(loadHomeNodeRef, true));
+        
+        // Consistent binary text
+        String contentUrlCheck = SpoofedTextContentReader.createContentUrl(Locale.ENGLISH, 0L, 16L);
+        ContentReader readerCheck = new SpoofedTextContentReader(contentUrlCheck);
+        String textCheck = readerCheck.getContentString();
+        
+        // Size should be default
+        List<FileInfo> fileInfos = fileFolderService.list(loadHomeNodeRef);
+        for (FileInfo fileInfo : fileInfos)
+        {
+            NodeRef fileNodeRef = fileInfo.getNodeRef();
+            ContentReader reader = fileFolderService.getReader(fileNodeRef);
+            // Expect storage in store
+            assertTrue(reader.getContentUrl().startsWith(FileContentStore.STORE_PROTOCOL));
+            // Check text
+            String text = reader.getContentString();
+            assertEquals("Text not the same.", textCheck, text);
+        }
+    }
+}

source/test-java/org/alfresco/repo/web/scripts/RepositoryContainerTest.java

@@ -18,12 +18,8 @@
  */
 package org.alfresco.repo.web.scripts;
 
-import static org.mockito.Matchers.any;
-import static org.springframework.extensions.webscripts.Status.STATUS_OK;
-
 import java.sql.SQLException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.regex.Pattern;
@@ -52,6 +48,9 @@ import org.springframework.extensions.webscripts.WebScriptException;
 import org.springframework.extensions.webscripts.WebScriptRequest;
 import org.springframework.extensions.webscripts.WebScriptResponse;
 
+import static org.mockito.Matchers.any;
+import static org.springframework.extensions.webscripts.Status.STATUS_OK;
+
 /**
  * Unit test to test runas function
  * 

source/test-java/org/alfresco/repo/web/scripts/WebScriptTestSuite.java

@@ -21,6 +21,7 @@ package org.alfresco.repo.web.scripts;
 import junit.framework.Test;
 import junit.framework.TestSuite;
 
+import org.alfresco.repo.model.filefolder.RemoteFileFolderLoaderTest;
 import org.alfresco.repo.web.scripts.action.RunningActionRestApiTest;
 import org.alfresco.repo.web.scripts.activities.feed.control.FeedControlTest;
 import org.alfresco.repo.web.scripts.admin.AdminWebScriptTest;
@@ -107,7 +108,7 @@ public class WebScriptTestSuite extends TestSuite
         suite.addTestSuite( SlingshotContentGetTest.class);
         suite.addTestSuite( XssVulnerabilityTest.class );
         suite.addTestSuite( LinksRestApiTest.class );
-
+        suite.addTestSuite( RemoteFileFolderLoaderTest.class );
         // This uses a slightly different context
         // As such, we can't run it in the same suite as the others,
         //  due to finalisers closing caches when we're not looking