Further extensions to FileFolderService loader

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@2918 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/model/filefolder/FileFolderPerformanceTester.java

@@ -17,13 +17,15 @@
 package org.alfresco.repo.model.filefolder;
 
 import java.io.File;
-import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import junit.framework.TestCase;
 
 import org.alfresco.model.ContentModel;
 import org.alfresco.repo.content.transform.AbstractContentTransformerTest;
+import org.alfresco.repo.security.authentication.AuthenticationComponent;
 import org.alfresco.repo.transaction.TransactionUtil;
 import org.alfresco.repo.transaction.TransactionUtil.TransactionWork;
 import org.alfresco.service.ServiceRegistry;
@@ -38,6 +40,8 @@ import org.alfresco.service.namespace.QName;
 import org.alfresco.service.transaction.TransactionService;
 import org.alfresco.util.ApplicationContextHelper;
 import org.alfresco.util.GUID;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.springframework.context.ApplicationContext;
 
 /**
@@ -53,9 +57,12 @@ import org.springframework.context.ApplicationContext;
  */
 public class FileFolderPerformanceTester extends TestCase
 {
+    private static Log logger = LogFactory.getLog(FileFolderPerformanceTester.class);
+    
     private static ApplicationContext ctx = ApplicationContextHelper.getApplicationContext();
     
     private TransactionService transactionService;
+    private AuthenticationComponent authenticationComponent;
     private NodeService nodeService;
     private FileFolderService fileFolderService;
     private StoreRef storeRef;
@@ -67,9 +74,13 @@ public class FileFolderPerformanceTester extends TestCase
     {
         ServiceRegistry serviceRegistry = (ServiceRegistry) ctx.getBean(ServiceRegistry.SERVICE_REGISTRY);
         transactionService = serviceRegistry.getTransactionService();
+        authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent");
         nodeService = serviceRegistry.getNodeService();
         fileFolderService = serviceRegistry.getFileFolderService();
         
+        // authenticate
+        authenticationComponent.setSystemUserAsCurrentUser();
+        
         // create a folder root to work in
         storeRef = nodeService.createStore(StoreRef.PROTOCOL_WORKSPACE, getName() + "_" + System.currentTimeMillis());
         NodeRef rootNodeRef = nodeService.getRootNode(storeRef);
@@ -92,38 +103,82 @@ public class FileFolderPerformanceTester extends TestCase
      * are added one to each folder until each folder has the presribed number of files within it.
      * This can therefore be used to test the performance when the L2 cache sizes are exceeded.
      * <p>
-     * Each creation (file or folder) uses the <b>REQUIRES_NEW</b> transaction declaration.
+     * Each creation (file or folder) uses the <b>PROPAGATION REQUIRED</b> transaction declaration.
      * 
      * @param parentNodeRef the level zero parent
+     * @param randomOrder true if each thread must put the children into the folders in a random order
      * @return Returns the average time (ms) to create the <b>files only</b>
      */
-    private double buildStructure(final NodeRef parentNodeRef, final int folderCount, final int fileCount)
+    private void buildStructure(
+            final NodeRef parentNodeRef,
+            final int threadCount,
+            final boolean randomOrder,
+            final int folderCount,
+            final int fileCount,
+            final double[] dumpPoints)
     {
-        List<NodeRef> folders = new ArrayList<NodeRef>(folderCount);
+        TransactionWork<NodeRef[]> createFoldersWork = new TransactionWork<NodeRef[]>()
+        {
+            public NodeRef[] doWork() throws Exception
+            {
+                NodeRef[] folders = new NodeRef[folderCount];
                 for (int i = 0; i < folderCount; i++)
-        {
-            TransactionWork<FileInfo> createFolderWork = new TransactionWork<FileInfo>()
-            {
-                public FileInfo doWork() throws Exception
                 {
                     FileInfo folderInfo = fileFolderService.create(
                             parentNodeRef,
                             GUID.generate(),
                             ContentModel.TYPE_FOLDER);
-                    // done
+                    // keep the reference
-                    return folderInfo;
+                    folders[i] = folderInfo.getNodeRef();
+                }
+                return folders;
             }
         };
-            FileInfo folderInfo = TransactionUtil.executeInUserTransaction(transactionService, createFolderWork);
+        final NodeRef[] folders = TransactionUtil.executeInUserTransaction(
-            // keep the reference
+                transactionService,
-            folders.add(folderInfo.getNodeRef());
+                createFoldersWork);
-        }
+        // the worker that will load the files into the folders
-        // now progress around the folders until they have been populated
+        Runnable runnable = new Runnable()
-        long start = System.currentTimeMillis();
+        {
+            private long start;
+            public void run()
+            {
+                // authenticate
+                authenticationComponent.setSystemUserAsCurrentUser();
+                
+                // progress around the folders until they have been populated
+                start = System.currentTimeMillis();
+                int nextDumpNumber = 0;
                 for (int i = 0; i < fileCount; i++)
                 {
-            for (final NodeRef folderRef : folders)
+                    // must we dump results
+                    double completedCount = (double) i;
+                    double nextDumpCount = (dumpPoints == null || dumpPoints.length == 0 || nextDumpNumber >= dumpPoints.length)
+                                           ? -1.0
+                                           : (double) fileCount * dumpPoints[nextDumpNumber];
+                    if ((nextDumpCount - 0.5) < completedCount && completedCount < (nextDumpCount + 0.5))
                     {
+                        dumpResults(i);
+                        nextDumpNumber++;
+                    }
+                    // shuffle folders if required
+                    List<NodeRef> foldersList = Arrays.asList(folders);
+                    if (randomOrder)
+                    {
+                        // shuffle folder list
+                        Collections.shuffle(foldersList);
+                    }
+                    for (int j = 0; j < folders.length; j++)
+                    {
+                        if (logger.isDebugEnabled())
+                        {
+                            String msg = String.format(
+                                    "Thread %s loading file %4d into folder %4d",
+                                    Thread.currentThread().getName(),
+                                    i, j);
+                            logger.debug(msg);
+                        }
+                        final NodeRef folderRef = folders[j];
                         TransactionWork<FileInfo> createFileWork = new TransactionWork<FileInfo>()
                         {
                             public FileInfo doWork() throws Exception
@@ -143,34 +198,71 @@ public class FileFolderPerformanceTester extends TestCase
                         TransactionUtil.executeInUserTransaction(transactionService, createFileWork);
                     }
                 }
+                dumpResults(fileCount);
+            }
+            private void dumpResults(int currentFileCount)
+            {
                 long end = System.currentTimeMillis();
                 long time = (end - start);
-        double average = (double) time / (double) (folderCount * fileCount);
+                double average = (double) time / (double) (folderCount * currentFileCount);
-        // done
+                double percentComplete = (double) currentFileCount / (double) fileCount * 100.0;
-        return average;
-    }
-
-    private void timeBuildStructure(NodeRef parentNodeRef, int folderCount, int fileCount)
-    {
-        System.out.println("Starting load of " + fileCount + " files in each of " + folderCount + " folders");
-        double average = buildStructure(parentNodeRef, folderCount, fileCount);
                 System.out.println(
-                "[" + getName() + "] \n" +
+                        "[" + Thread.currentThread().getName() + "] \n" +
-                "   Created " + fileCount + " files in each of " + folderCount + " folders: \n" +
+                        "   Created " + currentFileCount + " files in each of " + folderCount + " folders: \n" +
+                        "   Progress: " + String.format("%9.2f", percentComplete) +  " percent complete \n" +
                         "   Average: " + String.format("%10.2f", average) + " ms per file \n" +
                         "   Average: " + String.format("%10.2f", 1000.0/average) + " files per second");
             }
+        };
+
+        // kick off the required number of threads
+        System.out.println(
+                "Starting " + threadCount +
+                " threads loading " + fileCount +
+                " files in each of " + folderCount +
+                " folders (" +
+                (randomOrder ? "shuffled" : "in order") + ").");
+        ThreadGroup threadGroup = new ThreadGroup(getName());
+        Thread[] threads = new Thread[threadCount];
+        for (int i = 0; i < threadCount; i++)
+        {
+            threads[i] = new Thread(threadGroup, runnable, String.format("FileLoader-%02d", i));
+            threads[i].start();
+        }
+        // join each thread so that we wait for them all to finish
+        for (int i = 0; i < threads.length; i++)
+        {
+            try
+            {
+                threads[i].join();
+            }
+            catch (InterruptedException e)
+            {
+                // not too serious - the worker threads are non-daemon
+            }
+        }
+    }
     
     public void test1Folder10Children() throws Exception
     {
-        timeBuildStructure(rootFolderRef, 1, 10);
+        buildStructure(rootFolderRef, 2, false, 1, 10, null);
     }
     
     public void test10Folders100ChildrenMultiTxn() throws Exception
     {
-        timeBuildStructure(rootFolderRef, 10, 100);
+        buildStructure(rootFolderRef, 2, false, 10, 100, new double[] {0.50});
     }
 //    
+//    public void test10Folders100ChildrenMultiTxnMultiThread() throws Exception
+//    {
+//        buildStructure(4, rootFolderRef, 10, 100);
+//    }
+//
+//    public void test1000Folders1000ChildrenMultiTxnMultiThread() throws Exception
+//    {
+//        buildStructure(rootFolderRef, 4, true, 1000, 1000);
+//    }
+//    
 //    public void test100Folders1Child() throws Exception
 //    {
 //        timeBuildStructure(rootFolderRef, 100, 1);