More efficient loading test.

Attempted implmentation for 'listFolders' test, but there are issues.
Implementation of 'totals' test to help keep track of repo size.


git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@6806 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261

source/java/org/alfresco/repo/model/filefolder/loader/AbstractLoaderThread.java

@@ -65,7 +65,7 @@ public abstract class AbstractLoaderThread extends Thread
         this.session = session;
         this.loaderName = loaderName;
         this.testPeriod = testPeriod;
-        this.testTotal = testTotal;
+        this.testTotal = testTotal < 1 ? Integer.MAX_VALUE : testTotal;
         this.testLoadDepth = testLoadDepth;
         
         this.mustStop = new AtomicBoolean(false);
@@ -84,7 +84,22 @@ public abstract class AbstractLoaderThread extends Thread
         mustStop.set(true);
     }
     
-    public abstract String getSummary();
+    /**
+     * <pre>
+     * name, average, 
+     * </pre>
+     * 
+     * @return          Returns the summary of the results, in the same format as the verbose output
+     */
+    public String getSummary()
+    {
+        // Summarize the results
+        StringBuilder sb = new StringBuilder();
+        sb.append(loaderName).append("\t")
+          .append(String.format("%5.1f", statAverageMs)).append("\t")
+          .append("");
+        return sb.toString();
+    }
     
     @Override
     public void run()
@@ -123,7 +138,7 @@ public abstract class AbstractLoaderThread extends Thread
                 
                 // Do we wait or continue immediately
                 long duration = endTime - startTime;
-                long mustWait = (testPeriod * 1000L) - duration;
+                long mustWait = (testPeriod * 1000L) - (long)(duration / 1000.0 / 1000.0);
                 if (mustWait >= 5)
                 {
                     synchronized(this)
@@ -134,7 +149,7 @@ public abstract class AbstractLoaderThread extends Thread
             }
             catch (Throwable e)
             {
-                session.logError(e.getMessage());
+                session.logError("Loading error on '" + loaderName + "': " + e.getMessage(), e);
             }
         }
     }
@@ -143,7 +158,7 @@ public abstract class AbstractLoaderThread extends Thread
     {
         statCount++;
         // Calculate the delta in milliseconds
-        double delta = ((double)(endTime - startTime) / 1000.0);
+        double delta = ((double)(endTime - startTime) / 1000.0 / 1000.0);
         // Now recalculate the average
         statTotalMs += delta;
         statAverageMs = (statTotalMs / (double)statCount);
@@ -151,11 +166,11 @@ public abstract class AbstractLoaderThread extends Thread
     
     private void logVerbose(long startTime, long endTime, String msg)
     {
-        double delta = ((double)(endTime - startTime) / 1000.0);
+        double delta = ((double)(endTime - startTime) / 1000.0 / 1000.0 );
         
         StringBuilder sb = new StringBuilder();
-        sb.append(loaderName).append(", ")
-          .append(String.format("%5.1d", (int)delta)).append(", ")
+        sb.append(loaderName).append("\t")
+          .append(String.format("%5.1f", delta)).append("\t")
           .append(msg);
         session.logVerbose(sb.toString());
     }

source/java/org/alfresco/repo/model/filefolder/loader/FileFolderRemoteLoader.java

@@ -74,6 +74,18 @@ public class FileFolderRemoteLoader
         }
         session = FileFolderRemoteLoader.makeSession(username, password, properties);
         threads = FileFolderRemoteLoader.makeThreads(session, properties);
+        
+        // Log the initial summaries
+        String summary = session.getSummary();
+        session.logVerbose(summary);
+        session.logSummary(summary);
+        session.logError(summary);
+        
+        // Header the outputs
+        session.logSummary(LoaderSession.getLineEnding());
+        session.logSummary("NAME\tTIME\tDESCRIPTION");
+        session.logVerbose(LoaderSession.getLineEnding());
+        session.logVerbose("NAME\tTIME\tDESCRIPTION");
     }
     
     public synchronized void start()
@@ -82,6 +94,7 @@ public class FileFolderRemoteLoader
         {
             throw new AlfrescoRuntimeException("Application not initialized");
         }
+        
         // Fire up the threads
         for (Thread thread : threads)
         {
@@ -99,19 +112,35 @@ public class FileFolderRemoteLoader
         // Now join each thread to make sure they all finish their current operation
         for (AbstractLoaderThread thread : threads)
         {
+            // Notify any waits
+            synchronized(thread)
+            {
+                thread.notifyAll();
+            }
             try
             {
                 thread.join();
             }
             catch (InterruptedException e) {}
         }
-        // Get each one's summary
+        // Log each thread's summary
         for (AbstractLoaderThread thread : threads)
         {
             String summary = thread.getSummary();
             session.logSummary(summary);
         }
     }
+    
+    public void dumpThreadSummaries()
+    {
+        System.out.println("");
+        // Dump each thread's summary
+        for (AbstractLoaderThread thread : threads)
+        {
+            String summary = thread.getSummary();
+            System.out.println(summary);
+        }
+    }
 
     public static final String PROP_SESSION_NAME = "session.name";
     public static final String PROP_SESSION_VERBOSE = "session.verbose";
@@ -260,6 +289,14 @@ public class FileFolderRemoteLoader
                 {
                     thread = new LoaderUploadThread(session, name, testPeriod, testTotal, testDepth);
                 }
+                else if (type.equals("totals"))
+                {
+                    thread = new LoaderTotalsThread(session, name, testPeriod, testTotal, testDepth);
+                }
+                else if (type.equals("listFolders"))
+                {
+                    thread = new LoaderListFoldersThread(session, name, testPeriod, testTotal, testDepth);
+                }
                 else
                 {
                     throw new LoaderClientException("Unknown test type: " + name);
@@ -325,7 +362,9 @@ public class FileFolderRemoteLoader
             Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
             
             // Wait for a quit signal
-            System.out.println("Running test " + app.session.getName() + ".  Enter 'q' to quit");
+            System.out.println("Running test " + app.session.getName() + ".");
+            System.out.println("   Enter 'q' to quit.");
+            System.out.println("   Enter 's' to dump a thread summary.");
             while (true)
             {
                 int keyPress = System.in.read();
@@ -333,6 +372,10 @@ public class FileFolderRemoteLoader
                 {
                     break;
                 }
+                else if (keyPress == 'S' || keyPress == 's')
+                {
+                    app.dumpThreadSummaries();
+                }
                 else if (System.in.available() > 0)
                 {
                     // Don't wait, just process

source/java/org/alfresco/repo/model/filefolder/loader/LoaderListFoldersThread.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2005-2007 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing"
+ */
+package org.alfresco.repo.model.filefolder.loader;
+
+import java.util.List;
+
+import org.alfresco.service.cmr.model.FileInfo;
+import org.alfresco.service.cmr.repository.NodeRef;
+
+/**
+ * A loader thread that retrieves the folders beneath each directory from
+ * the root.  This is an expensive process but should reach a stable execution time
+ * once the folders in the profile have all been created.
+ * @since 2.2
+ * 
+ * @author Derek Hulley
+ */
+public class LoaderListFoldersThread extends AbstractLoaderThread
+{
+    public LoaderListFoldersThread(
+            LoaderSession session,
+            String loaderName,
+            int testPeriod,
+            int testTotal,
+            int testLoadDepth)
+    {
+        super(session, loaderName, testPeriod, testTotal, testLoadDepth);
+    }
+
+    /**
+     * Go to a directory and get a listing of the folders beneath it.
+     */
+    @Override
+    protected String doLoading(LoaderServerProxy serverProxy, NodeRef workingRootNodeRef) throws Exception
+    {
+        int count = listFoldersRecursive(serverProxy, workingRootNodeRef, 0);
+        
+        // Done
+        String msg = String.format("Found %d folders below node %s", count, workingRootNodeRef.toString());
+        return msg;
+    }
+
+    /**
+     * Recursive method to list all folders in the hierarchy.
+     * @return          Returns the number of folders listed
+     */
+    private int listFoldersRecursive(LoaderServerProxy serverProxy, NodeRef parentNodeRef, int count)
+    {
+        List<FileInfo> fileInfos = serverProxy.fileFolderRemote.listFolders(
+                serverProxy.ticket,
+                parentNodeRef);
+        for (FileInfo info : fileInfos)
+        {
+            if (!info.isFolder())
+            {
+                // Ooops
+                continue;
+            }
+            count += listFoldersRecursive(serverProxy, info.getNodeRef(), count);
+        }
+        return count;
+    }
+}

source/java/org/alfresco/repo/model/filefolder/loader/LoaderSession.java

@@ -27,13 +27,13 @@ package org.alfresco.repo.model.filefolder.loader;
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
-import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
 import org.alfresco.error.AlfrescoRuntimeException;
+import org.alfresco.error.StackTraceUtil;
 import org.alfresco.model.ContentModel;
 import org.alfresco.repo.remote.FileFolderRemoteClient;
 import org.alfresco.service.cmr.model.FileInfo;
@@ -155,9 +155,9 @@ public class LoaderSession
         
         // Construct output and error files
         long time = System.currentTimeMillis();
-        File fileVerbose = new File("./LoaderSession-" + name + "-"+ time + "-verbose.txt");
-        File fileSummary = new File("./LoaderSession-" + name + "-"+ time + "-summary.txt");
-        File fileError = new File("./LoaderSession-" + name + "-"+ time + "-error.txt");
+        File fileVerbose = new File("./LoaderSession-" + name + "-"+ time + "-verbose.tsv");
+        File fileSummary = new File("./LoaderSession-" + name + "-"+ time + "-summary.tsv");
+        File fileError = new File("./LoaderSession-" + name + "-"+ time + "-error.tsv");
         outVerbose = new BufferedOutputStream(new FileOutputStream(fileVerbose));
         outSummary = new BufferedOutputStream(new FileOutputStream(fileSummary));
         outError = new BufferedOutputStream(new FileOutputStream(fileError));
@@ -343,18 +343,23 @@ public class LoaderSession
         }
     }
     
-    private void writeLineEnding(OutputStream out) throws IOException
+    public static String getLineEnding()
     {
-        if (File.separatorChar == '/')
+        try
         {
-            // It's unix
-            out.write('\n');
+            if (File.separatorChar == '/')
+            {
+                // It's unix
+                return "\n";
+            }
+            else
+            {
+                return "\r\n";
+            }
         }
-        else
+        catch (Throwable e)
         {
-            // Windows
-            out.write('\r');
-            out.write('\n');
+            return "\n";
         }
     }
     
@@ -368,7 +373,7 @@ public class LoaderSession
         {
             byte[] bytes = msg.getBytes("UTF-8");
             outVerbose.write(bytes);
-            writeLineEnding(outVerbose);
+            outVerbose.write(getLineEnding().getBytes("UTF-8"));
             outVerbose.flush();
         }
         catch (Throwable e)
@@ -387,7 +392,7 @@ public class LoaderSession
         {
             byte[] bytes = msg.getBytes("UTF-8");
             outSummary.write(bytes);
-            writeLineEnding(outSummary);
+            outSummary.write(getLineEnding().getBytes("UTF-8"));
             outSummary.flush();
         }
         catch (Throwable e)
@@ -405,13 +410,50 @@ public class LoaderSession
         try
         {
             byte[] bytes = msg.getBytes("UTF-8");
-            outSummary.write(bytes);
-            writeLineEnding(outError);
-            outSummary.flush();
+            outError.write(bytes);
+            outError.write(getLineEnding().getBytes("UTF-8"));
+            outError.flush();
         }
         catch (Throwable e)
         {
             System.err.println("Failed to write message to error file: " + e.getMessage());
         }
     }
+    
+    public synchronized void logError(String msg, Throwable e)
+    {
+        if (outSummary == null)
+        {
+            return;
+        }
+        try
+        {
+            StringBuilder sb = new StringBuilder(1024);
+            StackTraceUtil.buildStackTrace(msg, e.getStackTrace(), sb, 50);
+            byte[] bytes = sb.toString().getBytes("UTF-8");
+            outError.write(bytes);
+            outError.write(getLineEnding().getBytes("UTF-8"));
+            outError.flush();
+        }
+        catch (Throwable ee)
+        {
+            System.err.println("Failed to write message to error file: " + e.getMessage());
+        }
+    }
+    
+    public String getSummary()
+    {
+        List<Integer> folderProfilesAsList = new ArrayList<Integer>(10);
+        for (int folderProfile : folderProfiles)
+        {
+            folderProfilesAsList.add(Integer.valueOf(folderProfile));
+        }
+        StringBuilder sb = new StringBuilder();
+        sb.append("Session name:     ").append(name).append(getLineEnding())
+          .append("RMI URLS:         ").append(rmiUrls).append(getLineEnding())
+          .append("Store References: ").append(storeRefs).append(getLineEnding())
+          .append("Verbose:          ").append(Boolean.toString(verbose)).append(getLineEnding())
+          .append("Folder Profiles:  ").append(folderProfilesAsList);
+        return sb.toString();
+    }
 }

source/java/org/alfresco/repo/model/filefolder/loader/LoaderTotalsThread.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2005-2007 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of 
+ * the GPL, you may redistribute this Program in connection with Free/Libre 
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's 
+ * FLOSS exception.  You should have recieved a copy of the text describing 
+ * the FLOSS exception, and it is also available here: 
+ * http://www.alfresco.com/legal/licensing"
+ */
+package org.alfresco.repo.model.filefolder.loader;
+
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.cmr.repository.StoreRef;
+
+/**
+ * A loader thread that merely reports the size of the remote repository.
+ * @since 2.2
+ * 
+ * @author Derek Hulley
+ */
+public class LoaderTotalsThread extends AbstractLoaderThread
+{
+    public LoaderTotalsThread(
+            LoaderSession session,
+            String loaderName,
+            int testPeriod,
+            int testTotal,
+            int testLoadDepth)
+    {
+        super(session, loaderName, testPeriod, testTotal, testLoadDepth);
+    }
+
+    /**
+     * Gets the remote repository sizes and dumps those.
+     */
+    @Override
+    protected String doLoading(LoaderServerProxy serverProxy, NodeRef workingRootNodeRef) throws Exception
+    {
+        return getTotalsMessage();
+    }
+
+    @Override
+    public String getSummary()
+    {
+        return super.getSummary() + getTotalsMessage();
+    }
+    
+    private String getTotalsMessage()
+    {
+        LoaderServerProxy serverProxy = session.getRemoteServers().get(0);
+        StringBuilder sb = new StringBuilder();
+        // Get total
+        int totalNodeCount = serverProxy.loaderRemote.getNodeCount(
+                serverProxy.ticket);
+        sb.append(String.format("Total=%d", totalNodeCount));
+        // Get totals for each store
+        for (NodeRef nodeRef : session.getWorkingRootNodeRefs())
+        {
+            StoreRef storeRef = nodeRef.getStoreRef();
+            int storeNodeCount = serverProxy.loaderRemote.getNodeCount(
+                    serverProxy.ticket,
+                    storeRef);
+            sb.append(", ").append(storeRef.getIdentifier()).append("=").append(storeNodeCount);
+        }
+        // Done
+        return sb.toString();
+    }
+}

source/java/org/alfresco/repo/model/filefolder/loader/LoaderUploadThread.java

@@ -25,21 +25,39 @@
 package org.alfresco.repo.model.filefolder.loader;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.List;
 
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+
 import org.alfresco.model.ContentModel;
+import org.alfresco.repo.cache.EhCacheAdapter;
 import org.alfresco.service.cmr.model.FileInfo;
 import org.alfresco.service.cmr.repository.NodeRef;
 import org.alfresco.util.GUID;
 import org.springframework.util.FileCopyUtils;
 
 /**
- * A description of what the remote loader should do.
+ * Loader thread that puts documents to the remote repository.
  * 
  * @author Derek Hulley
+ * @since 2.2
  */
 public class LoaderUploadThread extends AbstractLoaderThread
 {
+    private static EhCacheAdapter<String, NodeRef> pathCache;
+    
+    static
+    {
+        CacheManager cacheManager = new CacheManager();
+        Cache cache = new Cache("LoaderUploadThread-PathCache", 10000, false, false, 300, 300);
+        cacheManager.addCache(cache);
+        
+        pathCache = new EhCacheAdapter<String, NodeRef>();
+        pathCache.setCache(cache);
+    }
+    
     public LoaderUploadThread(
             LoaderSession session,
             String loaderName,
@@ -50,17 +68,52 @@ public class LoaderUploadThread extends AbstractLoaderThread
         super(session, loaderName, testPeriod, testTotal, testLoadDepth);
     }
 
+    /**
+     * Creates or find the folders based on caching.
+     */
+    private NodeRef makeFolders(
+            String ticket,
+            LoaderServerProxy serverProxy,
+            NodeRef workingRootNodeRef,
+            List<String> folderPath) throws Exception
+    {
+        // Iterate down the path, checking the cache and populating it as necessary
+        List<String> currentPath = new ArrayList<String>();
+        NodeRef currentParentNodeRef = workingRootNodeRef;
+        for (String pathElement : currentPath)
+        {
+            currentPath.add(pathElement);
+            String key = currentPath.toString();
+            // Is this there?
+            NodeRef nodeRef = pathCache.get(key);
+            if (nodeRef != null)
+            {
+                // Found it
+                currentParentNodeRef = nodeRef;
+                // Step into the next level
+                continue;
+            }
+            // It is not there, so create it
+            FileInfo folderInfo = serverProxy.fileFolderRemote.makeFolders(
+                    serverProxy.ticket,
+                    currentParentNodeRef,
+                    currentPath,
+                    ContentModel.TYPE_FOLDER);
+            currentParentNodeRef = folderInfo.getNodeRef();
+            // Cache the new node
+            pathCache.put(key, currentParentNodeRef);
+        }
+        // Done
+        return currentParentNodeRef;
+    }
+    
     @Override
     protected String doLoading(LoaderServerProxy serverProxy, NodeRef workingRootNodeRef) throws Exception
     {
         // Get a random folder
         List<String> folderPath = super.chooseFolderPath();
         // Make sure the folder exists
-        FileInfo folderInfo = serverProxy.fileFolderRemote.makeFolders(
-                serverProxy.ticket,
-                workingRootNodeRef,
-                folderPath,
-                ContentModel.TYPE_FOLDER);
+        NodeRef folderNodeRef = makeFolders(serverProxy.ticket, serverProxy, workingRootNodeRef, folderPath);
 
         // Get a random file
         File file = getFile();
@@ -75,8 +128,6 @@ public class LoaderUploadThread extends AbstractLoaderThread
         }
         
         // Upload it
-        NodeRef folderNodeRef = folderInfo.getNodeRef();
-        
         FileInfo fileInfo = serverProxy.fileFolderRemote.create(
                 serverProxy.ticket,
                 folderNodeRef,
@@ -87,21 +138,6 @@ public class LoaderUploadThread extends AbstractLoaderThread
         
         // Done
         String msg = String.format("Uploaded %s to folder: %s", filename, folderPath.toString());
-        session.logVerbose(msg);
-        
         return msg;
-        
-//        int totalNodeCount = serverProxy.loaderRemote.getNodeCount(
-//                serverProxy.ticket);
-//        int storeNodeCount = serverProxy.loaderRemote.getNodeCount(
-//                serverProxy.ticket,
-//                workingRootNodeRef.getStoreRef());
-//        session.logVerbose("Nodes: " + totalNodeCount + ".  Store Nodes: " + storeNodeCount);
-    }
-
-    @Override
-    public String getSummary()
-    {
-        return "Tanker";
     }
 }