diff --git a/source/java/org/alfresco/repo/model/filefolder/loader/AbstractLoaderThread.java b/source/java/org/alfresco/repo/model/filefolder/loader/AbstractLoaderThread.java
index c7ba7af28b..13e41889b5 100644
--- a/source/java/org/alfresco/repo/model/filefolder/loader/AbstractLoaderThread.java
+++ b/source/java/org/alfresco/repo/model/filefolder/loader/AbstractLoaderThread.java
@@ -65,7 +65,7 @@ public abstract class AbstractLoaderThread extends Thread
this.session = session;
this.loaderName = loaderName;
this.testPeriod = testPeriod;
- this.testTotal = testTotal;
+ this.testTotal = testTotal < 1 ? Integer.MAX_VALUE : testTotal;
this.testLoadDepth = testLoadDepth;
this.mustStop = new AtomicBoolean(false);
@@ -84,7 +84,22 @@ public abstract class AbstractLoaderThread extends Thread
mustStop.set(true);
}
- public abstract String getSummary();
+ /**
+ *
+ * name, average,
+ *
+ *
+ * @return Returns the summary of the results, in the same format as the verbose output
+ */
+ public String getSummary()
+ {
+ // Summarize the results
+ StringBuilder sb = new StringBuilder();
+ sb.append(loaderName).append("\t")
+ .append(String.format("%5.1f", statAverageMs)).append("\t")
+ .append("");
+ return sb.toString();
+ }
@Override
public void run()
@@ -123,7 +138,7 @@ public abstract class AbstractLoaderThread extends Thread
// Do we wait or continue immediately
long duration = endTime - startTime;
- long mustWait = (testPeriod * 1000L) - duration;
+ long mustWait = (testPeriod * 1000L) - (long)(duration / 1000.0 / 1000.0);
if (mustWait >= 5)
{
synchronized(this)
@@ -134,7 +149,7 @@ public abstract class AbstractLoaderThread extends Thread
}
catch (Throwable e)
{
- session.logError(e.getMessage());
+ session.logError("Loading error on '" + loaderName + "': " + e.getMessage(), e);
}
}
}
@@ -143,7 +158,7 @@ public abstract class AbstractLoaderThread extends Thread
{
statCount++;
// Calculate the delta in milliseconds
- double delta = ((double)(endTime - startTime) / 1000.0);
+ double delta = ((double)(endTime - startTime) / 1000.0 / 1000.0);
// Now recalculate the average
statTotalMs += delta;
statAverageMs = (statTotalMs / (double)statCount);
@@ -151,11 +166,11 @@ public abstract class AbstractLoaderThread extends Thread
private void logVerbose(long startTime, long endTime, String msg)
{
- double delta = ((double)(endTime - startTime) / 1000.0);
+ double delta = ((double)(endTime - startTime) / 1000.0 / 1000.0 );
StringBuilder sb = new StringBuilder();
- sb.append(loaderName).append(", ")
- .append(String.format("%5.1d", (int)delta)).append(", ")
+ sb.append(loaderName).append("\t")
+ .append(String.format("%5.1f", delta)).append("\t")
.append(msg);
session.logVerbose(sb.toString());
}
diff --git a/source/java/org/alfresco/repo/model/filefolder/loader/FileFolderRemoteLoader.java b/source/java/org/alfresco/repo/model/filefolder/loader/FileFolderRemoteLoader.java
index 89bc63b20c..381997d260 100644
--- a/source/java/org/alfresco/repo/model/filefolder/loader/FileFolderRemoteLoader.java
+++ b/source/java/org/alfresco/repo/model/filefolder/loader/FileFolderRemoteLoader.java
@@ -74,6 +74,18 @@ public class FileFolderRemoteLoader
}
session = FileFolderRemoteLoader.makeSession(username, password, properties);
threads = FileFolderRemoteLoader.makeThreads(session, properties);
+
+ // Log the initial summaries
+ String summary = session.getSummary();
+ session.logVerbose(summary);
+ session.logSummary(summary);
+ session.logError(summary);
+
+ // Header the outputs
+ session.logSummary(LoaderSession.getLineEnding());
+ session.logSummary("NAME\tTIME\tDESCRIPTION");
+ session.logVerbose(LoaderSession.getLineEnding());
+ session.logVerbose("NAME\tTIME\tDESCRIPTION");
}
public synchronized void start()
@@ -82,6 +94,7 @@ public class FileFolderRemoteLoader
{
throw new AlfrescoRuntimeException("Application not initialized");
}
+
// Fire up the threads
for (Thread thread : threads)
{
@@ -99,19 +112,35 @@ public class FileFolderRemoteLoader
// Now join each thread to make sure they all finish their current operation
for (AbstractLoaderThread thread : threads)
{
+ // Notify any waits
+ synchronized(thread)
+ {
+ thread.notifyAll();
+ }
try
{
thread.join();
}
catch (InterruptedException e) {}
}
- // Get each one's summary
+ // Log each thread's summary
for (AbstractLoaderThread thread : threads)
{
String summary = thread.getSummary();
session.logSummary(summary);
}
}
+
+ public void dumpThreadSummaries()
+ {
+ System.out.println("");
+ // Dump each thread's summary
+ for (AbstractLoaderThread thread : threads)
+ {
+ String summary = thread.getSummary();
+ System.out.println(summary);
+ }
+ }
public static final String PROP_SESSION_NAME = "session.name";
public static final String PROP_SESSION_VERBOSE = "session.verbose";
@@ -260,6 +289,14 @@ public class FileFolderRemoteLoader
{
thread = new LoaderUploadThread(session, name, testPeriod, testTotal, testDepth);
}
+ else if (type.equals("totals"))
+ {
+ thread = new LoaderTotalsThread(session, name, testPeriod, testTotal, testDepth);
+ }
+ else if (type.equals("listFolders"))
+ {
+ thread = new LoaderListFoldersThread(session, name, testPeriod, testTotal, testDepth);
+ }
else
{
throw new LoaderClientException("Unknown test type: " + name);
@@ -325,7 +362,9 @@ public class FileFolderRemoteLoader
Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
// Wait for a quit signal
- System.out.println("Running test " + app.session.getName() + ". Enter 'q' to quit");
+ System.out.println("Running test " + app.session.getName() + ".");
+ System.out.println(" Enter 'q' to quit.");
+ System.out.println(" Enter 's' to dump a thread summary.");
while (true)
{
int keyPress = System.in.read();
@@ -333,6 +372,10 @@ public class FileFolderRemoteLoader
{
break;
}
+ else if (keyPress == 'S' || keyPress == 's')
+ {
+ app.dumpThreadSummaries();
+ }
else if (System.in.available() > 0)
{
// Don't wait, just process
diff --git a/source/java/org/alfresco/repo/model/filefolder/loader/LoaderListFoldersThread.java b/source/java/org/alfresco/repo/model/filefolder/loader/LoaderListFoldersThread.java
new file mode 100644
index 0000000000..121a4c5ead
--- /dev/null
+++ b/source/java/org/alfresco/repo/model/filefolder/loader/LoaderListFoldersThread.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2005-2007 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of
+ * the GPL, you may redistribute this Program in connection with Free/Libre
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's
+ * FLOSS exception. You should have recieved a copy of the text describing
+ * the FLOSS exception, and it is also available here:
+ * http://www.alfresco.com/legal/licensing"
+ */
+package org.alfresco.repo.model.filefolder.loader;
+
+import java.util.List;
+
+import org.alfresco.service.cmr.model.FileInfo;
+import org.alfresco.service.cmr.repository.NodeRef;
+
+/**
+ * A loader thread that retrieves the folders beneath each directory from
+ * the root. This is an expensive process but should reach a stable execution time
+ * once the folders in the profile have all been created.
+ * @since 2.2
+ *
+ * @author Derek Hulley
+ */
+public class LoaderListFoldersThread extends AbstractLoaderThread
+{
+ public LoaderListFoldersThread(
+ LoaderSession session,
+ String loaderName,
+ int testPeriod,
+ int testTotal,
+ int testLoadDepth)
+ {
+ super(session, loaderName, testPeriod, testTotal, testLoadDepth);
+ }
+
+ /**
+ * Go to a directory and get a listing of the folders beneath it.
+ */
+ @Override
+ protected String doLoading(LoaderServerProxy serverProxy, NodeRef workingRootNodeRef) throws Exception
+ {
+ int count = listFoldersRecursive(serverProxy, workingRootNodeRef, 0);
+
+ // Done
+ String msg = String.format("Found %d folders below node %s", count, workingRootNodeRef.toString());
+ return msg;
+ }
+
+ /**
+ * Recursive method to list all folders in the hierarchy.
+ * @return Returns the number of folders listed
+ */
+ private int listFoldersRecursive(LoaderServerProxy serverProxy, NodeRef parentNodeRef, int count)
+ {
+ List fileInfos = serverProxy.fileFolderRemote.listFolders(
+ serverProxy.ticket,
+ parentNodeRef);
+ for (FileInfo info : fileInfos)
+ {
+ if (!info.isFolder())
+ {
+ // Ooops
+ continue;
+ }
+ count += listFoldersRecursive(serverProxy, info.getNodeRef(), count);
+ }
+ return count;
+ }
+}
diff --git a/source/java/org/alfresco/repo/model/filefolder/loader/LoaderSession.java b/source/java/org/alfresco/repo/model/filefolder/loader/LoaderSession.java
index 15d5359c73..9137d82159 100644
--- a/source/java/org/alfresco/repo/model/filefolder/loader/LoaderSession.java
+++ b/source/java/org/alfresco/repo/model/filefolder/loader/LoaderSession.java
@@ -27,13 +27,13 @@ package org.alfresco.repo.model.filefolder.loader;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.alfresco.error.AlfrescoRuntimeException;
+import org.alfresco.error.StackTraceUtil;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.remote.FileFolderRemoteClient;
import org.alfresco.service.cmr.model.FileInfo;
@@ -155,9 +155,9 @@ public class LoaderSession
// Construct output and error files
long time = System.currentTimeMillis();
- File fileVerbose = new File("./LoaderSession-" + name + "-"+ time + "-verbose.txt");
- File fileSummary = new File("./LoaderSession-" + name + "-"+ time + "-summary.txt");
- File fileError = new File("./LoaderSession-" + name + "-"+ time + "-error.txt");
+ File fileVerbose = new File("./LoaderSession-" + name + "-"+ time + "-verbose.tsv");
+ File fileSummary = new File("./LoaderSession-" + name + "-"+ time + "-summary.tsv");
+ File fileError = new File("./LoaderSession-" + name + "-"+ time + "-error.tsv");
outVerbose = new BufferedOutputStream(new FileOutputStream(fileVerbose));
outSummary = new BufferedOutputStream(new FileOutputStream(fileSummary));
outError = new BufferedOutputStream(new FileOutputStream(fileError));
@@ -343,18 +343,23 @@ public class LoaderSession
}
}
- private void writeLineEnding(OutputStream out) throws IOException
+ public static String getLineEnding()
{
- if (File.separatorChar == '/')
+ try
{
- // It's unix
- out.write('\n');
+ if (File.separatorChar == '/')
+ {
+ // It's unix
+ return "\n";
+ }
+ else
+ {
+ return "\r\n";
+ }
}
- else
+ catch (Throwable e)
{
- // Windows
- out.write('\r');
- out.write('\n');
+ return "\n";
}
}
@@ -368,7 +373,7 @@ public class LoaderSession
{
byte[] bytes = msg.getBytes("UTF-8");
outVerbose.write(bytes);
- writeLineEnding(outVerbose);
+ outVerbose.write(getLineEnding().getBytes("UTF-8"));
outVerbose.flush();
}
catch (Throwable e)
@@ -387,7 +392,7 @@ public class LoaderSession
{
byte[] bytes = msg.getBytes("UTF-8");
outSummary.write(bytes);
- writeLineEnding(outSummary);
+ outSummary.write(getLineEnding().getBytes("UTF-8"));
outSummary.flush();
}
catch (Throwable e)
@@ -405,13 +410,50 @@ public class LoaderSession
try
{
byte[] bytes = msg.getBytes("UTF-8");
- outSummary.write(bytes);
- writeLineEnding(outError);
- outSummary.flush();
+ outError.write(bytes);
+ outError.write(getLineEnding().getBytes("UTF-8"));
+ outError.flush();
}
catch (Throwable e)
{
System.err.println("Failed to write message to error file: " + e.getMessage());
}
}
+
+ public synchronized void logError(String msg, Throwable e)
+ {
+ if (outSummary == null)
+ {
+ return;
+ }
+ try
+ {
+ StringBuilder sb = new StringBuilder(1024);
+ StackTraceUtil.buildStackTrace(msg, e.getStackTrace(), sb, 50);
+ byte[] bytes = sb.toString().getBytes("UTF-8");
+ outError.write(bytes);
+ outError.write(getLineEnding().getBytes("UTF-8"));
+ outError.flush();
+ }
+ catch (Throwable ee)
+ {
+ System.err.println("Failed to write message to error file: " + e.getMessage());
+ }
+ }
+
+ public String getSummary()
+ {
+ List folderProfilesAsList = new ArrayList(10);
+ for (int folderProfile : folderProfiles)
+ {
+ folderProfilesAsList.add(Integer.valueOf(folderProfile));
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append("Session name: ").append(name).append(getLineEnding())
+ .append("RMI URLS: ").append(rmiUrls).append(getLineEnding())
+ .append("Store References: ").append(storeRefs).append(getLineEnding())
+ .append("Verbose: ").append(Boolean.toString(verbose)).append(getLineEnding())
+ .append("Folder Profiles: ").append(folderProfilesAsList);
+ return sb.toString();
+ }
}
diff --git a/source/java/org/alfresco/repo/model/filefolder/loader/LoaderTotalsThread.java b/source/java/org/alfresco/repo/model/filefolder/loader/LoaderTotalsThread.java
new file mode 100644
index 0000000000..b79b41e4ff
--- /dev/null
+++ b/source/java/org/alfresco/repo/model/filefolder/loader/LoaderTotalsThread.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2005-2007 Alfresco Software Limited.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+ * As a special exception to the terms and conditions of version 2.0 of
+ * the GPL, you may redistribute this Program in connection with Free/Libre
+ * and Open Source Software ("FLOSS") applications as described in Alfresco's
+ * FLOSS exception. You should have recieved a copy of the text describing
+ * the FLOSS exception, and it is also available here:
+ * http://www.alfresco.com/legal/licensing"
+ */
+package org.alfresco.repo.model.filefolder.loader;
+
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.cmr.repository.StoreRef;
+
+/**
+ * A loader thread that merely reports the size of the remote repository.
+ * @since 2.2
+ *
+ * @author Derek Hulley
+ */
+public class LoaderTotalsThread extends AbstractLoaderThread
+{
+ public LoaderTotalsThread(
+ LoaderSession session,
+ String loaderName,
+ int testPeriod,
+ int testTotal,
+ int testLoadDepth)
+ {
+ super(session, loaderName, testPeriod, testTotal, testLoadDepth);
+ }
+
+ /**
+ * Gets the remote repository sizes and dumps those.
+ */
+ @Override
+ protected String doLoading(LoaderServerProxy serverProxy, NodeRef workingRootNodeRef) throws Exception
+ {
+ return getTotalsMessage();
+ }
+
+ @Override
+ public String getSummary()
+ {
+ return super.getSummary() + getTotalsMessage();
+ }
+
+ private String getTotalsMessage()
+ {
+ LoaderServerProxy serverProxy = session.getRemoteServers().get(0);
+ StringBuilder sb = new StringBuilder();
+ // Get total
+ int totalNodeCount = serverProxy.loaderRemote.getNodeCount(
+ serverProxy.ticket);
+ sb.append(String.format("Total=%d", totalNodeCount));
+ // Get totals for each store
+ for (NodeRef nodeRef : session.getWorkingRootNodeRefs())
+ {
+ StoreRef storeRef = nodeRef.getStoreRef();
+ int storeNodeCount = serverProxy.loaderRemote.getNodeCount(
+ serverProxy.ticket,
+ storeRef);
+ sb.append(", ").append(storeRef.getIdentifier()).append("=").append(storeNodeCount);
+ }
+ // Done
+ return sb.toString();
+ }
+}
diff --git a/source/java/org/alfresco/repo/model/filefolder/loader/LoaderUploadThread.java b/source/java/org/alfresco/repo/model/filefolder/loader/LoaderUploadThread.java
index 0b7950b4e0..b539a11545 100644
--- a/source/java/org/alfresco/repo/model/filefolder/loader/LoaderUploadThread.java
+++ b/source/java/org/alfresco/repo/model/filefolder/loader/LoaderUploadThread.java
@@ -25,21 +25,39 @@
package org.alfresco.repo.model.filefolder.loader;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+
import org.alfresco.model.ContentModel;
+import org.alfresco.repo.cache.EhCacheAdapter;
import org.alfresco.service.cmr.model.FileInfo;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.util.GUID;
import org.springframework.util.FileCopyUtils;
/**
- * A description of what the remote loader should do.
+ * Loader thread that puts documents to the remote repository.
*
* @author Derek Hulley
+ * @since 2.2
*/
public class LoaderUploadThread extends AbstractLoaderThread
{
+ private static EhCacheAdapter pathCache;
+
+ static
+ {
+ CacheManager cacheManager = new CacheManager();
+ Cache cache = new Cache("LoaderUploadThread-PathCache", 10000, false, false, 300, 300);
+ cacheManager.addCache(cache);
+
+ pathCache = new EhCacheAdapter();
+ pathCache.setCache(cache);
+ }
+
public LoaderUploadThread(
LoaderSession session,
String loaderName,
@@ -50,17 +68,52 @@ public class LoaderUploadThread extends AbstractLoaderThread
super(session, loaderName, testPeriod, testTotal, testLoadDepth);
}
+ /**
+ * Creates or find the folders based on caching.
+ */
+ private NodeRef makeFolders(
+ String ticket,
+ LoaderServerProxy serverProxy,
+ NodeRef workingRootNodeRef,
+ List folderPath) throws Exception
+ {
+ // Iterate down the path, checking the cache and populating it as necessary
+ List currentPath = new ArrayList();
+ NodeRef currentParentNodeRef = workingRootNodeRef;
+ for (String pathElement : currentPath)
+ {
+ currentPath.add(pathElement);
+ String key = currentPath.toString();
+ // Is this there?
+ NodeRef nodeRef = pathCache.get(key);
+ if (nodeRef != null)
+ {
+ // Found it
+ currentParentNodeRef = nodeRef;
+ // Step into the next level
+ continue;
+ }
+ // It is not there, so create it
+ FileInfo folderInfo = serverProxy.fileFolderRemote.makeFolders(
+ serverProxy.ticket,
+ currentParentNodeRef,
+ currentPath,
+ ContentModel.TYPE_FOLDER);
+ currentParentNodeRef = folderInfo.getNodeRef();
+ // Cache the new node
+ pathCache.put(key, currentParentNodeRef);
+ }
+ // Done
+ return currentParentNodeRef;
+ }
+
@Override
protected String doLoading(LoaderServerProxy serverProxy, NodeRef workingRootNodeRef) throws Exception
{
// Get a random folder
List folderPath = super.chooseFolderPath();
// Make sure the folder exists
- FileInfo folderInfo = serverProxy.fileFolderRemote.makeFolders(
- serverProxy.ticket,
- workingRootNodeRef,
- folderPath,
- ContentModel.TYPE_FOLDER);
+ NodeRef folderNodeRef = makeFolders(serverProxy.ticket, serverProxy, workingRootNodeRef, folderPath);
// Get a random file
File file = getFile();
@@ -75,8 +128,6 @@ public class LoaderUploadThread extends AbstractLoaderThread
}
// Upload it
- NodeRef folderNodeRef = folderInfo.getNodeRef();
-
FileInfo fileInfo = serverProxy.fileFolderRemote.create(
serverProxy.ticket,
folderNodeRef,
@@ -87,21 +138,6 @@ public class LoaderUploadThread extends AbstractLoaderThread
// Done
String msg = String.format("Uploaded %s to folder: %s", filename, folderPath.toString());
- session.logVerbose(msg);
-
return msg;
-
-// int totalNodeCount = serverProxy.loaderRemote.getNodeCount(
-// serverProxy.ticket);
-// int storeNodeCount = serverProxy.loaderRemote.getNodeCount(
-// serverProxy.ticket,
-// workingRootNodeRef.getStoreRef());
-// session.logVerbose("Nodes: " + totalNodeCount + ". Store Nodes: " + storeNodeCount);
- }
-
- @Override
- public String getSummary()
- {
- return "Tanker";
}
}