inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-06-09 17:45:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Further extensions to FileFolderService loader

Browse Source 
git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@2918 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Derek Hulley 2006-05-18 13:53:24 +00:00
parent 2d8d764421
commit dcd3c78955
1 changed files with 142 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
source/java/org/alfresco/repo/model/filefolder/FileFolderPerformanceTester.java
View File

@ -17,13 +17,15 @@
package org.alfresco.repo.model.filefolder;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import junit.framework.TestCase;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.content.transform.AbstractContentTransformerTest;
import org.alfresco.repo.security.authentication.AuthenticationComponent;
import org.alfresco.repo.transaction.TransactionUtil;
import org.alfresco.repo.transaction.TransactionUtil.TransactionWork;
import org.alfresco.service.ServiceRegistry;
@ -38,6 +40,8 @@ import org.alfresco.service.namespace.QName;
import org.alfresco.service.transaction.TransactionService;
import org.alfresco.util.ApplicationContextHelper;
import org.alfresco.util.GUID;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.ApplicationContext;
/**
@ -53,9 +57,12 @@ import org.springframework.context.ApplicationContext;
*/
public class FileFolderPerformanceTester extends TestCase
{
private static Log logger = LogFactory.getLog(FileFolderPerformanceTester.class);
private static ApplicationContext ctx = ApplicationContextHelper.getApplicationContext();
private TransactionService transactionService;
private AuthenticationComponent authenticationComponent;
private NodeService nodeService;
private FileFolderService fileFolderService;
private StoreRef storeRef;
@ -67,9 +74,13 @@ public class FileFolderPerformanceTester extends TestCase
{
ServiceRegistry serviceRegistry = (ServiceRegistry) ctx.getBean(ServiceRegistry.SERVICE_REGISTRY);
transactionService = serviceRegistry.getTransactionService();
authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent");
nodeService = serviceRegistry.getNodeService();
fileFolderService = serviceRegistry.getFileFolderService();
// authenticate
authenticationComponent.setSystemUserAsCurrentUser();
// create a folder root to work in
storeRef = nodeService.createStore(StoreRef.PROTOCOL_WORKSPACE, getName() + "_" + System.currentTimeMillis());
NodeRef rootNodeRef = nodeService.getRootNode(storeRef);
@ -92,85 +103,166 @@ public class FileFolderPerformanceTester extends TestCase
* are added one to each folder until each folder has the presribed number of files within it.
* This can therefore be used to test the performance when the L2 cache sizes are exceeded.
* <p>
* Each creation (file or folder) uses the <b>REQUIRES_NEW</b> transaction declaration.
* Each creation (file or folder) uses the <b>PROPAGATION REQUIRED</b> transaction declaration.
*
* @param parentNodeRef the level zero parent
* @param randomOrder true if each thread must put the children into the folders in a random order
* @return Returns the average time (ms) to create the <b>files only</b>
*/
private double buildStructure(final NodeRef parentNodeRef, final int folderCount, final int fileCount)
private void buildStructure(
final NodeRef parentNodeRef,
final int threadCount,
final boolean randomOrder,
final int folderCount,
final int fileCount,
final double[] dumpPoints)
{
List<NodeRef> folders = new ArrayList<NodeRef>(folderCount);
for (int i = 0; i < folderCount; i++)
TransactionWork<NodeRef[]> createFoldersWork = new TransactionWork<NodeRef[]>()
{
TransactionWork<FileInfo> createFolderWork = new TransactionWork<FileInfo>()
public NodeRef[] doWork() throws Exception
{
public FileInfo doWork() throws Exception
NodeRef[] folders = new NodeRef[folderCount];
for (int i = 0; i < folderCount; i++)
{
FileInfo folderInfo = fileFolderService.create(
parentNodeRef,
GUID.generate(),
ContentModel.TYPE_FOLDER);
// done
return folderInfo;
// keep the reference
folders[i] = folderInfo.getNodeRef();
}
};
FileInfo folderInfo = TransactionUtil.executeInUserTransaction(transactionService, createFolderWork);
// keep the reference
folders.add(folderInfo.getNodeRef());
}
// now progress around the folders until they have been populated
long start = System.currentTimeMillis();
for (int i = 0; i < fileCount; i++)
return folders;
}
};
final NodeRef[] folders = TransactionUtil.executeInUserTransaction(
transactionService,
createFoldersWork);
// the worker that will load the files into the folders
Runnable runnable = new Runnable()
{
for (final NodeRef folderRef : folders)
private long start;
public void run()
{
TransactionWork<FileInfo> createFileWork = new TransactionWork<FileInfo>()
// authenticate
authenticationComponent.setSystemUserAsCurrentUser();
// progress around the folders until they have been populated
start = System.currentTimeMillis();
int nextDumpNumber = 0;
for (int i = 0; i < fileCount; i++)
{
public FileInfo doWork() throws Exception
// must we dump results
double completedCount = (double) i;
double nextDumpCount = (dumpPoints == null || dumpPoints.length == 0 || nextDumpNumber >= dumpPoints.length)
? -1.0
: (double) fileCount * dumpPoints[nextDumpNumber];
if ((nextDumpCount - 0.5) < completedCount && completedCount < (nextDumpCount + 0.5))
{
FileInfo fileInfo = fileFolderService.create(
folderRef,
GUID.generate(),
ContentModel.TYPE_CONTENT);
NodeRef nodeRef = fileInfo.getNodeRef();
// write the content
ContentWriter writer = fileFolderService.getWriter(nodeRef);
writer.putContent(dataFile);
// done
return fileInfo;
dumpResults(i);
nextDumpNumber++;
}
};
TransactionUtil.executeInUserTransaction(transactionService, createFileWork);
// shuffle folders if required
List<NodeRef> foldersList = Arrays.asList(folders);
if (randomOrder)
{
// shuffle folder list
Collections.shuffle(foldersList);
}
for (int j = 0; j < folders.length; j++)
{
if (logger.isDebugEnabled())
{
String msg = String.format(
"Thread %s loading file %4d into folder %4d",
Thread.currentThread().getName(),
i, j);
logger.debug(msg);
}
final NodeRef folderRef = folders[j];
TransactionWork<FileInfo> createFileWork = new TransactionWork<FileInfo>()
{
public FileInfo doWork() throws Exception
{
FileInfo fileInfo = fileFolderService.create(
folderRef,
GUID.generate(),
ContentModel.TYPE_CONTENT);
NodeRef nodeRef = fileInfo.getNodeRef();
// write the content
ContentWriter writer = fileFolderService.getWriter(nodeRef);
writer.putContent(dataFile);
// done
return fileInfo;
}
};
TransactionUtil.executeInUserTransaction(transactionService, createFileWork);
}
}
dumpResults(fileCount);
}
private void dumpResults(int currentFileCount)
{
long end = System.currentTimeMillis();
long time = (end - start);
double average = (double) time / (double) (folderCount * currentFileCount);
double percentComplete = (double) currentFileCount / (double) fileCount * 100.0;
System.out.println(
"[" + Thread.currentThread().getName() + "] \n" +
" Created " + currentFileCount + " files in each of " + folderCount + " folders: \n" +
" Progress: " + String.format("%9.2f", percentComplete) + " percent complete \n" +
" Average: " + String.format("%10.2f", average) + " ms per file \n" +
" Average: " + String.format("%10.2f", 1000.0/average) + " files per second");
}
};
// kick off the required number of threads
System.out.println(
"Starting " + threadCount +
" threads loading " + fileCount +
" files in each of " + folderCount +
" folders (" +
(randomOrder ? "shuffled" : "in order") + ").");
ThreadGroup threadGroup = new ThreadGroup(getName());
Thread[] threads = new Thread[threadCount];
for (int i = 0; i < threadCount; i++)
{
threads[i] = new Thread(threadGroup, runnable, String.format("FileLoader-%02d", i));
threads[i].start();
}
// join each thread so that we wait for them all to finish
for (int i = 0; i < threads.length; i++)
{
try
{
threads[i].join();
}
catch (InterruptedException e)
{
// not too serious - the worker threads are non-daemon
}
}
long end = System.currentTimeMillis();
long time = (end - start);
double average = (double) time / (double) (folderCount * fileCount);
// done
return average;
}
private void timeBuildStructure(NodeRef parentNodeRef, int folderCount, int fileCount)
{
System.out.println("Starting load of " + fileCount + " files in each of " + folderCount + " folders");
double average = buildStructure(parentNodeRef, folderCount, fileCount);
System.out.println(
"[" + getName() + "] \n" +
" Created " + fileCount + " files in each of " + folderCount + " folders: \n" +
" Average: " + String.format("%10.2f", average) + "ms per file \n" +
" Average: " + String.format("%10.2f", 1000.0/average) + " files per second");
}
public void test1Folder10Children() throws Exception
{
timeBuildStructure(rootFolderRef, 1, 10);
buildStructure(rootFolderRef, 2, false, 1, 10, null);
}
public void test10Folders100ChildrenMultiTxn() throws Exception
{
timeBuildStructure(rootFolderRef, 10, 100);
buildStructure(rootFolderRef, 2, false, 10, 100, new double[] {0.50});
}
//
// public void test10Folders100ChildrenMultiTxnMultiThread() throws Exception
// {
// buildStructure(4, rootFolderRef, 10, 100);
// }
//
// public void test1000Folders1000ChildrenMultiTxnMultiThread() throws Exception
// {
// buildStructure(rootFolderRef, 4, true, 1000, 1000);
// }
//
// public void test100Folders1Child() throws Exception
// {
// timeBuildStructure(rootFolderRef, 100, 1);