inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-06-09 17:45:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

FileImporterTest now has the option to load each file/folder in a new transaction

Browse Source 
git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@2873 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Derek Hulley 2006-05-12 13:17:23 +00:00
parent aa99aeae3a
commit 76535b9afb
2 changed files with 149 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
source/java/org/alfresco/repo/importer/FileImporterImpl.java
View File

@ -26,6 +26,8 @@ import java.util.HashMap;
import java.util.Map;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.transaction.TransactionUtil;
import org.alfresco.repo.transaction.TransactionUtil.TransactionWork;
import org.alfresco.service.cmr.dictionary.DictionaryService;
import org.alfresco.service.cmr.repository.ChildAssociationRef;
import org.alfresco.service.cmr.repository.ContentData;
@ -38,6 +40,7 @@ import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.security.AuthenticationService;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.service.transaction.TransactionService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -55,6 +58,8 @@ public class FileImporterImpl implements FileImporter
private DictionaryService dictionaryService;
private ContentService contentService;
private MimetypeService mimetypeService;
private TransactionService transactionService;
private boolean txnPerFile = false;
public FileImporterImpl()
{
@ -63,30 +68,75 @@ public class FileImporterImpl implements FileImporter
public int loadFile(NodeRef container, File file, boolean recurse) throws FileImporterException
{
Counter counter = new Counter();
create(counter, container, file, null, recurse, null);
return counter.getCount();
try
{
Counter counter = new Counter();
create(counter, container, file, null, recurse, null);
return counter.getCount();
}
catch (Throwable e)
{
throw new FileImporterException("Failed to load file: \n" +
" container: " + container + "\n" +
" file: " + file + "\n" +
" recurse: " + recurse,
e);
}
}
public int loadNamedFile(NodeRef container, File file, boolean recurse, String name) throws FileImporterException
{
Counter counter = new Counter();
create(counter, container, file, null, recurse, name);
return counter.getCount();
try
{
Counter counter = new Counter();
create(counter, container, file, null, recurse, name);
return counter.getCount();
}
catch (Throwable e)
{
throw new FileImporterException("Failed to load file: \n" +
" container: " + container + "\n" +
" file: " + file + "\n" +
" name: " + name + "\n" +
" recurse: " + recurse,
e);
}
}
public int loadFile(NodeRef container, File file, FileFilter filter, boolean recurse) throws FileImporterException
{
Counter counter = new Counter();
create(counter, container, file, filter, recurse, null);
return counter.getCount();
try
{
Counter counter = new Counter();
create(counter, container, file, filter, recurse, null);
return counter.getCount();
}
catch (Throwable e)
{
throw new FileImporterException("Failed to load file: \n" +
" container: " + container + "\n" +
" file: " + file + "\n" +
" filter: " + filter + "\n" +
" recurse: " + recurse,
e);
}
}
public int loadFile(NodeRef container, File file) throws FileImporterException
{
Counter counter = new Counter();
create(counter, container, file, null, false, null);
return counter.getCount();
try
{
Counter counter = new Counter();
create(counter, container, file, null, false, null);
return counter.getCount();
}
catch (Throwable e)
{
throw new FileImporterException("Failed to load file: \n" +
" container: " + container + "\n" +
" file: " + file,
e);
}
}
/** Helper class for mutable int */
@ -103,9 +153,15 @@ public class FileImporterImpl implements FileImporter
}
}
private NodeRef create(Counter counter, NodeRef container, File file, FileFilter filter, boolean recurse, String containerName)
private NodeRef create(
Counter counter,
final NodeRef container,
final File file,
FileFilter filter,
boolean recurse,
String containerName) throws Exception
{
if(containerName != null)
if (containerName != null)
{
NodeRef newContainer = createDirectory(container, containerName, containerName);
return create(counter, newContainer, file, filter, recurse, null);
@ -113,10 +169,27 @@ public class FileImporterImpl implements FileImporter
}
if (file.isDirectory())
{
NodeRef directoryNodeRef = createDirectory(container, file);
counter.increment();
TransactionWork<NodeRef> createDirectoryWork = new TransactionWork<NodeRef>()
{
public NodeRef doWork() throws Exception
{
return createDirectory(container, file);
}
};
NodeRef directoryNodeRef = null;
if (txnPerFile)
{
directoryNodeRef = TransactionUtil.executeInUserTransaction(
transactionService,
createDirectoryWork);
}
else
{
directoryNodeRef = createDirectoryWork.doWork();
}
if(recurse)
if (recurse)
{
File[] files = ((filter == null) ? file.listFiles() : file.listFiles(filter));
for(int i = 0; i < files.length; i++)
@ -130,7 +203,25 @@ public class FileImporterImpl implements FileImporter
else
{
counter.increment();
return createFile(container, file);
TransactionWork<NodeRef> createFileWork = new TransactionWork<NodeRef>()
{
public NodeRef doWork() throws Exception
{
return createFile(container, file);
}
};
NodeRef fileNodeRef = null;
if (txnPerFile)
{
fileNodeRef = TransactionUtil.executeInUserTransaction(
transactionService,
createFileWork);
}
else
{
fileNodeRef = createFileWork.doWork();
}
return fileNodeRef;
}
}
@ -291,4 +382,18 @@ public class FileImporterImpl implements FileImporter
{
this.dictionaryService = dictionaryService;
}
public void setTransactionService(TransactionService transactionService)
{
this.transactionService = transactionService;
}
/**
* @param txnPerFile true to force each file or directory creation to be in its
* own file
*/
public void setTxnPerFile(boolean txnPerFile)
{
this.txnPerFile = txnPerFile;
}
}

91
source/java/org/alfresco/repo/importer/FileImporterTest.java
View File

@ -31,7 +31,6 @@ import javax.transaction.UserTransaction;
import junit.framework.TestCase;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.content.transform.AbstractContentTransformerTest;
import org.alfresco.repo.security.authentication.AuthenticationComponent;
import org.alfresco.service.ServiceRegistry;
@ -41,8 +40,6 @@ import org.alfresco.service.cmr.repository.MimetypeService;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.search.ResultSet;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.service.cmr.search.SearchService;
import org.alfresco.service.cmr.security.AuthenticationService;
import org.alfresco.service.cmr.security.PermissionService;
@ -64,6 +61,7 @@ public class FileImporterTest extends TestCase
private PermissionService permissionService;
private MimetypeService mimetypeService;
private NamespaceService namespaceService;
private TransactionService transactionService;
private ServiceRegistry serviceRegistry;
private NodeRef rootNodeRef;
@ -91,13 +89,14 @@ public class FileImporterTest extends TestCase
permissionService = serviceRegistry.getPermissionService();
mimetypeService = serviceRegistry.getMimetypeService();
namespaceService = serviceRegistry.getNamespaceService();
transactionService = serviceRegistry.getTransactionService();
authenticationComponent.setSystemUserAsCurrentUser();
StoreRef storeRef = nodeService.createStore(StoreRef.PROTOCOL_WORKSPACE, "Test_" + System.currentTimeMillis());
rootNodeRef = nodeService.getRootNode(storeRef);
}
private FileImporter createFileImporter()
private FileImporter createFileImporter(boolean txnPerFile)
{
FileImporterImpl fileImporter = new FileImporterImpl();
fileImporter.setAuthenticationService(authenticationService);
@ -105,19 +104,21 @@ public class FileImporterTest extends TestCase
fileImporter.setMimetypeService(mimetypeService);
fileImporter.setNodeService(nodeService);
fileImporter.setDictionaryService(dictionaryService);
fileImporter.setTransactionService(transactionService);
fileImporter.setTxnPerFile(txnPerFile);
return fileImporter;
}
public void testCreateFile() throws Exception
{
FileImporter fileImporter = createFileImporter();
FileImporter fileImporter = createFileImporter(false);
File file = AbstractContentTransformerTest.loadQuickTestFile("xml");
fileImporter.loadFile(rootNodeRef, file);
}
public void testLoadRootNonRecursive1()
{
FileImporter fileImporter = createFileImporter();
FileImporter fileImporter = createFileImporter(false);
URL url = this.getClass().getClassLoader().getResource("");
File rootFile = new File(url.getFile());
int count = fileImporter.loadFile(rootNodeRef, rootFile);
@ -126,7 +127,7 @@ public class FileImporterTest extends TestCase
public void testLoadRootNonRecursive2()
{
FileImporter fileImporter = createFileImporter();
FileImporter fileImporter = createFileImporter(false);
URL url = this.getClass().getClassLoader().getResource("");
File root = new File(url.getFile());
int count = fileImporter.loadFile(rootNodeRef, root, null, false);
@ -135,7 +136,7 @@ public class FileImporterTest extends TestCase
public void testLoadXMLFiles()
{
FileImporter fileImporter = createFileImporter();
FileImporter fileImporter = createFileImporter(false);
URL url = this.getClass().getClassLoader().getResource("");
FileFilter filter = new XMLFileFilter();
fileImporter.loadFile(rootNodeRef, new File(url.getFile()), filter, true);
@ -143,7 +144,7 @@ public class FileImporterTest extends TestCase
public void testLoadSourceTestResources()
{
FileImporter fileImporter = createFileImporter();
FileImporter fileImporter = createFileImporter(false);
URL url = this.getClass().getClassLoader().getResource("quick");
FileFilter filter = new QuickFileFilter();
fileImporter.loadFile(rootNodeRef, new File(url.getFile()), filter, true);
@ -173,6 +174,7 @@ public class FileImporterTest extends TestCase
* <li>String: Directory to use as source (e.g. c:/temp)
* <li>String: New name to give the source. It may have a suffix added (e.g. upload_xxx)
* <li>Integer: Number of times to repeat the load.
* <li>Boolean: (optional - default 'false') Create each file/folder in a new transaction
* <li>String: (optional) user to authenticate as
* <li>String: (optional) password for authentication
* </ol>
@ -191,8 +193,9 @@ public class FileImporterTest extends TestCase
File sourceFile = new File(args[2]);
String baseName = args[3];
int target = Integer.parseInt(args[4]);
String userName = args.length > 5 ? args[5] : null;
String userPwd = args.length > 6 ? args[6] : "";
Boolean txnPerFile = args.length > 5 ? Boolean.parseBoolean(args[5]) : false;
String userName = args.length > 6 ? args[6] : null;
String userPwd = args.length > 7 ? args[7] : "";
while (count < target)
{
count++;
@ -240,13 +243,19 @@ public class FileImporterTest extends TestCase
userPwd,
test.authenticationService,
test.authenticationComponent);
tx.commit();
}
tx.commit();
// only begin if we are doing it all in one transaction
if (!txnPerFile)
{
tx = transactionService.getUserTransaction();
tx.begin();
}
long start = System.nanoTime();
int importCount = test.createFileImporter().loadNamedFile(
FileImporter importer = test.createFileImporter(txnPerFile);
int importCount = importer.loadNamedFile(
importLocation,
sourceFile,
true,
@ -257,7 +266,10 @@ public class FileImporterTest extends TestCase
System.out.println("Created in: " + ((end - start) / 1000000.0) + "ms");
start = System.nanoTime();
tx.commit();
if (!txnPerFile)
{
tx.commit();
}
end = System.nanoTime();
long second = end-start;
System.out.println("Committed in: " + ((end - start) / 1000000.0) + "ms");
@ -265,55 +277,6 @@ public class FileImporterTest extends TestCase
System.out.println("Grand Total: "+ grandTotal);
System.out.println("Imported: " + importCount + " files or directories");
System.out.println("Average: " + (importCount / (total / 1000.0)) + " files per second");
tx = transactionService.getUserTransaction();
tx.begin();
SearchParameters sp = new SearchParameters();
sp.setLanguage("lucene");
sp.setQuery("ISNODE:T");
sp.addStore(spacesStore);
start = System.nanoTime();
ResultSet rs = test.searchService.query(sp);
end = System.nanoTime();
System.out.println("Find all in: " + ((end - start) / 1000000.0) + "ms");
System.out.println(" = "+rs.length() +"\n\n");
rs.close();
sp = new SearchParameters();
sp.setLanguage("lucene");
sp.setQuery("TEXT:\"andy\"");
sp.addStore(spacesStore);
start = System.nanoTime();
rs = test.searchService.query(sp);
end = System.nanoTime();
System.out.println("Find andy in: " + ((end - start) / 1000000.0) + "ms");
System.out.println(" = "+rs.length() +"\n\n");
rs.close();
sp = new SearchParameters();
sp.setLanguage("lucene");
sp.setQuery("TYPE:\"" + ContentModel.TYPE_CONTENT.toString() + "\"");
sp.addStore(spacesStore);
start = System.nanoTime();
rs = test.searchService.query(sp);
end = System.nanoTime();
System.out.println("Find content in: " + ((end - start) / 1000000.0) + "ms");
System.out.println(" = "+rs.length() +"\n\n");
rs.close();
sp = new SearchParameters();
sp.setLanguage("lucene");
sp.setQuery("PATH:\"/*/*/*\"");
sp.addStore(spacesStore);
start = System.nanoTime();
rs = test.searchService.query(sp);
end = System.nanoTime();
System.out.println("Find /*/*/* in: " + ((end - start) / 1000000.0) + "ms");
System.out.println(" = "+rs.length() +"\n\n");
rs.close();
tx.commit();
}
catch (Throwable e)
{