inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-08-21 18:09:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit for ALF-10419:

Browse Source 
- Bulk Filesystem Importer
- Adapted from work by Peter Monks (see http://code.google.com/p/alfresco-bulk-filesystem-import) and Romain Guinot
- Refactored to limit repeated code
- Refactored to use the Bulk Processor
- Currently limited to one bulk import at a time (enforced by lock service)
- Unit tests added
- Some fixes applied
- Performance tests; tried different filesystem "walking" strategies
- Still to do: yui dependencies - are these necessary?

git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@31100 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261
This commit is contained in:
Steven Glover
2011-10-10 18:45:00 +00:00
parent 0866d3eb0b
commit 87967a9447
18 changed files with 1704 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
backup/bulkload/AbstractBulkFileSystemImportWebScript.java Normal file
View File

@@ -0,0 +1,208 @@
/*
* Copyright (C) 2005-2011 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* As a special exception to the terms and conditions of version 2.0 of
* the GPL, you may redistribute this Program in connection with Free/Libre
* and Open Source Software ("FLOSS") applications as described in Alfresco's
* FLOSS exception. You should have received a copy of the text describing
* the FLOSS exception, and it is also available here:
* http://www.alfresco.com/legal/licensing"
*/
package org.alfresco.repo.web.scripts.node.bulkload;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.Date;
import org.alfresco.repo.model.Repository;
import org.alfresco.service.cmr.model.FileFolderService;
import org.alfresco.service.cmr.model.FileInfo;
import org.alfresco.service.cmr.model.FileNotFoundException;
import org.alfresco.service.cmr.repository.NodeRef;
import org.springframework.extensions.webscripts.DeclarativeWebScript;
import org.springframework.extensions.webscripts.WebScriptException;
/**
* contains common fields and methods for the import web scripts.
*/
public class AbstractBulkFileSystemImportWebScript extends DeclarativeWebScript
{
protected static final String WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS = "/bulk/import/filesystem/status";
protected static final String PARAMETER_TARGET_NODEREF = "targetNodeRef";
protected static final String PARAMETER_TARGET_PATH = "targetPath";
protected static final String COMPANY_HOME_NAME = "Company Home";
protected static final String COMPANY_HOME_PATH = "/" + COMPANY_HOME_NAME;
// Web scripts parameters (common)
protected static final String PARAMETER_REPLACE_EXISTING = "replaceExisting";
protected static final String PARAMETER_VALUE_REPLACE_EXISTING = "replaceExisting";
protected static final String PARAMETER_SOURCE_DIRECTORY = "sourceDirectory";
protected static final String IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY = "importInProgress";
protected static final String IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY ="bfsit.error.importAlreadyInProgress";
protected FileFolderService fileFolderService;
protected Repository repository;
protected volatile boolean importInProgress;
protected NodeRef getTargetNodeRef(String targetNodeRefStr, String targetPath) throws FileNotFoundException
{
NodeRef targetNodeRef;
if (targetNodeRefStr == null || targetNodeRefStr.trim().length() == 0)
{
if (targetPath == null || targetPath.trim().length() == 0)
{
throw new WebScriptException("Error: neither parameter '" + PARAMETER_TARGET_NODEREF +
"' nor parameter '" + PARAMETER_TARGET_PATH +
"' was provided, but at least one is required !");
}
targetNodeRef = convertPathToNodeRef(targetPath.trim());
}
else
targetNodeRef = new NodeRef(targetNodeRefStr.trim());
return targetNodeRef;
}
protected NodeRef convertPathToNodeRef(String targetPath) throws FileNotFoundException
{
NodeRef result = null;
NodeRef companyHome = repository.getCompanyHome();
String cleanTargetPath = targetPath.replaceAll("/+", "/");
if (cleanTargetPath.startsWith(COMPANY_HOME_PATH))
cleanTargetPath = cleanTargetPath.substring(COMPANY_HOME_PATH.length());
if (cleanTargetPath.startsWith("/"))
cleanTargetPath = cleanTargetPath.substring(1);
if (cleanTargetPath.endsWith("/"))
cleanTargetPath = cleanTargetPath.substring(0, cleanTargetPath.length() - 1);
if (cleanTargetPath.length() == 0)
result = companyHome;
else
{
FileInfo info = fileFolderService.resolveNamePath(companyHome, Arrays.asList(cleanTargetPath.split("/")));
if(info == null)
throw new WebScriptException("could not determine NodeRef for path :'"+cleanTargetPath+"'");
result = info.getNodeRef();
}
return(result);
}
protected String buildTextMessage(Throwable t)
{
StringBuffer result = new StringBuffer();
String timeOfFailure = (new Date()).toString();
String hostName = null;
String ipAddress = null;
try
{
hostName = InetAddress.getLocalHost().getHostName();
ipAddress = InetAddress.getLocalHost().getHostAddress();
}
catch (UnknownHostException uhe)
{
hostName = "unknown";
ipAddress = "unknown";
}
result.append("\nTime of failure: " + timeOfFailure);
result.append("\nHost where failure occurred: " + hostName + " (" + ipAddress + ")");
if (t != null)
{
result.append("\nRoot exception:");
result.append(renderExceptionStackAsText(t));
}
else
{
result.append("\nNo exception was provided.");
}
return(result.toString());
}
private String renderExceptionStackAsText( Throwable t)
{
StringBuffer result = new StringBuffer();
if (t != null)
{
String message = t.getMessage();
Throwable cause = t.getCause();
if (cause != null)
{
result.append(renderExceptionStackAsText(cause));
result.append("\nWrapped by:");
}
if (message == null)
{
message = "";
}
result.append("\n");
result.append(t.getClass().getName());
result.append(": ");
result.append(message);
result.append("\n");
result.append(renderStackTraceElements(t.getStackTrace()));
}
return(result.toString());
}
private String renderStackTraceElements(StackTraceElement[] elements)
{
StringBuffer result = new StringBuffer();
if (elements != null)
{
for (int i = 0; i < elements.length; i++)
{
result.append("\tat " + elements[i].toString() + "\n");
}
}
return(result.toString());
}
// boilerplate setters
public void setFileFolderService(FileFolderService fileFolderService)
{
this.fileFolderService = fileFolderService;
}
public void setRepository(Repository repository)
{
this.repository = repository;
}
}

73
backup/bulkload/BulkFilesystemImportStatusWebScript.java Normal file
View File

@@ -0,0 +1,73 @@
/*
* Copyright (C) 2005-2011 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* As a special exception to the terms and conditions of version 2.0 of
* the GPL, you may redistribute this Program in connection with Free/Libre
* and Open Source Software ("FLOSS") applications as described in Alfresco's
* FLOSS exception. You should have received a copy of the text describing
* the FLOSS exception, and it is also available here:
* http://www.alfresco.com/legal/licensing"
*/
package org.alfresco.repo.web.scripts.node.bulkload;
import java.util.HashMap;
import java.util.Map;
import org.alfresco.repo.bulkimport.BulkFilesystemImporter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.extensions.webscripts.Cache;
import org.springframework.extensions.webscripts.DeclarativeWebScript;
import org.springframework.extensions.webscripts.Status;
import org.springframework.extensions.webscripts.WebScriptRequest;
/**
* Web Script class that provides status information on the bulk filesystem import process.
*
* @since 4.0
*/
public class BulkFilesystemImportStatusWebScript extends DeclarativeWebScript
{
private final static Log logger = LogFactory.getLog(BulkFilesystemImportStatusWebScript.class);
// Output parameters (for Freemarker)
private final static String RESULT_IMPORT_STATUS = "importStatus";
// Attributes
private BulkFilesystemImporter importer;
public void setImporter(BulkFilesystemImporter importer)
{
this.importer = importer;
}
/**
* @see org.alfresco.web.scripts.DeclarativeWebScript#executeImpl(org.alfresco.web.scripts.WebScriptRequest, org.alfresco.web.scripts.Status, org.alfresco.web.scripts.Cache)
*/
@Override
protected Map<String, Object> executeImpl(WebScriptRequest request, Status status, Cache cache)
{
Map<String, Object> result = new HashMap<String, Object>();
cache.setNeverCache(true);
result.put(RESULT_IMPORT_STATUS, importer.getStatus());
return(result);
}
}

147
backup/bulkload/copy/BulkFilesystemImportWebScript.java Normal file
View File

@@ -0,0 +1,147 @@
/*
* Copyright (C) 2005-2011 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* As a special exception to the terms and conditions of version 2.0 of
* the GPL, you may redistribute this Program in connection with Free/Libre
* and Open Source Software ("FLOSS") applications as described in Alfresco's
* FLOSS exception. You should have received a copy of the text describing
* the FLOSS exception, and it is also available here:
* http://www.alfresco.com/legal/licensing"
*/
package org.alfresco.repo.web.scripts.node.bulkload.copy;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import org.alfresco.repo.bulkimport.BulkFilesystemImporter;
import org.alfresco.repo.bulkimport.impl.FilesystemBulkImportSource;
import org.alfresco.repo.bulkimport.impl.StreamingBulkImportStrategy;
import org.alfresco.repo.web.scripts.node.bulkload.AbstractBulkFileSystemImportWebScript;
import org.alfresco.service.cmr.model.FileNotFoundException;
import org.alfresco.service.cmr.repository.NodeRef;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.extensions.surf.util.I18NUtil;
import org.springframework.extensions.webscripts.Cache;
import org.springframework.extensions.webscripts.Status;
import org.springframework.extensions.webscripts.WebScriptException;
import org.springframework.extensions.webscripts.WebScriptRequest;
/**
* Web Script class that invokes a BulkFilesystemImporter implementation.
*
* @since 4.0
*/
public class BulkFilesystemImportWebScript extends AbstractBulkFileSystemImportWebScript
{
private final static Log logger = LogFactory.getLog(BulkFilesystemImportWebScript.class);
private BulkFilesystemImporter importer;
private StreamingBulkImportStrategy importStrategy;
/**
* @see org.springframework.extensions.webscripts.DeclarativeWebScript#executeImpl(org.springframework.extensions.webscripts.WebScriptRequest, org.springframework.extensions.webscripts.Status, org.springframework.extensions.webscripts.Cache)
*/
@Override
protected Map<String, Object> executeImpl(final WebScriptRequest request, final Status status, final Cache cache)
{
Map<String, Object> model = new HashMap<String, Object>();
String targetNodeRefStr = null;
String targetPath = null;
String sourceDirectoryStr = null;
String replaceExistingStr = null;
cache.setNeverCache(true);
try
{
if (!importer.getStatus().inProgress())
{
NodeRef targetNodeRef = null;
File sourceDirectory = null;
boolean replaceExisting = false;
// Retrieve, validate and convert parameters
targetNodeRefStr = request.getParameter(PARAMETER_TARGET_NODEREF);
targetPath = request.getParameter(PARAMETER_TARGET_PATH);
sourceDirectoryStr = request.getParameter(PARAMETER_SOURCE_DIRECTORY);
replaceExistingStr = request.getParameter(PARAMETER_REPLACE_EXISTING);
targetNodeRef = getTargetNodeRef(targetNodeRefStr, targetPath);
if (sourceDirectoryStr == null || sourceDirectoryStr.trim().length() == 0)
{
throw new RuntimeException("Error: mandatory parameter '" + PARAMETER_SOURCE_DIRECTORY + "' was not provided.");
}
sourceDirectory = new File(sourceDirectoryStr.trim());
if (replaceExistingStr != null && replaceExistingStr.trim().length() > 0)
{
replaceExisting = PARAMETER_VALUE_REPLACE_EXISTING.equals(replaceExistingStr);
}
// Initiate the import
importer.bulkImport(new FilesystemBulkImportSource(sourceDirectory), targetNodeRef, importStrategy, replaceExisting);
// redirect to the status Web Script
status.setCode(Status.STATUS_MOVED_TEMPORARILY);
status.setRedirect(true);
status.setLocation(request.getServiceContextPath() + WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS);
}
else
{
model.put(IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY, I18NUtil.getMessage(IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY));
}
}
catch (WebScriptException wse)
{
status.setCode(Status.STATUS_BAD_REQUEST, wse.getMessage());
status.setRedirect(true);
}
catch (FileNotFoundException fnfe)
{
status.setCode(Status.STATUS_BAD_REQUEST,"The repository path '" + targetPath + "' does not exist !");
status.setRedirect(true);
}
catch(IllegalArgumentException iae)
{
status.setCode(Status.STATUS_BAD_REQUEST,iae.getMessage());
status.setRedirect(true);
}
catch (Throwable t)
{
throw new WebScriptException(Status.STATUS_INTERNAL_SERVER_ERROR, buildTextMessage(t), t);
}
return model;
}
// boilerplate setters
public void setImporter(BulkFilesystemImporter importer)
{
this.importer = importer;
}
public void setImportStrategy(StreamingBulkImportStrategy importStrategy)
{
this.importStrategy = importStrategy;
}
}

167
backup/bulkload/inplace/BulkFilesystemImportWebScript.java Normal file
View File

@@ -0,0 +1,167 @@
/*
* Copyright (C) 2005-2011 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* As a special exception to the terms and conditions of version 2.0 of
* the GPL, you may redistribute this Program in connection with Free/Libre
* and Open Source Software ("FLOSS") applications as described in Alfresco's
* FLOSS exception. You should have received a copy of the text describing
* the FLOSS exception, and it is also available here:
* http://www.alfresco.com/legal/licensing"
*/
package org.alfresco.repo.web.scripts.node.bulkload.inplace;
import java.util.HashMap;
import java.util.Map;
import org.alfresco.repo.bulkimport.BulkFilesystemImporter;
import org.alfresco.repo.bulkimport.BulkImportSource;
import org.alfresco.repo.bulkimport.ContentStoreMapProvider;
import org.alfresco.repo.bulkimport.impl.InPlaceBulkImportStrategy;
import org.alfresco.repo.bulkimport.impl.StoreBulkImportSource;
import org.alfresco.repo.content.ContentStore;
import org.alfresco.repo.web.scripts.node.bulkload.AbstractBulkFileSystemImportWebScript;
import org.alfresco.service.cmr.model.FileNotFoundException;
import org.alfresco.service.cmr.repository.NodeRef;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.extensions.surf.util.I18NUtil;
import org.springframework.extensions.webscripts.Cache;
import org.springframework.extensions.webscripts.Status;
import org.springframework.extensions.webscripts.WebScriptException;
import org.springframework.extensions.webscripts.WebScriptRequest;
/**
* Web Script class that invokes a BulkFilesystemImporter implementation.
*
* @since 4.0
*
* Copied by Romain Guinot from {@link BulkFilesystemImportWebScript},
* and expanded to support "in-place" importing of existing content.
*
*/
public class BulkFilesystemImportWebScript extends AbstractBulkFileSystemImportWebScript
{
private static final Log logger = LogFactory.getLog(BulkFilesystemImportWebScript.class);
// Web Script parameters (non-inherited)
private static final String PARAMETER_CONTENT_STORE = "contentStore";
private BulkFilesystemImporter importer;
private ContentStoreMapProvider storeMapProvider;
private InPlaceBulkImportStrategy importStrategy;
/**
* @see org.springframework.extensions.webscripts.DeclarativeWebScript#executeImpl(org.springframework.extensions.webscripts.WebScriptRequest, org.springframework.extensions.webscripts.Status, org.springframework.extensions.webscripts.Cache)
*/
@Override
protected Map<String, Object> executeImpl(WebScriptRequest request, Status status, Cache cache)
{
Map<String, Object> model = new HashMap<String, Object>();
String relativeSourceDirectory = null;
String destinationContentStoreName = null;
String targetNodeRefStr = null;
String targetPath = null;
String replaceExistingStr = null;
cache.setNeverCache(true);
try
{
importInProgress = importer.getStatus().inProgress();
if (!importInProgress)
{
NodeRef targetNodeRef = null;
boolean replaceExisting = false;
// Retrieve, validate and convert parameters
relativeSourceDirectory = request.getParameter(PARAMETER_SOURCE_DIRECTORY);
destinationContentStoreName = request.getParameter(PARAMETER_CONTENT_STORE);
targetNodeRefStr = request.getParameter(PARAMETER_TARGET_NODEREF);
targetPath = request.getParameter(PARAMETER_TARGET_PATH);
replaceExistingStr = request.getParameter(PARAMETER_REPLACE_EXISTING);
targetNodeRef = getTargetNodeRef(targetNodeRefStr, targetPath);
if (relativeSourceDirectory == null || relativeSourceDirectory.trim().length() == 0)
{
throw new WebScriptException("Error: mandatory parameter '" + PARAMETER_SOURCE_DIRECTORY + "' was not provided !");
}
if (replaceExistingStr != null && replaceExistingStr.trim().length() > 0)
{
replaceExisting = PARAMETER_VALUE_REPLACE_EXISTING.equals(replaceExistingStr);
}
ContentStore store = storeMapProvider.checkAndGetStore(destinationContentStoreName);
// Initiate the import
// ImportStrategy importContext = new InPlaceBulkImportStrategy(destinationContentStoreName, store, relativeSourceDirectory);
//importer.bulkImport(targetNodeRef, new Triple<String, ContentStore, String> (destinationContentStoreName, store, relativeSourceDirectory), replaceExisting);
BulkImportSource importSource = new StoreBulkImportSource(destinationContentStoreName, store, relativeSourceDirectory);
importer.bulkImport(importSource, targetNodeRef, importStrategy, replaceExisting);
// redirect to the status Web Script
status.setCode(Status.STATUS_MOVED_TEMPORARILY);
status.setRedirect(true);
status.setLocation(request.getServiceContextPath() + WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS);
}
else
{
model.put(IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY, I18NUtil.getMessage(IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY));
}
}
catch (WebScriptException wse)
{
status.setCode(Status.STATUS_BAD_REQUEST, wse.getMessage());
status.setRedirect(true);
}
catch (FileNotFoundException fnfe)
{
status.setCode(Status.STATUS_BAD_REQUEST,"The repository path '" + targetPath + "' does not exist !");
status.setRedirect(true);
}
catch(IllegalArgumentException iae)
{
status.setCode(Status.STATUS_BAD_REQUEST,iae.getMessage());
status.setRedirect(true);
}
catch (Throwable t)
{
throw new WebScriptException(Status.STATUS_INTERNAL_SERVER_ERROR, buildTextMessage(t), t);
}
return model;
}
// boilerplate setters
public void setImporter(BulkFilesystemImporter importer)
{
this.importer = importer;
}
public void setStoreMapProvider(ContentStoreMapProvider storeMapProvider)
{
this.storeMapProvider = storeMapProvider;
}
public void setImportStrategy(InPlaceBulkImportStrategy importStrategy)
{
this.importStrategy = importStrategy;
}
}

11
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.desc.xml Normal file
View File

@@ -0,0 +1,11 @@
<webscript>
<shortname>Bulk Filesystem Import - AJAX Auto Suggest for Space Names</shortname>
<description>Web Script that returns a list of spaces for the AJAX auto-suggest control in the bulk import initiation form.</description>
<url>/bulk/import/filesystem/ajax/suggest/spaces?query={query}</url>
<authentication>admin</authentication>
<format default="json">any</format>
<family>Bulk Filesystem Import</family>
<cache>
<never>true</never>
</cache>
</webscript>

24
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.json.ftl Normal file
View File

@@ -0,0 +1,24 @@
[#ftl]
[#-- TODO: Filter out Data Dictionary and all sub-folders --]
[#macro fullPath node]
[@compress single_line=true]
[#if node == companyhome]
/Company Home
[#else]
[@fullPath node=node.parent/]/${node.properties.name}
[/#if]
[/@compress]
[/#macro]
[#assign luceneQuery = "@cm\\:name:\"" + args.query + "*\" AND TYPE:\\{http\\://www.alfresco.org/model/content/1.0\\}folder AND NOT TYPE:\\{http\\://www.alfresco.org/model/wcmappmodel/1.0\\}webfolder"]
[#assign matches = companyhome.childrenByLuceneSearch[luceneQuery]]
{
"data" :
[
[#list matches as match]
{
"path" : "[@fullPath node=match/]",
"nodeRef" : "${match.nodeRef}"
}[#if match != matches?last],[/#if]
[/#list]
]
}

8
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.text.ftl Normal file
View File

@@ -0,0 +1,8 @@
[#ftl]
[#-- TODO: Filter out Data Dictionary and all sub-folders --]
[#macro fullPath node][#if node == companyhome]/Company Home[#else][@fullPath node=node.parent/]/${node.properties.name}[/#if][/#macro]
[#assign luceneQuery = "@cm\\:name:\"" + args.query + "*\" AND TYPE:\\{http\\://www.alfresco.org/model/content/1.0\\}folder AND NOT TYPE:\\{http\\://www.alfresco.org/model/wcmappmodel/1.0\\}webfolder"]
[#assign matches = companyhome.childrenByLuceneSearch[luceneQuery]]
[#list matches as match]
[@fullPath node=match/],${match.nodeRef}
[/#list]

11
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.desc.xml Normal file
View File

@@ -0,0 +1,11 @@
<webscript>
<shortname>Bulk Filesystem Import</shortname>
<description>Web Script that initiates a bulk filesystem import.</description>
<url>/bulk/import/filesystem/initiate</url>
<authentication>admin</authentication>
<transaction>none</transaction>
<family>Bulk Filesystem Import</family>
<cache>
<never>true</never>
</cache>
</webscript>

1
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.html.ftl Normal file
View File

@@ -0,0 +1 @@
<b>${importInProgress}</b>

11
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.desc.xml Normal file
View File

@@ -0,0 +1,11 @@
<webscript>
<shortname>Bulk Filesystem Import Status</shortname>
<description>Web Script that provides a simple UI for monitoring the status of a bulk filesystem import.</description>
<url>/bulk/import/filesystem/status</url>
<format default="html"/>
<authentication>admin</authentication>
<family>Bulk Filesystem Import</family>
<cache>
<never>true</never>
</cache>
</webscript>

315
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.html.ftl Normal file
View File

@@ -0,0 +1,315 @@
[#ftl]
[#macro formatDuration durationInNs]
[@compress single_line=true]
[#assign days = (durationInNs / (1000 * 1000 * 1000 * 60 * 60 * 24))?floor]
[#assign hours = (durationInNs / (1000 * 1000 * 1000 * 60 * 60))?floor % 24]
[#assign minutes = (durationInNs / (1000 * 1000 * 1000 * 60))?floor % 60]
[#assign seconds = (durationInNs / (1000 * 1000 * 1000))?floor % 60]
[#assign milliseconds = (durationInNs / (1000 * 1000)) % 1000]
[#assign microseconds = (durationInNs / (1000)) % 1000]
${days}d ${hours}h ${minutes}m ${seconds}s ${milliseconds}.${microseconds}ms
[/@compress]
[/#macro]
[#macro formatBytes bytes]
[@compress single_line=true]
[#if bytes > 1000000000]${(bytes / 1000000000)?string("#,##0.00")}GB
[#elseif bytes > 1000000]${(bytes / 1000000)?string("#,##0.00")}MB
[#elseif bytes > 1000]${(bytes / 1000)?string("#,##0.00")}kB
[#else]${bytes?string("#,##0")}B
[/#if]
[/@compress]
[/#macro]
[#assign refreshIntervalInSeconds = 5]
<!DOCTYPE HTML>
<html>
<head>
<title>Bulk Filesystem Import Status</title>
<link rel="stylesheet" href="/alfresco/css/main.css" type="text/css"/>
[#if importStatus.inProgress()]
<meta http-equiv="refresh" content="${refreshIntervalInSeconds}" />
[/#if]
</head>
<body>
<table>
<tr>
<td><img src="${url.context}/images/logo/AlfrescoLogo32.png" alt="Alfresco" /></td>
<td><nobr>Bulk Filesystem Import Tool Status</nobr></td>
</tr>
<tr><td><td>Alfresco ${server.edition} v${server.version}
</table>
<blockquote>
<p>
<table border="1" cellspacing="0" cellpadding="1" width="80%">
<tr>
<td colspan="2"><strong>General Statistics</strong></td>
</tr>
<tr>
<td width="25%">Current status:</td>
<td width="75%">
[#if importStatus.inProgress()]
<span style="color:red">In progress</span>
[#else]
<span style="color:green">Idle</span>
[/#if]
</td>
</tr>
<tr>
<td>Successful:</td>
[#if importStatus.inProgress() || !importStatus.endDate??]
<td>n/a</td>
[#elseif importStatus.lastExceptionAsString??]
<td style="color:red">No</td>
[#else]
<td style="color:green">Yes</td>
[/#if]
</tr>
<tr>
<td>Batch Size:</td>
<td>${importStatus.batchSize}</td>
</tr>
<tr>
<td>Number of threads:</td>
<td>${importStatus.numThreads}</td>
</tr>
<tr>
<td>Source Directory:</td>
<td>
[#if importStatus.sourceDirectory??]
${importStatus.sourceDirectory}
[#else]
n/a
[/#if]
</td>
</tr>
<tr>
<td>Target Space:</td>
<td>
[#if importStatus.targetSpace??]
${importStatus.targetSpace}
[#else]
n/a
[/#if]
</td>
</tr>
<tr>
<td>Start Date:</td>
<td>
[#if importStatus.startDate??]
${importStatus.startDate?datetime?string("yyyy-MM-dd hh:mm:ss.SSSaa")}
[#else]
n/a
[/#if]
</td>
</tr>
<tr>
<td>End Date:</td>
<td>
[#if importStatus.endDate??]
${importStatus.endDate?datetime?string("yyyy-MM-dd hh:mm:ss.SSSaa")}</td>
[#else]
n/a
[/#if]
</td>
</tr>
<tr>
<td>
[#if importStatus.inProgress()]
Elapsed Time:
[#else]
Duration:
[/#if]
</td>
<td>
[#if importStatus.durationInNs??]
[@formatDuration durationInNs = importStatus.durationInNs/]
[#else]
n/a
[/#if]
</td>
</tr>
<tr>
<td>Number of Completed Batches:</td>
<td>${importStatus.numberOfBatchesCompleted}</td>
</tr>
<tr>
<td colspan="2"><strong>Source (read) Statistics</strong></td>
</tr>
<!--
<tr>
<td>Last folder or file processed:</td>
<td>${importStatus.currentFileBeingProcessed!"n/a"}</td>
</tr>
-->
<tr>
<td>Scanned:</td>
<td>
<table border="1" cellspacing="0" cellpadding="1">
<tr>
<td>Folders</td>
<td>Files</td>
<td>Unreadable</td>
</tr>
<tr>
<td>${importStatus.numberOfFoldersScanned}</td>
<td>${importStatus.numberOfFilesScanned}</td>
<td>${importStatus.numberOfUnreadableEntries}</td>
</tr>
</table>
</td>
</tr>
<tr>
<td>Read:</td>
<td>
<table border="1" cellspacing="0" cellpadding="1">
<tr>
<td>Content</td>
<td>Metadata</td>
<td>Content Versions</td>
<td>Metadata Versions</td>
</tr>
<tr>
<td>${importStatus.numberOfContentFilesRead} ([@formatBytes importStatus.numberOfContentBytesRead/])</td>
<td>${importStatus.numberOfMetadataFilesRead} ([@formatBytes importStatus.numberOfMetadataBytesRead/])</td>
<td>${importStatus.numberOfContentVersionFilesRead} ([@formatBytes importStatus.numberOfContentVersionBytesRead/])</td>
<td>${importStatus.numberOfMetadataVersionFilesRead} ([@formatBytes importStatus.numberOfMetadataVersionBytesRead/])</td>
</tr>
</tr>
</table>
</td>
</tr>
<tr>
<td>Throughput:</td>
<td>
[#if importStatus.durationInNs?? && importStatus.durationInNs > 0]
[#if importStatus.entriesScannedPerSecond??]
${importStatus.entriesScannedPerSecond} entries scanned / sec<br/>
[/#if]
[#if importStatus.filesReadPerSecond??]
${importStatus.filesReadPerSecond} files read / sec<br/>
[/#if]
[#if importStatus.bytesReadPerSecond??]
[@formatBytes importStatus.bytesReadPerSecond /] / sec
[/#if]
[#else]
n/a
[/#if]
</td>
</tr>
<tr>
<td colspan="2"><strong>Target (write) Statistics</strong></td>
</tr>
<tr>
<td>Space Nodes:</td>
<td>
<table border="1" cellspacing="0" cellpadding="1">
<tr>
<td># Created</td>
<td># Replaced</td>
<td># Skipped</td>
<td># Properties</td>
</tr>
<tr>
<td>${importStatus.numberOfSpaceNodesCreated}</td>
<td>${importStatus.numberOfSpaceNodesReplaced}</td>
<td>${importStatus.numberOfSpaceNodesSkipped}</td>
<td>${importStatus.numberOfSpacePropertiesWritten}</td>
</tr>
</table>
</td>
</tr>
<tr>
<td>Content Nodes:</td>
<td>
<table border="1" cellspacing="0" cellpadding="1">
<tr>
<td># Created</td>
<td># Replaced</td>
<td># Skipped</td>
<td>Data Written</td>
<td># Properties</td>
</tr>
<tr>
<td>${importStatus.numberOfContentNodesCreated}</td>
<td>${importStatus.numberOfContentNodesReplaced}</td>
<td>${importStatus.numberOfContentNodesSkipped}</td>
<td>[@formatBytes importStatus.numberOfContentBytesWritten/]</td>
<td>${importStatus.numberOfContentPropertiesWritten}</td>
</tr>
</table>
</td>
</tr>
<tr>
<td>Content Versions:</td>
<td>
<table border="1" cellspacing="0" cellpadding="1">
<tr>
<td># Created</td>
<td>Data Written</td>
<td># Properties</td>
</tr>
</tr>
<td>${importStatus.numberOfContentVersionsCreated}</td>
<td>[@formatBytes importStatus.numberOfContentVersionBytesWritten/]</td>
<td>${importStatus.numberOfContentVersionPropertiesWritten}</td>
</tr>
</table>
</td>
<tr>
<tr>
<td>Throughput (write):</td>
<td>
[#if importStatus.durationInNs?? && importStatus.durationInNs > 0]
[#if importStatus.nodesCreatedPerSecond??]
${importStatus.nodesCreatedPerSecond?string("#0")} nodes / sec<br/>
[/#if]
[#if importStatus.bytesWrittenPerSecond??]
[@formatBytes importStatus.bytesWrittenPerSecond /] / sec
[/#if]
[#else]
n/a
[/#if]
</td>
</tr>
[#if importStatus.lastExceptionAsString??]
<tr>
<td colspan="2"><strong>Error Information From Last Run</strong></td>
</tr>
<tr>
<td>File that failed:</td>
<td>${importStatus.currentFileBeingProcessed!"n/a"}</td>
</tr>
<tr>
<td>Exception:</td>
<td><pre>${importStatus.lastExceptionAsString}</pre></td>
</tr>
[/#if]
</table>
</p>
<p>
[#if importStatus.inProgress()]
This page will automatically refresh in <span id="countdownTimer">${refreshIntervalInSeconds}</span> seconds.
[#else]
<a href="${url.serviceContext}/bulk/import/filesystem">Initiate another import</a><br><br>
<a href="${url.serviceContext}/bulk/import/filesystem/inplace">Initiate another in-place import</a>
[/#if]
</p>
</blockquote>
<script type="text/javascript">
var seconds = ${refreshIntervalInSeconds} + 1;
function display()
{
seconds -= 1;
var countdownTimerElement = document.getElementById("countdownTimer");
if(countdownTimerElement)
{
countdownTimerElement.textContent = seconds;
}
setTimeout("display()", 1000);
}
display()
</script>
</body>
</html>

71
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.xml.ftl Normal file
View File

@@ -0,0 +1,71 @@
[#ftl]
<?xml version='1.0' encoding='UTF-8'?>
<BulkFilesystemImportStatus>
<CurrentStatus>[@compress single_line=true]
[#if importStatus.inProgress()]
In progress
[#else]
Idle
[/#if]
[/@compress]</CurrentStatus>
<ResultOfLastExecution>[@compress single_line=true]
[#if importStatus.lastExceptionAsString??]
Failed
[#else]
Succeeded
[/#if]
[/@compress]</ResultOfLastExecution>
[#if importStatus.sourceDirectory??]
<SourceDirectory>${importStatus.sourceDirectory}</SourceDirectory>
[/#if]
[#if importStatus.targetSpace??]
<TargetSpace>${importStatus.targetSpace}</TargetSpace>
[/#if]
<BatchWeight>${importStatus.batchWeight?c}</BatchWeight>
[#if importStatus.startDate??]
<StartDate>${importStatus.startDate?datetime?string("yyyy-MM-dd'T'HH:mm:ss.SSS")}</StartDate>
[/#if]
[#if importStatus.endDate??]
<EndDate>${importStatus.endDate?datetime?string("yyyy-MM-dd'T'HH:mm:ss.SSS")}</EndDate>
[/#if]
[#if importStatus.durationInNs??]
<DurationInNS>${importStatus.durationInNs?c}</DurationInNS>
[/#if]
<CompletedBatches>${importStatus.numberOfBatchesCompleted}</CompletedBatches>
[#if !importStatus.inProgress() && importStatus.endDate??]
[/#if]
<SourceStatistics>
<LastFileOrFolderProcessed>${importStatus.currentFileBeingProcessed!"n/a"}</LastFileOrFolderProcessed>
<FilesScanned>${importStatus.numberOfFilesScanned?c}</FilesScanned>
<FoldersScanned>${importStatus.numberOfFoldersScanned?c}</FoldersScanned>
<UnreadableEntries>${importStatus.numberOfUnreadableEntries?c}</UnreadableEntries>
<ContentFilesRead>${importStatus.numberOfContentFilesRead?c}</ContentFilesRead>
<ContentBytesRead>${importStatus.numberOfContentBytesRead?c}</ContentBytesRead>
<MetadataFilesRead>${importStatus.numberOfMetadataFilesRead?c}</MetadataFilesRead>
<MetadataBytesRead>${importStatus.numberOfMetadataBytesRead?c}</MetadataBytesRead>
<ContentVersionFilesRead>${importStatus.numberOfContentVersionFilesRead?c}</ContentVersionFilesRead>
<ContentVersionBytesRead>${importStatus.numberOfContentVersionBytesRead?c}</ContentVersionBytesRead>
<MetadataVersionFilesRead>${importStatus.numberOfMetadataVersionFilesRead?c}</MetadataVersionFilesRead>
<MetadataVersionBytesRead>${importStatus.numberOfMetadataVersionBytesRead?c}</MetadataVersionBytesRead>
</SourceStatistics>
<TargetStatistics>
<SpaceNodesCreated>${importStatus.numberOfSpaceNodesCreated?c}</SpaceNodesCreated>
<SpaceNodesReplaced>${importStatus.numberOfSpaceNodesReplaced?c}</SpaceNodesReplaced>
<SpaceNodesSkipped>${importStatus.numberOfSpaceNodesSkipped?c}</SpaceNodesSkipped>
<SpacePropertiesWritten>${importStatus.numberOfSpacePropertiesWritten?c}</SpacePropertiesWritten>
<ContentNodesCreated>${importStatus.numberOfContentNodesCreated?c}</ContentNodesCreated>
<ContentNodesReplaced>${importStatus.numberOfContentNodesReplaced?c}</ContentNodesReplaced>
<ContentNodesSkipped>${importStatus.numberOfContentNodesSkipped?c}</ContentNodesSkipped>
<ContentBytesWritten>${importStatus.numberOfContentBytesWritten?c}</ContentBytesWritten>
<ContentPropertiesWritten>${importStatus.numberOfContentPropertiesWritten?c}</ContentPropertiesWritten>
<ContentVersionsCreated>${importStatus.numberOfContentVersionsCreated?c}</ContentVersionsCreated>
<ContentVersionsBytesWritten>${importStatus.numberOfContentVersionBytesWritten?c}</ContentVersionsBytesWritten>
<ContentVersionsPropertiesWritten>${importStatus.numberOfContentVersionPropertiesWritten?c}</ContentVersionsPropertiesWritten>
</TargetStatistics>
[#if importStatus.lastExceptionAsString??]
<ErrorInformation>
<FileThatFailed>${importStatus.currentFileBeingProcessed!"n/a"}</FileThatFailed>
<Exception>${importStatus.lastExceptionAsString}</Exception>
</ErrorInformation>
[/#if]
</BulkFilesystemImportStatus>

11
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.desc.xml Normal file
View File

@@ -0,0 +1,11 @@
<webscript>
<shortname>Bulk Filesystem Import</shortname>
<description>Web Script that provides a simple UI for initiating a bulk filesystem import.</description>
<url>/bulk/import/filesystem</url>
<format default="html"/>
<authentication>admin</authentication>
<family>Bulk Filesystem Import</family>
<cache>
<never>true</never>
</cache>
</webscript>

118
config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.html.ftl Normal file
View File

@@ -0,0 +1,118 @@
[#ftl]
<!DOCTYPE HTML>
<html>
<head>
<title>Bulk Filesystem Import Tool</title>
<link rel="stylesheet" href="${url.context}/css/main.css" TYPE="text/css">
<!-- YUI 3.x -->
<link rel="stylesheet" type="text/css" href="${url.context}/css/yui-3.3.0-dependencies.css">
<style type="text/css">
.yui3-aclist-content {
background-color : white;
border : 1px solid darkgrey;
box-shadow : 3px 3px 4px lightgrey;
-webkit-box-shadow : 3px 3px 4px lightgrey; /* Safari and Chrome */
}
</style>
<script type="text/javascript" src="${url.context}/scripts/yui-3.3.0-dependencies.js"></script>
<!-- Validation functions -->
<script type="text/javascript">
function validateRequired(field, errorMessageElement, errorMessage)
{
var result = true;
if (field.value == null || field.value == "")
{
errorMessageElement.textContent = errorMessage;
result = false;
}
else
{
errorMessageElement.textContent = "";
}
return result;
}
function validateForm(form)
{
var result = true;
result = validateRequired(form.sourceDirectory, document.getElementById("sourceDirectoryMessage"), "Source directory is mandatory.");
if (result)
{
result = validateRequired(form.targetPath, document.getElementById("targetPathMessage"), "Target space is mandatory.");
}
return result;
}
</script>
</head>
<body class="yui-skin-sam">
<table>
<tr>
<td><img src="${url.context}/images/logo/AlfrescoLogo32.png" alt="Alfresco" /></td>
<td><nobr>Bulk Filesystem Import Tool</nobr></td>
</tr>
<tr><td><td>Alfresco ${server.edition} v${server.version}
</table>
<form action="${url.service}/initiate" method="post" enctype="multipart/form-data" charset="utf-8" onsubmit="return validateForm(this);">
<table>
<tr>
<td>Import directory:</td><td><input type="text" name="sourceDirectory" size="128" /></td><td id="sourceDirectoryMessage" style="color:red"></td>
</tr>
<tr>
<!-- TODO i18n for this string -->
<td><br/><label for="targetPath">Target space (NodeRef or Path):</label></td>
<td>
<div id="targetNodeRefAutoComplete">
<input id="targetPath" type="text" name="targetPath" size="128" />
<div id="targetPathAutoSuggestContainer"></div>
</div>
</td>
<td id="targetPathMessage" style="color:red"></td>
</tr>
<tr>
<td colspan="3">&nbsp;</td>
</tr>
<tr>
<td><label for="replaceExisting">Replace existing files:</label></td><td><input type="checkbox" id="replaceExisting" name="replaceExisting" value="replaceExisting" unchecked/> (unchecked means skip files that already exist in the repository)</td><td></td>
</tr>
<tr>
<td>Batch Size:</td>
<td colspan="2"><input type="text" name="batchSize" size="5"></td>
</tr>
<tr>
<td>Number of Threads:</td>
<td colspan="2"><input type="text" name="numThreads" size="5"></td>
</tr>
<tr>
<td colspan="3">&nbsp;</td>
</tr>
<tr>
<td colspan="3"><input type="submit" name="submit" value="Initiate Bulk Import"></td>
</tr>
</table>
<br/>
</form>
<script type="text/javascript">
YUI().use("autocomplete", "autocomplete-highlighters", "datasource-get", function(Y)
{
Y.one('#targetPath').plug(Y.Plugin.AutoComplete,
{
source : '${url.service}/ajax/suggest/spaces.json?query={query}',
maxResults : 25,
resultHighlighter : 'phraseMatch',
resultListLocator : 'data',
resultTextLocator : 'path'
});
});
</script>
</body>
</html>

48
config/alfresco/web-scripts-application-context.xml
View File

@@ -1753,4 +1753,52 @@
parent="abstractCalendarWebScript">
</bean>
<!-- Java-backed Web Scripts for the bulk filesystem importer -->
<!-- work only with the default contentStore. -->
<bean id="streamingStoreMapProvider" class="org.alfresco.repo.bulkimport.impl.stores.DefaultContentStoreMapProvider">
<property name="contentStore">
<ref bean="fileContentStore" />
</property>
<property name="defaultStoreName" value="default" />
</bean>
<!-- Custom javascript root object that interfaces to batch import. -->
<bean id="bfsiScript" parent="baseJavaScriptExtension" class="org.alfresco.repo.bulkimport.script.BulkImport">
<property name="extensionName" value="bulkFSImport" />
<property name="storeMapProvider" ref="streamingStoreMapProvider" />
</bean>
<bean id="bfsiAbstractImportWebscript"
class="org.alfresco.repo.web.scripts.bulkimport.AbstractBulkFileSystemImportWebScript"
parent="webscript" abstract="true">
<property name="fileFolderService" ref="fileFolderService" />
<property name="repository" ref="repositoryHelper" />
</bean>
<!-- content streaming -->
<bean id="webscript.org.alfresco.repository.bulkfilesystemimport.initiate.post"
class="org.alfresco.repo.web.scripts.bulkimport.copy.BulkFilesystemImportWebScript"
parent="bfsiAbstractImportWebscript">
<property name="bulkImporter" ref="bulkFilesystemImporter" />
<property name="nodeImporterFactory" ref="streamingNodeImporterFactory" />
</bean>
<!-- status -->
<bean id="webscript.org.alfresco.repository.bulkfilesystemimport.status.get"
class="org.alfresco.repo.web.scripts.bulkimport.BulkFilesystemImportStatusWebScript"
parent="webscript">
<property name="bulkImporter" ref="bulkFilesystemImporter" />
</bean>
<!-- i18n bundles for the templates -->
<bean id="bfsi.i18n.bundles" class="org.alfresco.i18n.ResourceBundleBootstrapComponent">
<property name="resourceBundles">
<list>
<value>alfresco.messages.ui-inplace</value>
<value>alfresco.messages.initiate-inplace</value>
</list>
</property>
</bean>
</beans>

213
source/java/org/alfresco/repo/web/scripts/bulkimport/AbstractBulkFileSystemImportWebScript.java Normal file
View File

@@ -0,0 +1,213 @@
/*
* Copyright (C) 2005-2011 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* As a special exception to the terms and conditions of version 2.0 of
* the GPL, you may redistribute this Program in connection with Free/Libre
* and Open Source Software ("FLOSS") applications as described in Alfresco's
* FLOSS exception. You should have received a copy of the text describing
* the FLOSS exception, and it is also available here:
* http://www.alfresco.com/legal/licensing"
*/
package org.alfresco.repo.web.scripts.bulkimport;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.Date;
import org.alfresco.repo.model.Repository;
import org.alfresco.service.cmr.model.FileFolderService;
import org.alfresco.service.cmr.model.FileInfo;
import org.alfresco.service.cmr.model.FileNotFoundException;
import org.alfresco.service.cmr.repository.NodeRef;
import org.springframework.extensions.webscripts.DeclarativeWebScript;
import org.springframework.extensions.webscripts.WebScriptException;
/**
* contains common fields and methods for the import web scripts.
*/
public class AbstractBulkFileSystemImportWebScript extends DeclarativeWebScript
{
protected static final String WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS = "/bulk/import/filesystem/status";
protected static final String PARAMETER_TARGET_NODEREF = "targetNodeRef";
protected static final String PARAMETER_TARGET_PATH = "targetPath";
protected static final String COMPANY_HOME_NAME = "Company Home";
protected static final String COMPANY_HOME_PATH = "/" + COMPANY_HOME_NAME;
// Web scripts parameters (common)
protected static final String PARAMETER_REPLACE_EXISTING = "replaceExisting";
protected static final String PARAMETER_VALUE_REPLACE_EXISTING = "replaceExisting";
protected static final String PARAMETER_SOURCE_DIRECTORY = "sourceDirectory";
protected static final String IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY = "importInProgress";
protected static final String IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY ="bfsit.error.importAlreadyInProgress";
protected static final String PARAMETER_BATCH_SIZE = "batchSize";
protected static final String PARAMETER_NUM_THREADS = "numThreads";
protected FileFolderService fileFolderService;
protected Repository repository;
protected volatile boolean importInProgress;
protected NodeRef getTargetNodeRef(String targetNodeRefStr, String targetPath) throws FileNotFoundException
{
NodeRef targetNodeRef;
if (targetNodeRefStr == null || targetNodeRefStr.trim().length() == 0)
{
if (targetPath == null || targetPath.trim().length() == 0)
{
throw new WebScriptException("Error: neither parameter '" + PARAMETER_TARGET_NODEREF +
"' nor parameter '" + PARAMETER_TARGET_PATH +
"' was provided, but at least one is required !");
}
targetNodeRef = convertPathToNodeRef(targetPath.trim());
}
else
{
targetNodeRef = new NodeRef(targetNodeRefStr.trim());
}
return targetNodeRef;
}
protected NodeRef convertPathToNodeRef(String targetPath) throws FileNotFoundException
{
NodeRef result = null;
NodeRef companyHome = repository.getCompanyHome();
String cleanTargetPath = targetPath.replaceAll("/+", "/");
if (cleanTargetPath.startsWith(COMPANY_HOME_PATH))
cleanTargetPath = cleanTargetPath.substring(COMPANY_HOME_PATH.length());
if (cleanTargetPath.startsWith("/"))
cleanTargetPath = cleanTargetPath.substring(1);
if (cleanTargetPath.endsWith("/"))
cleanTargetPath = cleanTargetPath.substring(0, cleanTargetPath.length() - 1);
if (cleanTargetPath.length() == 0)
result = companyHome;
else
{
FileInfo info = fileFolderService.resolveNamePath(companyHome, Arrays.asList(cleanTargetPath.split("/")));
if(info == null)
throw new WebScriptException("could not determine NodeRef for path :'"+cleanTargetPath+"'");
result = info.getNodeRef();
}
return(result);
}
protected String buildTextMessage(Throwable t)
{
StringBuffer result = new StringBuffer();
String timeOfFailure = (new Date()).toString();
String hostName = null;
String ipAddress = null;
try
{
hostName = InetAddress.getLocalHost().getHostName();
ipAddress = InetAddress.getLocalHost().getHostAddress();
}
catch (UnknownHostException uhe)
{
hostName = "unknown";
ipAddress = "unknown";
}
result.append("\nTime of failure: " + timeOfFailure);
result.append("\nHost where failure occurred: " + hostName + " (" + ipAddress + ")");
if (t != null)
{
result.append("\nRoot exception:");
result.append(renderExceptionStackAsText(t));
}
else
{
result.append("\nNo exception was provided.");
}
return(result.toString());
}
private String renderExceptionStackAsText( Throwable t)
{
StringBuffer result = new StringBuffer();
if (t != null)
{
String message = t.getMessage();
Throwable cause = t.getCause();
if (cause != null)
{
result.append(renderExceptionStackAsText(cause));
result.append("\nWrapped by:");
}
if (message == null)
{
message = "";
}
result.append("\n");
result.append(t.getClass().getName());
result.append(": ");
result.append(message);
result.append("\n");
result.append(renderStackTraceElements(t.getStackTrace()));
}
return(result.toString());
}
private String renderStackTraceElements(StackTraceElement[] elements)
{
StringBuffer result = new StringBuffer();
if (elements != null)
{
for (int i = 0; i < elements.length; i++)
{
result.append("\tat " + elements[i].toString() + "\n");
}
}
return(result.toString());
}
// boilerplate setters
public void setFileFolderService(FileFolderService fileFolderService)
{
this.fileFolderService = fileFolderService;
}
public void setRepository(Repository repository)
{
this.repository = repository;
}
}

73
source/java/org/alfresco/repo/web/scripts/bulkimport/BulkFilesystemImportStatusWebScript.java Normal file
View File

@@ -0,0 +1,73 @@
/*
* Copyright (C) 2005-2011 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* As a special exception to the terms and conditions of version 2.0 of
* the GPL, you may redistribute this Program in connection with Free/Libre
* and Open Source Software ("FLOSS") applications as described in Alfresco's
* FLOSS exception. You should have received a copy of the text describing
* the FLOSS exception, and it is also available here:
* http://www.alfresco.com/legal/licensing"
*/
package org.alfresco.repo.web.scripts.bulkimport;
import java.util.HashMap;
import java.util.Map;
import org.alfresco.repo.bulkimport.BulkFilesystemImporter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.extensions.webscripts.Cache;
import org.springframework.extensions.webscripts.DeclarativeWebScript;
import org.springframework.extensions.webscripts.Status;
import org.springframework.extensions.webscripts.WebScriptRequest;
/**
* Web Script class that provides status information on the bulk filesystem import process.
*
* @since 4.0
*/
public class BulkFilesystemImportStatusWebScript extends DeclarativeWebScript
{
private final static Log logger = LogFactory.getLog(BulkFilesystemImportStatusWebScript.class);
// Output parameters (for Freemarker)
private final static String RESULT_IMPORT_STATUS = "importStatus";
// Attributes
private BulkFilesystemImporter bulkImporter;
public void setBulkImporter(BulkFilesystemImporter bulkImporter)
{
this.bulkImporter = bulkImporter;
}
/**
* @see org.alfresco.web.scripts.DeclarativeWebScript#executeImpl(org.alfresco.web.scripts.WebScriptRequest, org.alfresco.web.scripts.Status, org.alfresco.web.scripts.Cache)
*/
@Override
protected Map<String, Object> executeImpl(WebScriptRequest request, Status status, Cache cache)
{
Map<String, Object> result = new HashMap<String, Object>();
cache.setNeverCache(true);
result.put(RESULT_IMPORT_STATUS, bulkImporter.getStatus());
return(result);
}
}

194
source/java/org/alfresco/repo/web/scripts/bulkimport/copy/BulkFilesystemImportWebScript.java Normal file
View File

@@ -0,0 +1,194 @@
/*
* Copyright (C) 2005-2011 Alfresco Software Limited.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* As a special exception to the terms and conditions of version 2.0 of
* the GPL, you may redistribute this Program in connection with Free/Libre
* and Open Source Software ("FLOSS") applications as described in Alfresco's
* FLOSS exception. You should have received a copy of the text describing
* the FLOSS exception, and it is also available here:
* http://www.alfresco.com/legal/licensing"
*/
package org.alfresco.repo.web.scripts.bulkimport.copy;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import org.alfresco.repo.bulkimport.BulkImportParameters;
import org.alfresco.repo.bulkimport.NodeImporter;
import org.alfresco.repo.bulkimport.impl.MultiThreadedBulkFilesystemImporter;
import org.alfresco.repo.bulkimport.impl.StreamingNodeImporterFactory;
import org.alfresco.repo.web.scripts.bulkimport.AbstractBulkFileSystemImportWebScript;
import org.alfresco.service.cmr.model.FileNotFoundException;
import org.alfresco.service.cmr.repository.NodeRef;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.extensions.surf.util.I18NUtil;
import org.springframework.extensions.webscripts.Cache;
import org.springframework.extensions.webscripts.Status;
import org.springframework.extensions.webscripts.WebScriptException;
import org.springframework.extensions.webscripts.WebScriptRequest;
/**
* Web Script class that invokes a BulkFilesystemImporter implementation.
*
* @since 4.0
*/
public class BulkFilesystemImportWebScript extends AbstractBulkFileSystemImportWebScript
{
private final static Log logger = LogFactory.getLog(BulkFilesystemImportWebScript.class);
private MultiThreadedBulkFilesystemImporter bulkImporter;
private StreamingNodeImporterFactory nodeImporterFactory;
public void setBulkImporter(MultiThreadedBulkFilesystemImporter bulkImporter)
{
this.bulkImporter = bulkImporter;
}
public void setNodeImporterFactory(StreamingNodeImporterFactory nodeImporterFactory)
{
this.nodeImporterFactory = nodeImporterFactory;
}
/**
* @see org.springframework.extensions.webscripts.DeclarativeWebScript#executeImpl(org.springframework.extensions.webscripts.WebScriptRequest, org.springframework.extensions.webscripts.Status, org.springframework.extensions.webscripts.Cache)
*/
@Override
protected Map<String, Object> executeImpl(final WebScriptRequest request, final Status status, final Cache cache)
{
Map<String, Object> model = new HashMap<String, Object>();
String targetNodeRefStr = null;
String targetPath = null;
String sourceDirectoryStr = null;
String replaceExistingStr = null;
String batchSizeStr = null;
String numThreadsStr = null;
cache.setNeverCache(true);
try
{
if(!bulkImporter.getStatus().inProgress())
{
NodeRef targetNodeRef = null;
File sourceDirectory = null;
boolean replaceExisting = false;
int batchSize = bulkImporter.getDefaultBatchSize();
int numThreads = bulkImporter.getDefaultNumThreads();
// Retrieve, validate and convert parameters
targetNodeRefStr = request.getParameter(PARAMETER_TARGET_NODEREF);
targetPath = request.getParameter(PARAMETER_TARGET_PATH);
sourceDirectoryStr = request.getParameter(PARAMETER_SOURCE_DIRECTORY);
replaceExistingStr = request.getParameter(PARAMETER_REPLACE_EXISTING);
batchSizeStr = request.getParameter(PARAMETER_BATCH_SIZE);
numThreadsStr = request.getParameter(PARAMETER_NUM_THREADS);
targetNodeRef = getTargetNodeRef(targetNodeRefStr, targetPath);
if (sourceDirectoryStr == null || sourceDirectoryStr.trim().length() == 0)
{
throw new RuntimeException("Error: mandatory parameter '" + PARAMETER_SOURCE_DIRECTORY + "' was not provided.");
}
sourceDirectory = new File(sourceDirectoryStr.trim());
if (replaceExistingStr != null && replaceExistingStr.trim().length() > 0)
{
replaceExisting = PARAMETER_VALUE_REPLACE_EXISTING.equals(replaceExistingStr);
}
// Initiate the import
NodeImporter nodeImporter = nodeImporterFactory.getNodeImporter(sourceDirectory);
//bulkImporter.asyncBulkImport(targetNodeRef, nodeImporter, replaceExisting);
BulkImportParameters bulkImportParameters = new BulkImportParameters();
if (numThreadsStr != null && numThreadsStr.trim().length() > 0)
{
try
{
numThreads = Integer.parseInt(numThreadsStr);
if(numThreads < 1)
{
throw new RuntimeException("Error: parameter '" + PARAMETER_NUM_THREADS + "' must be an integer > 0.");
}
bulkImportParameters.setNumThreads(numThreads);
}
catch(NumberFormatException e)
{
throw new RuntimeException("Error: parameter '" + PARAMETER_NUM_THREADS + "' must be an integer > 0.");
}
}
if (batchSizeStr != null && batchSizeStr.trim().length() > 0)
{
try
{
batchSize = Integer.parseInt(batchSizeStr);
if(batchSize < 1)
{
throw new RuntimeException("Error: parameter '" + PARAMETER_BATCH_SIZE + "' must be an integer > 0.");
}
bulkImportParameters.setBatchSize(batchSize);
}
catch(NumberFormatException e)
{
throw new RuntimeException("Error: parameter '" + PARAMETER_BATCH_SIZE + "' must be an integer > 0.");
}
}
bulkImportParameters.setReplaceExisting(replaceExisting);
bulkImportParameters.setTarget(targetNodeRef);
bulkImporter.asyncBulkImport(bulkImportParameters, nodeImporter);
// redirect to the status Web Script
status.setCode(Status.STATUS_MOVED_TEMPORARILY);
status.setRedirect(true);
status.setLocation(request.getServiceContextPath() + WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS);
}
else
{
model.put(IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY, I18NUtil.getMessage(IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY));
}
}
catch (WebScriptException wse)
{
status.setCode(Status.STATUS_BAD_REQUEST, wse.getMessage());
status.setRedirect(true);
}
catch (FileNotFoundException fnfe)
{
status.setCode(Status.STATUS_BAD_REQUEST,"The repository path '" + targetPath + "' does not exist !");
status.setRedirect(true);
}
catch(IllegalArgumentException iae)
{
status.setCode(Status.STATUS_BAD_REQUEST,iae.getMessage());
status.setRedirect(true);
}
catch (Throwable t)
{
throw new WebScriptException(Status.STATUS_INTERNAL_SERVER_ERROR, buildTextMessage(t), t);
}
return model;
}
}