From 87967a9447e6b9badf479df86ed5800aacfc3d73 Mon Sep 17 00:00:00 2001 From: Steven Glover Date: Mon, 10 Oct 2011 18:45:00 +0000 Subject: [PATCH] Initial commit for ALF-10419: - Bulk Filesystem Importer - Adapted from work by Peter Monks (see http://code.google.com/p/alfresco-bulk-filesystem-import) and Romain Guinot - Refactored to limit repeated code - Refactored to use the Bulk Processor - Currently limited to one bulk import at a time (enforced by lock service) - Unit tests added - Some fixes applied - Performance tests; tried different filesystem "walking" strategies - Still to do: yui dependencies - are these necessary? git-svn-id: https://svn.alfresco.com/repos/alfresco-enterprise/alfresco/HEAD/root@31100 c4b6b30b-aa2e-2d43-bbcb-ca4b014f7261 --- ...AbstractBulkFileSystemImportWebScript.java | 208 ++++++++++++ .../BulkFilesystemImportStatusWebScript.java | 73 ++++ .../copy/BulkFilesystemImportWebScript.java | 147 ++++++++ .../BulkFilesystemImportWebScript.java | 167 ++++++++++ .../ajaxspace.get.desc.xml | 11 + .../ajaxspace.get.json.ftl | 24 ++ .../ajaxspace.get.text.ftl | 8 + .../initiate.post.desc.xml | 11 + .../initiate.post.html.ftl | 1 + .../bulkfilesystemimport/status.get.desc.xml | 11 + .../bulkfilesystemimport/status.get.html.ftl | 315 ++++++++++++++++++ .../bulkfilesystemimport/status.get.xml.ftl | 71 ++++ .../bulkfilesystemimport/ui.get.desc.xml | 11 + .../bulkfilesystemimport/ui.get.html.ftl | 118 +++++++ .../web-scripts-application-context.xml | 48 +++ ...AbstractBulkFileSystemImportWebScript.java | 213 ++++++++++++ .../BulkFilesystemImportStatusWebScript.java | 73 ++++ .../copy/BulkFilesystemImportWebScript.java | 194 +++++++++++ 18 files changed, 1704 insertions(+) create mode 100644 backup/bulkload/AbstractBulkFileSystemImportWebScript.java create mode 100644 backup/bulkload/BulkFilesystemImportStatusWebScript.java create mode 100644 backup/bulkload/copy/BulkFilesystemImportWebScript.java create mode 100644 backup/bulkload/inplace/BulkFilesystemImportWebScript.java create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.desc.xml create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.json.ftl create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.text.ftl create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.desc.xml create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.html.ftl create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.desc.xml create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.html.ftl create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.xml.ftl create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.desc.xml create mode 100644 config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.html.ftl create mode 100644 source/java/org/alfresco/repo/web/scripts/bulkimport/AbstractBulkFileSystemImportWebScript.java create mode 100644 source/java/org/alfresco/repo/web/scripts/bulkimport/BulkFilesystemImportStatusWebScript.java create mode 100644 source/java/org/alfresco/repo/web/scripts/bulkimport/copy/BulkFilesystemImportWebScript.java diff --git a/backup/bulkload/AbstractBulkFileSystemImportWebScript.java b/backup/bulkload/AbstractBulkFileSystemImportWebScript.java new file mode 100644 index 0000000000..5600cf455e --- /dev/null +++ b/backup/bulkload/AbstractBulkFileSystemImportWebScript.java @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2005-2011 Alfresco Software Limited. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * As a special exception to the terms and conditions of version 2.0 of + * the GPL, you may redistribute this Program in connection with Free/Libre + * and Open Source Software ("FLOSS") applications as described in Alfresco's + * FLOSS exception. You should have received a copy of the text describing + * the FLOSS exception, and it is also available here: + * http://www.alfresco.com/legal/licensing" + */ + +package org.alfresco.repo.web.scripts.node.bulkload; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Date; + +import org.alfresco.repo.model.Repository; +import org.alfresco.service.cmr.model.FileFolderService; +import org.alfresco.service.cmr.model.FileInfo; +import org.alfresco.service.cmr.model.FileNotFoundException; +import org.alfresco.service.cmr.repository.NodeRef; +import org.springframework.extensions.webscripts.DeclarativeWebScript; +import org.springframework.extensions.webscripts.WebScriptException; + +/** + * contains common fields and methods for the import web scripts. + */ +public class AbstractBulkFileSystemImportWebScript extends DeclarativeWebScript +{ + protected static final String WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS = "/bulk/import/filesystem/status"; + + protected static final String PARAMETER_TARGET_NODEREF = "targetNodeRef"; + protected static final String PARAMETER_TARGET_PATH = "targetPath"; + + protected static final String COMPANY_HOME_NAME = "Company Home"; + protected static final String COMPANY_HOME_PATH = "/" + COMPANY_HOME_NAME; + + // Web scripts parameters (common) + protected static final String PARAMETER_REPLACE_EXISTING = "replaceExisting"; + protected static final String PARAMETER_VALUE_REPLACE_EXISTING = "replaceExisting"; + protected static final String PARAMETER_SOURCE_DIRECTORY = "sourceDirectory"; + + protected static final String IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY = "importInProgress"; + protected static final String IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY ="bfsit.error.importAlreadyInProgress"; + + protected FileFolderService fileFolderService; + protected Repository repository; + + protected volatile boolean importInProgress; + + protected NodeRef getTargetNodeRef(String targetNodeRefStr, String targetPath) throws FileNotFoundException + { + NodeRef targetNodeRef; + + if (targetNodeRefStr == null || targetNodeRefStr.trim().length() == 0) + { + if (targetPath == null || targetPath.trim().length() == 0) + { + throw new WebScriptException("Error: neither parameter '" + PARAMETER_TARGET_NODEREF + + "' nor parameter '" + PARAMETER_TARGET_PATH + + "' was provided, but at least one is required !"); + } + targetNodeRef = convertPathToNodeRef(targetPath.trim()); + } + else + targetNodeRef = new NodeRef(targetNodeRefStr.trim()); + + return targetNodeRef; + } + + protected NodeRef convertPathToNodeRef(String targetPath) throws FileNotFoundException + { + NodeRef result = null; + NodeRef companyHome = repository.getCompanyHome(); + String cleanTargetPath = targetPath.replaceAll("/+", "/"); + + if (cleanTargetPath.startsWith(COMPANY_HOME_PATH)) + cleanTargetPath = cleanTargetPath.substring(COMPANY_HOME_PATH.length()); + + if (cleanTargetPath.startsWith("/")) + cleanTargetPath = cleanTargetPath.substring(1); + + if (cleanTargetPath.endsWith("/")) + cleanTargetPath = cleanTargetPath.substring(0, cleanTargetPath.length() - 1); + + if (cleanTargetPath.length() == 0) + result = companyHome; + else + { + FileInfo info = fileFolderService.resolveNamePath(companyHome, Arrays.asList(cleanTargetPath.split("/"))); + if(info == null) + throw new WebScriptException("could not determine NodeRef for path :'"+cleanTargetPath+"'"); + + result = info.getNodeRef(); + } + + return(result); + } + + protected String buildTextMessage(Throwable t) + { + StringBuffer result = new StringBuffer(); + String timeOfFailure = (new Date()).toString(); + String hostName = null; + String ipAddress = null; + + try + { + hostName = InetAddress.getLocalHost().getHostName(); + ipAddress = InetAddress.getLocalHost().getHostAddress(); + } + catch (UnknownHostException uhe) + { + hostName = "unknown"; + ipAddress = "unknown"; + } + + result.append("\nTime of failure: " + timeOfFailure); + result.append("\nHost where failure occurred: " + hostName + " (" + ipAddress + ")"); + + if (t != null) + { + result.append("\nRoot exception:"); + result.append(renderExceptionStackAsText(t)); + } + else + { + result.append("\nNo exception was provided."); + } + + return(result.toString()); + } + + private String renderExceptionStackAsText( Throwable t) + { + StringBuffer result = new StringBuffer(); + + if (t != null) + { + String message = t.getMessage(); + Throwable cause = t.getCause(); + + if (cause != null) + { + result.append(renderExceptionStackAsText(cause)); + result.append("\nWrapped by:"); + } + + if (message == null) + { + message = ""; + } + + result.append("\n"); + result.append(t.getClass().getName()); + result.append(": "); + result.append(message); + result.append("\n"); + result.append(renderStackTraceElements(t.getStackTrace())); + } + + return(result.toString()); + } + + private String renderStackTraceElements(StackTraceElement[] elements) + { + StringBuffer result = new StringBuffer(); + + if (elements != null) + { + for (int i = 0; i < elements.length; i++) + { + result.append("\tat " + elements[i].toString() + "\n"); + } + } + + return(result.toString()); + } + + // boilerplate setters + + public void setFileFolderService(FileFolderService fileFolderService) + { + this.fileFolderService = fileFolderService; + } + + public void setRepository(Repository repository) + { + this.repository = repository; + } + +} \ No newline at end of file diff --git a/backup/bulkload/BulkFilesystemImportStatusWebScript.java b/backup/bulkload/BulkFilesystemImportStatusWebScript.java new file mode 100644 index 0000000000..caf3b31e22 --- /dev/null +++ b/backup/bulkload/BulkFilesystemImportStatusWebScript.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2005-2011 Alfresco Software Limited. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * As a special exception to the terms and conditions of version 2.0 of + * the GPL, you may redistribute this Program in connection with Free/Libre + * and Open Source Software ("FLOSS") applications as described in Alfresco's + * FLOSS exception. You should have received a copy of the text describing + * the FLOSS exception, and it is also available here: + * http://www.alfresco.com/legal/licensing" + */ + +package org.alfresco.repo.web.scripts.node.bulkload; + +import java.util.HashMap; +import java.util.Map; + +import org.alfresco.repo.bulkimport.BulkFilesystemImporter; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.extensions.webscripts.Cache; +import org.springframework.extensions.webscripts.DeclarativeWebScript; +import org.springframework.extensions.webscripts.Status; +import org.springframework.extensions.webscripts.WebScriptRequest; + +/** + * Web Script class that provides status information on the bulk filesystem import process. + * + * @since 4.0 + */ +public class BulkFilesystemImportStatusWebScript extends DeclarativeWebScript +{ + private final static Log logger = LogFactory.getLog(BulkFilesystemImportStatusWebScript.class); + + // Output parameters (for Freemarker) + private final static String RESULT_IMPORT_STATUS = "importStatus"; + + // Attributes + private BulkFilesystemImporter importer; + + public void setImporter(BulkFilesystemImporter importer) + { + this.importer = importer; + } + + /** + * @see org.alfresco.web.scripts.DeclarativeWebScript#executeImpl(org.alfresco.web.scripts.WebScriptRequest, org.alfresco.web.scripts.Status, org.alfresco.web.scripts.Cache) + */ + @Override + protected Map executeImpl(WebScriptRequest request, Status status, Cache cache) + { + Map result = new HashMap(); + + cache.setNeverCache(true); + + result.put(RESULT_IMPORT_STATUS, importer.getStatus()); + + return(result); + } +} diff --git a/backup/bulkload/copy/BulkFilesystemImportWebScript.java b/backup/bulkload/copy/BulkFilesystemImportWebScript.java new file mode 100644 index 0000000000..8b28883d8f --- /dev/null +++ b/backup/bulkload/copy/BulkFilesystemImportWebScript.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2005-2011 Alfresco Software Limited. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * As a special exception to the terms and conditions of version 2.0 of + * the GPL, you may redistribute this Program in connection with Free/Libre + * and Open Source Software ("FLOSS") applications as described in Alfresco's + * FLOSS exception. You should have received a copy of the text describing + * the FLOSS exception, and it is also available here: + * http://www.alfresco.com/legal/licensing" + */ + +package org.alfresco.repo.web.scripts.node.bulkload.copy; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.alfresco.repo.bulkimport.BulkFilesystemImporter; +import org.alfresco.repo.bulkimport.impl.FilesystemBulkImportSource; +import org.alfresco.repo.bulkimport.impl.StreamingBulkImportStrategy; +import org.alfresco.repo.web.scripts.node.bulkload.AbstractBulkFileSystemImportWebScript; +import org.alfresco.service.cmr.model.FileNotFoundException; +import org.alfresco.service.cmr.repository.NodeRef; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.extensions.surf.util.I18NUtil; +import org.springframework.extensions.webscripts.Cache; +import org.springframework.extensions.webscripts.Status; +import org.springframework.extensions.webscripts.WebScriptException; +import org.springframework.extensions.webscripts.WebScriptRequest; + +/** + * Web Script class that invokes a BulkFilesystemImporter implementation. + * + * @since 4.0 + */ +public class BulkFilesystemImportWebScript extends AbstractBulkFileSystemImportWebScript +{ + private final static Log logger = LogFactory.getLog(BulkFilesystemImportWebScript.class); + + private BulkFilesystemImporter importer; + private StreamingBulkImportStrategy importStrategy; + + /** + * @see org.springframework.extensions.webscripts.DeclarativeWebScript#executeImpl(org.springframework.extensions.webscripts.WebScriptRequest, org.springframework.extensions.webscripts.Status, org.springframework.extensions.webscripts.Cache) + */ + @Override + protected Map executeImpl(final WebScriptRequest request, final Status status, final Cache cache) + { + Map model = new HashMap(); + String targetNodeRefStr = null; + String targetPath = null; + String sourceDirectoryStr = null; + String replaceExistingStr = null; + + cache.setNeverCache(true); + + try + { + if (!importer.getStatus().inProgress()) + { + NodeRef targetNodeRef = null; + File sourceDirectory = null; + boolean replaceExisting = false; + + // Retrieve, validate and convert parameters + targetNodeRefStr = request.getParameter(PARAMETER_TARGET_NODEREF); + targetPath = request.getParameter(PARAMETER_TARGET_PATH); + sourceDirectoryStr = request.getParameter(PARAMETER_SOURCE_DIRECTORY); + replaceExistingStr = request.getParameter(PARAMETER_REPLACE_EXISTING); + + targetNodeRef = getTargetNodeRef(targetNodeRefStr, targetPath); + + if (sourceDirectoryStr == null || sourceDirectoryStr.trim().length() == 0) + { + throw new RuntimeException("Error: mandatory parameter '" + PARAMETER_SOURCE_DIRECTORY + "' was not provided."); + } + + sourceDirectory = new File(sourceDirectoryStr.trim()); + + if (replaceExistingStr != null && replaceExistingStr.trim().length() > 0) + { + replaceExisting = PARAMETER_VALUE_REPLACE_EXISTING.equals(replaceExistingStr); + } + + // Initiate the import + importer.bulkImport(new FilesystemBulkImportSource(sourceDirectory), targetNodeRef, importStrategy, replaceExisting); + + // redirect to the status Web Script + status.setCode(Status.STATUS_MOVED_TEMPORARILY); + status.setRedirect(true); + status.setLocation(request.getServiceContextPath() + WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS); + } + else + { + model.put(IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY, I18NUtil.getMessage(IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY)); + } + } + catch (WebScriptException wse) + { + status.setCode(Status.STATUS_BAD_REQUEST, wse.getMessage()); + status.setRedirect(true); + } + catch (FileNotFoundException fnfe) + { + status.setCode(Status.STATUS_BAD_REQUEST,"The repository path '" + targetPath + "' does not exist !"); + status.setRedirect(true); + } + catch(IllegalArgumentException iae) + { + status.setCode(Status.STATUS_BAD_REQUEST,iae.getMessage()); + status.setRedirect(true); + } + catch (Throwable t) + { + throw new WebScriptException(Status.STATUS_INTERNAL_SERVER_ERROR, buildTextMessage(t), t); + } + + return model; + } + + // boilerplate setters + + public void setImporter(BulkFilesystemImporter importer) + { + this.importer = importer; + } + + public void setImportStrategy(StreamingBulkImportStrategy importStrategy) + { + this.importStrategy = importStrategy; + } +} diff --git a/backup/bulkload/inplace/BulkFilesystemImportWebScript.java b/backup/bulkload/inplace/BulkFilesystemImportWebScript.java new file mode 100644 index 0000000000..71cf4f5f76 --- /dev/null +++ b/backup/bulkload/inplace/BulkFilesystemImportWebScript.java @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2005-2011 Alfresco Software Limited. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * As a special exception to the terms and conditions of version 2.0 of + * the GPL, you may redistribute this Program in connection with Free/Libre + * and Open Source Software ("FLOSS") applications as described in Alfresco's + * FLOSS exception. You should have received a copy of the text describing + * the FLOSS exception, and it is also available here: + * http://www.alfresco.com/legal/licensing" + */ + +package org.alfresco.repo.web.scripts.node.bulkload.inplace; + +import java.util.HashMap; +import java.util.Map; + +import org.alfresco.repo.bulkimport.BulkFilesystemImporter; +import org.alfresco.repo.bulkimport.BulkImportSource; +import org.alfresco.repo.bulkimport.ContentStoreMapProvider; +import org.alfresco.repo.bulkimport.impl.InPlaceBulkImportStrategy; +import org.alfresco.repo.bulkimport.impl.StoreBulkImportSource; +import org.alfresco.repo.content.ContentStore; +import org.alfresco.repo.web.scripts.node.bulkload.AbstractBulkFileSystemImportWebScript; +import org.alfresco.service.cmr.model.FileNotFoundException; +import org.alfresco.service.cmr.repository.NodeRef; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.extensions.surf.util.I18NUtil; +import org.springframework.extensions.webscripts.Cache; +import org.springframework.extensions.webscripts.Status; +import org.springframework.extensions.webscripts.WebScriptException; +import org.springframework.extensions.webscripts.WebScriptRequest; + +/** + * Web Script class that invokes a BulkFilesystemImporter implementation. + * + * @since 4.0 + * + * Copied by Romain Guinot from {@link BulkFilesystemImportWebScript}, + * and expanded to support "in-place" importing of existing content. + * + */ +public class BulkFilesystemImportWebScript extends AbstractBulkFileSystemImportWebScript +{ + private static final Log logger = LogFactory.getLog(BulkFilesystemImportWebScript.class); + + // Web Script parameters (non-inherited) + private static final String PARAMETER_CONTENT_STORE = "contentStore"; + + private BulkFilesystemImporter importer; + private ContentStoreMapProvider storeMapProvider; + private InPlaceBulkImportStrategy importStrategy; + + /** + * @see org.springframework.extensions.webscripts.DeclarativeWebScript#executeImpl(org.springframework.extensions.webscripts.WebScriptRequest, org.springframework.extensions.webscripts.Status, org.springframework.extensions.webscripts.Cache) + */ + @Override + protected Map executeImpl(WebScriptRequest request, Status status, Cache cache) + { + Map model = new HashMap(); + + String relativeSourceDirectory = null; + String destinationContentStoreName = null; + String targetNodeRefStr = null; + String targetPath = null; + String replaceExistingStr = null; + + cache.setNeverCache(true); + try + { + importInProgress = importer.getStatus().inProgress(); + if (!importInProgress) + { + NodeRef targetNodeRef = null; + boolean replaceExisting = false; + + // Retrieve, validate and convert parameters + relativeSourceDirectory = request.getParameter(PARAMETER_SOURCE_DIRECTORY); + destinationContentStoreName = request.getParameter(PARAMETER_CONTENT_STORE); + targetNodeRefStr = request.getParameter(PARAMETER_TARGET_NODEREF); + targetPath = request.getParameter(PARAMETER_TARGET_PATH); + replaceExistingStr = request.getParameter(PARAMETER_REPLACE_EXISTING); + + targetNodeRef = getTargetNodeRef(targetNodeRefStr, targetPath); + + if (relativeSourceDirectory == null || relativeSourceDirectory.trim().length() == 0) + { + throw new WebScriptException("Error: mandatory parameter '" + PARAMETER_SOURCE_DIRECTORY + "' was not provided !"); + } + + if (replaceExistingStr != null && replaceExistingStr.trim().length() > 0) + { + replaceExisting = PARAMETER_VALUE_REPLACE_EXISTING.equals(replaceExistingStr); + } + + ContentStore store = storeMapProvider.checkAndGetStore(destinationContentStoreName); + + // Initiate the import +// ImportStrategy importContext = new InPlaceBulkImportStrategy(destinationContentStoreName, store, relativeSourceDirectory); + //importer.bulkImport(targetNodeRef, new Triple (destinationContentStoreName, store, relativeSourceDirectory), replaceExisting); + BulkImportSource importSource = new StoreBulkImportSource(destinationContentStoreName, store, relativeSourceDirectory); + importer.bulkImport(importSource, targetNodeRef, importStrategy, replaceExisting); + + // redirect to the status Web Script + status.setCode(Status.STATUS_MOVED_TEMPORARILY); + status.setRedirect(true); + status.setLocation(request.getServiceContextPath() + WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS); + } + else + { + model.put(IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY, I18NUtil.getMessage(IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY)); + } + } + catch (WebScriptException wse) + { + status.setCode(Status.STATUS_BAD_REQUEST, wse.getMessage()); + status.setRedirect(true); + } + catch (FileNotFoundException fnfe) + { + status.setCode(Status.STATUS_BAD_REQUEST,"The repository path '" + targetPath + "' does not exist !"); + status.setRedirect(true); + } + catch(IllegalArgumentException iae) + { + status.setCode(Status.STATUS_BAD_REQUEST,iae.getMessage()); + status.setRedirect(true); + } + catch (Throwable t) + { + throw new WebScriptException(Status.STATUS_INTERNAL_SERVER_ERROR, buildTextMessage(t), t); + } + + return model; + } + + // boilerplate setters + + public void setImporter(BulkFilesystemImporter importer) + { + this.importer = importer; + } + + public void setStoreMapProvider(ContentStoreMapProvider storeMapProvider) + { + this.storeMapProvider = storeMapProvider; + } + + public void setImportStrategy(InPlaceBulkImportStrategy importStrategy) + { + this.importStrategy = importStrategy; + } +} \ No newline at end of file diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.desc.xml b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.desc.xml new file mode 100644 index 0000000000..c7046c98e3 --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.desc.xml @@ -0,0 +1,11 @@ + + Bulk Filesystem Import - AJAX Auto Suggest for Space Names + Web Script that returns a list of spaces for the AJAX auto-suggest control in the bulk import initiation form. + /bulk/import/filesystem/ajax/suggest/spaces?query={query} + admin + any + Bulk Filesystem Import + + true + + diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.json.ftl b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.json.ftl new file mode 100644 index 0000000000..d81cff9131 --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.json.ftl @@ -0,0 +1,24 @@ +[#ftl] +[#-- TODO: Filter out Data Dictionary and all sub-folders --] +[#macro fullPath node] + [@compress single_line=true] + [#if node == companyhome] +/Company Home + [#else] +[@fullPath node=node.parent/]/${node.properties.name} + [/#if] + [/@compress] +[/#macro] +[#assign luceneQuery = "@cm\\:name:\"" + args.query + "*\" AND TYPE:\\{http\\://www.alfresco.org/model/content/1.0\\}folder AND NOT TYPE:\\{http\\://www.alfresco.org/model/wcmappmodel/1.0\\}webfolder"] +[#assign matches = companyhome.childrenByLuceneSearch[luceneQuery]] +{ + "data" : + [ +[#list matches as match] + { + "path" : "[@fullPath node=match/]", + "nodeRef" : "${match.nodeRef}" + }[#if match != matches?last],[/#if] +[/#list] + ] +} \ No newline at end of file diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.text.ftl b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.text.ftl new file mode 100644 index 0000000000..f564320820 --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ajaxspace.get.text.ftl @@ -0,0 +1,8 @@ +[#ftl] +[#-- TODO: Filter out Data Dictionary and all sub-folders --] +[#macro fullPath node][#if node == companyhome]/Company Home[#else][@fullPath node=node.parent/]/${node.properties.name}[/#if][/#macro] +[#assign luceneQuery = "@cm\\:name:\"" + args.query + "*\" AND TYPE:\\{http\\://www.alfresco.org/model/content/1.0\\}folder AND NOT TYPE:\\{http\\://www.alfresco.org/model/wcmappmodel/1.0\\}webfolder"] +[#assign matches = companyhome.childrenByLuceneSearch[luceneQuery]] +[#list matches as match] +[@fullPath node=match/],${match.nodeRef} +[/#list] diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.desc.xml b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.desc.xml new file mode 100644 index 0000000000..4f4429fed4 --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.desc.xml @@ -0,0 +1,11 @@ + + Bulk Filesystem Import + Web Script that initiates a bulk filesystem import. + /bulk/import/filesystem/initiate + admin + none + Bulk Filesystem Import + + true + + diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.html.ftl b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.html.ftl new file mode 100644 index 0000000000..f56029e3ac --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/initiate.post.html.ftl @@ -0,0 +1 @@ +${importInProgress} \ No newline at end of file diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.desc.xml b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.desc.xml new file mode 100644 index 0000000000..3407c740d6 --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.desc.xml @@ -0,0 +1,11 @@ + + Bulk Filesystem Import Status + Web Script that provides a simple UI for monitoring the status of a bulk filesystem import. + /bulk/import/filesystem/status + + admin + Bulk Filesystem Import + + true + + diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.html.ftl b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.html.ftl new file mode 100644 index 0000000000..81d0dd74b3 --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.html.ftl @@ -0,0 +1,315 @@ +[#ftl] +[#macro formatDuration durationInNs] + [@compress single_line=true] + [#assign days = (durationInNs / (1000 * 1000 * 1000 * 60 * 60 * 24))?floor] + [#assign hours = (durationInNs / (1000 * 1000 * 1000 * 60 * 60))?floor % 24] + [#assign minutes = (durationInNs / (1000 * 1000 * 1000 * 60))?floor % 60] + [#assign seconds = (durationInNs / (1000 * 1000 * 1000))?floor % 60] + [#assign milliseconds = (durationInNs / (1000 * 1000)) % 1000] + [#assign microseconds = (durationInNs / (1000)) % 1000] + ${days}d ${hours}h ${minutes}m ${seconds}s ${milliseconds}.${microseconds}ms + [/@compress] +[/#macro] +[#macro formatBytes bytes] + [@compress single_line=true] + [#if bytes > 1000000000]${(bytes / 1000000000)?string("#,##0.00")}GB + [#elseif bytes > 1000000]${(bytes / 1000000)?string("#,##0.00")}MB + [#elseif bytes > 1000]${(bytes / 1000)?string("#,##0.00")}kB + [#else]${bytes?string("#,##0")}B + [/#if] + [/@compress] +[/#macro] +[#assign refreshIntervalInSeconds = 5] + + + + Bulk Filesystem Import Status + +[#if importStatus.inProgress()] + +[/#if] + + + + + + + +
AlfrescoBulk Filesystem Import Tool Status
Alfresco ${server.edition} v${server.version} +
+
+

+ + + + + + + + + + +[#if importStatus.inProgress() || !importStatus.endDate??] + +[#elseif importStatus.lastExceptionAsString??] + +[#else] + +[/#if] + + + + + + + + + + + + + + + + + + + + + + + + +[#else] + n/a +[/#if] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +[#if importStatus.lastExceptionAsString??] + + + + + + + + + + + +[/#if] +
General Statistics
Current status: +[#if importStatus.inProgress()] + In progress +[#else] + Idle +[/#if] +
Successful:n/aNoYes
Batch Size:${importStatus.batchSize}
Number of threads:${importStatus.numThreads}
Source Directory: +[#if importStatus.sourceDirectory??] + ${importStatus.sourceDirectory} +[#else] + n/a +[/#if] +
Target Space: +[#if importStatus.targetSpace??] + ${importStatus.targetSpace} +[#else] + n/a +[/#if] +
Start Date: +[#if importStatus.startDate??] + ${importStatus.startDate?datetime?string("yyyy-MM-dd hh:mm:ss.SSSaa")} +[#else] + n/a +[/#if] +
End Date: +[#if importStatus.endDate??] + ${importStatus.endDate?datetime?string("yyyy-MM-dd hh:mm:ss.SSSaa")}
+[#if importStatus.inProgress()] + Elapsed Time: +[#else] + Duration: +[/#if] + +[#if importStatus.durationInNs??] + [@formatDuration durationInNs = importStatus.durationInNs/] +[#else] + n/a +[/#if] +
Number of Completed Batches:${importStatus.numberOfBatchesCompleted}
Source (read) Statistics
Scanned: + + + + + + + + + + + +
FoldersFilesUnreadable
${importStatus.numberOfFoldersScanned}${importStatus.numberOfFilesScanned}${importStatus.numberOfUnreadableEntries}
+
Read: + + + + + + + + + + + + + + +
ContentMetadataContent VersionsMetadata Versions
${importStatus.numberOfContentFilesRead} ([@formatBytes importStatus.numberOfContentBytesRead/])${importStatus.numberOfMetadataFilesRead} ([@formatBytes importStatus.numberOfMetadataBytesRead/])${importStatus.numberOfContentVersionFilesRead} ([@formatBytes importStatus.numberOfContentVersionBytesRead/])${importStatus.numberOfMetadataVersionFilesRead} ([@formatBytes importStatus.numberOfMetadataVersionBytesRead/])
+
Throughput: + [#if importStatus.durationInNs?? && importStatus.durationInNs > 0] + [#if importStatus.entriesScannedPerSecond??] + ${importStatus.entriesScannedPerSecond} entries scanned / sec
+ [/#if] + [#if importStatus.filesReadPerSecond??] + ${importStatus.filesReadPerSecond} files read / sec
+ [/#if] + [#if importStatus.bytesReadPerSecond??] + [@formatBytes importStatus.bytesReadPerSecond /] / sec + [/#if] + [#else] + n/a + [/#if] +
Target (write) Statistics
Space Nodes: + + + + + + + + + + + + + +
# Created# Replaced# Skipped# Properties
${importStatus.numberOfSpaceNodesCreated}${importStatus.numberOfSpaceNodesReplaced}${importStatus.numberOfSpaceNodesSkipped}${importStatus.numberOfSpacePropertiesWritten}
+
Content Nodes: + + + + + + + + + + + + + + + +
# Created# Replaced# SkippedData Written# Properties
${importStatus.numberOfContentNodesCreated}${importStatus.numberOfContentNodesReplaced}${importStatus.numberOfContentNodesSkipped}[@formatBytes importStatus.numberOfContentBytesWritten/]${importStatus.numberOfContentPropertiesWritten}
+
Content Versions: + + + + + + + + + + + +
# CreatedData Written# Properties
${importStatus.numberOfContentVersionsCreated}[@formatBytes importStatus.numberOfContentVersionBytesWritten/]${importStatus.numberOfContentVersionPropertiesWritten}
+
Throughput (write): + [#if importStatus.durationInNs?? && importStatus.durationInNs > 0] + [#if importStatus.nodesCreatedPerSecond??] + ${importStatus.nodesCreatedPerSecond?string("#0")} nodes / sec
+ [/#if] + [#if importStatus.bytesWrittenPerSecond??] + [@formatBytes importStatus.bytesWrittenPerSecond /] / sec + [/#if] + [#else] + n/a + [/#if] +
Error Information From Last Run
File that failed:${importStatus.currentFileBeingProcessed!"n/a"}
Exception:
${importStatus.lastExceptionAsString}
+

+

+[#if importStatus.inProgress()] + This page will automatically refresh in ${refreshIntervalInSeconds} seconds. +[#else] + Initiate another import

+ Initiate another in-place import +[/#if] +

+
+ + + diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.xml.ftl b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.xml.ftl new file mode 100644 index 0000000000..31b06f7da4 --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/status.get.xml.ftl @@ -0,0 +1,71 @@ +[#ftl] + + + [@compress single_line=true] +[#if importStatus.inProgress()] + In progress +[#else] + Idle +[/#if] +[/@compress] + [@compress single_line=true] +[#if importStatus.lastExceptionAsString??] + Failed +[#else] + Succeeded +[/#if] +[/@compress] +[#if importStatus.sourceDirectory??] + ${importStatus.sourceDirectory} +[/#if] +[#if importStatus.targetSpace??] + ${importStatus.targetSpace} +[/#if] + ${importStatus.batchWeight?c} +[#if importStatus.startDate??] + ${importStatus.startDate?datetime?string("yyyy-MM-dd'T'HH:mm:ss.SSS")} +[/#if] +[#if importStatus.endDate??] + ${importStatus.endDate?datetime?string("yyyy-MM-dd'T'HH:mm:ss.SSS")} +[/#if] +[#if importStatus.durationInNs??] + ${importStatus.durationInNs?c} +[/#if] + ${importStatus.numberOfBatchesCompleted} +[#if !importStatus.inProgress() && importStatus.endDate??] +[/#if] + + ${importStatus.currentFileBeingProcessed!"n/a"} + ${importStatus.numberOfFilesScanned?c} + ${importStatus.numberOfFoldersScanned?c} + ${importStatus.numberOfUnreadableEntries?c} + ${importStatus.numberOfContentFilesRead?c} + ${importStatus.numberOfContentBytesRead?c} + ${importStatus.numberOfMetadataFilesRead?c} + ${importStatus.numberOfMetadataBytesRead?c} + ${importStatus.numberOfContentVersionFilesRead?c} + ${importStatus.numberOfContentVersionBytesRead?c} + ${importStatus.numberOfMetadataVersionFilesRead?c} + ${importStatus.numberOfMetadataVersionBytesRead?c} + + + ${importStatus.numberOfSpaceNodesCreated?c} + ${importStatus.numberOfSpaceNodesReplaced?c} + ${importStatus.numberOfSpaceNodesSkipped?c} + ${importStatus.numberOfSpacePropertiesWritten?c} + ${importStatus.numberOfContentNodesCreated?c} + ${importStatus.numberOfContentNodesReplaced?c} + ${importStatus.numberOfContentNodesSkipped?c} + ${importStatus.numberOfContentBytesWritten?c} + ${importStatus.numberOfContentPropertiesWritten?c} + ${importStatus.numberOfContentVersionsCreated?c} + ${importStatus.numberOfContentVersionBytesWritten?c} + ${importStatus.numberOfContentVersionPropertiesWritten?c} + +[#if importStatus.lastExceptionAsString??] + + ${importStatus.currentFileBeingProcessed!"n/a"} + ${importStatus.lastExceptionAsString} + +[/#if] + diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.desc.xml b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.desc.xml new file mode 100644 index 0000000000..b33039114a --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.desc.xml @@ -0,0 +1,11 @@ + + Bulk Filesystem Import + Web Script that provides a simple UI for initiating a bulk filesystem import. + /bulk/import/filesystem + + admin + Bulk Filesystem Import + + true + + diff --git a/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.html.ftl b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.html.ftl new file mode 100644 index 0000000000..c56de0568b --- /dev/null +++ b/config/alfresco/templates/webscripts/org/alfresco/repository/bulkfilesystemimport/ui.get.html.ftl @@ -0,0 +1,118 @@ +[#ftl] + + + + Bulk Filesystem Import Tool + + + + + + + + + + + + + + + + + +
AlfrescoBulk Filesystem Import Tool
Alfresco ${server.edition} v${server.version} +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Import directory:

 +
+ +
+
+
 
(unchecked means skip files that already exist in the repository)
Batch Size:
Number of Threads:
 
+
+
+ + + diff --git a/config/alfresco/web-scripts-application-context.xml b/config/alfresco/web-scripts-application-context.xml index 734a15d6ec..91d9119e56 100644 --- a/config/alfresco/web-scripts-application-context.xml +++ b/config/alfresco/web-scripts-application-context.xml @@ -1752,5 +1752,53 @@ class="org.alfresco.repo.web.scripts.calendar.UserCalendarEntriesGet" parent="abstractCalendarWebScript"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + alfresco.messages.ui-inplace + alfresco.messages.initiate-inplace + + + diff --git a/source/java/org/alfresco/repo/web/scripts/bulkimport/AbstractBulkFileSystemImportWebScript.java b/source/java/org/alfresco/repo/web/scripts/bulkimport/AbstractBulkFileSystemImportWebScript.java new file mode 100644 index 0000000000..05fac888b1 --- /dev/null +++ b/source/java/org/alfresco/repo/web/scripts/bulkimport/AbstractBulkFileSystemImportWebScript.java @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2005-2011 Alfresco Software Limited. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * As a special exception to the terms and conditions of version 2.0 of + * the GPL, you may redistribute this Program in connection with Free/Libre + * and Open Source Software ("FLOSS") applications as described in Alfresco's + * FLOSS exception. You should have received a copy of the text describing + * the FLOSS exception, and it is also available here: + * http://www.alfresco.com/legal/licensing" + */ + +package org.alfresco.repo.web.scripts.bulkimport; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Date; + +import org.alfresco.repo.model.Repository; +import org.alfresco.service.cmr.model.FileFolderService; +import org.alfresco.service.cmr.model.FileInfo; +import org.alfresco.service.cmr.model.FileNotFoundException; +import org.alfresco.service.cmr.repository.NodeRef; +import org.springframework.extensions.webscripts.DeclarativeWebScript; +import org.springframework.extensions.webscripts.WebScriptException; + +/** + * contains common fields and methods for the import web scripts. + */ +public class AbstractBulkFileSystemImportWebScript extends DeclarativeWebScript +{ + protected static final String WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS = "/bulk/import/filesystem/status"; + + protected static final String PARAMETER_TARGET_NODEREF = "targetNodeRef"; + protected static final String PARAMETER_TARGET_PATH = "targetPath"; + + protected static final String COMPANY_HOME_NAME = "Company Home"; + protected static final String COMPANY_HOME_PATH = "/" + COMPANY_HOME_NAME; + + // Web scripts parameters (common) + protected static final String PARAMETER_REPLACE_EXISTING = "replaceExisting"; + protected static final String PARAMETER_VALUE_REPLACE_EXISTING = "replaceExisting"; + protected static final String PARAMETER_SOURCE_DIRECTORY = "sourceDirectory"; + + protected static final String IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY = "importInProgress"; + protected static final String IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY ="bfsit.error.importAlreadyInProgress"; + + protected static final String PARAMETER_BATCH_SIZE = "batchSize"; + protected static final String PARAMETER_NUM_THREADS = "numThreads"; + + protected FileFolderService fileFolderService; + protected Repository repository; + + protected volatile boolean importInProgress; + + protected NodeRef getTargetNodeRef(String targetNodeRefStr, String targetPath) throws FileNotFoundException + { + NodeRef targetNodeRef; + + if (targetNodeRefStr == null || targetNodeRefStr.trim().length() == 0) + { + if (targetPath == null || targetPath.trim().length() == 0) + { + throw new WebScriptException("Error: neither parameter '" + PARAMETER_TARGET_NODEREF + + "' nor parameter '" + PARAMETER_TARGET_PATH + + "' was provided, but at least one is required !"); + } + targetNodeRef = convertPathToNodeRef(targetPath.trim()); + } + else + { + targetNodeRef = new NodeRef(targetNodeRefStr.trim()); + } + + return targetNodeRef; + } + + protected NodeRef convertPathToNodeRef(String targetPath) throws FileNotFoundException + { + NodeRef result = null; + NodeRef companyHome = repository.getCompanyHome(); + String cleanTargetPath = targetPath.replaceAll("/+", "/"); + + if (cleanTargetPath.startsWith(COMPANY_HOME_PATH)) + cleanTargetPath = cleanTargetPath.substring(COMPANY_HOME_PATH.length()); + + if (cleanTargetPath.startsWith("/")) + cleanTargetPath = cleanTargetPath.substring(1); + + if (cleanTargetPath.endsWith("/")) + cleanTargetPath = cleanTargetPath.substring(0, cleanTargetPath.length() - 1); + + if (cleanTargetPath.length() == 0) + result = companyHome; + else + { + FileInfo info = fileFolderService.resolveNamePath(companyHome, Arrays.asList(cleanTargetPath.split("/"))); + if(info == null) + throw new WebScriptException("could not determine NodeRef for path :'"+cleanTargetPath+"'"); + + result = info.getNodeRef(); + } + + return(result); + } + + protected String buildTextMessage(Throwable t) + { + StringBuffer result = new StringBuffer(); + String timeOfFailure = (new Date()).toString(); + String hostName = null; + String ipAddress = null; + + try + { + hostName = InetAddress.getLocalHost().getHostName(); + ipAddress = InetAddress.getLocalHost().getHostAddress(); + } + catch (UnknownHostException uhe) + { + hostName = "unknown"; + ipAddress = "unknown"; + } + + result.append("\nTime of failure: " + timeOfFailure); + result.append("\nHost where failure occurred: " + hostName + " (" + ipAddress + ")"); + + if (t != null) + { + result.append("\nRoot exception:"); + result.append(renderExceptionStackAsText(t)); + } + else + { + result.append("\nNo exception was provided."); + } + + return(result.toString()); + } + + private String renderExceptionStackAsText( Throwable t) + { + StringBuffer result = new StringBuffer(); + + if (t != null) + { + String message = t.getMessage(); + Throwable cause = t.getCause(); + + if (cause != null) + { + result.append(renderExceptionStackAsText(cause)); + result.append("\nWrapped by:"); + } + + if (message == null) + { + message = ""; + } + + result.append("\n"); + result.append(t.getClass().getName()); + result.append(": "); + result.append(message); + result.append("\n"); + result.append(renderStackTraceElements(t.getStackTrace())); + } + + return(result.toString()); + } + + private String renderStackTraceElements(StackTraceElement[] elements) + { + StringBuffer result = new StringBuffer(); + + if (elements != null) + { + for (int i = 0; i < elements.length; i++) + { + result.append("\tat " + elements[i].toString() + "\n"); + } + } + + return(result.toString()); + } + + // boilerplate setters + + public void setFileFolderService(FileFolderService fileFolderService) + { + this.fileFolderService = fileFolderService; + } + + public void setRepository(Repository repository) + { + this.repository = repository; + } + +} \ No newline at end of file diff --git a/source/java/org/alfresco/repo/web/scripts/bulkimport/BulkFilesystemImportStatusWebScript.java b/source/java/org/alfresco/repo/web/scripts/bulkimport/BulkFilesystemImportStatusWebScript.java new file mode 100644 index 0000000000..65c50228bb --- /dev/null +++ b/source/java/org/alfresco/repo/web/scripts/bulkimport/BulkFilesystemImportStatusWebScript.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2005-2011 Alfresco Software Limited. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * As a special exception to the terms and conditions of version 2.0 of + * the GPL, you may redistribute this Program in connection with Free/Libre + * and Open Source Software ("FLOSS") applications as described in Alfresco's + * FLOSS exception. You should have received a copy of the text describing + * the FLOSS exception, and it is also available here: + * http://www.alfresco.com/legal/licensing" + */ + +package org.alfresco.repo.web.scripts.bulkimport; + +import java.util.HashMap; +import java.util.Map; + +import org.alfresco.repo.bulkimport.BulkFilesystemImporter; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.extensions.webscripts.Cache; +import org.springframework.extensions.webscripts.DeclarativeWebScript; +import org.springframework.extensions.webscripts.Status; +import org.springframework.extensions.webscripts.WebScriptRequest; + +/** + * Web Script class that provides status information on the bulk filesystem import process. + * + * @since 4.0 + */ +public class BulkFilesystemImportStatusWebScript extends DeclarativeWebScript +{ + private final static Log logger = LogFactory.getLog(BulkFilesystemImportStatusWebScript.class); + + // Output parameters (for Freemarker) + private final static String RESULT_IMPORT_STATUS = "importStatus"; + + // Attributes + private BulkFilesystemImporter bulkImporter; + + public void setBulkImporter(BulkFilesystemImporter bulkImporter) + { + this.bulkImporter = bulkImporter; + } + + /** + * @see org.alfresco.web.scripts.DeclarativeWebScript#executeImpl(org.alfresco.web.scripts.WebScriptRequest, org.alfresco.web.scripts.Status, org.alfresco.web.scripts.Cache) + */ + @Override + protected Map executeImpl(WebScriptRequest request, Status status, Cache cache) + { + Map result = new HashMap(); + + cache.setNeverCache(true); + + result.put(RESULT_IMPORT_STATUS, bulkImporter.getStatus()); + + return(result); + } +} diff --git a/source/java/org/alfresco/repo/web/scripts/bulkimport/copy/BulkFilesystemImportWebScript.java b/source/java/org/alfresco/repo/web/scripts/bulkimport/copy/BulkFilesystemImportWebScript.java new file mode 100644 index 0000000000..6ab39babf4 --- /dev/null +++ b/source/java/org/alfresco/repo/web/scripts/bulkimport/copy/BulkFilesystemImportWebScript.java @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2005-2011 Alfresco Software Limited. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * As a special exception to the terms and conditions of version 2.0 of + * the GPL, you may redistribute this Program in connection with Free/Libre + * and Open Source Software ("FLOSS") applications as described in Alfresco's + * FLOSS exception. You should have received a copy of the text describing + * the FLOSS exception, and it is also available here: + * http://www.alfresco.com/legal/licensing" + */ + +package org.alfresco.repo.web.scripts.bulkimport.copy; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.alfresco.repo.bulkimport.BulkImportParameters; +import org.alfresco.repo.bulkimport.NodeImporter; +import org.alfresco.repo.bulkimport.impl.MultiThreadedBulkFilesystemImporter; +import org.alfresco.repo.bulkimport.impl.StreamingNodeImporterFactory; +import org.alfresco.repo.web.scripts.bulkimport.AbstractBulkFileSystemImportWebScript; +import org.alfresco.service.cmr.model.FileNotFoundException; +import org.alfresco.service.cmr.repository.NodeRef; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.extensions.surf.util.I18NUtil; +import org.springframework.extensions.webscripts.Cache; +import org.springframework.extensions.webscripts.Status; +import org.springframework.extensions.webscripts.WebScriptException; +import org.springframework.extensions.webscripts.WebScriptRequest; + +/** + * Web Script class that invokes a BulkFilesystemImporter implementation. + * + * @since 4.0 + */ +public class BulkFilesystemImportWebScript extends AbstractBulkFileSystemImportWebScript +{ + private final static Log logger = LogFactory.getLog(BulkFilesystemImportWebScript.class); + + private MultiThreadedBulkFilesystemImporter bulkImporter; + private StreamingNodeImporterFactory nodeImporterFactory; + + public void setBulkImporter(MultiThreadedBulkFilesystemImporter bulkImporter) + { + this.bulkImporter = bulkImporter; + } + + public void setNodeImporterFactory(StreamingNodeImporterFactory nodeImporterFactory) + { + this.nodeImporterFactory = nodeImporterFactory; + } + + /** + * @see org.springframework.extensions.webscripts.DeclarativeWebScript#executeImpl(org.springframework.extensions.webscripts.WebScriptRequest, org.springframework.extensions.webscripts.Status, org.springframework.extensions.webscripts.Cache) + */ + @Override + protected Map executeImpl(final WebScriptRequest request, final Status status, final Cache cache) + { + Map model = new HashMap(); + String targetNodeRefStr = null; + String targetPath = null; + String sourceDirectoryStr = null; + String replaceExistingStr = null; + String batchSizeStr = null; + String numThreadsStr = null; + + cache.setNeverCache(true); + + try + { + if(!bulkImporter.getStatus().inProgress()) + { + NodeRef targetNodeRef = null; + File sourceDirectory = null; + boolean replaceExisting = false; + int batchSize = bulkImporter.getDefaultBatchSize(); + int numThreads = bulkImporter.getDefaultNumThreads(); + + // Retrieve, validate and convert parameters + targetNodeRefStr = request.getParameter(PARAMETER_TARGET_NODEREF); + targetPath = request.getParameter(PARAMETER_TARGET_PATH); + sourceDirectoryStr = request.getParameter(PARAMETER_SOURCE_DIRECTORY); + replaceExistingStr = request.getParameter(PARAMETER_REPLACE_EXISTING); + batchSizeStr = request.getParameter(PARAMETER_BATCH_SIZE); + numThreadsStr = request.getParameter(PARAMETER_NUM_THREADS); + + targetNodeRef = getTargetNodeRef(targetNodeRefStr, targetPath); + + if (sourceDirectoryStr == null || sourceDirectoryStr.trim().length() == 0) + { + throw new RuntimeException("Error: mandatory parameter '" + PARAMETER_SOURCE_DIRECTORY + "' was not provided."); + } + + sourceDirectory = new File(sourceDirectoryStr.trim()); + + if (replaceExistingStr != null && replaceExistingStr.trim().length() > 0) + { + replaceExisting = PARAMETER_VALUE_REPLACE_EXISTING.equals(replaceExistingStr); + } + + // Initiate the import + NodeImporter nodeImporter = nodeImporterFactory.getNodeImporter(sourceDirectory); + //bulkImporter.asyncBulkImport(targetNodeRef, nodeImporter, replaceExisting); + BulkImportParameters bulkImportParameters = new BulkImportParameters(); + + if (numThreadsStr != null && numThreadsStr.trim().length() > 0) + { + try + { + numThreads = Integer.parseInt(numThreadsStr); + if(numThreads < 1) + { + throw new RuntimeException("Error: parameter '" + PARAMETER_NUM_THREADS + "' must be an integer > 0."); + } + bulkImportParameters.setNumThreads(numThreads); + } + catch(NumberFormatException e) + { + throw new RuntimeException("Error: parameter '" + PARAMETER_NUM_THREADS + "' must be an integer > 0."); + } + } + + if (batchSizeStr != null && batchSizeStr.trim().length() > 0) + { + try + { + batchSize = Integer.parseInt(batchSizeStr); + if(batchSize < 1) + { + throw new RuntimeException("Error: parameter '" + PARAMETER_BATCH_SIZE + "' must be an integer > 0."); + } + bulkImportParameters.setBatchSize(batchSize); + } + catch(NumberFormatException e) + { + throw new RuntimeException("Error: parameter '" + PARAMETER_BATCH_SIZE + "' must be an integer > 0."); + } + } + + bulkImportParameters.setReplaceExisting(replaceExisting); + bulkImportParameters.setTarget(targetNodeRef); + + bulkImporter.asyncBulkImport(bulkImportParameters, nodeImporter); + + // redirect to the status Web Script + status.setCode(Status.STATUS_MOVED_TEMPORARILY); + status.setRedirect(true); + status.setLocation(request.getServiceContextPath() + WEB_SCRIPT_URI_BULK_FILESYSTEM_IMPORT_STATUS); + } + else + { + model.put(IMPORT_ALREADY_IN_PROGRESS_MODEL_KEY, I18NUtil.getMessage(IMPORT_ALREADY_IN_PROGRESS_ERROR_KEY)); + } + } + catch (WebScriptException wse) + { + status.setCode(Status.STATUS_BAD_REQUEST, wse.getMessage()); + status.setRedirect(true); + } + catch (FileNotFoundException fnfe) + { + status.setCode(Status.STATUS_BAD_REQUEST,"The repository path '" + targetPath + "' does not exist !"); + status.setRedirect(true); + } + catch(IllegalArgumentException iae) + { + status.setCode(Status.STATUS_BAD_REQUEST,iae.getMessage()); + status.setRedirect(true); + } + catch (Throwable t) + { + throw new WebScriptException(Status.STATUS_INTERNAL_SERVER_ERROR, buildTextMessage(t), t); + } + + return model; + } + +}