diff --git a/demo-shell/src/app.config.json b/demo-shell/src/app.config.json index 28d7504421..1ea970f269 100644 --- a/demo-shell/src/app.config.json +++ b/demo-shell/src/app.config.json @@ -12,6 +12,9 @@ "auth": { "withCredentials": false }, + "upload": { + "threads": 1 + }, "oauth2": { "host": "{protocol}//{hostname}{:port}/auth/realms/alfresco", "clientId": "alfresco", diff --git a/docs/core/services/upload.service.md b/docs/core/services/upload.service.md index 91928e2ced..128d570efc 100644 --- a/docs/core/services/upload.service.md +++ b/docs/core/services/upload.service.md @@ -119,3 +119,18 @@ It is also possible to provide the `versioningEnabled` value as part of the [`Fi > Note: When creating a new node using multipart/form-data by default versioning is enabled and set to MAJOR Version. > Since Alfresco 6.2.3 versioningEnabled flag was introduced offering better control over the new node Versioning. + +### Concurrent Uploads + +By default, the Upload Service processes one file at a time. +You can increase the number of concurrent threads by changing the `upload.threads` configuration parameter: + +**app.config.json** + +```json +{ + "upload": { + "threads": 2 + } +} +``` diff --git a/e2e/protractor.excludes.json b/e2e/protractor.excludes.json index eb2c4fb0cb..b644f70f85 100644 --- a/e2e/protractor.excludes.json +++ b/e2e/protractor.excludes.json @@ -4,5 +4,6 @@ "C260188": "https://alfresco.atlassian.net/browse/ADF-5470", "C260192": "https://alfresco.atlassian.net/browse/ADF-5470", "C260193": "https://alfresco.atlassian.net/browse/ADF-5470", - "C216426": "https://alfresco.atlassian.net/browse/ADF-5470" + "C216426": "https://alfresco.atlassian.net/browse/ADF-5470", + "C362241": "https://alfresco.atlassian.net/browse/ADF-5470" } diff --git a/lib/content-services/src/lib/upload/components/upload-drag-area.component.spec.ts b/lib/content-services/src/lib/upload/components/upload-drag-area.component.spec.ts index e1d9e6db66..7a391e13b7 100644 --- a/lib/content-services/src/lib/upload/components/upload-drag-area.component.spec.ts +++ b/lib/content-services/src/lib/upload/components/upload-drag-area.component.spec.ts @@ -108,6 +108,8 @@ describe('UploadDragAreaComponent', () => { component = fixture.componentInstance; fixture.detectChanges(); + + uploadService.clearCache(); }); afterEach(() => { diff --git a/lib/core/.eslintrc.json b/lib/core/.eslintrc.json index 387680af7d..ed6272b132 100644 --- a/lib/core/.eslintrc.json +++ b/lib/core/.eslintrc.json @@ -24,7 +24,7 @@ "@typescript-eslint/naming-convention": "warn", "@typescript-eslint/consistent-type-assertions": "warn", "@typescript-eslint/prefer-for-of": "warn", - "no-underscore-dangle": "warn", + "no-underscore-dangle": ["warn", { "allowAfterThis": true }], "no-shadow": "warn", "quote-props": "warn", "object-shorthand": "warn", diff --git a/lib/core/services/upload.service.spec.ts b/lib/core/services/upload.service.spec.ts index 40d53c6c4b..f866d5f8f0 100644 --- a/lib/core/services/upload.service.spec.ts +++ b/lib/core/services/upload.service.spec.ts @@ -76,7 +76,7 @@ describe('UploadService', () => { service = TestBed.inject(UploadService); service.queue = []; - service.activeTask = null; + service.clearCache(); uploadFileSpy = spyOn(service.uploadApi, 'uploadFile').and.callThrough(); diff --git a/lib/core/services/upload.service.ts b/lib/core/services/upload.service.ts index bad306078b..45f01cac00 100644 --- a/lib/core/services/upload.service.ts +++ b/lib/core/services/upload.service.ts @@ -38,20 +38,7 @@ const MAX_CANCELLABLE_FILE_PERCENTAGE = 50; providedIn: 'root' }) export class UploadService { - private cache: { [key: string]: any } = {}; - private totalComplete: number = 0; - private totalAborted: number = 0; - private totalError: number = 0; - private excludedFileList: string[] = []; - private excludedFoldersList: string[] = []; - private matchingOptions: any = null; - private folderMatchingOptions: any = null; - private abortedFile: string; - private isThumbnailGenerationEnabled: boolean; - - activeTask: Promise = null; queue: FileModel[] = []; - queueChanged: Subject = new Subject(); fileUpload: Subject = new Subject(); fileUploadStarting: Subject = new Subject(); @@ -63,19 +50,30 @@ export class UploadService { fileUploadDeleted: Subject = new Subject(); fileDeleted: Subject = new Subject(); - _uploadApi: UploadApi; + private cache: { [key: string]: any } = {}; + private totalComplete: number = 0; + private totalAborted: number = 0; + private totalError: number = 0; + private excludedFileList: string[] = []; + private excludedFoldersList: string[] = []; + private matchingOptions: any = null; + private folderMatchingOptions: any = null; + private abortedFile: string; + private isThumbnailGenerationEnabled: boolean; + + private _uploadApi: UploadApi; get uploadApi(): UploadApi { this._uploadApi = this._uploadApi ?? new UploadApi(this.apiService.getInstance()); return this._uploadApi; } - _nodesApi: NodesApi; + private _nodesApi: NodesApi; get nodesApi(): NodesApi { this._nodesApi = this._nodesApi ?? new NodesApi(this.apiService.getInstance()); return this._nodesApi; } - _versionsApi: VersionsApi; + private _versionsApi: VersionsApi; get versionsApi(): VersionsApi { this._versionsApi = this._versionsApi ?? new VersionsApi(this.apiService.getInstance()); return this._versionsApi; @@ -92,19 +90,32 @@ export class UploadService { }); } + clearCache() { + this.cache = {}; + } + + /** + * Returns the number of concurrent threads for uploading. + * + * @returns Number of concurrent threads (default 1) + */ + getThreadsCount(): number { + return this.appConfigService.get('upload.threads', 1); + } + /** * Checks whether the service still has files uploading or awaiting upload. + * * @returns True if files in the queue are still uploading, false otherwise */ isUploading(): boolean { const finishedFileStates = [FileUploadStatus.Complete, FileUploadStatus.Cancelled, FileUploadStatus.Aborted, FileUploadStatus.Error, FileUploadStatus.Deleted]; - return this.queue.reduce((stillUploading: boolean, currentFile: FileModel) => { - return stillUploading || finishedFileStates.indexOf(currentFile.status) === -1; - }, false); + return this.queue.reduce((stillUploading: boolean, currentFile: FileModel) => stillUploading || finishedFileStates.indexOf(currentFile.status) === -1, false); } /** * Gets the file Queue + * * @returns Array of files that form the queue */ getQueue(): FileModel[] { @@ -113,6 +124,7 @@ export class UploadService { /** * Adds files to the uploading queue to be uploaded + * * @param files One or more separate parameters or an array of files to queue * @returns Array of files that were not blocked from upload by the ignore list */ @@ -125,69 +137,23 @@ export class UploadService { return allowedFiles; } - private filterElement(file: FileModel) { - this.excludedFileList = this.appConfigService.get('files.excluded'); - this.excludedFoldersList = this.appConfigService.get('folders.excluded'); - let isAllowed = true; - - if (this.excludedFileList) { - this.matchingOptions = this.appConfigService.get('files.match-options'); - isAllowed = this.isFileNameAllowed(file); - } - - if (isAllowed && this.excludedFoldersList) { - this.folderMatchingOptions = this.appConfigService.get('folders.match-options'); - isAllowed = this.isParentFolderAllowed(file); - } - return isAllowed; - } - - private isParentFolderAllowed(file: FileModel): boolean { - let isAllowed: boolean = true; - const currentFile: any = file.file; - const fileRelativePath = currentFile.webkitRelativePath ? currentFile.webkitRelativePath : file.options.path; - if (currentFile && fileRelativePath) { - isAllowed = - this.excludedFoldersList.filter((folderToExclude) => { - return fileRelativePath - .split('/') - .some((pathElement) => { - const minimatch = new Minimatch(folderToExclude, this.folderMatchingOptions); - return minimatch.match(pathElement); - }); - }).length === 0; - } - return isAllowed; - } - - private isFileNameAllowed(file: FileModel): boolean { - return ( - this.excludedFileList.filter((pattern) => { - const minimatch = new Minimatch(pattern, this.matchingOptions); - return minimatch.match(file.name); - }).length === 0 - ); - } - /** * Finds all the files in the queue that are not yet uploaded and uploads them into the directory folder. + * * @param successEmitter Emitter to invoke on file success status change * @param errorEmitter Emitter to invoke on file error status change */ uploadFilesInTheQueue(successEmitter?: EventEmitter, errorEmitter?: EventEmitter): void { - if (!this.activeTask) { - const file = this.queue.find( - (currentFile) => currentFile.status === FileUploadStatus.Pending - ); - if (file) { + const files = this.getFilesToUpload(); + + if (files && files.length > 0) { + for (const file of files) { this.onUploadStarting(file); const promise = this.beginUpload(file, successEmitter, errorEmitter); - this.activeTask = promise; this.cache[file.name] = promise; const next = () => { - this.activeTask = null; setTimeout(() => this.uploadFilesInTheQueue(successEmitter, errorEmitter), 100); }; @@ -205,6 +171,7 @@ export class UploadService { * Cancels uploading of files. * If the file is smaller than 1 MB the file will be uploaded and then the node deleted * to prevent having files that were aborted but still uploaded. + * * @param files One or more separate parameters or an array of files specifying uploads to cancel */ cancelUpload(...files: FileModel[]) { @@ -234,6 +201,7 @@ export class UploadService { /** * Gets an upload promise for a file. + * * @param file The target file * @returns Promise that is resolved if the upload is successful or error otherwise */ @@ -264,7 +232,7 @@ export class UploadService { } if (file.id) { - return this.nodesApi.updateNodeContent(file.id, file.file, opts); + return this.nodesApi.updateNodeContent(file.id, file.file as any, opts); } else { const nodeBody = { ... file.options }; delete nodeBody['versioningEnabled']; @@ -279,6 +247,21 @@ export class UploadService { } } + private getFilesToUpload(): FileModel[] { + const cached = Object.keys(this.cache); + const threadsCount = this.getThreadsCount(); + + if (cached.length >= threadsCount) { + return []; + } + + const files = this.queue + .filter(toUpload => !cached.includes(toUpload.name) && toUpload.status === FileUploadStatus.Pending) + .slice(0, threadsCount); + + return files; + } + private beginUpload(file: FileModel, successEmitter?: EventEmitter, errorEmitter?: EventEmitter): any { const promise = this.getUploadPromise(file); promise @@ -444,4 +427,46 @@ export class UploadService { file.progress.percent < MAX_CANCELLABLE_FILE_PERCENTAGE ); } + + private filterElement(file: FileModel) { + this.excludedFileList = this.appConfigService.get('files.excluded'); + this.excludedFoldersList = this.appConfigService.get('folders.excluded'); + let isAllowed = true; + + if (this.excludedFileList) { + this.matchingOptions = this.appConfigService.get('files.match-options'); + isAllowed = this.isFileNameAllowed(file); + } + + if (isAllowed && this.excludedFoldersList) { + this.folderMatchingOptions = this.appConfigService.get('folders.match-options'); + isAllowed = this.isParentFolderAllowed(file); + } + return isAllowed; + } + + private isParentFolderAllowed(file: FileModel): boolean { + let isAllowed: boolean = true; + const currentFile: any = file.file; + const fileRelativePath = currentFile.webkitRelativePath ? currentFile.webkitRelativePath : file.options.path; + if (currentFile && fileRelativePath) { + isAllowed = + this.excludedFoldersList.filter((folderToExclude) => fileRelativePath + .split('/') + .some((pathElement) => { + const minimatch = new Minimatch(folderToExclude, this.folderMatchingOptions); + return minimatch.match(pathElement); + })).length === 0; + } + return isAllowed; + } + + private isFileNameAllowed(file: FileModel): boolean { + return ( + this.excludedFileList.filter((pattern) => { + const minimatch = new Minimatch(pattern, this.matchingOptions); + return minimatch.match(file.name); + }).length === 0 + ); + } }