inteligr8/alfresco-ng2-components
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-ng2-components.git synced 2025-08-14 17:58:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

[MNT-22649] Support for concurrent uploads and configurable thread count (#7496)

Browse Source 
* eslint fixes

* try remove upload restrictions

* disable unit test

* support threading count for upload

* update docs

* remove comment

* fix unit test

* remove fdescribe

* make 1 thread by default

* exclude e2e
This commit is contained in:
Denys Vuika
2022-02-14 09:38:24 +00:00
committed by GitHub
parent 5ee4482c60
commit 1a6746ff3c
7 changed files with 120 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
demo-shell/src/app.config.json
View File

@@ -12,6 +12,9 @@
"auth": {
"withCredentials": false
},
"upload": {
"threads": 1
},
"oauth2": {
"host": "{protocol}//{hostname}{:port}/auth/realms/alfresco",
"clientId": "alfresco",

15
docs/core/services/upload.service.md
View File

@@ -119,3 +119,18 @@ It is also possible to provide the `versioningEnabled` value as part of the [`Fi
> Note: When creating a new node using multipart/form-data by default versioning is enabled and set to MAJOR Version.
> Since Alfresco 6.2.3 versioningEnabled flag was introduced offering better control over the new node Versioning.
### Concurrent Uploads
By default, the Upload Service processes one file at a time.
You can increase the number of concurrent threads by changing the `upload.threads` configuration parameter:
**app.config.json**
```json
{
"upload": {
"threads": 2
}
}
```

3
e2e/protractor.excludes.json
View File

@@ -4,5 +4,6 @@
"C260188": "https://alfresco.atlassian.net/browse/ADF-5470",
"C260192": "https://alfresco.atlassian.net/browse/ADF-5470",
"C260193": "https://alfresco.atlassian.net/browse/ADF-5470",
"C216426": "https://alfresco.atlassian.net/browse/ADF-5470"
"C216426": "https://alfresco.atlassian.net/browse/ADF-5470",
"C362241": "https://alfresco.atlassian.net/browse/ADF-5470"
}

2
lib/content-services/src/lib/upload/components/upload-drag-area.component.spec.ts
View File

@@ -108,6 +108,8 @@ describe('UploadDragAreaComponent', () => {
component = fixture.componentInstance;
fixture.detectChanges();
uploadService.clearCache();
});
afterEach(() => {

2
lib/core/.eslintrc.json
View File

@@ -24,7 +24,7 @@
"@typescript-eslint/naming-convention": "warn",
"@typescript-eslint/consistent-type-assertions": "warn",
"@typescript-eslint/prefer-for-of": "warn",
"no-underscore-dangle": "warn",
"no-underscore-dangle": ["warn", { "allowAfterThis": true }],
"no-shadow": "warn",
"quote-props": "warn",
"object-shorthand": "warn",

2
lib/core/services/upload.service.spec.ts
View File

@@ -76,7 +76,7 @@ describe('UploadService', () => {
service = TestBed.inject(UploadService);
service.queue = [];
service.activeTask = null;
service.clearCache();
uploadFileSpy = spyOn(service.uploadApi, 'uploadFile').and.callThrough();

167
lib/core/services/upload.service.ts
View File

@@ -38,20 +38,7 @@ const MAX_CANCELLABLE_FILE_PERCENTAGE = 50;
providedIn: 'root'
})
export class UploadService {
private cache: { [key: string]: any } = {};
private totalComplete: number = 0;
private totalAborted: number = 0;
private totalError: number = 0;
private excludedFileList: string[] = [];
private excludedFoldersList: string[] = [];
private matchingOptions: any = null;
private folderMatchingOptions: any = null;
private abortedFile: string;
private isThumbnailGenerationEnabled: boolean;
activeTask: Promise<any> = null;
queue: FileModel[] = [];
queueChanged: Subject<FileModel[]> = new Subject<FileModel[]>();
fileUpload: Subject<FileUploadEvent> = new Subject<FileUploadEvent>();
fileUploadStarting: Subject<FileUploadEvent> = new Subject<FileUploadEvent>();
@@ -63,19 +50,30 @@ export class UploadService {
fileUploadDeleted: Subject<FileUploadDeleteEvent> = new Subject<FileUploadDeleteEvent>();
fileDeleted: Subject<string> = new Subject<string>();
_uploadApi: UploadApi;
private cache: { [key: string]: any } = {};
private totalComplete: number = 0;
private totalAborted: number = 0;
private totalError: number = 0;
private excludedFileList: string[] = [];
private excludedFoldersList: string[] = [];
private matchingOptions: any = null;
private folderMatchingOptions: any = null;
private abortedFile: string;
private isThumbnailGenerationEnabled: boolean;
private _uploadApi: UploadApi;
get uploadApi(): UploadApi {
this._uploadApi = this._uploadApi ?? new UploadApi(this.apiService.getInstance());
return this._uploadApi;
}
_nodesApi: NodesApi;
private _nodesApi: NodesApi;
get nodesApi(): NodesApi {
this._nodesApi = this._nodesApi ?? new NodesApi(this.apiService.getInstance());
return this._nodesApi;
}
_versionsApi: VersionsApi;
private _versionsApi: VersionsApi;
get versionsApi(): VersionsApi {
this._versionsApi = this._versionsApi ?? new VersionsApi(this.apiService.getInstance());
return this._versionsApi;
@@ -92,19 +90,32 @@ export class UploadService {
});
}
clearCache() {
this.cache = {};
}
/**
* Returns the number of concurrent threads for uploading.
*
* @returns Number of concurrent threads (default 1)
*/
getThreadsCount(): number {
return this.appConfigService.get<number>('upload.threads', 1);
}
/**
* Checks whether the service still has files uploading or awaiting upload.
*
* @returns True if files in the queue are still uploading, false otherwise
*/
isUploading(): boolean {
const finishedFileStates = [FileUploadStatus.Complete, FileUploadStatus.Cancelled, FileUploadStatus.Aborted, FileUploadStatus.Error, FileUploadStatus.Deleted];
return this.queue.reduce((stillUploading: boolean, currentFile: FileModel) => {
return stillUploading || finishedFileStates.indexOf(currentFile.status) === -1;
}, false);
return this.queue.reduce((stillUploading: boolean, currentFile: FileModel) => stillUploading || finishedFileStates.indexOf(currentFile.status) === -1, false);
}
/**
* Gets the file Queue
*
* @returns Array of files that form the queue
*/
getQueue(): FileModel[] {
@@ -113,6 +124,7 @@ export class UploadService {
/**
* Adds files to the uploading queue to be uploaded
*
* @param files One or more separate parameters or an array of files to queue
* @returns Array of files that were not blocked from upload by the ignore list
*/
@@ -125,69 +137,23 @@ export class UploadService {
return allowedFiles;
}
private filterElement(file: FileModel) {
this.excludedFileList = <string[]> this.appConfigService.get('files.excluded');
this.excludedFoldersList = <string[]> this.appConfigService.get('folders.excluded');
let isAllowed = true;
if (this.excludedFileList) {
this.matchingOptions = this.appConfigService.get('files.match-options');
isAllowed = this.isFileNameAllowed(file);
}
if (isAllowed && this.excludedFoldersList) {
this.folderMatchingOptions = this.appConfigService.get('folders.match-options');
isAllowed = this.isParentFolderAllowed(file);
}
return isAllowed;
}
private isParentFolderAllowed(file: FileModel): boolean {
let isAllowed: boolean = true;
const currentFile: any = file.file;
const fileRelativePath = currentFile.webkitRelativePath ? currentFile.webkitRelativePath : file.options.path;
if (currentFile && fileRelativePath) {
isAllowed =
this.excludedFoldersList.filter((folderToExclude) => {
return fileRelativePath
.split('/')
.some((pathElement) => {
const minimatch = new Minimatch(folderToExclude, this.folderMatchingOptions);
return minimatch.match(pathElement);
});
}).length === 0;
}
return isAllowed;
}
private isFileNameAllowed(file: FileModel): boolean {
return (
this.excludedFileList.filter((pattern) => {
const minimatch = new Minimatch(pattern, this.matchingOptions);
return minimatch.match(file.name);
}).length === 0
);
}
/**
* Finds all the files in the queue that are not yet uploaded and uploads them into the directory folder.
*
* @param successEmitter Emitter to invoke on file success status change
* @param errorEmitter Emitter to invoke on file error status change
*/
uploadFilesInTheQueue(successEmitter?: EventEmitter<any>, errorEmitter?: EventEmitter<any>): void {
if (!this.activeTask) {
const file = this.queue.find(
(currentFile) => currentFile.status === FileUploadStatus.Pending
);
if (file) {
const files = this.getFilesToUpload();
if (files && files.length > 0) {
for (const file of files) {
this.onUploadStarting(file);
const promise = this.beginUpload(file, successEmitter, errorEmitter);
this.activeTask = promise;
this.cache[file.name] = promise;
const next = () => {
this.activeTask = null;
setTimeout(() => this.uploadFilesInTheQueue(successEmitter, errorEmitter), 100);
};
@@ -205,6 +171,7 @@ export class UploadService {
* Cancels uploading of files.
* If the file is smaller than 1 MB the file will be uploaded and then the node deleted
* to prevent having files that were aborted but still uploaded.
*
* @param files One or more separate parameters or an array of files specifying uploads to cancel
*/
cancelUpload(...files: FileModel[]) {
@@ -234,6 +201,7 @@ export class UploadService {
/**
* Gets an upload promise for a file.
*
* @param file The target file
* @returns Promise that is resolved if the upload is successful or error otherwise
*/
@@ -264,7 +232,7 @@ export class UploadService {
}
if (file.id) {
return this.nodesApi.updateNodeContent(file.id, <any> file.file, opts);
return this.nodesApi.updateNodeContent(file.id, file.file as any, opts);
} else {
const nodeBody = { ... file.options };
delete nodeBody['versioningEnabled'];
@@ -279,6 +247,21 @@ export class UploadService {
}
}
private getFilesToUpload(): FileModel[] {
const cached = Object.keys(this.cache);
const threadsCount = this.getThreadsCount();
if (cached.length >= threadsCount) {
return [];
}
const files = this.queue
.filter(toUpload => !cached.includes(toUpload.name) && toUpload.status === FileUploadStatus.Pending)
.slice(0, threadsCount);
return files;
}
private beginUpload(file: FileModel, successEmitter?: EventEmitter<any>, errorEmitter?: EventEmitter<any>): any {
const promise = this.getUploadPromise(file);
promise
@@ -444,4 +427,46 @@ export class UploadService {
file.progress.percent < MAX_CANCELLABLE_FILE_PERCENTAGE
);
}
private filterElement(file: FileModel) {
this.excludedFileList = this.appConfigService.get<string[]>('files.excluded');
this.excludedFoldersList = this.appConfigService.get<string[]>('folders.excluded');
let isAllowed = true;
if (this.excludedFileList) {
this.matchingOptions = this.appConfigService.get('files.match-options');
isAllowed = this.isFileNameAllowed(file);
}
if (isAllowed && this.excludedFoldersList) {
this.folderMatchingOptions = this.appConfigService.get('folders.match-options');
isAllowed = this.isParentFolderAllowed(file);
}
return isAllowed;
}
private isParentFolderAllowed(file: FileModel): boolean {
let isAllowed: boolean = true;
const currentFile: any = file.file;
const fileRelativePath = currentFile.webkitRelativePath ? currentFile.webkitRelativePath : file.options.path;
if (currentFile && fileRelativePath) {
isAllowed =
this.excludedFoldersList.filter((folderToExclude) => fileRelativePath
.split('/')
.some((pathElement) => {
const minimatch = new Minimatch(folderToExclude, this.folderMatchingOptions);
return minimatch.match(pathElement);
})).length === 0;
}
return isAllowed;
}
private isFileNameAllowed(file: FileModel): boolean {
return (
this.excludedFileList.filter((pattern) => {
const minimatch = new Minimatch(pattern, this.matchingOptions);
return minimatch.match(file.name);
}).length === 0
);
}
}