hotfix(dataset): enhance file upload and update functionality
- Added file upload functionality to the dataset update form. - Implemented file size validation and aggregated upload limit. - Added temporary file storage and cleanup to handle large file uploads. - Added a clear button to the file upload component. - Added the ability to sort files in the file upload component. - Fixed an issue where the file upload component was not correctly updating the model value. - Updated the dataset edit form to use the new file upload component. - Added the ability to sort files in the file upload component. - Added a global declaration for the `sort_order` property on the `File` interface. - Added helper functions for byte size parsing, configuration retrieval, and temporary file path generation.
This commit is contained in:
parent
8fbda9fc64
commit
10d159a57a
5 changed files with 177 additions and 84 deletions
|
|
@ -45,11 +45,7 @@ import { pipeline } from 'node:stream/promises';
|
|||
import { createWriteStream } from 'node:fs';
|
||||
import type { Multipart } from '@adonisjs/bodyparser';
|
||||
import * as fs from 'fs';
|
||||
import { join, isAbsolute } from 'node:path';
|
||||
import type { BodyParserConfig } from '#models/types';
|
||||
import { createId } from '@paralleldrive/cuid2';
|
||||
import { tmpdir } from 'node:os';
|
||||
import config from '@adonisjs/core/services/config';
|
||||
import { parseBytesSize, getConfigFor, getTmpPath, formatBytes } from '#app/utils/utility-functions';
|
||||
|
||||
interface Dictionary {
|
||||
[index: string]: string;
|
||||
|
|
@ -60,7 +56,7 @@ export default class DatasetController {
|
|||
/**
|
||||
* Bodyparser config
|
||||
*/
|
||||
config: BodyParserConfig = config.get('bodyparser');
|
||||
// config: BodyParserConfig = config.get('bodyparser');
|
||||
|
||||
public async index({ auth, request, inertia }: HttpContext) {
|
||||
const user = (await User.find(auth.user?.id)) as User;
|
||||
|
|
@ -272,6 +268,7 @@ export default class DatasetController {
|
|||
}
|
||||
return response.redirect().back();
|
||||
}
|
||||
|
||||
public async thirdStep({ request, response }: HttpContext) {
|
||||
const newDatasetSchema = vine.object({
|
||||
// first step
|
||||
|
|
@ -420,60 +417,23 @@ export default class DatasetController {
|
|||
return response.redirect().back();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the tmp path for storing the files temporarly
|
||||
*/
|
||||
private getTmpPath(config: BodyParserConfig['multipart']): string {
|
||||
if (typeof config.tmpFileName === 'function') {
|
||||
const tmpPath = config.tmpFileName();
|
||||
return isAbsolute(tmpPath) ? tmpPath : join(tmpdir(), tmpPath);
|
||||
}
|
||||
|
||||
return join(tmpdir(), createId());
|
||||
}
|
||||
/**
|
||||
* Returns config for a given type
|
||||
*/
|
||||
private getConfigFor<K extends keyof BodyParserConfig>(type: K): BodyParserConfig[K] {
|
||||
const config = this.config[type];
|
||||
return config;
|
||||
}
|
||||
|
||||
private parseBytesSize(size: string): number {
|
||||
const units = {
|
||||
kb: 1024,
|
||||
mb: 1024 * 1024,
|
||||
gb: 1024 * 1024 * 1024,
|
||||
tb: 1024 * 1024 * 1024 * 1024,
|
||||
};
|
||||
|
||||
const match = size.match(/^(\d+)(kb|mb|gb|tb)$/i); // Regex to match size format
|
||||
|
||||
if (!match) {
|
||||
throw new Error('Invalid size format');
|
||||
}
|
||||
|
||||
const [, value, unit] = match;
|
||||
return parseInt(value) * units[unit.toLowerCase()];
|
||||
}
|
||||
|
||||
public async store({ auth, request, response, session }: HttpContext) {
|
||||
// At the top of the store() method, declare an array to hold temporary file paths
|
||||
const uploadedTmpFiles: string[] = [];
|
||||
// Aggregated limit example (adjust as needed)
|
||||
const multipartConfig = this.getConfigFor('multipart');
|
||||
const aggregatedLimit = multipartConfig.limit ? this.parseBytesSize(multipartConfig.limit) : 100 * 1024 * 1024;
|
||||
const multipartConfig = getConfigFor('multipart');
|
||||
const aggregatedLimit = multipartConfig.limit ? parseBytesSize(multipartConfig.limit) : 100 * 1024 * 1024;
|
||||
// const aggregatedLimit = 200 * 1024 * 1024;
|
||||
let totalUploadedSize = 0;
|
||||
|
||||
// Helper function to format bytes as human-readable text
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
// // Helper function to format bytes as human-readable text
|
||||
// function formatBytes(bytes: number): string {
|
||||
// if (bytes === 0) return '0 Bytes';
|
||||
// const k = 1024;
|
||||
// const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
// const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
// return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
// }
|
||||
// const enabledExtensions = await this.getEnabledExtensions();
|
||||
const multipart: Multipart = request.multipart;
|
||||
|
||||
|
|
@ -529,7 +489,7 @@ export default class DatasetController {
|
|||
|
||||
// part.file.sortOrder = part.file.sortOrder;
|
||||
|
||||
const tmpPath = this.getTmpPath(multipartConfig);
|
||||
const tmpPath = getTmpPath(multipartConfig);
|
||||
(part.file as any).tmpPath = tmpPath;
|
||||
|
||||
const writeStream = createWriteStream(tmpPath);
|
||||
|
|
@ -1054,20 +1014,82 @@ export default class DatasetController {
|
|||
}
|
||||
|
||||
public async update({ request, response, session }: HttpContext) {
|
||||
try {
|
||||
// await request.validate(UpdateDatasetValidator);
|
||||
await request.validateUsing(updateDatasetValidator);
|
||||
} catch (error) {
|
||||
// - Handle errors
|
||||
// return response.badRequest(error.messages);
|
||||
throw error;
|
||||
// return response.badRequest(error.messages);
|
||||
}
|
||||
// await request.validate(UpdateDatasetValidator);
|
||||
const id = request.param('id');
|
||||
// Get the dataset id from the route parameter
|
||||
const datasetId = request.param('id');
|
||||
// Retrieve the dataset and load its existing files
|
||||
const dataset = await Dataset.findOrFail(datasetId);
|
||||
await dataset.load('files');
|
||||
// Accumulate the size of the already related files
|
||||
const preExistingFileSize = dataset.files.reduce((acc, file) => acc + file.fileSize, 0);
|
||||
|
||||
const uploadedTmpFiles: string[] = [];
|
||||
// Only process multipart if the request has a multipart content type
|
||||
const contentType = request.request.headers['content-type'] || '';
|
||||
if (contentType.includes('multipart/form-data')) {
|
||||
const multipart: Multipart = request.multipart;
|
||||
// Aggregated limit example (adjust as needed)
|
||||
const multipartConfig = getConfigFor('multipart');
|
||||
const aggregatedLimit = multipartConfig.limit ? parseBytesSize(multipartConfig.limit) : 100 * 1024 * 1024;
|
||||
// Initialize totalUploadedSize with the size of existing files
|
||||
let totalUploadedSize = preExistingFileSize;
|
||||
|
||||
multipart.onFile('files', { deferValidations: true }, async (part) => {
|
||||
let fileUploadedSize = 0;
|
||||
|
||||
part.on('data', (chunk) => {
|
||||
fileUploadedSize += chunk.length;
|
||||
});
|
||||
|
||||
part.on('end', () => {
|
||||
totalUploadedSize += fileUploadedSize;
|
||||
part.file.size = fileUploadedSize;
|
||||
if (part.file.tmpPath) {
|
||||
uploadedTmpFiles.push(part.file.tmpPath);
|
||||
}
|
||||
if (totalUploadedSize > aggregatedLimit) {
|
||||
uploadedTmpFiles.forEach((tmpPath) => {
|
||||
try {
|
||||
fs.unlinkSync(tmpPath);
|
||||
} catch (cleanupError) {
|
||||
console.error('Error cleaning up temporary file:', cleanupError);
|
||||
}
|
||||
});
|
||||
const error = new errors.E_VALIDATION_ERROR({
|
||||
'upload error': `Aggregated upload limit of ${formatBytes(aggregatedLimit)} exceeded. The total size of files being uploaded would exceed the limit.`,
|
||||
});
|
||||
request.multipart.abort(error);
|
||||
}
|
||||
});
|
||||
|
||||
part.on('error', (error) => {
|
||||
request.multipart.abort(error);
|
||||
});
|
||||
|
||||
try {
|
||||
const fileNameWithoutParams = part.file.clientName.split('?')[0];
|
||||
const ext = path.extname(fileNameWithoutParams).replace('.', '');
|
||||
part.file.extname = ext;
|
||||
const tmpPath = getTmpPath(multipartConfig);
|
||||
(part.file as any).tmpPath = tmpPath;
|
||||
const writeStream = createWriteStream(tmpPath);
|
||||
await pipeline(part, writeStream);
|
||||
} catch (error) {
|
||||
request.multipart.abort(new errors.E_VALIDATION_ERROR({ 'upload error': error.message }));
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
await multipart.process();
|
||||
} catch (error) {
|
||||
session.flash('errors', error.messages);
|
||||
return response.redirect().back();
|
||||
}
|
||||
}
|
||||
|
||||
const id = request.param('id');
|
||||
let trx: TransactionClientContract | null = null;
|
||||
try {
|
||||
await request.validateUsing(updateDatasetValidator);
|
||||
trx = await db.transaction();
|
||||
// const user = (await User.find(auth.user?.id)) as User;
|
||||
// await this.createDatasetAndAssociations(user, request, trx);
|
||||
|
|
@ -1175,9 +1197,9 @@ export default class DatasetController {
|
|||
// handle new uploaded files:
|
||||
const uploadedFiles: MultipartFile[] = request.files('files');
|
||||
if (Array.isArray(uploadedFiles) && uploadedFiles.length > 0) {
|
||||
for (const [index, fileData] of uploadedFiles.entries()) {
|
||||
for (const [index, file] of uploadedFiles.entries()) {
|
||||
try {
|
||||
await this.scanFileForViruses(fileData.tmpPath); //, 'gitea.lan', 3310);
|
||||
await this.scanFileForViruses(file.tmpPath); //, 'gitea.lan', 3310);
|
||||
// await this.scanFileForViruses("/tmp/testfile.txt");
|
||||
} catch (error) {
|
||||
// If the file is infected or there's an error scanning the file, throw a validation exception
|
||||
|
|
@ -1185,29 +1207,29 @@ export default class DatasetController {
|
|||
}
|
||||
|
||||
// move to disk:
|
||||
const fileName = `file-${cuid()}.${fileData.extname}`; //'file-ls0jyb8xbzqtrclufu2z2e0c.pdf'
|
||||
const fileName = this.generateFilename(file.extname as string);
|
||||
const datasetFolder = `files/${dataset.id}`; // 'files/307'
|
||||
const datasetFullPath = path.join(`${datasetFolder}`, fileName);
|
||||
// await fileData.moveToDisk(datasetFolder, { name: fileName, overwrite: true }, 'local');
|
||||
// await fileData.move(drive.makePath(datasetFolder), {
|
||||
// await file.moveToDisk(datasetFolder, { name: fileName, overwrite: true }, 'local');
|
||||
// await file.move(drive.makePath(datasetFolder), {
|
||||
// name: fileName,
|
||||
// overwrite: true, // overwrite in case of conflict
|
||||
// });
|
||||
await fileData.moveToDisk(datasetFullPath, 'local', {
|
||||
await file.moveToDisk(datasetFullPath, 'local', {
|
||||
name: fileName,
|
||||
overwrite: true, // overwrite in case of conflict
|
||||
disk: 'local',
|
||||
});
|
||||
|
||||
//save to db:
|
||||
const { clientFileName, sortOrder } = this.extractVariableNameAndSortOrder(fileData.clientName);
|
||||
const mimeType = fileData.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
|
||||
const { clientFileName, sortOrder } = this.extractVariableNameAndSortOrder(file.clientName);
|
||||
const mimeType = file.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
|
||||
const newFile = await dataset
|
||||
.useTransaction(trx)
|
||||
.related('files')
|
||||
.create({
|
||||
pathName: `${datasetFolder}/${fileName}`,
|
||||
fileSize: fileData.size,
|
||||
fileSize: file.size,
|
||||
mimeType,
|
||||
label: clientFileName,
|
||||
sortOrder: sortOrder || index,
|
||||
|
|
@ -1253,10 +1275,18 @@ export default class DatasetController {
|
|||
// return response.redirect().toRoute('user.index');
|
||||
return response.redirect().toRoute('dataset.edit', [dataset.id]);
|
||||
} catch (error) {
|
||||
// Clean up temporary files if validation or later steps fail
|
||||
uploadedTmpFiles.forEach((tmpPath) => {
|
||||
try {
|
||||
fs.unlinkSync(tmpPath);
|
||||
} catch (cleanupError) {
|
||||
console.error('Error cleaning up temporary file:', cleanupError);
|
||||
}
|
||||
});
|
||||
if (trx !== null) {
|
||||
await trx.rollback();
|
||||
}
|
||||
console.error('Failed to create dataset and related models:', error);
|
||||
console.error('Failed to update dataset and related models:', error);
|
||||
// throw new ValidationException(true, { 'upload error': `failed to create dataset and related models. ${error}` });
|
||||
throw error;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
import { join, isAbsolute } from 'node:path';
|
||||
import type { BodyParserConfig } from '#models/types';
|
||||
import { createId } from '@paralleldrive/cuid2';
|
||||
import { tmpdir } from 'node:os';
|
||||
import config from '@adonisjs/core/services/config';
|
||||
|
||||
export function sum(a: number, b: number): number {
|
||||
return a + b;
|
||||
}
|
||||
|
|
@ -24,3 +30,51 @@ export function preg_match(regex: RegExp, str: string) {
|
|||
const result: boolean = regex.test(str);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the tmp path for storing the files temporarly
|
||||
*/
|
||||
export function getTmpPath(config: BodyParserConfig['multipart']): string {
|
||||
if (typeof config.tmpFileName === 'function') {
|
||||
const tmpPath = config.tmpFileName();
|
||||
return isAbsolute(tmpPath) ? tmpPath : join(tmpdir(), tmpPath);
|
||||
}
|
||||
|
||||
return join(tmpdir(), createId());
|
||||
}
|
||||
/**
|
||||
* Returns config for a given type
|
||||
*/
|
||||
export function getConfigFor<K extends keyof BodyParserConfig>(type: K): BodyParserConfig[K] {
|
||||
const bodyParserConfig: BodyParserConfig = config.get('bodyparser');
|
||||
const configType = bodyParserConfig[type];
|
||||
return configType;
|
||||
}
|
||||
|
||||
export function parseBytesSize(size: string): number {
|
||||
const units: Record<string, number> = {
|
||||
kb: 1024,
|
||||
mb: 1024 * 1024,
|
||||
gb: 1024 * 1024 * 1024,
|
||||
tb: 1024 * 1024 * 1024 * 1024,
|
||||
};
|
||||
|
||||
const match = size.match(/^(\d+)(kb|mb|gb|tb)$/i); // Regex to match size format
|
||||
|
||||
if (!match) {
|
||||
throw new Error('Invalid size format');
|
||||
}
|
||||
|
||||
const [, value, unit] = match;
|
||||
return parseInt(value) * units[unit.toLowerCase()];
|
||||
}
|
||||
|
||||
// Helper function to format bytes as human-readable text
|
||||
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ allowedMethods: ['POST', 'PUT', 'PATCH', 'DELETE'],
|
|||
| projects/:id/file
|
||||
| ```
|
||||
*/
|
||||
processManually: ['/submitter/dataset/submit'],
|
||||
processManually: ['/submitter/dataset/submit', '/submitter/dataset/:id/update'],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -42,7 +42,8 @@
|
|||
<span class="font-semibold">Click to upload</span> or drag and drop
|
||||
</p>
|
||||
</div>
|
||||
<input id="dropzone-file" type="file" class="hidden" @click="showSpinner" @change="onChangeFile" @cancel="cancelSpinner" multiple="true" />
|
||||
<input id="dropzone-file" type="file" class="hidden" @click="showSpinner" @change="onChangeFile"
|
||||
@cancel="cancelSpinner" multiple="true" />
|
||||
</label>
|
||||
</header>
|
||||
|
||||
|
|
@ -190,7 +191,7 @@
|
|||
|
||||
<!-- sticky footer -->
|
||||
<footer class="flex justify-end px-8 pb-8 pt-4">
|
||||
<button id="cancel"
|
||||
<button v-if="showClearButton" id="cancel"
|
||||
class="ml-3 rounded-sm px-3 py-1 hover:bg-gray-300 focus:shadow-outline focus:outline-none"
|
||||
@click="clearAllFiles">
|
||||
Clear
|
||||
|
|
@ -267,6 +268,12 @@ class FileUploadComponent extends Vue {
|
|||
})
|
||||
filesToDelete: Array<TethysFile>;
|
||||
|
||||
@Prop({
|
||||
type: Boolean,
|
||||
default: true,
|
||||
})
|
||||
showClearButton: boolean;
|
||||
|
||||
// // deletetFiles: Array<TethysFile> = [];
|
||||
get deletetFiles(): Array<TethysFile> {
|
||||
return this.filesToDelete;
|
||||
|
|
@ -451,8 +458,10 @@ class FileUploadComponent extends Vue {
|
|||
|
||||
public clearAllFiles(event: Event) {
|
||||
event.preventDefault();
|
||||
if (this.showClearButton == true) {
|
||||
this.items.splice(0);
|
||||
}
|
||||
}
|
||||
|
||||
public removeFile(key: number) {
|
||||
// Check if the key is within the bounds of the items array
|
||||
|
|
|
|||
|
|
@ -447,7 +447,7 @@
|
|||
</select> -->
|
||||
</div>
|
||||
|
||||
<FileUploadComponent v-model:files="form.files" v-model:filesToDelete="form.filesToDelete"></FileUploadComponent>
|
||||
<FileUploadComponent v-model:files="form.files" v-model:filesToDelete="form.filesToDelete" :showClearButton="false"></FileUploadComponent>
|
||||
|
||||
<div class="text-red-400 text-sm" v-if="form.errors['file'] && Array.isArray(form.errors['files'])">
|
||||
{{ form.errors['files'].join(', ') }}
|
||||
|
|
@ -700,7 +700,7 @@ const submit = async (): Promise<void> => {
|
|||
// const file = new File([obj.blob], `${obj.label}?sortOrder=${obj.sort_order}`, options);
|
||||
// const metadata = JSON.stringify({ sort_order: obj.sort_order });
|
||||
// const metadataBlob = new Blob([metadata + '\n'], { type: 'application/json' });
|
||||
const file = new File([obj.blob], `${obj.label}`, options,);
|
||||
const file = new File([obj.blob], `${obj.label}?sortorder=${obj.sort_order}`, options,);
|
||||
|
||||
// const file = new File([obj.blob], `${obj.label}`, options);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue