hotfix(dataset): enhance file upload and update functionality

- Added file upload functionality to the dataset update form.
- Implemented file size validation and aggregated upload limit.
- Added temporary file storage and cleanup to handle large file uploads.
- Added a clear button to the file upload component.
- Added the ability to sort files in the file upload component.
- Fixed an issue where the file upload component was not correctly updating the model value.
- Updated the dataset edit form to use the new file upload component.
- Added the ability to sort files in the file upload component.
- Added a global declaration for the `sort_order` property on the `File` interface.
- Added helper functions for byte size parsing, configuration retrieval, and temporary file path generation.

app/Controllers/Http/Submitter/DatasetController.ts

@@ -45,11 +45,7 @@ import { pipeline } from 'node:stream/promises';
 import { createWriteStream } from 'node:fs';
 import type { Multipart } from '@adonisjs/bodyparser';
 import * as fs from 'fs';
-import { join, isAbsolute } from 'node:path';
+import { parseBytesSize, getConfigFor, getTmpPath, formatBytes } from '#app/utils/utility-functions';
-import type { BodyParserConfig } from '#models/types';
-import { createId } from '@paralleldrive/cuid2';
-import { tmpdir } from 'node:os';
-import config from '@adonisjs/core/services/config';
 
 interface Dictionary {
     [index: string]: string;
@@ -60,7 +56,7 @@ export default class DatasetController {
     /**
      * Bodyparser config
      */
-    config: BodyParserConfig = config.get('bodyparser');
+    // config: BodyParserConfig = config.get('bodyparser');
 
     public async index({ auth, request, inertia }: HttpContext) {
         const user = (await User.find(auth.user?.id)) as User;
@@ -272,6 +268,7 @@ export default class DatasetController {
         }
         return response.redirect().back();
     }
+
     public async thirdStep({ request, response }: HttpContext) {
         const newDatasetSchema = vine.object({
             // first step
@@ -420,60 +417,23 @@ export default class DatasetController {
         return response.redirect().back();
     }
 
-    /**
-     * Returns the tmp path for storing the files temporarly
-     */
-    private getTmpPath(config: BodyParserConfig['multipart']): string {
-        if (typeof config.tmpFileName === 'function') {
-            const tmpPath = config.tmpFileName();
-            return isAbsolute(tmpPath) ? tmpPath : join(tmpdir(), tmpPath);
-        }
-
-        return join(tmpdir(), createId());
-    }
-    /**
-     * Returns config for a given type
-     */
-    private getConfigFor<K extends keyof BodyParserConfig>(type: K): BodyParserConfig[K] {
-        const config = this.config[type];
-        return config;
-    }
-
-    private parseBytesSize(size: string): number {
-        const units = {
-            kb: 1024,
-            mb: 1024 * 1024,
-            gb: 1024 * 1024 * 1024,
-            tb: 1024 * 1024 * 1024 * 1024,
-        };
-
-        const match = size.match(/^(\d+)(kb|mb|gb|tb)$/i); // Regex to match size format
-
-        if (!match) {
-            throw new Error('Invalid size format');
-        }
-
-        const [, value, unit] = match;
-        return parseInt(value) * units[unit.toLowerCase()];
-    }
-
     public async store({ auth, request, response, session }: HttpContext) {
         // At the top of the store() method, declare an array to hold temporary file paths
         const uploadedTmpFiles: string[] = [];
         // Aggregated limit example (adjust as needed)
-        const multipartConfig = this.getConfigFor('multipart');
+        const multipartConfig = getConfigFor('multipart');
-        const aggregatedLimit = multipartConfig.limit ? this.parseBytesSize(multipartConfig.limit) : 100 * 1024 * 1024;
+        const aggregatedLimit = multipartConfig.limit ? parseBytesSize(multipartConfig.limit) : 100 * 1024 * 1024;
         // const aggregatedLimit = 200 * 1024 * 1024;
         let totalUploadedSize = 0;
 
-        // Helper function to format bytes as human-readable text
+        // // Helper function to format bytes as human-readable text
-        function formatBytes(bytes: number): string {
+        // function formatBytes(bytes: number): string {
-            if (bytes === 0) return '0 Bytes';
+        //     if (bytes === 0) return '0 Bytes';
-            const k = 1024;
+        //     const k = 1024;
-            const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+        //     const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
-            const i = Math.floor(Math.log(bytes) / Math.log(k));
+        //     const i = Math.floor(Math.log(bytes) / Math.log(k));
-            return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+        //     return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
-        }
+        // }
         // const enabledExtensions = await this.getEnabledExtensions();
         const multipart: Multipart = request.multipart;
 
@@ -529,7 +489,7 @@ export default class DatasetController {
 
                 // part.file.sortOrder = part.file.sortOrder;
 
-                const tmpPath = this.getTmpPath(multipartConfig);
+                const tmpPath = getTmpPath(multipartConfig);
                 (part.file as any).tmpPath = tmpPath;
 
                 const writeStream = createWriteStream(tmpPath);
@@ -1054,20 +1014,82 @@ export default class DatasetController {
     }
 
     public async update({ request, response, session }: HttpContext) {
-        try {
+        // Get the dataset id from the route parameter
-            // await request.validate(UpdateDatasetValidator);
+        const datasetId = request.param('id');
-            await request.validateUsing(updateDatasetValidator);
+        // Retrieve the dataset and load its existing files
-        } catch (error) {
+        const dataset = await Dataset.findOrFail(datasetId);
-            // - Handle errors
+        await dataset.load('files');
-            // return response.badRequest(error.messages);
+        // Accumulate the size of the already related files
-            throw error;
+        const preExistingFileSize = dataset.files.reduce((acc, file) => acc + file.fileSize, 0);
-            // return response.badRequest(error.messages);
-        }
-        // await request.validate(UpdateDatasetValidator);
-        const id = request.param('id');
 
+        const uploadedTmpFiles: string[] = [];
+        // Only process multipart if the request has a multipart content type
+        const contentType = request.request.headers['content-type'] || '';
+        if (contentType.includes('multipart/form-data')) {
+            const multipart: Multipart = request.multipart;
+            // Aggregated limit example (adjust as needed)
+            const multipartConfig = getConfigFor('multipart');
+            const aggregatedLimit = multipartConfig.limit ? parseBytesSize(multipartConfig.limit) : 100 * 1024 * 1024;
+            // Initialize totalUploadedSize with the size of existing files
+            let totalUploadedSize = preExistingFileSize;
+
+            multipart.onFile('files', { deferValidations: true }, async (part) => {
+                let fileUploadedSize = 0;
+
+                part.on('data', (chunk) => {
+                    fileUploadedSize += chunk.length;
+                });
+
+                part.on('end', () => {
+                    totalUploadedSize += fileUploadedSize;
+                    part.file.size = fileUploadedSize;
+                    if (part.file.tmpPath) {
+                        uploadedTmpFiles.push(part.file.tmpPath);
+                    }
+                    if (totalUploadedSize > aggregatedLimit) {
+                        uploadedTmpFiles.forEach((tmpPath) => {
+                            try {
+                                fs.unlinkSync(tmpPath);
+                            } catch (cleanupError) {
+                                console.error('Error cleaning up temporary file:', cleanupError);
+                            }
+                        });
+                        const error = new errors.E_VALIDATION_ERROR({
+                            'upload error': `Aggregated upload limit of ${formatBytes(aggregatedLimit)} exceeded. The total size of files being uploaded would exceed the limit.`,
+                        });
+                        request.multipart.abort(error);
+                    }
+                });
+
+                part.on('error', (error) => {
+                    request.multipart.abort(error);
+                });
+
+                try {
+                    const fileNameWithoutParams = part.file.clientName.split('?')[0];
+                    const ext = path.extname(fileNameWithoutParams).replace('.', '');
+                    part.file.extname = ext;
+                    const tmpPath = getTmpPath(multipartConfig);
+                    (part.file as any).tmpPath = tmpPath;
+                    const writeStream = createWriteStream(tmpPath);
+                    await pipeline(part, writeStream);
+                } catch (error) {
+                    request.multipart.abort(new errors.E_VALIDATION_ERROR({ 'upload error': error.message }));
+                }
+            });
+
+            try {
+                await multipart.process();
+            } catch (error) {
+                session.flash('errors', error.messages);
+                return response.redirect().back();
+            }
+        }
+
+        const id = request.param('id');
         let trx: TransactionClientContract | null = null;
         try {
+            await request.validateUsing(updateDatasetValidator);
             trx = await db.transaction();
             // const user = (await User.find(auth.user?.id)) as User;
             // await this.createDatasetAndAssociations(user, request, trx);
@@ -1175,9 +1197,9 @@ export default class DatasetController {
             // handle new uploaded files:
             const uploadedFiles: MultipartFile[] = request.files('files');
             if (Array.isArray(uploadedFiles) && uploadedFiles.length > 0) {
-                for (const [index, fileData] of uploadedFiles.entries()) {
+                for (const [index, file] of uploadedFiles.entries()) {
                     try {
-                        await this.scanFileForViruses(fileData.tmpPath); //, 'gitea.lan', 3310);
+                        await this.scanFileForViruses(file.tmpPath); //, 'gitea.lan', 3310);
                         // await this.scanFileForViruses("/tmp/testfile.txt");
                     } catch (error) {
                         // If the file is infected or there's an error scanning the file, throw a validation exception
@@ -1185,29 +1207,29 @@ export default class DatasetController {
                     }
 
                     // move to disk:
-                    const fileName = `file-${cuid()}.${fileData.extname}`; //'file-ls0jyb8xbzqtrclufu2z2e0c.pdf'
+                    const fileName = this.generateFilename(file.extname as string);
                     const datasetFolder = `files/${dataset.id}`; // 'files/307'
                     const datasetFullPath = path.join(`${datasetFolder}`, fileName);
-                    // await fileData.moveToDisk(datasetFolder, { name: fileName, overwrite: true }, 'local');
+                    // await file.moveToDisk(datasetFolder, { name: fileName, overwrite: true }, 'local');
-                    // await fileData.move(drive.makePath(datasetFolder), {
+                    // await file.move(drive.makePath(datasetFolder), {
                     //     name: fileName,
                     //     overwrite: true, // overwrite in case of conflict
                     // });
-                    await fileData.moveToDisk(datasetFullPath, 'local', {
+                    await file.moveToDisk(datasetFullPath, 'local', {
                         name: fileName,
                         overwrite: true, // overwrite in case of conflict
                         disk: 'local',
                     });
 
                     //save to db:
-                    const { clientFileName, sortOrder } = this.extractVariableNameAndSortOrder(fileData.clientName);
+                    const { clientFileName, sortOrder } = this.extractVariableNameAndSortOrder(file.clientName);
-                    const mimeType = fileData.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
+                    const mimeType = file.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
                     const newFile = await dataset
                         .useTransaction(trx)
                         .related('files')
                         .create({
                             pathName: `${datasetFolder}/${fileName}`,
-                            fileSize: fileData.size,
+                            fileSize: file.size,
                             mimeType,
                             label: clientFileName,
                             sortOrder: sortOrder || index,
@@ -1253,10 +1275,18 @@ export default class DatasetController {
             // return response.redirect().toRoute('user.index');
             return response.redirect().toRoute('dataset.edit', [dataset.id]);
         } catch (error) {
+            // Clean up temporary files if validation or later steps fail
+            uploadedTmpFiles.forEach((tmpPath) => {
+                try {
+                    fs.unlinkSync(tmpPath);
+                } catch (cleanupError) {
+                    console.error('Error cleaning up temporary file:', cleanupError);
+                }
+            });
             if (trx !== null) {
                 await trx.rollback();
             }
-            console.error('Failed to create dataset and related models:', error);
+            console.error('Failed to update dataset and related models:', error);
             // throw new ValidationException(true, { 'upload error': `failed to create dataset and related models. ${error}` });
             throw error;
         }

app/utils/utility-functions.ts

@@ -1,3 +1,9 @@
+import { join, isAbsolute } from 'node:path';
+import type { BodyParserConfig } from '#models/types';
+import { createId } from '@paralleldrive/cuid2';
+import { tmpdir } from 'node:os';
+import config from '@adonisjs/core/services/config';
+
 export function sum(a: number, b: number): number {
     return a + b;
 }
@@ -24,3 +30,51 @@ export function preg_match(regex: RegExp, str: string) {
     const result: boolean = regex.test(str);
     return result;
 }
+
+/**
+ * Returns the tmp path for storing the files temporarly
+ */
+export function getTmpPath(config: BodyParserConfig['multipart']): string {
+    if (typeof config.tmpFileName === 'function') {
+        const tmpPath = config.tmpFileName();
+        return isAbsolute(tmpPath) ? tmpPath : join(tmpdir(), tmpPath);
+    }
+
+    return join(tmpdir(), createId());
+}
+/**
+ * Returns config for a given type
+ */
+export function getConfigFor<K extends keyof BodyParserConfig>(type: K): BodyParserConfig[K] {
+    const bodyParserConfig: BodyParserConfig = config.get('bodyparser');
+    const configType = bodyParserConfig[type];
+    return configType;
+}
+
+export function parseBytesSize(size: string): number {
+    const units: Record<string, number> = {
+        kb: 1024,
+        mb: 1024 * 1024,
+        gb: 1024 * 1024 * 1024,
+        tb: 1024 * 1024 * 1024 * 1024,
+    };
+
+    const match = size.match(/^(\d+)(kb|mb|gb|tb)$/i); // Regex to match size format
+
+    if (!match) {
+        throw new Error('Invalid size format');
+    }
+
+    const [, value, unit] = match;
+    return parseInt(value) * units[unit.toLowerCase()];
+}
+
+// Helper function to format bytes as human-readable text
+
+export function formatBytes(bytes: number): string {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+}

config/bodyparser.ts

@@ -128,7 +128,7 @@ allowedMethods: ['POST', 'PUT', 'PATCH', 'DELETE'],
 | projects/:id/file
 | ```
 */
-        processManually: ['/submitter/dataset/submit'],
+        processManually: ['/submitter/dataset/submit', '/submitter/dataset/:id/update'],
 
         /*
 |--------------------------------------------------------------------------

resources/js/Components/FileUpload.vue

@@ -42,7 +42,8 @@
                             <span class="font-semibold">Click to upload</span> or drag and drop
                         </p>
                     </div>
-                    <input id="dropzone-file" type="file" class="hidden" @click="showSpinner" @change="onChangeFile" @cancel="cancelSpinner" multiple="true" />
+                    <input id="dropzone-file" type="file" class="hidden" @click="showSpinner" @change="onChangeFile"
+                        @cancel="cancelSpinner" multiple="true" />
                 </label>
             </header>
 
@@ -190,7 +191,7 @@
 
         <!-- sticky footer -->
         <footer class="flex justify-end px-8 pb-8 pt-4">
-            <button id="cancel"
+            <button v-if="showClearButton" id="cancel"
                 class="ml-3 rounded-sm px-3 py-1 hover:bg-gray-300 focus:shadow-outline focus:outline-none"
                 @click="clearAllFiles">
                 Clear
@@ -267,6 +268,12 @@ class FileUploadComponent extends Vue {
     })
     filesToDelete: Array<TethysFile>;
 
+    @Prop({
+        type: Boolean,
+        default: true,
+    })
+    showClearButton: boolean;
+
     // // deletetFiles: Array<TethysFile> = [];
     get deletetFiles(): Array<TethysFile> {
         return this.filesToDelete;
@@ -451,8 +458,10 @@ class FileUploadComponent extends Vue {
 
     public clearAllFiles(event: Event) {
         event.preventDefault();
+        if (this.showClearButton == true) {
             this.items.splice(0);
         }
+    }
 
     public removeFile(key: number) {
         // Check if the key is within the bounds of the items array

resources/js/Pages/Submitter/Dataset/Edit.vue

@@ -447,7 +447,7 @@
                     </select> -->
                 </div>
 
-                <FileUploadComponent v-model:files="form.files" v-model:filesToDelete="form.filesToDelete"></FileUploadComponent>
+                <FileUploadComponent v-model:files="form.files" v-model:filesToDelete="form.filesToDelete" :showClearButton="false"></FileUploadComponent>
 
                 <div class="text-red-400 text-sm" v-if="form.errors['file'] && Array.isArray(form.errors['files'])">
                     {{ form.errors['files'].join(', ') }}
@@ -700,7 +700,7 @@ const submit = async (): Promise<void> => {
                 // const file = new File([obj.blob], `${obj.label}?sortOrder=${obj.sort_order}`, options);
                 // const metadata = JSON.stringify({ sort_order: obj.sort_order });
                 // const metadataBlob = new Blob([metadata + '\n'], { type: 'application/json' });
-                const file = new File([obj.blob], `${obj.label}`, options,);
+                const file = new File([obj.blob], `${obj.label}?sortorder=${obj.sort_order}`, options,);
             
                 // const file = new File([obj.blob], `${obj.label}`, options);