- addes @adonisjs/redis fo saving session into redis with redis.ts contract and config

- npm updated
- added createHashValues and dlete inside File.ts
- added dataset_count property inside Subject.ts
- corrected rotes.ts with correct permissions

app/Controllers/Http/Submitter/DatasetController.ts

@@ -34,6 +34,7 @@ import ClamScan from 'clamscan';
 import { ValidationException } from '@ioc:Adonis/Core/Validator';
 import Drive from '@ioc:Adonis/Core/Drive';
 import { Exception } from '@adonisjs/core/build/standalone';
+import { MultipartFileContract } from '@ioc:Adonis/Core/BodyParser';
 
 export default class DatasetController {
     public async index({ auth, request, inertia }: HttpContextContract) {
@@ -335,8 +336,8 @@ export default class DatasetController {
         }
 
         session.flash('message', 'Dataset has been created successfully');
-        // return response.redirect().toRoute('user.index');
-        return response.redirect().back();
+        return response.redirect().toRoute('user.index');
+        // return response.redirect().back();
     }
 
     private async createDatasetAndAssociations(user: User, request: HttpContextContract['request'], trx: TransactionClientContract) {
@@ -691,7 +692,10 @@ export default class DatasetController {
             .preload('licenses')
             .preload('authors')
             .preload('contributors')
-            .preload('subjects')
+            // .preload('subjects')
+            .preload('subjects', (builder) => {
+                builder.orderBy('id', 'asc').withCount('datasets');
+            })
             .preload('references')
             .preload('files');
 
@@ -779,6 +783,7 @@ export default class DatasetController {
             throw error;
             // return response.badRequest(error.messages);
         }
+        // await request.validate(UpdateDatasetValidator);
         const id = request.param('id');
 
         let trx: TransactionClientContract | null = null;
@@ -843,6 +848,25 @@ export default class DatasetController {
                 }
             }
 
+            // await dataset.useTransaction(trx).related('subjects').sync([]);
+            const keywords = request.input('subjects');
+            for (const keywordData of keywords) {
+                if (keywordData.id) {
+                    const subject = await Subject.findOrFail(keywordData.id);
+                    // await dataset.useTransaction(trx).related('subjects').attach([keywordData.id]);
+                    subject.value = keywordData.value;
+                    subject.type = keywordData.type;
+                    subject.external_key = keywordData.external_key;
+                    if (subject.$isDirty) {
+                        await subject.save();
+                    }
+                } else {
+                    const keyword = new Subject();
+                    keyword.fill(keywordData);
+                    await dataset.useTransaction(trx).related('subjects').save(keyword, false);
+                }
+            }
+
             // Save already existing files
             const files = request.input('fileInputs', []);
             for (const fileData of files) {
@@ -857,43 +881,57 @@ export default class DatasetController {
             }
 
             // handle new uploaded files:
-            const uploadedFiles = request.files('files');
+            const uploadedFiles: MultipartFileContract[] = request.files('files');
             if (Array.isArray(uploadedFiles) && uploadedFiles.length > 0) {
-                // let index = 1;
-                // for (const key in files) {
-                //     const formFile = files[key]
-                // for (const fileData of files) {
                 for (const [index, fileData] of uploadedFiles.entries()) {
-                    // const uploads = request.file('uploads');
-                    // const fileIndex = formFile.file;
-                    // const file = uploads[fileIndex];
-
                     const fileName = `file-${cuid()}.${fileData.extname}`;
-                    const mimeType = fileData.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
                     const datasetFolder = `files/${dataset.id}`;
-                    await fileData.moveToDisk(
-                        datasetFolder,
-                        {
-                            name: fileName,
-                            overwrite: true, // overwrite in case of conflict
-                        },
-                        'local',
-                    );
-                    // save file metadata into db
-                    const newFile = new File();
-                    newFile.pathName = `${datasetFolder}/${fileName}`;
-                    newFile.fileSize = fileData.size;
-                    newFile.mimeType = mimeType;
-                    newFile.label = fileData.clientName;
-                    newFile.sortOrder = index;
-                    newFile.visibleInFrontdoor = true;
-                    newFile.visibleInOai = true;
+
+                    await fileData.moveToDisk(datasetFolder, { name: fileName, overwrite: true }, 'local');
                     // let path = coverImage.filePath;
-                    await dataset.useTransaction(trx).related('files').save(newFile);
-                    await newFile.createHashValues();
+
+                    const { clientFileName, sortOrder } = this.extractVariableNameAndSortOrder(fileData.clientName);
+                    const mimeType = fileData.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
+                    // save file metadata into db
+                    // const newFile = new File();
+                    // newFile.pathName = `${datasetFolder}/${fileName}`;
+                    // newFile.fileSize = fileData.size;
+                    // newFile.mimeType = mimeType;
+                    // newFile.label = clientFileName;
+                    // newFile.sortOrder = sortOrder ? sortOrder : index;
+                    // newFile.visibleInFrontdoor = true;
+                    // newFile.visibleInOai = true;
+
+                    const newFile = await dataset
+                        .useTransaction(trx)
+                        .related('files')
+                        .create({
+                            pathName: `${datasetFolder}/${fileName}`,
+                            fileSize: fileData.size,
+                            mimeType,
+                            label: clientFileName,
+                            sortOrder: sortOrder || index,
+                            visibleInFrontdoor: true,
+                            visibleInOai: true,
+                        });
+
+                    // save many related HashValue Instances to the file:
+                    await newFile.createHashValues(trx);
                 }
             }
 
+            // save collection
+            // const collection: Collection | null = await Collection.query().where('id', 21).first();
+            // collection && (await dataset.useTransaction(trx).related('collections').attach([collection.id]));
+
+            // // Save coverage
+            // if (data.coverage && !this.containsOnlyNull(data.coverage)) {
+            //     const formCoverage = request.input('coverage');
+            //     const coverage = await dataset.related('coverage').updateOrCreate({ dataset_id: dataset.id }, formCoverage);
+            // } else if (data.coverage && this.containsOnlyNull(data.coverage) && !dataset.coverage) {
+            //     await dataset.coverage().delete();
+            // }
+
             const input = request.only(['project_id', 'embargo_date', 'language', 'type', 'creating_corporation']);
             // dataset.type = request.input('type');
             dataset.merge(input);
@@ -911,11 +949,30 @@ export default class DatasetController {
             throw error;
         }
 
-        session.flash('message', 'Dataset has been created successfully');
+        session.flash('message', 'Dataset has been updated successfully');
         // return response.redirect().toRoute('user.index');
         return response.redirect().back();
     }
 
+    private extractVariableNameAndSortOrder(inputString: string): { clientFileName: string; sortOrder?: number } {
+        const regex = /^([^?]+)(?:\?([^=]+)=([^&]+))?/;
+        const match = inputString.match(regex);
+
+        if (match) {
+            const clientFileName = match[1];
+
+            const param = match[2];
+            let sortOrder;
+            if (param && param.toLowerCase() === 'sortorder') {
+                sortOrder = parseInt(match[3], 10);
+            }
+
+            return { clientFileName, sortOrder };
+        } else {
+            return { clientFileName: '', sortOrder: undefined }; // Or handle as needed for no match
+        }
+    }
+
     public async delete({ request, inertia, response, session }) {
         const id = request.param('id');
         try {
@@ -923,8 +980,8 @@ export default class DatasetController {
                 .preload('user', (builder) => {
                     builder.select('id', 'login');
                 })
-                .preload('files')
                 .where('id', id)
+                .preload('files')
                 .firstOrFail();
             const validStates = ['inprogress', 'rejected_editor'];
             if (!validStates.includes(dataset.server_state)) {
@@ -958,21 +1015,27 @@ export default class DatasetController {
             if (validStates.includes(dataset.server_state)) {
                 if (dataset.files && dataset.files.length > 0) {
                     for (const file of dataset.files) {
-                        if (file.pathName) {
-                            // delete file from filesystem
-                            await Drive.delete(file.pathName);
-                        }
+                        // overwriten delete method also delets file on filespace
+                        await file.delete();
                     }
                 }
-                // delete dataset wirh relation from db
-                await dataset.delete();
-                session.flash({ message: 'You have deleted 1 dataset!' });
-                return response.redirect().toRoute('dataset.list');
-            } else {
-                session.flash({
-                    warning: `You cannot delete this dataset! The status of this dataset is "${dataset.server_state}"!`,
-                });
-                return response.redirect().back();
+                const datasetFolder = `files/${params.id}`;
+                const folderExists = await Drive.exists(datasetFolder);
+                if (folderExists) {
+                    const folderContents = await Drive.list(datasetFolder).toArray();
+                    if (folderContents.length === 0) {
+                        await Drive.delete(datasetFolder);
+                    }
+                    // delete dataset wirh relation from db
+                    await dataset.delete();
+                    session.flash({ message: 'You have deleted 1 dataset!' });
+                    return response.redirect().toRoute('dataset.list');
+                } else {
+                    session.flash({
+                        warning: `You cannot delete this dataset! Invalid server_state: "${dataset.server_state}"!`,
+                    });
+                    return response.status(400).redirect().back();
+                }
             }
         } catch (error) {
             if (error instanceof ValidationException) {

app/Models/Dataset.ts

@@ -203,7 +203,7 @@ export default class Dataset extends DatasetExtension {
         pivotForeignKey: 'document_id',
         pivotRelatedForeignKey: 'person_id',
         pivotTable: 'link_documents_persons',
-        pivotColumns: ['role', 'sort_order', 'allow_email_contact'],
+        pivotColumns: ['role', 'sort_order', 'allow_email_contact', 'contributor_type'],
         onQuery(query) {
             query.wherePivot('role', 'contributor');
         },

app/Models/File.ts

@@ -7,6 +7,7 @@ import BaseModel from './BaseModel';
 import * as fs from 'fs';
 import crypto from 'crypto';
 import { TransactionClientContract } from '@ioc:Adonis/Lucid/Database';
+import Drive from '@ioc:Adonis/Core/Drive';
 
 export default class File extends BaseModel {
     // private readonly _data: Uint8Array;
@@ -116,21 +117,20 @@ export default class File extends BaseModel {
         serializeAs: 'fileData',
     })
     public get fileData(): string {
-        // return this.fileData;
-        // const fileData = fs.readFileSync(path.resolve(__dirname, this.filePath));
-        // const fileData = fs.readFileSync(this.filePath);
-        const fileContent: Buffer = fs.readFileSync(this.filePath);
-        // Create a Blob from the file content
-        // const blob = new Blob([fileContent], { type: this.type }); // Adjust
-        // let fileSrc = URL.createObjectURL(blob);
-        // return fileSrc;
+        try {
+            const fileContent: Buffer = fs.readFileSync(this.filePath);
+            // Create a Blob from the file content
+            // const blob = new Blob([fileContent], { type: this.type }); // Adjust
+            // let fileSrc = URL.createObjectURL(blob);
+            // return fileSrc;
 
-        // return Buffer.from(fileContent);
-        // get the buffer from somewhere
-        // const buff = fs.readFileSync('./test.bin');
-        // create a JSON string that contains the data in the property "blob"
-        const json = JSON.stringify({ blob: fileContent.toString('base64') });
-        return json;
+            // create a JSON string that contains the data in the property "blob"
+            const json = JSON.stringify({ blob: fileContent.toString('base64') });
+            return json;
+        } catch (err) {
+            // console.error(`Error reading file: ${err}`);
+            return '';
+        }
     }
 
     public async createHashValues(trx?: TransactionClientContract) {
@@ -139,7 +139,7 @@ export default class File extends BaseModel {
         for (const type of hashtypes) {
             const hash = new HashValue();
             hash.type = type;
-            const hashString = await this.checksumFile(this.filePath, type); // Assuming getRealHash is a method in the same model
+            const hashString = await this._checksumFile(this.filePath, type); // Assuming getRealHash is a method in the same model
             hash.value = hashString;
 
             // https://github.com/adonisjs/core/discussions/1872#discussioncomment-132289
@@ -152,7 +152,17 @@ export default class File extends BaseModel {
         }
     }
 
-    private async checksumFile(path, hashName = 'md5'): Promise<string> {
+    public async delete() {
+        if (this.pathName) {
+            // Delete file from additional storage
+            await Drive.delete(this.pathName);
+        }
+
+        // Call the original delete method of the BaseModel to remove the record from the database
+        await super.delete();
+    }
+
+    private async _checksumFile(path, hashName = 'md5'): Promise<string> {
         return new Promise((resolve, reject) => {
             const hash = crypto.createHash(hashName);
             const stream = fs.createReadStream(path);

app/Models/Person.ts

@@ -69,6 +69,12 @@ export default class Person extends BaseModel {
         return stock;
     }
 
+    @computed()
+    public get pivot_contributor_type() {
+        const contributor_type = this.$extras.pivot_contributor_type; //my pivot column name was "stock"
+        return contributor_type;
+    }
+
     @manyToMany(() => Dataset, {
         pivotForeignKey: 'person_id',
         pivotRelatedForeignKey: 'document_id',

app/Models/Subject.ts

@@ -1,4 +1,4 @@
-import { column, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, manyToMany, ManyToMany, computed} from '@ioc:Adonis/Lucid/Orm';
 import BaseModel from './BaseModel';
 
 import { DateTime } from 'luxon';
@@ -44,28 +44,33 @@ export default class Subject extends BaseModel {
     })
     public updated_at: DateTime;
 
-    @beforeCreate()
-    @beforeUpdate()
-    public static async resetDate(role) {
-        role.created_at = this.formatDateTime(role.created_at);
-        role.updated_at = this.formatDateTime(role.updated_at);
-    }
+    // @beforeCreate()
+    // @beforeUpdate()
+    // public static async resetDate(role) {
+    //     role.created_at = this.formatDateTime(role.created_at);
+    //     role.updated_at = this.formatDateTime(role.updated_at);
+    // }
 
-    private static formatDateTime(datetime) {
-        let value = new Date(datetime);
-        return datetime
-            ? value.getFullYear() +
-                  '-' +
-                  (value.getMonth() + 1) +
-                  '-' +
-                  value.getDate() +
-                  ' ' +
-                  value.getHours() +
-                  ':' +
-                  value.getMinutes() +
-                  ':' +
-                  value.getSeconds()
-            : datetime;
+    // private static formatDateTime(datetime) {
+    //     let value = new Date(datetime);
+    //     return datetime
+    //         ? value.getFullYear() +
+    //               '-' +
+    //               (value.getMonth() + 1) +
+    //               '-' +
+    //               value.getDate() +
+    //               ' ' +
+    //               value.getHours() +
+    //               ':' +
+    //               value.getMinutes() +
+    //               ':' +
+    //               value.getSeconds()
+    //         : datetime;
+    // }
+    @computed()
+    public get dataset_count() : number{
+        const count = this.$extras.datasets_count; //my pivot column name was "stock"
+        return count;
     }
 
     @manyToMany(() => Dataset, {

app/Validators/CreateDatasetValidator.ts

@@ -135,7 +135,7 @@ export default class CreateDatasetValidator {
         'required': '{{ field }} is required',
         'unique': '{{ field }} must be unique, and this value is already taken',
         // 'confirmed': '{{ field }} is not correct',
-        'licenses.minLength': 'at least {{ options.minLength }} permission must be defined',
+        'licenses.minLength': 'at least {{ options.minLength }} licenses must be defined',
         'licenses.*.number': 'Define licences as valid numbers',
         'rights.equalTo': 'you must agree to continue',
 

app/Validators/UpdateDatasetValidator.ts

@@ -136,7 +136,7 @@ export default class UpdateDatasetValidator {
         'required': '{{ field }} is required',
         'unique': '{{ field }} must be unique, and this value is already taken',
         // 'confirmed': '{{ field }} is not correct',
-        'licenses.minLength': 'at least {{ options.minLength }} permission must be defined',
+        'licenses.minLength': 'at least {{ options.minLength }} licenses must be defined',
         'licenses.*.number': 'Define licences as valid numbers',
         'rights.equalTo': 'you must agree to continue',