- validate all file-upload via clamdscan (clamav), throw ValidationException in case of an error

- add @types/clamscan and clamscan for node
- package clamav-daemon and clamav-frehshclam for docker
- add API Controller: HomeController.ts for /api/years and /api/sitelinks/{year}
 change root path of file storage from '/storage/app/public/files' to '/storage/app/public'
 - adapt dockerfile to use node:18-bookworm-slim

app/Controllers/Http/Api/DatasetController.ts

@@ -1,6 +1,7 @@
 import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext';
 // import Person from 'App/Models/Person';
 import Dataset from 'App/Models/Dataset';
+import { StatusCodes } from 'http-status-codes';
 
 // node ace make:controller Author
 export default class DatasetController {
@@ -12,4 +13,50 @@ export default class DatasetController {
 
         return datasets;
     }
+
+    public async findAll({ response }: HttpContextContract) {       
+        try {
+            const datasets = await Dataset.query()
+                .where('server_state', 'published')
+                .orWhere('server_state', 'deleted')
+                .preload('descriptions') // Preload any relationships you need
+                .orderBy('server_date_published');
+            return response.status(StatusCodes.OK).json(datasets);
+        } catch (error) {
+            return response.status(500).json({
+                message: error.message || 'Some error occurred while retrieving datasets.',
+            });
+        }
+    }
+
+    public async findOne({ params }: HttpContextContract) {
+        const datasets = await Dataset.query()
+            .where('publish_id', params.publish_id)
+            .preload('titles')
+            .preload('descriptions')
+            .preload('user')
+            .preload('authors', (builder) => {
+                builder.orderBy('pivot_sort_order', 'asc');
+            })
+            .preload('contributors', (builder) => {
+                builder.orderBy('pivot_sort_order', 'asc');
+            })
+            .preload('subjects')
+            .preload('coverage')
+            .preload('licenses')
+            .preload('references')
+            .preload('project')
+            .preload('referenced_by', (builder) => {
+                builder.preload('dataset', (builder) => {
+                    builder.preload('identifier');
+                });
+            })
+            .preload('files', (builder) => {
+                builder.preload('hashvalues');
+            })
+            .preload('identifier')
+            .firstOrFail();
+
+        return datasets;
+    }
 }

app/Controllers/Http/Api/HomeController.ts

@@ -0,0 +1,64 @@
+import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext';
+import Database from '@ioc:Adonis/Lucid/Database';
+import { StatusCodes } from 'http-status-codes';
+
+export default class HomeController {
+    public async findDocumentsPerYear({ response, params }: HttpContextContract) {
+        const year = params.year;
+        const from = parseInt(year);
+        const serverState = 'published';
+        try {
+            // Database.raw(`date_part('year', server_date_published) as pub_year`)
+            // const datasets = await Dataset.query()
+            //     .select(['id', 'publish_id', 'server_date_published', ])
+            //     .where('server_state', serverState)
+            //     .andWhereRaw(`date_part('year', server_date_published) = ?`, [from])
+            //     .preload('titles')
+            //     .preload('authors')
+            //     .orderBy('server_date_published');
+
+            const datasets = await Database.from('documents as doc')
+                .select([                   
+                    'publish_id',
+                    'server_date_published',
+                    Database.raw(`date_part('year', server_date_published) as pub_year`)
+                ],
+                    // Database
+                    //   .raw('select "ip_address" from "user_logins" where "users.id" = "user_logins.user_id" limit 1')
+                    //   .wrap('(', ')')
+                )
+                .where('server_state', serverState)
+                .innerJoin('link_documents_persons as ba', 'doc.id', 'ba.document_id')
+                .andWhereRaw(`date_part('year', server_date_published) = ?`, [from])
+                .orderBy('server_date_published');
+
+            return response.json(datasets);
+        } catch (error) {
+            return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
+                message: error.message || 'Some error occurred while retrieving datasets.',
+            });
+        }
+    }
+
+    public async findYears({ response }: HttpContextContract) {
+        const serverState = 'published';
+        // Use raw SQL queries to select all cars which belongs to the user
+        try {
+            const datasets = await Database.rawQuery(
+                'SELECT distinct EXTRACT(YEAR FROM server_date_published) as published_date FROM gba.documents WHERE server_state = ?',
+                [serverState],
+            );
+
+            // Pluck the ids of the cars
+            const years = datasets.rows.map((dataset) => dataset.published_date);
+            // check if the cars is returned
+            // if (years.length > 0) {
+            return response.status(StatusCodes.OK).json(years);
+            // }
+        } catch (error) {
+            return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
+                message: 'An error occurred while retrieving the list of publication years from the Tethys repository.',
+            });
+        }
+    }
+}

app/Controllers/Http/Submitter/DatasetController.ts

@@ -20,6 +20,10 @@ import CreateDatasetValidator from 'App/Validators/CreateDatasetValidator';
 import { TitleTypes, DescriptionTypes, ContributorTypes, PersonNameTypes, ReferenceIdentifierTypes, RelationTypes } from 'Contracts/enums';
 import type { ModelQueryBuilderContract } from '@ioc:Adonis/Lucid/Orm';
 import DatasetReference from 'App/Models/DatasetReference';
+import { cuid } from '@ioc:Adonis/Core/Helpers';
+import File from 'App/Models/File';
+import ClamScan from 'clamscan';
+import { ValidationException } from '@ioc:Adonis/Core/Validator';
 
 export default class DatasetController {
     public async index({ auth, request, inertia }: HttpContextContract) {
@@ -50,12 +54,11 @@ export default class DatasetController {
         }
 
         // const results = await Database
-        // .query()  
+        // .query()
         // .select(Database.raw("CONCAT('https://doi.org/', b.value) AS concatenated_value"))
         // .from('documents as doc')
         // .innerJoin('dataset_identifiers as b', 'doc.id', 'b.dataset_id')
         // .groupBy('a.id').toQuery();
-      
 
         // const users = await User.query().orderBy('login').paginate(page, limit);
         const myDatasets = await datasets
@@ -91,7 +94,7 @@ export default class DatasetController {
     }
 
     public async create({ inertia }: HttpContextContract) {
-        const licenses = await License.query().select('id', 'name_long').pluck('name_long', 'id');
+        const licenses = await License.query().select('id', 'name_long').where('active', 'true').pluck('name_long', 'id');
 
         const projects = await Project.query().pluck('label', 'id');
 
@@ -103,6 +106,7 @@ export default class DatasetController {
             gis: 'GIS',
             models: 'Models',
             mixedtype: 'Mixed Type',
+            vocabulary: 'Vocabulary'
         };
 
         // const titletypes = {
@@ -313,33 +317,10 @@ export default class DatasetController {
                 await trx.rollback();
             }
             console.error('Failed to create dataset and related models:', error);
-
-            // Handle the error and exit the controller code accordingly
-            // session.flash('message', 'Failed to create dataset and relaed models');
-            // return response.redirect().back();
+            // throw new ValidationException(true, { 'upload error': `failed to create dataset and related models. ${error}` });
             throw error;
         }
 
-        // save data files:
-        const coverImage = request.files('files')[0];
-        if (coverImage) {
-            // clientName: 'Gehaltsschema.png'
-            // extname: 'png'
-            // fieldName: 'file'
-            // size: 135624
-
-            // await coverImage.moveToDisk('./')
-            await coverImage.moveToDisk(
-                '/test_dataset2',
-                {
-                    name: 'renamed-file-name.jpg',
-                    overwrite: true, // overwrite in case of conflict
-                },
-                'local',
-            );
-            // let path = coverImage.filePath;
-        }
-
         session.flash('message', 'Dataset has been created successfully');
         // return response.redirect().toRoute('user.index');
         return response.redirect().back();
@@ -424,6 +405,80 @@ export default class DatasetController {
             // await coverage.dataset().associate(dataset).save();
             // await coverage.useTransaction(trx).related('dataset').associate(dataset);
         }
+
+        // save data files
+        const uploadedFiles = request.files('files');
+        for (const [index, file] of uploadedFiles.entries()) {
+            try {
+                await this.scanFileForViruses(file.tmpPath); //, 'gitea.lan', 3310);
+                // await this.scanFileForViruses("/tmp/testfile.txt");
+            } catch (error) {
+                // If the file is infected or there's an error scanning the file, throw a validation exception
+                throw error;
+            }
+            // clientName: 'Gehaltsschema.png'
+            // extname: 'png'
+            // fieldName: 'file'
+            const fileName = `file-${cuid()}.${file.extname}`;
+            const mimeType = file.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
+            const datasetFolder = `files/${dataset.id}`;
+            // const size = file.size;
+            await file.moveToDisk(
+                datasetFolder,
+                {
+                    name: fileName,
+                    overwrite: true, // overwrite in case of conflict
+                },
+                'local',
+            );
+            // save file metadata into db
+            const newFile = new File();
+            newFile.pathName = `${datasetFolder}/${fileName}`;
+            newFile.fileSize = file.size;
+            newFile.mimeType = mimeType;
+            newFile.label = file.clientName;
+            newFile.sortOrder = index;
+            newFile.visibleInFrontdoor = true;
+            newFile.visibleInOai = true;
+            // let path = coverImage.filePath;
+            await dataset.useTransaction(trx).related('files').save(newFile);
+            // await newFile.createHashValues();
+        }
+    }
+
+    private async scanFileForViruses(filePath, host?: string, port?: number): Promise<void> {
+        // const clamscan = await (new ClamScan().init());
+        const opts: ClamScan.Options = {
+            removeInfected: true, // If true, removes infected files
+            debugMode: false, // Whether or not to log info/debug/error msgs to the console
+            scanRecursively: true, // If true, deep scan folders recursively
+            clamdscan: {
+                active: true, // If true, this module will consider using the clamdscan binary
+                host,
+                port,
+                multiscan: true, // Scan using all available cores! Yay!
+            },
+            preference: 'clamdscan', // If clamdscan is found and active, it will be used by default
+        };
+
+        return new Promise(async (resolve, reject) => {
+            try {
+                const clamscan = await new ClamScan().init(opts);
+                // You can re-use the `clamscan` object as many times as you want
+                // const version = await clamscan.getVersion();
+                // console.log(`ClamAV Version: ${version}`);
+                const { file, isInfected, viruses } = await clamscan.isInfected(filePath);
+                if (isInfected) {
+                    console.log(`${file} is infected with ${viruses}!`);
+                    reject(new ValidationException(true, { 'upload error': `File ${file} is infected!` }));
+                } else {
+                    resolve();
+                }
+            } catch (error) {
+                // If there's an error scanning the file, throw a validation exception
+                reject(new ValidationException(true, { 'upload error': `${error.message}` }));
+            }
+        });
     }
 
     private async savePersons(dataset: Dataset, persons: any[], role: string, trx: TransactionClientContract) {
@@ -459,8 +514,8 @@ export default class DatasetController {
         'required': '{{ field }} is required',
         'unique': '{{ field }} must be unique, and this value is already taken',
         // 'confirmed': '{{ field }} is not correct',
-        'licences.minLength': 'at least {{ options.minLength }} permission must be defined',
-        'licences.*.number': 'Define roles as valid numbers',
+        'licenses.minLength': 'at least {{ options.minLength }} permission must be defined',
+        'licenses.*.number': 'Define roles as valid numbers',
         'rights.equalTo': 'you must agree to continue',
 
         'titles.0.value.minLength': 'Main Title  must be at least {{ options.minLength }} characters long',

app/Models/Dataset.ts

@@ -22,9 +22,10 @@ import License from './License';
 import Subject from './Subject';
 import File from './File';
 import Coverage from './Coverage';
-import DatasetReference from './DatasetReference'; 
+import DatasetReference from './DatasetReference';
 import Collection from './Collection';
 import DatasetIdentifier from './DatasetIdentifier';
+import Project from './Project';
 
 export default class Dataset extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();
@@ -53,6 +54,12 @@ export default class Dataset extends BaseModel {
     @column({})
     public language: string;
 
+    @column({})
+    public publish_id: number | null = null;
+
+    @column({})
+    public project_id: number | null = null;
+
     @column({})
     public account_id: number | null = null;
 
@@ -62,7 +69,6 @@ export default class Dataset extends BaseModel {
     @column({})
     public reviewer_id: number | null = null;
 
-    
     @column({})
     public reject_editor_note: string | null;
 
@@ -107,6 +113,11 @@ export default class Dataset extends BaseModel {
     })
     public user: BelongsTo<typeof User>;
 
+    @belongsTo(() => Project, {
+        foreignKey: 'project_id',
+    })
+    public project: BelongsTo<typeof Project>;
+
     @hasMany(() => Title, {
         foreignKey: 'document_id',
     })
@@ -146,11 +157,15 @@ export default class Dataset extends BaseModel {
     })
     public references: HasMany<typeof DatasetReference>;
 
-    // public function collections()
-    // {
-    //     return $this
-    //         ->belongsToMany(Collection::class, 'link_documents_collections', 'document_id', 'collection_id');
-    // }
+    // Dataset.hasMany(Reference, {
+    //     foreignKey: "related_document_id",
+    //     as: "referenced_by",
+    // });
+    @hasMany(() => DatasetReference, {
+        foreignKey: 'related_document_id',
+    })
+    public referenced_by: HasMany<typeof DatasetReference>;
+
     @manyToMany(() => Collection, {
         pivotForeignKey: 'document_id',
         pivotRelatedForeignKey: 'collection_id',
@@ -159,11 +174,10 @@ export default class Dataset extends BaseModel {
     public collections: ManyToMany<typeof Collection>;
 
     @hasOne(() => DatasetIdentifier, {
-        foreignKey: 'document_id',
+        foreignKey: 'dataset_id',
     })
     public identifier: HasOne<typeof DatasetIdentifier>;
 
-
     @computed({
         serializeAs: 'main_title',
     })
@@ -172,4 +186,26 @@ export default class Dataset extends BaseModel {
         const mainTitle = this.titles?.find((title) => title.type === 'Main');
         return mainTitle ? mainTitle.value : null;
     }
+
+    @manyToMany(() => Person, {
+        pivotForeignKey: 'document_id',
+        pivotRelatedForeignKey: 'person_id',
+        pivotTable: 'link_documents_persons',
+        pivotColumns: ['role', 'sort_order', 'allow_email_contact'],
+        onQuery(query) {
+            query.wherePivot('role', 'author');
+        },
+    })
+    public authors: ManyToMany<typeof Person>;
+   
+    @manyToMany(() => Person, {
+        pivotForeignKey: 'document_id',
+        pivotRelatedForeignKey: 'person_id',
+        pivotTable: 'link_documents_persons',
+        pivotColumns: ['role', 'sort_order', 'allow_email_contact'],
+        onQuery(query) {
+            query.wherePivot('role', 'contributor');
+        },
+    })
+    public contributors: ManyToMany<typeof Person>;
 }

app/Models/DatasetIdentifier.ts

@@ -14,7 +14,7 @@ export default class DatasetIdentifier extends BaseModel {
     public id: number;
 
     @column({})
-    public document_id: number; 
+    public dataset_id: number; 
 
     @column({})
     public type: string;
@@ -34,7 +34,7 @@ export default class DatasetIdentifier extends BaseModel {
     public updated_at?: DateTime;
 
     @belongsTo(() => Dataset, {
-        foreignKey: 'document_id',
+        foreignKey: 'dataset_id',
     })
     public dataset: BelongsTo<typeof Dataset>;
 }

app/Models/DatasetReference.ts

@@ -46,4 +46,16 @@ export default class DatasetReference extends BaseModel {
         foreignKey: 'document_id',
     })
     public dataset: BelongsTo<typeof Dataset>;
+
+    // Reference.belongsTo(Dataset, {
+    //     foreignKey: "related_document_id",
+    //     as: "new_dataset",
+    //     include: "identifier"
+    // });
+    @belongsTo(() => Dataset, {
+        foreignKey: 'related_document_id',
+    })
+    public new_dataset: BelongsTo<typeof Dataset>;
+
+
 }

app/Models/File.ts

@@ -37,17 +37,20 @@ export default class File extends BaseModel {
     public comment: string;
 
     @column()
-    public mimetype: string;
+    public mimeType: string;
 
     @column()
     public language: string;
 
     @column()
-    public fileSize: bigint;
+    public fileSize: number;
 
     @column()
     public visibleInOai: boolean;
 
+    @column()
+    public visibleInFrontdoor: boolean;
+
     @column()
     public sortOrder: number;
 

app/Validators/CreateDatasetValidator.ts

@@ -129,8 +129,8 @@ export default class CreateDatasetValidator {
         'required': '{{ field }} is required',
         'unique': '{{ field }} must be unique, and this value is already taken',
         // 'confirmed': '{{ field }} is not correct',
-        'licences.minLength': 'at least {{ options.minLength }} permission must be defined',
-        'licences.*.number': 'Define roles as valid numbers',
+        'licenses.minLength': 'at least {{ options.minLength }} permission must be defined',
+        'licenses.*.number': 'Define roles as valid numbers',
         'rights.equalTo': 'you must agree to continue',
 
         'titles.0.value.minLength': 'Main Title  must be at least {{ options.minLength }} characters long',