barmar/tethys.backend
1
0
Fork 0
forked from geolba/tethys.backend
Code Pull requests Activity

- validate all file-upload via clamdscan (clamav), throw ValidationException in case of an error

Browse source 
- add @types/clamscan and clamscan for node
- package clamav-daemon and clamav-frehshclam for docker
- add API Controller: HomeController.ts for /api/years and /api/sitelinks/{year}
 change root path of file storage from '/storage/app/public/files' to '/storage/app/public'
 - adapt dockerfile to use node:18-bookworm-slim
This commit is contained in:
Kaimbacher 2023-09-04 13:24:58 +02:00
parent 5f8fe1c16d
commit b6b1c90ff8
20 changed files with 941 additions and 278 deletions
Download patch file Download diff file Expand all files Collapse all files

113
app/Controllers/Http/Submitter/DatasetController.ts
View file

@ -20,6 +20,10 @@ import CreateDatasetValidator from 'App/Validators/CreateDatasetValidator';
import { TitleTypes, DescriptionTypes, ContributorTypes, PersonNameTypes, ReferenceIdentifierTypes, RelationTypes } from 'Contracts/enums';
import type { ModelQueryBuilderContract } from '@ioc:Adonis/Lucid/Orm';
import DatasetReference from 'App/Models/DatasetReference';
import { cuid } from '@ioc:Adonis/Core/Helpers';
import File from 'App/Models/File';
import ClamScan from 'clamscan';
import { ValidationException } from '@ioc:Adonis/Core/Validator';
export default class DatasetController {
public async index({ auth, request, inertia }: HttpContextContract) {
@ -50,12 +54,11 @@ export default class DatasetController {
}
// const results = await Database
// .query()
// .query()
// .select(Database.raw("CONCAT('https://doi.org/', b.value) AS concatenated_value"))
// .from('documents as doc')
// .innerJoin('dataset_identifiers as b', 'doc.id', 'b.dataset_id')
// .groupBy('a.id').toQuery();
// const users = await User.query().orderBy('login').paginate(page, limit);
const myDatasets = await datasets
@ -91,7 +94,7 @@ export default class DatasetController {
}
public async create({ inertia }: HttpContextContract) {
const licenses = await License.query().select('id', 'name_long').pluck('name_long', 'id');
const licenses = await License.query().select('id', 'name_long').where('active', 'true').pluck('name_long', 'id');
const projects = await Project.query().pluck('label', 'id');
@ -103,6 +106,7 @@ export default class DatasetController {
gis: 'GIS',
models: 'Models',
mixedtype: 'Mixed Type',
vocabulary: 'Vocabulary'
};
// const titletypes = {
@ -313,33 +317,10 @@ export default class DatasetController {
await trx.rollback();
}
console.error('Failed to create dataset and related models:', error);
// Handle the error and exit the controller code accordingly
// session.flash('message', 'Failed to create dataset and relaed models');
// return response.redirect().back();
// throw new ValidationException(true, { 'upload error': `failed to create dataset and related models. ${error}` });
throw error;
}
// save data files:
const coverImage = request.files('files')[0];
if (coverImage) {
// clientName: 'Gehaltsschema.png'
// extname: 'png'
// fieldName: 'file'
// size: 135624
// await coverImage.moveToDisk('./')
await coverImage.moveToDisk(
'/test_dataset2',
{
name: 'renamed-file-name.jpg',
overwrite: true, // overwrite in case of conflict
},
'local',
);
// let path = coverImage.filePath;
}
session.flash('message', 'Dataset has been created successfully');
// return response.redirect().toRoute('user.index');
return response.redirect().back();
@ -424,6 +405,80 @@ export default class DatasetController {
// await coverage.dataset().associate(dataset).save();
// await coverage.useTransaction(trx).related('dataset').associate(dataset);
}
// save data files
const uploadedFiles = request.files('files');
for (const [index, file] of uploadedFiles.entries()) {
try {
await this.scanFileForViruses(file.tmpPath); //, 'gitea.lan', 3310);
// await this.scanFileForViruses("/tmp/testfile.txt");
} catch (error) {
// If the file is infected or there's an error scanning the file, throw a validation exception
throw error;
}
// clientName: 'Gehaltsschema.png'
// extname: 'png'
// fieldName: 'file'
const fileName = `file-${cuid()}.${file.extname}`;
const mimeType = file.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
const datasetFolder = `files/${dataset.id}`;
// const size = file.size;
await file.moveToDisk(
datasetFolder,
{
name: fileName,
overwrite: true, // overwrite in case of conflict
},
'local',
);
// save file metadata into db
const newFile = new File();
newFile.pathName = `${datasetFolder}/${fileName}`;
newFile.fileSize = file.size;
newFile.mimeType = mimeType;
newFile.label = file.clientName;
newFile.sortOrder = index;
newFile.visibleInFrontdoor = true;
newFile.visibleInOai = true;
// let path = coverImage.filePath;
await dataset.useTransaction(trx).related('files').save(newFile);
// await newFile.createHashValues();
}
}
private async scanFileForViruses(filePath, host?: string, port?: number): Promise<void> {
// const clamscan = await (new ClamScan().init());
const opts: ClamScan.Options = {
removeInfected: true, // If true, removes infected files
debugMode: false, // Whether or not to log info/debug/error msgs to the console
scanRecursively: true, // If true, deep scan folders recursively
clamdscan: {
active: true, // If true, this module will consider using the clamdscan binary
host,
port,
multiscan: true, // Scan using all available cores! Yay!
},
preference: 'clamdscan', // If clamdscan is found and active, it will be used by default
};
return new Promise(async (resolve, reject) => {
try {
const clamscan = await new ClamScan().init(opts);
// You can re-use the `clamscan` object as many times as you want
// const version = await clamscan.getVersion();
// console.log(`ClamAV Version: ${version}`);
const { file, isInfected, viruses } = await clamscan.isInfected(filePath);
if (isInfected) {
console.log(`${file} is infected with ${viruses}!`);
reject(new ValidationException(true, { 'upload error': `File ${file} is infected!` }));
} else {
resolve();
}
} catch (error) {
// If there's an error scanning the file, throw a validation exception
reject(new ValidationException(true, { 'upload error': `${error.message}` }));
}
});
}
private async savePersons(dataset: Dataset, persons: any[], role: string, trx: TransactionClientContract) {
@ -459,8 +514,8 @@ export default class DatasetController {
'required': '{{ field }} is required',
'unique': '{{ field }} must be unique, and this value is already taken',
// 'confirmed': '{{ field }} is not correct',
'licences.minLength': 'at least {{ options.minLength }} permission must be defined',
'licences.*.number': 'Define roles as valid numbers',
'licenses.minLength': 'at least {{ options.minLength }} permission must be defined',
'licenses.*.number': 'Define roles as valid numbers',
'rights.equalTo': 'you must agree to continue',
'titles.0.value.minLength': 'Main Title must be at least {{ options.minLength }} characters long',