barmar/tethys.backend
1
0
Fork 0
forked from geolba/tethys.backend
Code Pull requests Activity

- validate all file-upload via clamdscan (clamav), throw ValidationException in case of an error

Browse source 
- add @types/clamscan and clamscan for node
- package clamav-daemon and clamav-frehshclam for docker
- add API Controller: HomeController.ts for /api/years and /api/sitelinks/{year}
 change root path of file storage from '/storage/app/public/files' to '/storage/app/public'
 - adapt dockerfile to use node:18-bookworm-slim
This commit is contained in:
Kaimbacher 2023-09-04 13:24:58 +02:00
parent 5f8fe1c16d
commit b6b1c90ff8
20 changed files with 941 additions and 278 deletions
Download patch file Download diff file Expand all files Collapse all files

47
app/Controllers/Http/Api/DatasetController.ts
View file

@ -1,6 +1,7 @@
import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext';
// import Person from 'App/Models/Person';
import Dataset from 'App/Models/Dataset';
import { StatusCodes } from 'http-status-codes';
// node ace make:controller Author
export default class DatasetController {
@ -12,4 +13,50 @@ export default class DatasetController {
return datasets;
}
public async findAll({ response }: HttpContextContract) {
try {
const datasets = await Dataset.query()
.where('server_state', 'published')
.orWhere('server_state', 'deleted')
.preload('descriptions') // Preload any relationships you need
.orderBy('server_date_published');
return response.status(StatusCodes.OK).json(datasets);
} catch (error) {
return response.status(500).json({
message: error.message || 'Some error occurred while retrieving datasets.',
});
}
}
public async findOne({ params }: HttpContextContract) {
const datasets = await Dataset.query()
.where('publish_id', params.publish_id)
.preload('titles')
.preload('descriptions')
.preload('user')
.preload('authors', (builder) => {
builder.orderBy('pivot_sort_order', 'asc');
})
.preload('contributors', (builder) => {
builder.orderBy('pivot_sort_order', 'asc');
})
.preload('subjects')
.preload('coverage')
.preload('licenses')
.preload('references')
.preload('project')
.preload('referenced_by', (builder) => {
builder.preload('dataset', (builder) => {
builder.preload('identifier');
});
})
.preload('files', (builder) => {
builder.preload('hashvalues');
})
.preload('identifier')
.firstOrFail();
return datasets;
}
}

64
app/Controllers/Http/Api/HomeController.ts Normal file
View file

@ -0,0 +1,64 @@
import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext';
import Database from '@ioc:Adonis/Lucid/Database';
import { StatusCodes } from 'http-status-codes';
export default class HomeController {
public async findDocumentsPerYear({ response, params }: HttpContextContract) {
const year = params.year;
const from = parseInt(year);
const serverState = 'published';
try {
// Database.raw(`date_part('year', server_date_published) as pub_year`)
// const datasets = await Dataset.query()
// .select(['id', 'publish_id', 'server_date_published', ])
// .where('server_state', serverState)
// .andWhereRaw(`date_part('year', server_date_published) = ?`, [from])
// .preload('titles')
// .preload('authors')
// .orderBy('server_date_published');
const datasets = await Database.from('documents as doc')
.select([
'publish_id',
'server_date_published',
Database.raw(`date_part('year', server_date_published) as pub_year`)
],
// Database
// .raw('select "ip_address" from "user_logins" where "users.id" = "user_logins.user_id" limit 1')
// .wrap('(', ')')
)
.where('server_state', serverState)
.innerJoin('link_documents_persons as ba', 'doc.id', 'ba.document_id')
.andWhereRaw(`date_part('year', server_date_published) = ?`, [from])
.orderBy('server_date_published');
return response.json(datasets);
} catch (error) {
return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
message: error.message || 'Some error occurred while retrieving datasets.',
});
}
}
public async findYears({ response }: HttpContextContract) {
const serverState = 'published';
// Use raw SQL queries to select all cars which belongs to the user
try {
const datasets = await Database.rawQuery(
'SELECT distinct EXTRACT(YEAR FROM server_date_published) as published_date FROM gba.documents WHERE server_state = ?',
[serverState],
);
// Pluck the ids of the cars
const years = datasets.rows.map((dataset) => dataset.published_date);
// check if the cars is returned
// if (years.length > 0) {
return response.status(StatusCodes.OK).json(years);
// }
} catch (error) {
return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
message: 'An error occurred while retrieving the list of publication years from the Tethys repository.',
});
}
}
}

113
app/Controllers/Http/Submitter/DatasetController.ts
View file

@ -20,6 +20,10 @@ import CreateDatasetValidator from 'App/Validators/CreateDatasetValidator';
import { TitleTypes, DescriptionTypes, ContributorTypes, PersonNameTypes, ReferenceIdentifierTypes, RelationTypes } from 'Contracts/enums';
import type { ModelQueryBuilderContract } from '@ioc:Adonis/Lucid/Orm';
import DatasetReference from 'App/Models/DatasetReference';
import { cuid } from '@ioc:Adonis/Core/Helpers';
import File from 'App/Models/File';
import ClamScan from 'clamscan';
import { ValidationException } from '@ioc:Adonis/Core/Validator';
export default class DatasetController {
public async index({ auth, request, inertia }: HttpContextContract) {
@ -50,12 +54,11 @@ export default class DatasetController {
}
// const results = await Database
// .query()
// .query()
// .select(Database.raw("CONCAT('https://doi.org/', b.value) AS concatenated_value"))
// .from('documents as doc')
// .innerJoin('dataset_identifiers as b', 'doc.id', 'b.dataset_id')
// .groupBy('a.id').toQuery();
// const users = await User.query().orderBy('login').paginate(page, limit);
const myDatasets = await datasets
@ -91,7 +94,7 @@ export default class DatasetController {
}
public async create({ inertia }: HttpContextContract) {
const licenses = await License.query().select('id', 'name_long').pluck('name_long', 'id');
const licenses = await License.query().select('id', 'name_long').where('active', 'true').pluck('name_long', 'id');
const projects = await Project.query().pluck('label', 'id');
@ -103,6 +106,7 @@ export default class DatasetController {
gis: 'GIS',
models: 'Models',
mixedtype: 'Mixed Type',
vocabulary: 'Vocabulary'
};
// const titletypes = {
@ -313,33 +317,10 @@ export default class DatasetController {
await trx.rollback();
}
console.error('Failed to create dataset and related models:', error);
// Handle the error and exit the controller code accordingly
// session.flash('message', 'Failed to create dataset and relaed models');
// return response.redirect().back();
// throw new ValidationException(true, { 'upload error': `failed to create dataset and related models. ${error}` });
throw error;
}
// save data files:
const coverImage = request.files('files')[0];
if (coverImage) {
// clientName: 'Gehaltsschema.png'
// extname: 'png'
// fieldName: 'file'
// size: 135624
// await coverImage.moveToDisk('./')
await coverImage.moveToDisk(
'/test_dataset2',
{
name: 'renamed-file-name.jpg',
overwrite: true, // overwrite in case of conflict
},
'local',
);
// let path = coverImage.filePath;
}
session.flash('message', 'Dataset has been created successfully');
// return response.redirect().toRoute('user.index');
return response.redirect().back();
@ -424,6 +405,80 @@ export default class DatasetController {
// await coverage.dataset().associate(dataset).save();
// await coverage.useTransaction(trx).related('dataset').associate(dataset);
}
// save data files
const uploadedFiles = request.files('files');
for (const [index, file] of uploadedFiles.entries()) {
try {
await this.scanFileForViruses(file.tmpPath); //, 'gitea.lan', 3310);
// await this.scanFileForViruses("/tmp/testfile.txt");
} catch (error) {
// If the file is infected or there's an error scanning the file, throw a validation exception
throw error;
}
// clientName: 'Gehaltsschema.png'
// extname: 'png'
// fieldName: 'file'
const fileName = `file-${cuid()}.${file.extname}`;
const mimeType = file.headers['content-type'] || 'application/octet-stream'; // Fallback to a default MIME type
const datasetFolder = `files/${dataset.id}`;
// const size = file.size;
await file.moveToDisk(
datasetFolder,
{
name: fileName,
overwrite: true, // overwrite in case of conflict
},
'local',
);
// save file metadata into db
const newFile = new File();
newFile.pathName = `${datasetFolder}/${fileName}`;
newFile.fileSize = file.size;
newFile.mimeType = mimeType;
newFile.label = file.clientName;
newFile.sortOrder = index;
newFile.visibleInFrontdoor = true;
newFile.visibleInOai = true;
// let path = coverImage.filePath;
await dataset.useTransaction(trx).related('files').save(newFile);
// await newFile.createHashValues();
}
}
private async scanFileForViruses(filePath, host?: string, port?: number): Promise<void> {
// const clamscan = await (new ClamScan().init());
const opts: ClamScan.Options = {
removeInfected: true, // If true, removes infected files
debugMode: false, // Whether or not to log info/debug/error msgs to the console
scanRecursively: true, // If true, deep scan folders recursively
clamdscan: {
active: true, // If true, this module will consider using the clamdscan binary
host,
port,
multiscan: true, // Scan using all available cores! Yay!
},
preference: 'clamdscan', // If clamdscan is found and active, it will be used by default
};
return new Promise(async (resolve, reject) => {
try {
const clamscan = await new ClamScan().init(opts);
// You can re-use the `clamscan` object as many times as you want
// const version = await clamscan.getVersion();
// console.log(`ClamAV Version: ${version}`);
const { file, isInfected, viruses } = await clamscan.isInfected(filePath);
if (isInfected) {
console.log(`${file} is infected with ${viruses}!`);
reject(new ValidationException(true, { 'upload error': `File ${file} is infected!` }));
} else {
resolve();
}
} catch (error) {
// If there's an error scanning the file, throw a validation exception
reject(new ValidationException(true, { 'upload error': `${error.message}` }));
}
});
}
private async savePersons(dataset: Dataset, persons: any[], role: string, trx: TransactionClientContract) {
@ -459,8 +514,8 @@ export default class DatasetController {
'required': '{{ field }} is required',
'unique': '{{ field }} must be unique, and this value is already taken',
// 'confirmed': '{{ field }} is not correct',
'licences.minLength': 'at least {{ options.minLength }} permission must be defined',
'licences.*.number': 'Define roles as valid numbers',
'licenses.minLength': 'at least {{ options.minLength }} permission must be defined',
'licenses.*.number': 'Define roles as valid numbers',
'rights.equalTo': 'you must agree to continue',
'titles.0.value.minLength': 'Main Title must be at least {{ options.minLength }} characters long',