geolba/tethys.backend
geolba/tethys.backend
2
0
Fork 2
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions

feat: Enhance background job settings UI and functionality
Some checks failed
build.yaml / feat: Enhance background job settings UI and functionality (push) Failing after 0s
Details

Browse source 
- Updated BackgroundJob.vue to improve the display of background job statuses, including missing cross-references and current job mode.
- Added auto-refresh functionality for background job status.
- Introduced success toast notifications for successful status refreshes.
- Modified the XML serialization process in DatasetXmlSerializer for better caching and performance.
- Implemented a new RuleProvider for managing custom validation rules.
- Improved error handling in routes for loading background job settings.
- Enhanced ClamScan configuration with socket support for virus scanning.
- Refactored dayjs utility to streamline locale management.
This commit is contained in:
Kaimbacher 2025-10-14 12:19:09 +02:00
commit b5bbe26ec2
27 changed files with 1221 additions and 603 deletions
Download patch file Download diff file Expand all files Collapse all files

134
commands/fix_dataset_cross_references.ts
View file

@ -9,6 +9,7 @@ import type { CommandOptions } from '@adonisjs/core/types/ace';
import { DateTime } from 'luxon';
import Dataset from '#models/dataset';
import DatasetReference from '#models/dataset_reference';
import AppConfig from '#models/appconfig';
// import env from '#start/env';
interface MissingCrossReference {
@ -22,6 +23,7 @@ interface MissingCrossReference {
relation: string;
doi: string | null;
reverseRelation: string;
sourceReferenceLabel: string | null;
}
export default class DetectMissingCrossReferences extends BaseCommand {
@ -50,7 +52,17 @@ export default class DetectMissingCrossReferences extends BaseCommand {
};
// Define the allowed relations that we want to process
private readonly ALLOWED_RELATIONS = ['IsNewVersionOf', 'IsPreviousVersionOf', 'IsVariantFormOf', 'IsOriginalFormOf'];
private readonly ALLOWED_RELATIONS = [
'IsNewVersionOf',
'IsPreviousVersionOf',
'IsVariantFormOf',
'IsOriginalFormOf',
'Continues',
'IsContinuedBy',
'HasPart',
'IsPartOf',
];
// private readonly ALLOWED_RELATIONS = ['IsPreviousVersionOf', 'IsOriginalFormOf'];
async run() {
this.logger.info('🔍 Detecting missing cross-references...');
@ -63,9 +75,18 @@ export default class DetectMissingCrossReferences extends BaseCommand {
try {
const missingReferences = await this.findMissingCrossReferences();
// Store count in AppConfig if not fixing and count >= 1
if (!this.fix && missingReferences.length >= 1) {
await this.storeMissingCrossReferencesCount(missingReferences.length);
}
if (missingReferences.length === 0) {
const filterMsg = this.publish_id ? ` for publish_id ${this.publish_id}` : '';
this.logger.success(`All cross-references are properly linked for the specified relations${filterMsg}!`);
// Clear the count if no missing references
if (!this.fix) {
await this.storeMissingCrossReferencesCount(0);
}
return;
}
@ -96,6 +117,8 @@ export default class DetectMissingCrossReferences extends BaseCommand {
if (this.fix) {
await this.fixMissingReferences(missingReferences);
// Clear the count after fixing
await this.storeMissingCrossReferencesCount(0);
this.logger.success('All missing cross-references have been fixed!');
} else {
if (this.verbose) {
@ -112,6 +135,24 @@ export default class DetectMissingCrossReferences extends BaseCommand {
}
}
private async storeMissingCrossReferencesCount(count: number): Promise<void> {
try {
await AppConfig.updateOrCreate(
{
appid: 'commands',
configkey: 'missing_cross_references_count',
},
{
configvalue: count.toString(),
},
);
this.logger.info(`📊 Stored missing cross-references count in database: ${count}`);
} catch (error) {
this.logger.error('Failed to store missing cross-references count:', error);
}
}
private async findMissingCrossReferences(): Promise<MissingCrossReference[]> {
const missingReferences: {
sourceDatasetId: number;
@ -124,6 +165,7 @@ export default class DetectMissingCrossReferences extends BaseCommand {
relation: string;
doi: string | null;
reverseRelation: string;
sourceReferenceLabel: string | null;
}[] = [];
this.logger.info('📊 Querying dataset references...');
@ -158,9 +200,9 @@ export default class DetectMissingCrossReferences extends BaseCommand {
for (const reference of tethysReferences) {
processedCount++;
if (this.verbose && processedCount % 10 === 0) {
this.logger.info(`📈 Processed ${processedCount}/${tethysReferences.length} references...`);
}
// if (this.verbose && processedCount % 10 === 0) {
// this.logger.info(`📈 Processed ${processedCount}/${tethysReferences.length} references...`);
// }
// Double-check that this relation is in our allowed list (safety check)
if (!this.ALLOWED_RELATIONS.includes(reference.relation)) {
@ -172,25 +214,41 @@ export default class DetectMissingCrossReferences extends BaseCommand {
}
// Extract dataset publish_id from DOI or URL
const targetDatasetPublish = this.extractDatasetPublishIdFromReference(reference.value);
// const targetDatasetPublish = this.extractDatasetPublishIdFromReference(reference.value);
// Extract DOI from reference URL
const doi = this.extractDoiFromReference(reference.value);
if (!targetDatasetPublish) {
// if (!targetDatasetPublish) {
// if (this.verbose) {
// this.logger.warning(`Could not extract publish ID from: ${reference.value}`);
// }
// continue;
// }
if (!doi) {
if (this.verbose) {
this.logger.warning(`⚠️ Could not extract publish ID from: ${reference.value}`);
this.logger.warning(`Could not extract DOI from: ${reference.value}`);
}
continue;
}
// Check if target dataset exists and is published
// // Check if target dataset exists and is published
// const targetDataset = await Dataset.query()
// .where('publish_id', targetDatasetPublish)
// .where('server_state', 'published')
// .preload('identifier')
// .first();
// Check if target dataset exists and is published by querying via identifier
const targetDataset = await Dataset.query()
.where('publish_id', targetDatasetPublish)
.where('server_state', 'published')
.whereHas('identifier', (query) => {
query.where('value', doi);
})
.preload('identifier')
.first();
if (!targetDataset) {
if (this.verbose) {
this.logger.warning(`⚠️ Target dataset with publish_id ${targetDatasetPublish} not found or not published`);
this.logger.warning(`⚠️ Target dataset with publish_id ${doi} not found or not published`);
}
continue;
}
@ -204,8 +262,9 @@ export default class DetectMissingCrossReferences extends BaseCommand {
// Check if reverse reference exists
const reverseReferenceExists = await this.checkReverseReferenceExists(
targetDataset.id,
// reference.document_id,
reference.document_id,
reference.relation,
reference.dataset.identifier.value
);
if (!reverseReferenceExists) {
@ -223,6 +282,7 @@ export default class DetectMissingCrossReferences extends BaseCommand {
reverseRelation: reverseRelation,
sourceDoi: reference.dataset.identifier ? reference.dataset.identifier.value : null,
targetDoi: targetDataset.identifier ? targetDataset.identifier.value : null,
sourceReferenceLabel: reference.label || null,
});
}
}
@ -232,6 +292,18 @@ export default class DetectMissingCrossReferences extends BaseCommand {
return missingReferences;
}
private extractDoiFromReference(reference: string): string | null {
// Match DOI pattern, with or without URL prefix
const doiPattern = /(?:https?:\/\/)?(?:doi\.org\/)?(.+)/i;
const match = reference.match(doiPattern);
if (match && match[1]) {
return match[1]; // Returns just "10.24341/tethys.99.2"
}
return null;
}
private extractDatasetPublishIdFromReference(value: string): number | null {
// Extract from DOI: https://doi.org/10.24341/tethys.107 -> 107
const doiMatch = value.match(/10\.24341\/tethys\.(\d+)/);
@ -248,7 +320,12 @@ export default class DetectMissingCrossReferences extends BaseCommand {
return null;
}
private async checkReverseReferenceExists(targetDatasetId: number, originalRelation: string): Promise<boolean> {
private async checkReverseReferenceExists(
targetDatasetId: number,
sourceDatasetId: number,
originalRelation: string,
sourceDatasetIdentifier: string | null,
): Promise<boolean> {
const reverseRelation = this.getReverseRelation(originalRelation);
if (!reverseRelation) {
@ -258,9 +335,10 @@ export default class DetectMissingCrossReferences extends BaseCommand {
// Only check for reverse references where the source dataset is also published
const reverseReference = await DatasetReference.query()
// We don't filter by source document_id here to find any incoming reference from any published dataset
// .where('document_id', sourceDatasetId)
.where('related_document_id', targetDatasetId)
.where('document_id', targetDatasetId)
// .where('related_document_id', sourceDatasetId) // Ensure it's an incoming reference
.where('relation', reverseRelation)
.where('value', 'like', `%${sourceDatasetIdentifier}`) // Basic check to ensure it points back to source dataset
.first();
return !!reverseReference;
@ -272,6 +350,10 @@ export default class DetectMissingCrossReferences extends BaseCommand {
IsPreviousVersionOf: 'IsNewVersionOf',
IsVariantFormOf: 'IsOriginalFormOf',
IsOriginalFormOf: 'IsVariantFormOf',
Continues: 'IsContinuedBy',
IsContinuedBy: 'Continues',
HasPart: 'IsPartOf',
IsPartOf: 'HasPart',
};
// Only return reverse relation if it exists in our map, otherwise return null
@ -316,6 +398,7 @@ export default class DetectMissingCrossReferences extends BaseCommand {
.where('id', missing.sourceDatasetId)
.where('server_state', 'published')
.preload('identifier')
.preload('titles') // Preload titles to get mainTitle
.first();
const targetDataset = await Dataset.query().where('id', missing.targetDatasetId).where('server_state', 'published').first();
@ -332,12 +415,27 @@ export default class DetectMissingCrossReferences extends BaseCommand {
continue;
}
// **NEW: Update the original reference if related_document_id is missing**
const originalReference = await DatasetReference.query()
.where('document_id', missing.sourceDatasetId)
.where('relation', missing.relation)
.where('value', 'like', `%${missing.targetDoi}%`)
.first();
if (originalReference && !originalReference.related_document_id) {
originalReference.related_document_id = missing.targetDatasetId;
await originalReference.save();
if (this.verbose) {
this.logger.info(`🔗 Updated original reference with related_document_id: ${missing.targetDatasetId}`);
}
}
// Create the reverse reference using the referenced_by relationship
// Example: If Dataset 297 IsNewVersionOf Dataset 144
// We create an incoming reference for Dataset 144 that shows Dataset 297 IsPreviousVersionOf it
const reverseReference = new DatasetReference();
// Don't set document_id - this creates an incoming reference via related_document_id
reverseReference.related_document_id = missing.targetDatasetId; // 144 (dataset receiving the incoming reference)
reverseReference.document_id = missing.targetDatasetId; //
reverseReference.related_document_id = missing.sourceDatasetId;
reverseReference.type = 'DOI';
reverseReference.relation = missing.reverseRelation;
@ -350,8 +448,12 @@ export default class DetectMissingCrossReferences extends BaseCommand {
}
// Use the source dataset's main title for the label
reverseReference.label = sourceDataset.mainTitle || `Dataset ${missing.sourceDatasetId}`;
//reverseReference.label = sourceDataset.mainTitle || `Dataset ${missing.sourceDatasetId}`;
// get label of forward reference
reverseReference.label = missing.sourceReferenceLabel || sourceDataset.mainTitle || `Dataset ${missing.sourceDatasetId}`;
// reverseReference.notes = `Auto-created by detect:missing-cross-references command on ${DateTime.now().toISO()} to fix missing bidirectional reference.`;
// Save the new reverse reference
// Also save 'server_date_modified' on target dataset to trigger any downstream updates (e.g. search index)
targetDataset.server_date_modified = DateTime.now();
await targetDataset.save();

17
commands/index_datasets.ts
View file

@ -4,7 +4,7 @@
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
import { create } from 'xmlbuilder2';
import Dataset from '#models/dataset';
import XmlModel from '#app/Library/XmlModel';
import XmlModel from '#app/Library/DatasetXmlSerializer';
import { readFileSync } from 'fs';
import SaxonJS from 'saxon-js';
import { Client } from '@opensearch-project/opensearch';
@ -151,19 +151,16 @@ export default class IndexDatasets extends BaseCommand {
}
private async getDatasetXmlDomNode(dataset: Dataset): Promise<XMLBuilder | null> {
const xmlModel = new XmlModel(dataset);
const serializer = new XmlModel(dataset).enableCaching().excludeEmptyFields();
// xmlModel.setModel(dataset);
xmlModel.excludeEmptyFields();
xmlModel.caching = true;
// const cache = dataset.xmlCache ? dataset.xmlCache : null;
// dataset.load('xmlCache');
if (dataset.xmlCache) {
xmlModel.xmlCache = dataset.xmlCache;
serializer.setCache(dataset.xmlCache);
}
// return cache.getDomDocument();
const domDocument: XMLBuilder | null = await xmlModel.getDomDocument();
return domDocument;
// return cache.toXmlDocument();
const xmlDocument: XMLBuilder | null = await serializer.toXmlDocument();
return xmlDocument;
}
private addSpecInformation(domNode: XMLBuilder, information: string) {