feat: Enhance background job settings UI and functionality
Some checks failed
build.yaml / feat: Enhance background job settings UI and functionality (push) Failing after 0s
Some checks failed
build.yaml / feat: Enhance background job settings UI and functionality (push) Failing after 0s
- Updated BackgroundJob.vue to improve the display of background job statuses, including missing cross-references and current job mode. - Added auto-refresh functionality for background job status. - Introduced success toast notifications for successful status refreshes. - Modified the XML serialization process in DatasetXmlSerializer for better caching and performance. - Implemented a new RuleProvider for managing custom validation rules. - Improved error handling in routes for loading background job settings. - Enhanced ClamScan configuration with socket support for virus scanning. - Refactored dayjs utility to streamline locale management.
This commit is contained in:
parent
6757bdb77c
commit
b5bbe26ec2
27 changed files with 1221 additions and 603 deletions
231
app/Library/DatasetXmlSerializer.ts
Normal file
231
app/Library/DatasetXmlSerializer.ts
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
import DocumentXmlCache from '#models/DocumentXmlCache';
|
||||
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
|
||||
import Dataset from '#models/dataset';
|
||||
import Strategy from './Strategy.js';
|
||||
import { builder } from 'xmlbuilder2';
|
||||
import logger from '@adonisjs/core/services/logger';
|
||||
|
||||
/**
|
||||
* Configuration for XML serialization
|
||||
*
|
||||
* @interface XmlSerializationConfig
|
||||
*/
|
||||
export interface XmlSerializationConfig {
|
||||
/** The dataset model to serialize */
|
||||
model: Dataset;
|
||||
/** DOM representation (if available) */
|
||||
dom?: XMLBuilder;
|
||||
/** Fields to exclude from serialization */
|
||||
excludeFields: Array<string>;
|
||||
/** Whether to exclude empty fields */
|
||||
excludeEmpty: boolean;
|
||||
/** Base URI for xlink:ref elements */
|
||||
baseUri: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for controlling serialization behavior
|
||||
*/
|
||||
export interface SerializationOptions {
|
||||
/** Enable XML caching */
|
||||
enableCaching?: boolean;
|
||||
/** Exclude empty fields from output */
|
||||
excludeEmptyFields?: boolean;
|
||||
/** Custom base URI */
|
||||
baseUri?: string;
|
||||
/** Fields to exclude */
|
||||
excludeFields?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* DatasetXmlSerializer
|
||||
*
|
||||
* Handles XML serialization of Dataset models with intelligent caching.
|
||||
* Generates XML representations and manages cache lifecycle to optimize performance.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const serializer = new DatasetXmlSerializer(dataset);
|
||||
* serializer.enableCaching();
|
||||
* serializer.excludeEmptyFields();
|
||||
*
|
||||
* const xmlDocument = await serializer.toXmlDocument();
|
||||
* ```
|
||||
*/
|
||||
export default class DatasetXmlSerializer {
|
||||
private readonly config: XmlSerializationConfig;
|
||||
private readonly strategy: Strategy;
|
||||
private cache: DocumentXmlCache | null = null;
|
||||
private cachingEnabled = false;
|
||||
|
||||
constructor(dataset: Dataset, options: SerializationOptions = {}) {
|
||||
this.config = {
|
||||
model: dataset,
|
||||
excludeEmpty: options.excludeEmptyFields ?? false,
|
||||
baseUri: options.baseUri ?? '',
|
||||
excludeFields: options.excludeFields ?? [],
|
||||
};
|
||||
|
||||
this.strategy = new Strategy({
|
||||
excludeEmpty: options.excludeEmptyFields ?? false,
|
||||
baseUri: options.baseUri ?? '',
|
||||
excludeFields: options.excludeFields ?? [],
|
||||
model: dataset,
|
||||
});
|
||||
|
||||
if (options.enableCaching) {
|
||||
this.cachingEnabled = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable caching for XML generation
|
||||
* When enabled, generated XML is stored in database for faster retrieval
|
||||
*/
|
||||
public enableCaching(): this {
|
||||
this.cachingEnabled = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable caching for XML generation
|
||||
*/
|
||||
public disableCaching(): this {
|
||||
this.cachingEnabled = false;
|
||||
return this;
|
||||
}
|
||||
|
||||
set model(model: Dataset) {
|
||||
this.config.model = model;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure to exclude empty fields from XML output
|
||||
*/
|
||||
public excludeEmptyFields(): this {
|
||||
this.config.excludeEmpty = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the cache instance directly (useful when preloading)
|
||||
* @param cache - The DocumentXmlCache instance
|
||||
*/
|
||||
public setCache(cache: DocumentXmlCache): this {
|
||||
this.cache = cache;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current cache instance
|
||||
*/
|
||||
public getCache(): DocumentXmlCache | null {
|
||||
return this.cache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get DOM document with intelligent caching
|
||||
* Returns cached version if valid, otherwise generates new document
|
||||
*/
|
||||
public async toXmlDocument(): Promise<XMLBuilder | null> {
|
||||
const dataset = this.config.model;
|
||||
|
||||
// Try to get from cache first
|
||||
let cachedDocument: XMLBuilder | null = await this.retrieveFromCache();
|
||||
|
||||
if (cachedDocument) {
|
||||
logger.debug(`Using cached XML for dataset ${dataset.id}`);
|
||||
return cachedDocument;
|
||||
}
|
||||
|
||||
// Generate fresh document
|
||||
logger.debug(`[DatasetXmlSerializer] Cache miss - generating fresh XML for dataset ${dataset.id}`);
|
||||
const freshDocument = await this.strategy.createDomDocument();
|
||||
|
||||
if (!freshDocument) {
|
||||
logger.error(`[DatasetXmlSerializer] Failed to generate XML for dataset ${dataset.id}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Cache if caching is enabled
|
||||
if (this.cachingEnabled) {
|
||||
await this.persistToCache(freshDocument, dataset);
|
||||
}
|
||||
|
||||
// Extract the dataset-specific node
|
||||
return this.extractDatasetNode(freshDocument);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate XML string representation
|
||||
* Convenience method that converts XMLBuilder to string
|
||||
*/
|
||||
public async toXmlString(): Promise<string | null> {
|
||||
const document = await this.toXmlDocument();
|
||||
return document ? document.end({ prettyPrint: false }) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist generated XML document to cache
|
||||
* Non-blocking - failures are logged but don't interrupt the flow
|
||||
*/
|
||||
private async persistToCache(domDocument: XMLBuilder, dataset: Dataset): Promise<void> {
|
||||
try {
|
||||
this.cache = this.cache || new DocumentXmlCache();
|
||||
this.cache.document_id = dataset.id;
|
||||
this.cache.xml_version = 1;
|
||||
this.cache.server_date_modified = dataset.server_date_modified.toFormat('yyyy-MM-dd HH:mm:ss');
|
||||
this.cache.xml_data = domDocument.end();
|
||||
|
||||
await this.cache.save();
|
||||
logger.debug(`Cached XML for dataset ${dataset.id}`);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to cache XML for dataset ${dataset.id}: ${error.message}`);
|
||||
// Don't throw - caching failure shouldn't break the flow
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the Rdr_Dataset node from full document
|
||||
*/
|
||||
private extractDatasetNode(domDocument: XMLBuilder): XMLBuilder | null {
|
||||
const node = domDocument.find((n) => n.node.nodeName === 'Rdr_Dataset', false, true)?.node;
|
||||
|
||||
if (node) {
|
||||
return builder({ version: '1.0', encoding: 'UTF-8', standalone: true }, node);
|
||||
}
|
||||
|
||||
return domDocument;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to retrieve valid cached XML document
|
||||
* Returns null if cache doesn't exist or is stale
|
||||
*/
|
||||
private async retrieveFromCache(): Promise<XMLBuilder | null> {
|
||||
const dataset: Dataset = this.config.model;
|
||||
if (!this.cache) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if cache is still valid
|
||||
const actuallyCached = await DocumentXmlCache.hasValidEntry(dataset.id, dataset.server_date_modified);
|
||||
|
||||
if (!actuallyCached) {
|
||||
logger.debug(`Cache invalid for dataset ${dataset.id}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
//cache is actual return cached document
|
||||
try {
|
||||
if (this.cache) {
|
||||
return this.cache.getDomDocument();
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Failed to retrieve cached document for dataset ${dataset.id}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import Dataset from '#models/dataset';
|
|||
import { Client } from '@opensearch-project/opensearch';
|
||||
import { create } from 'xmlbuilder2';
|
||||
import SaxonJS from 'saxon-js';
|
||||
import XmlModel from '#app/Library/XmlModel';
|
||||
import DatasetXmlSerializer from '#app/Library/DatasetXmlSerializer';
|
||||
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
|
||||
import logger from '@adonisjs/core/services/logger';
|
||||
import { readFileSync } from 'fs';
|
||||
|
|
@ -72,31 +72,42 @@ export default {
|
|||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Index a dataset document to OpenSearch/Elasticsearch
|
||||
*/
|
||||
async indexDocument(dataset: Dataset, index_name: string): Promise<void> {
|
||||
try {
|
||||
const proc = readFileSync('public/assets2/solr.sef.json');
|
||||
const doc: string = await this.getTransformedString(dataset, proc);
|
||||
// Load XSLT transformation file
|
||||
const xsltProc = readFileSync('public/assets2/solr.sef.json');
|
||||
|
||||
let document = JSON.parse(doc);
|
||||
// Transform dataset to JSON document
|
||||
const jsonDoc: string = await this.getTransformedString(dataset, xsltProc);
|
||||
|
||||
const document = JSON.parse(jsonDoc);
|
||||
|
||||
// Index document to OpenSearch with doument json body
|
||||
await this.client.index({
|
||||
id: dataset.publish_id?.toString(),
|
||||
index: index_name,
|
||||
body: document,
|
||||
refresh: true,
|
||||
refresh: true, // make immediately searchable
|
||||
});
|
||||
logger.info(`dataset with publish_id ${dataset.publish_id} successfully indexed`);
|
||||
logger.info(`Dataset ${dataset.publish_id} successfully indexed to ${index_name}`);
|
||||
} catch (error) {
|
||||
logger.error(`An error occurred while indexing datsaet with publish_id ${dataset.publish_id}.`);
|
||||
logger.error(`Failed to index dataset ${dataset.publish_id}: ${error.message}`);
|
||||
throw error; // Re-throw to allow caller to handle
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Transform dataset XML to JSON using XSLT
|
||||
*/
|
||||
async getTransformedString(dataset: Dataset, proc: Buffer): Promise<string> {
|
||||
let xml = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
|
||||
const datasetNode = xml.root().ele('Dataset');
|
||||
await createXmlRecord(dataset, datasetNode);
|
||||
const xmlString = xml.end({ prettyPrint: false });
|
||||
// Generate XML string from dataset
|
||||
const xmlString = await this.generateDatasetXml(dataset);
|
||||
|
||||
try {
|
||||
// Apply XSLT transformation
|
||||
const result = await SaxonJS.transform({
|
||||
stylesheetText: proc,
|
||||
destination: 'serialized',
|
||||
|
|
@ -108,6 +119,18 @@ export default {
|
|||
return '';
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Generate XML string from dataset model
|
||||
*/
|
||||
async generateDatasetXml(dataset: Dataset): Promise<string> {
|
||||
const xml = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
|
||||
const datasetNode = xml.root().ele('Dataset');
|
||||
|
||||
await createXmlRecord(dataset, datasetNode);
|
||||
|
||||
return xml.end({ prettyPrint: false });
|
||||
},
|
||||
};
|
||||
/**
|
||||
* Return the default global focus trap stack
|
||||
|
|
@ -115,74 +138,49 @@ export default {
|
|||
* @return {import('focus-trap').FocusTrap[]}
|
||||
*/
|
||||
|
||||
// export const indexDocument = async (dataset: Dataset, index_name: string, proc: Buffer): Promise<void> => {
|
||||
// try {
|
||||
// const doc = await getJsonString(dataset, proc);
|
||||
|
||||
// let document = JSON.parse(doc);
|
||||
// await client.index({
|
||||
// id: dataset.publish_id?.toString(),
|
||||
// index: index_name,
|
||||
// body: document,
|
||||
// refresh: true,
|
||||
// });
|
||||
// Logger.info(`dataset with publish_id ${dataset.publish_id} successfully indexed`);
|
||||
// } catch (error) {
|
||||
// Logger.error(`An error occurred while indexing datsaet with publish_id ${dataset.publish_id}.`);
|
||||
// }
|
||||
// };
|
||||
|
||||
// const getJsonString = async (dataset, proc): Promise<string> => {
|
||||
// let xml = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
|
||||
// const datasetNode = xml.root().ele('Dataset');
|
||||
// await createXmlRecord(dataset, datasetNode);
|
||||
// const xmlString = xml.end({ prettyPrint: false });
|
||||
|
||||
// try {
|
||||
// const result = await transform({
|
||||
// stylesheetText: proc,
|
||||
// destination: 'serialized',
|
||||
// sourceText: xmlString,
|
||||
// });
|
||||
// return result.principalResult;
|
||||
// } catch (error) {
|
||||
// Logger.error(`An error occurred while creating the user, error: ${error.message},`);
|
||||
// return '';
|
||||
// }
|
||||
// };
|
||||
|
||||
/**
|
||||
* Create complete XML record for dataset
|
||||
* Handles caching and metadata enrichment
|
||||
*/
|
||||
const createXmlRecord = async (dataset: Dataset, datasetNode: XMLBuilder): Promise<void> => {
|
||||
const domNode = await getDatasetXmlDomNode(dataset);
|
||||
if (domNode) {
|
||||
// add frontdoor url and data-type
|
||||
dataset.publish_id && addLandingPageAttribute(domNode, dataset.publish_id.toString());
|
||||
addSpecInformation(domNode, 'data-type:' + dataset.type);
|
||||
if (dataset.collections) {
|
||||
for (const coll of dataset.collections) {
|
||||
const collRole = coll.collectionRole;
|
||||
addSpecInformation(domNode, collRole.oai_name + ':' + coll.number);
|
||||
}
|
||||
}
|
||||
|
||||
datasetNode.import(domNode);
|
||||
if (!domNode) {
|
||||
throw new Error(`Failed to generate XML DOM node for dataset ${dataset.id}`);
|
||||
}
|
||||
|
||||
// Enrich with landing page URL
|
||||
if (dataset.publish_id) {
|
||||
addLandingPageAttribute(domNode, dataset.publish_id.toString());
|
||||
}
|
||||
|
||||
// Add data type specification
|
||||
addSpecInformation(domNode, `data-type:${dataset.type}`);
|
||||
|
||||
// Add collection information
|
||||
if (dataset.collections) {
|
||||
for (const coll of dataset.collections) {
|
||||
const collRole = coll.collectionRole;
|
||||
addSpecInformation(domNode, `${collRole.oai_name}:${coll.number}`);
|
||||
}
|
||||
}
|
||||
|
||||
datasetNode.import(domNode);
|
||||
};
|
||||
|
||||
const getDatasetXmlDomNode = async (dataset: Dataset): Promise<XMLBuilder | null> => {
|
||||
const xmlModel = new XmlModel(dataset);
|
||||
const serializer = new DatasetXmlSerializer(dataset).enableCaching().excludeEmptyFields();
|
||||
// xmlModel.setModel(dataset);
|
||||
xmlModel.excludeEmptyFields();
|
||||
xmlModel.caching = true;
|
||||
// const cache = dataset.xmlCache ? dataset.xmlCache : null;
|
||||
// dataset.load('xmlCache');
|
||||
|
||||
// Load cache relationship if not already loaded
|
||||
await dataset.load('xmlCache');
|
||||
if (dataset.xmlCache) {
|
||||
xmlModel.xmlCache = dataset.xmlCache;
|
||||
serializer.setCache(dataset.xmlCache);
|
||||
}
|
||||
|
||||
// return cache.getDomDocument();
|
||||
const domDocument: XMLBuilder | null = await xmlModel.getDomDocument();
|
||||
return domDocument;
|
||||
// Generate or retrieve cached DOM document
|
||||
const xmlDocument: XMLBuilder | null = await serializer.toXmlDocument();
|
||||
return xmlDocument;
|
||||
};
|
||||
|
||||
const addLandingPageAttribute = (domNode: XMLBuilder, dataid: string) => {
|
||||
|
|
@ -192,6 +190,6 @@ const addLandingPageAttribute = (domNode: XMLBuilder, dataid: string) => {
|
|||
domNode.att('landingpage', url);
|
||||
};
|
||||
|
||||
const addSpecInformation= (domNode: XMLBuilder, information: string) => {
|
||||
const addSpecInformation = (domNode: XMLBuilder, information: string) => {
|
||||
domNode.ele('SetSpec').att('Value', information);
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,129 +0,0 @@
|
|||
import DocumentXmlCache from '#models/DocumentXmlCache';
|
||||
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
|
||||
import Dataset from '#models/dataset';
|
||||
import Strategy from './Strategy.js';
|
||||
import { DateTime } from 'luxon';
|
||||
import { builder } from 'xmlbuilder2';
|
||||
|
||||
/**
|
||||
* This is the description of the interface
|
||||
*
|
||||
* @interface Conf
|
||||
* @member {Model} model holds the current dataset model
|
||||
* @member {XMLBuilder} dom holds the current DOM representation
|
||||
* @member {Array<string>} excludeFields List of fields to skip on serialization.
|
||||
* @member {boolean} excludeEmpty True, if empty fields get excluded from serialization.
|
||||
* @member {string} baseUri Base URI for xlink:ref elements
|
||||
*/
|
||||
export interface Conf {
|
||||
model: Dataset;
|
||||
dom?: XMLBuilder;
|
||||
excludeFields: Array<string>;
|
||||
excludeEmpty: boolean;
|
||||
baseUri: string;
|
||||
}
|
||||
|
||||
export default class XmlModel {
|
||||
private config: Conf;
|
||||
// private strategy = null;
|
||||
private cache: DocumentXmlCache | null = null;
|
||||
private _caching = false;
|
||||
private strategy: Strategy;
|
||||
|
||||
constructor(dataset: Dataset) {
|
||||
// $this->strategy = new Strategy();// Opus_Model_Xml_Version1;
|
||||
// $this->config = new Conf();
|
||||
// $this->strategy->setup($this->config);
|
||||
|
||||
this.config = {
|
||||
excludeEmpty: false,
|
||||
baseUri: '',
|
||||
excludeFields: [],
|
||||
model: dataset,
|
||||
};
|
||||
|
||||
this.strategy = new Strategy({
|
||||
excludeEmpty: true,
|
||||
baseUri: '',
|
||||
excludeFields: [],
|
||||
model: dataset,
|
||||
});
|
||||
}
|
||||
|
||||
set model(model: Dataset) {
|
||||
this.config.model = model;
|
||||
}
|
||||
|
||||
public excludeEmptyFields(): void {
|
||||
this.config.excludeEmpty = true;
|
||||
}
|
||||
|
||||
get xmlCache(): DocumentXmlCache | null {
|
||||
return this.cache;
|
||||
}
|
||||
|
||||
set xmlCache(cache: DocumentXmlCache) {
|
||||
this.cache = cache;
|
||||
}
|
||||
|
||||
get caching(): boolean {
|
||||
return this._caching;
|
||||
}
|
||||
set caching(caching: boolean) {
|
||||
this._caching = caching;
|
||||
}
|
||||
|
||||
public async getDomDocument(): Promise<XMLBuilder | null> {
|
||||
const dataset = this.config.model;
|
||||
|
||||
let domDocument: XMLBuilder | null = await this.getDomDocumentFromXmlCache();
|
||||
if (domDocument == null) {
|
||||
domDocument = await this.strategy.createDomDocument();
|
||||
// domDocument = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
|
||||
if (this._caching) {
|
||||
// caching is desired:
|
||||
this.cache = this.cache || new DocumentXmlCache();
|
||||
this.cache.document_id = dataset.id;
|
||||
this.cache.xml_version = 1; // (int)$this->strategy->getVersion();
|
||||
this.cache.server_date_modified = dataset.server_date_modified.toFormat('yyyy-MM-dd HH:mm:ss');
|
||||
this.cache.xml_data = domDocument.end();
|
||||
await this.cache.save();
|
||||
}
|
||||
const node = domDocument.find(
|
||||
(n) => {
|
||||
const test = n.node.nodeName == 'Rdr_Dataset';
|
||||
return test;
|
||||
},
|
||||
false,
|
||||
true,
|
||||
)?.node;
|
||||
if (node != undefined) {
|
||||
domDocument = builder({ version: '1.0', encoding: 'UTF-8', standalone: true }, node);
|
||||
}
|
||||
}
|
||||
return domDocument;
|
||||
}
|
||||
|
||||
private async getDomDocumentFromXmlCache(): Promise<XMLBuilder | null> {
|
||||
const dataset: Dataset = this.config.model;
|
||||
if (!this.cache) {
|
||||
return null;
|
||||
}
|
||||
//.toFormat('YYYY-MM-DD HH:mm:ss');
|
||||
let date: DateTime = dataset.server_date_modified;
|
||||
const actuallyCached: boolean = await DocumentXmlCache.hasValidEntry(dataset.id, date);
|
||||
if (!actuallyCached) {
|
||||
return null;
|
||||
}
|
||||
//cache is actual return it for oai:
|
||||
try {
|
||||
if (this.cache) {
|
||||
return this.cache.getDomDocument();
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue