- add classes inside app/library for creting Tethys xml: Field.ts, Strategy.ts, XmlModel.ts

- added model DocumentXmlCache.ts
- npm updates
- changed all models inside app/Models to use corrected BaseModel.ts
- added extra extension class DatasetExtension.ts for app/dataset.ts for caching internal and external fields

app/Models/BaseModel.ts

@@ -13,6 +13,7 @@ import { BaseModel as LucidBaseModel } from '@ioc:Adonis/Lucid/Orm';
 //   }
 // }
 
+
 /**
  * Helper to find if value is a valid Object or
  * not
@@ -21,7 +22,7 @@ export function isObject(value: any): boolean {
     return value !== null && typeof value === 'object' && !Array.isArray(value);
 }
 
-export default class BaseModel extends LucidBaseModel {
+export default class BaseModel extends LucidBaseModel {  
     /**
      * When `fill` method is called, then we may have a situation where it
      * removed the values which exists in `original` and hence the dirty
@@ -116,4 +117,12 @@ export default class BaseModel extends LucidBaseModel {
 
         return this;
     }
+
 }
+
+// export class DatasetRelatedBaseModel extends LucidBaseModel {
+//     public dataset: BelongsTo<typeof Dataset>;
+// }
+// export interface DatasetRelatedBaseModel {
+//     dataset: BelongsTo<typeof Dataset>;
+// }

app/Models/Collection.ts

@@ -1,5 +1,6 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, manyToMany, ManyToMany } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, manyToMany, ManyToMany } from '@ioc:Adonis/Lucid/Orm';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class Collection extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Coverage.ts

@@ -1,6 +1,7 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class Coverage extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Dataset.ts

@@ -1,6 +1,5 @@
 import {
     column,
-    BaseModel,
     SnakeCaseNamingStrategy,
     manyToMany,
     ManyToMany,
@@ -26,8 +25,10 @@ import DatasetReference from './DatasetReference';
 import Collection from './Collection';
 import DatasetIdentifier from './DatasetIdentifier';
 import Project from './Project';
+import DocumentXmlCache from './DocumentXmlCache';
+import DatasetExtension from 'App/Models/Traits/DatasetExtension'; // Adjust the import path
 
-export default class Dataset extends BaseModel {
+export default class Dataset extends DatasetExtension {
     public static namingStrategy = new SnakeCaseNamingStrategy();
     public static primaryKey = 'id';
     public static table = 'documents';
@@ -40,13 +41,13 @@ export default class Dataset extends BaseModel {
     public server_state: string;
 
     @column({})
-    public publisherName: string;
+    public publisher_name: string;
 
     @column({ columnName: 'creating_corporation' })
-    public creatingCorporation: string;
+    public creating_corporation: string;
 
     @column.dateTime({ columnName: 'embargo_date' })
-    public embargoDate: DateTime;
+    public embargo_date: DateTime;
 
     @column({})
     public type: string;
@@ -82,7 +83,7 @@ export default class Dataset extends BaseModel {
     public reject_reviewer_note: string | null;
 
     @column.dateTime({ columnName: 'server_date_published' })
-    public serverDatePublished: DateTime;
+    public server_date_published: DateTime;
 
     // @column.dateTime({ autoCreate: true, columnName: 'created_at' })
     @column.dateTime({
@@ -92,10 +93,10 @@ export default class Dataset extends BaseModel {
         autoCreate: true,
         columnName: 'created_at',
     })
-    public createdAt: DateTime;
+    public created_at: DateTime;
 
     @column.dateTime({ autoCreate: true, autoUpdate: true, columnName: 'server_date_modified' })
-    public updatedAt: DateTime;
+    public server_date_modified: DateTime;
 
     @manyToMany(() => Person, {
         pivotForeignKey: 'document_id',
@@ -208,4 +209,9 @@ export default class Dataset extends BaseModel {
         },
     })
     public contributors: ManyToMany<typeof Person>;
+
+    @hasOne(() => DocumentXmlCache, {
+        foreignKey: 'document_id',
+    })
+    public xmlCache: HasOne<typeof DocumentXmlCache>;
 }

app/Models/DatasetIdentifier.ts

@@ -1,6 +1,7 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class DatasetIdentifier extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/DatasetReference.ts

@@ -1,6 +1,7 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class DatasetReference extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/DocumentXmlCache.ts

@@ -0,0 +1,98 @@
+import { column, BaseModel, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
+import Dataset from './Dataset';
+import { builder, create } from 'xmlbuilder2';
+import { XMLBuilder } from 'xmlbuilder2/lib/interfaces';
+import Database from '@ioc:Adonis/Lucid/Database';
+import dayjs from 'dayjs';
+import { DateTime } from 'luxon';
+
+export default class DocumentXmlCache extends BaseModel {
+    public static namingStrategy = new SnakeCaseNamingStrategy();
+    public static table = 'document_xml_cache';
+    // public static fillable: string[] = ['value', 'label', 'type', 'relation'];
+    // public static primaryKey = false;
+    static primaryKey = ''; // Set primaryKey to null to indicate there is no primary key
+
+    @column({
+        isPrimary: true,
+    })
+    public document_id: number;
+
+    @column({})
+    public xml_version: number;
+
+    @column()
+    public server_date_modified?: string;
+
+    // @column.dateTime({
+    //     autoCreate: true,
+    //     autoUpdate: true,
+    // })
+    // public updated_at?: DateTime;
+
+    @column({})
+    public xml_data: string;
+
+    @belongsTo(() => Dataset, {
+        foreignKey: 'document_id',
+    })
+    public dataset: BelongsTo<typeof Dataset>;
+
+    /**
+     * Get dom document of 'xml_data' string
+     *
+     * @returns {XMLBuilder}
+     */
+    public getDomDocument(): XMLBuilder {
+        // const dom = xmlbuilder.create({ version: "1.0", encoding: "UTF-8", standalone: true });
+        let dom: XMLBuilder = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, this.xml_data);
+        // return dom.first();
+
+        const rdrDataset = dom.find(
+            (n) => {
+                const test = n.node.nodeName == 'Rdr_Dataset';
+                return test;
+            },
+            false,
+            true,
+        )?.node;
+
+        if (rdrDataset == undefined) {
+            return dom.first();
+        } else {
+            dom = builder({ version: '1.0', encoding: 'UTF-8', standalone: true }, rdrDataset);
+            return dom;
+        }
+    }
+
+    /**
+     * Check if a dataset in a specific xml version is already cached or not.
+     *
+     * @param mixed datasetId
+     * @param mixed serverDateModified
+     * @returns {Promise<boolean>} Returns true on cached hit else false.
+     */
+    // public static async hasValidEntry(datasetId: number, datasetServerDateModified: DateTime): Promise<boolean> {
+    //     // const formattedDate = dayjs(datasetServerDateModified).format('YYYY-MM-DD HH:mm:ss');
+
+    //     const query = Database.from(this.table)
+    //         .where('document_id', datasetId)
+    //         .where('server_date_modified', '2023-08-17 16:51:03')
+    //         .first();
+
+    //     const row = await query;
+    //     return !!row;
+    // }
+
+    // Assuming 'DocumentXmlCache' has a table with a 'server_date_modified' column in your database
+    public static async hasValidEntry(datasetId: number, datasetServerDateModified: DateTime): Promise<boolean> {
+        const serverDateModifiedString: string = datasetServerDateModified.toFormat('yyyy-MM-dd HH:mm:ss'); // Convert DateTime to ISO string
+        const query = Database.from(this.table)
+            .where('document_id', datasetId)
+            .where('server_date_modified', '>=', serverDateModifiedString) // Check if server_date_modified is newer or equal
+            .first();
+
+        const row = await query;
+        return !!row;
+    }
+}

app/Models/File.ts

@@ -1,7 +1,6 @@
 import { DateTime } from 'luxon';
 import {
     column,
-    BaseModel,
     hasMany,
     HasMany,
     belongsTo,
@@ -12,6 +11,7 @@ import {
 } from '@ioc:Adonis/Lucid/Orm';
 import HashValue from './HashValue';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class File extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Language.ts

@@ -1,4 +1,5 @@
-import { column, BaseModel, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import BaseModel from './BaseModel';
 // import { DateTime } from 'luxon';
 
 export default class Language extends BaseModel {

app/Models/License.ts

@@ -1,4 +1,5 @@
-import { column, BaseModel, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import BaseModel from './BaseModel';
 
 export default class License extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Permission.ts

@@ -1,7 +1,8 @@
-import { column, BaseModel, manyToMany, ManyToMany, SnakeCaseNamingStrategy, beforeUpdate, beforeCreate } from '@ioc:Adonis/Lucid/Orm';
+import { column, manyToMany, ManyToMany, SnakeCaseNamingStrategy, beforeUpdate, beforeCreate } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
 import dayjs from 'dayjs';
 import Role from 'App/Models/Role';
+import BaseModel from './BaseModel';
 
 export default class Permission extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Project.ts

@@ -1,5 +1,6 @@
-import { column, BaseModel, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
+import BaseModel from './BaseModel';
 
 export default class Project extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Role.ts

@@ -1,5 +1,5 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
-
+import { column, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
+import BaseModel from './BaseModel';
 import { DateTime } from 'luxon';
 // import moment from 'moment';
 import dayjs from 'dayjs';

app/Models/Subject.ts

@@ -1,4 +1,5 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
+import BaseModel from './BaseModel';
 
 import { DateTime } from 'luxon';
 import dayjs from 'dayjs';

app/Models/Title.ts

@@ -1,6 +1,7 @@
 import { column, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
 import Dataset from './Dataset';
 import BaseModel from './BaseModel';
+// import { DatasetRelatedBaseModel } from './BaseModel';
 
 export default class Title extends BaseModel {
     public static primaryKey = 'id';

app/Models/Traits/DatasetExtension.ts

@@ -0,0 +1,329 @@
+import Title from 'App/Models/Title';
+import Description from 'App/Models/Description';
+import License from 'App/Models/License';
+import Person from 'App/Models/Person';
+import DatasetReference from 'App/Models/DatasetReference';
+import DatasetIdentifier from 'App/Models/DatasetIdentifier';
+import Subject from 'App/Models/Subject';
+import File from 'App/Models/File';
+import Coverage from 'App/Models/Coverage';
+import Collection from 'App/Models/Collection';
+import { BaseModel as LucidBaseModel } from '@ioc:Adonis/Lucid/Orm';
+import Field from 'App/Library/Field';
+import { DateTime } from 'luxon';
+
+// @StaticImplements<LucidModel>()
+// class LucidDatasetModel extends BaseModel{
+//     @belongsTo(() => Dataset, {
+//         foreignKey: 'dataset_id',
+//     })
+//     dataset: BelongsTo<typeof Dataset>;
+// }
+
+export type DatasetRelatedModel =
+    | typeof Title
+    | typeof Description
+    | typeof Coverage
+    | typeof DatasetIdentifier
+    | typeof DatasetReference
+    | typeof Description
+    | typeof DatasetIdentifier
+    | typeof File;
+
+
+export default abstract class DatasetExtension extends LucidBaseModel {
+    public abstract id;
+    public externalFields: Record<string, any> = this.getExternalFields();
+    // which fields shoud#t be published
+    protected internalFields: Record<string, any> = {};
+    protected fields: Record<string, any> = {};
+
+    private getExternalFields(): Record<string, any> {
+        // External fields definition
+        return {
+            TitleMain: {
+                model: Title,
+                options: { type: ['Main'] },
+                fetch: 'eager',
+            },
+            TitleAdditional: {
+                model: Title,
+                options: { type: ['Alternative', 'Sub', 'Translated', 'Other'] },
+                fetch: 'eager',
+            },
+            TitleAbstract: {
+                model: Description,
+                options: { type: ['Abstract', 'Translated'] },
+                fetch: 'eager',
+            },
+            TitleAbstractAdditional: {
+                model: Description,
+                options: { type: ['Methods', 'Technical_info', 'Series_information', 'Other'] },
+                fetch: 'eager',
+            },
+            Licence: {
+                model: License,
+                through: 'link_documents_licences',
+                relation: 'licenses',
+                fetch: 'eager',
+            },
+            PersonAuthor: {
+                model: Person,
+                through: 'link_documents_persons',
+                pivot: { role: 'author', sort_order: 'sort_order', allow_email_contact: 'allow_email_contact' },
+                relation: 'persons',
+                fetch: 'eager',
+            },
+            PersonContributor: {
+                model: Person,
+                through: 'link_documents_persons',
+                pivot: {
+                    role: 'contributor',
+                    contributor_type: 'contributor_type',
+                    sort_order: 'sort_order',
+                    allow_email_contact: 'allow_email_contact',
+                },
+                relation: 'persons',
+                fetch: 'eager',
+            },
+            Reference: {
+                model: DatasetReference,
+                relation: 'references',
+                fetch: 'eager',
+            },
+            Identifier: {
+                model: DatasetIdentifier,
+                relation: 'identifier',
+                fetch: 'eager',
+            },
+            Subject: {
+                model: Subject,
+                through: 'link_dataset_subjects',
+                relation: 'subjects',
+                fetch: 'eager',
+            },
+            File: {
+                model: File,
+                relation: 'files',
+                fetch: 'eager',
+            },
+            Coverage: {
+                model: Coverage,
+                relation: 'coverage',
+                fetch: 'eager',
+            },
+            Collection: {
+                model: Collection,
+                through: 'link_documents_collections',
+                relation: 'collections',
+                fetch: 'eager',
+                //   'include': { 'model': CollectionRole, 'relation': 'collectionrole' }
+            },
+        };
+    }
+
+    public initFields(): void {
+        // Initialize internal fields
+        let fields = new Array<string>(
+            'Id',
+            'PublisherName',
+            'PublishId',
+            'ContributingCorporation',
+            'CreatingCorporation',
+            'Language',
+            'PublishedDate',
+            // 'PublishedYear',
+            'PublisherName',
+            // 'PublisherPlace',
+            'PublicationState',
+            'EmbargoDate',
+            'CreatedAt',
+            'ServerDateModified',
+            'ServerDatePublished',
+            'ServerDateDeleted',
+            'ServerState',
+            'Type',
+            'BelongsToBibliography',
+        );
+        fields.forEach((fieldname) => {
+            let field = new Field(fieldname);
+            this.addField(field);
+        });
+        // Initialize external fields
+        const fieldNames = Object.keys(this.externalFields);
+        for (const fieldName of fieldNames) {
+            // const field = this.externalFields[fieldName];
+            let field = new Field(fieldName);
+            field.setMultiplicity('*');
+            this.addField(field);
+        }
+
+        //      // Initialize available date fields and set up date validator
+        //     // if the particular field is present
+        let dateFields = new Array<string>('EmbargoDate', 'CreatedAt', 'ServerDatePublished', 'ServerDateDeleted');
+        dateFields.forEach((fieldname) => {
+            let dateField = this.getField(fieldname);
+            dateField instanceof Field && dateField.setValueModelClass(DateTime.now());
+        });
+    }
+
+    public async describe(): Promise<Array<string>> {
+        let length: number = Object.keys(this.fields).length;
+        if (length == 0) {
+            await this.fetchValues();
+        }
+        // Get an array of all field names in the 'fields' object
+        const allFields = Object.keys(this.fields);
+        // Get an array of all field names in the 'internalFields' array
+        const internalFields = Object.keys(this.internalFields);
+        // Use the `filter` method to find field names that are not in 'internalFields'
+        const filteredFields = allFields.filter((fieldName) => !internalFields.includes(fieldName));
+        return filteredFields;
+    }
+
+    private addField(field: Field): void {
+        // Add field
+        const fieldName = field.getName();
+        if (fieldName && this.externalFields[fieldName]) {
+            const options = this.externalFields[fieldName];
+
+            // Set ValueModelClass if a model option is given
+            if (options.model) {
+                field.setValueModelClass(options.model);
+            }
+        }
+
+        this.fields[field.getName()] = field;
+        // field.setOwningModelClass(this.constructor.name);
+    }
+
+    public getField(name: string): Field | null {
+        // Get field
+        return this.fields[name] !== undefined ? this.fields[name] : null;
+    }
+
+    public async fetchValues(): Promise<void> {
+        this.initFields();
+        await this.loadFieldValues();
+    }
+
+    private async loadFieldValues(): Promise<void> {
+        for (const [fieldname, field] of Object.entries(this.fields)) {
+            // extern fields via model relation
+            if (this.externalFields.hasOwnProperty(fieldname)) {
+                await this.loadExternal(fieldname);
+                // dataset attributes itself
+            } else {
+                // Field is not external and gets handled by simply reading. to snake_case
+                const property_name = this.convertFieldnameToColumn(fieldname); //id
+                const fieldVal = this[property_name]; //276
+
+                // Explicitly set null if the field represents a model except for dates.
+                if (field.getValueModelClass() !== null) {
+                    field.setValue(fieldVal === undefined || fieldVal === null ? null : fieldVal);
+                } else {
+                    field.setValue(fieldVal);
+                }
+            }
+        }
+    }
+
+    private async loadExternal(fieldname: string): Promise<void> {
+        const field = this.fields[fieldname];
+
+        // let modelclass: typeof Title | typeof Description;
+        let modelclass: DatasetRelatedModel = field.getValueModelClass();
+        let modelInstance = new modelclass();
+
+        // Create a query builder
+        const select = modelclass.query();
+
+        // If any declared constraints, add them to the query
+        if (this.externalFields[fieldname]?.options) {
+            const options: Array<string> = this.externalFields[fieldname].options;
+            for (const [column, value] of Object.entries(options)) {
+                if (Array.isArray(value)) {
+                    select.whereIn(column, value);
+                } else {
+                    select.where(column, value);
+                }
+            }
+        }
+
+        // Get dependent rows
+        const result: Record<string, any>[] = [];
+        const datasetId = this.id;
+        let rows: any[] = [];
+
+        if (this.externalFields[fieldname]?.through) {
+            const relation = this.externalFields[fieldname].relation;
+            // rows = this[relation];
+            rows = await this.related(relation).query();
+
+            if (this.externalFields[fieldname].pivot) {
+                const pivotArray = this.externalFields[fieldname].pivot;
+                const pivotValue = pivotArray.role;
+                // rows = this[relation]().wherePivot('role', pivotValue).get();
+                rows = await this.related(relation).query().wherePivot('role', pivotValue);
+            }
+        } else if (modelInstance.hasOwnProperty('dataset')) {
+            rows = await select
+                .whereHas('dataset', (q) => {
+                    q.where('id', datasetId);
+                })
+                .orderBy('id')
+                .select();
+        }
+
+        // 1 ..n relations
+        for (const row of rows) {
+            const attributes = Object.keys(row.$attributes);
+
+            if (this.externalFields[fieldname]?.pivot) {
+                const pivotArray = this.externalFields[fieldname].pivot;
+                const arrayKeys = Object.keys(pivotArray);
+                const extendedArrayKeys = arrayKeys.map((pivotAttribute) => {
+                    return `pivot_${pivotAttribute}`;
+                });
+                attributes.push(...extendedArrayKeys);
+            }
+
+            const objArray: Record<string, any> = {};
+
+            for (const property_name of attributes) {
+                let fieldName = this.convertColumnToFieldname(property_name);
+                let fieldval = '';
+
+                if (property_name.startsWith('pivot_')) {
+                    const str = property_name.replace('pivot_', '');
+                    fieldName = this.convertColumnToFieldname(str);
+                    // fieldval = row.$pivot[str];
+                    fieldval = row.$extras[property_name];
+                } else if (fieldName === 'Type') {
+                    fieldval = row[property_name]?.charAt(0).toUpperCase() + row[property_name]?.slice(1);
+                } else {
+                    fieldval = row[property_name];
+                }
+
+                objArray[fieldName] = fieldval;
+            }
+
+            result.push(objArray);
+        }
+
+        // Set the field value
+        field.setValue(result);
+    }
+
+    // to snakeCase
+    private convertFieldnameToColumn(fieldname: string): string {
+        return fieldname.replace(/([a-z\d])([A-Z])/g, '$1_$2').toLowerCase();
+    }
+
+    private convertColumnToFieldname(columnName: string): string {
+        return columnName
+            .split(/[-_]/)
+            .map((word) => (word.charAt(0).toUpperCase() + word.slice(1)))
+            .join('');
+    }
+}

app/Models/User.ts

@@ -1,10 +1,11 @@
 import { DateTime } from 'luxon';
-import { BaseModel, column, beforeSave, manyToMany, ManyToMany, hasMany, HasMany } from '@ioc:Adonis/Lucid/Orm';
+import { column, beforeSave, manyToMany, ManyToMany, hasMany, HasMany } from '@ioc:Adonis/Lucid/Orm';
 import Hash from '@ioc:Adonis/Core/Hash';
 import Role from './Role';
 import Database from '@ioc:Adonis/Lucid/Database';
 import Config from '@ioc:Adonis/Core/Config';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 // export default interface IUser {
 // 	id: number;