geolba/tethys.backend
geolba/tethys.backend
2
0
Fork 2
Code Issues Pull requests Projects Releases 4 Packages Wiki Activity Actions

Compare commits

base: geolba:master
Branches Tags
geolba:master
geolba:develop
kaiarn:develop
kaiarn:feat/checkReferenceType
kaiarn:master
geolba:v2.2.0
geolba:v2.1.0
geolba:v2.0.0-rc1
geolba:2.0.0-alpha
kaiarn:2.0.0-alpha
..
compare: kaiarn:master
Branches Tags
kaiarn:develop
kaiarn:feat/checkReferenceType
kaiarn:master
geolba:master
geolba:develop
kaiarn:2.0.0-alpha
geolba:v2.2.0
geolba:v2.1.0
geolba:v2.0.0-rc1
geolba:2.0.0-alpha

No commits in common. "master" and "master" have entirely different histories.
master ... master

227 changed files with 21106 additions and 32318 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.env.example
View file

@ -1,7 +1,7 @@
PORT=3333
HOST=0.0.0.0
NODE_ENV=development
APP_KEY=pvmU1vuAZDkSwarb7yh9pgZ-xxxxxx007
APP_KEY=pvmU1vuAZDkSwarb7yh9pgZ-RxaX4zS7
DRIVE_DISK=local
SESSION_DRIVER=cookie
CACHE_VIEWS=false
@ -17,6 +17,4 @@ REDIS_PORT=6379
REDIS_PASSWORD=
SMTP_HOST=
SMTP_PORT=
RESEND_API_KEY=
OPENSEARCH_HOST=http://localhost
OPENSEARCH_CORE=tethys-records
RESEND_API_KEY=

2
.gitea/workflows/build.yaml
View file

@ -13,7 +13,7 @@ jobs:
uses: actions/checkout@v3
- run: echo "The ${{ github.repository }} repository has been cloned to the runner."
- run: echo "The workflow is now ready to test your code on the runner."
- name: List files in the repository
- name: List files in the repository:
run: |
ls ${{ github.workspace }}
- run: echo "This job's status is ${{ job.status }}."

78
.gitea/workflows/checkReferenceType.yaml
View file

@ -1,78 +0,0 @@
# This is a Gitea Actions workflow configuration file for running CI tests on the `feat/checkReferenceType` branch.
# The workflow is named "CI" and runs on the latest Ubuntu environment using a Node.js 20 Docker container.
# It sets up a PostgreSQL service with specified environment variables and health checks.
# The workflow includes the following steps:
# 1. Checkout the repository using the actions/checkout@v3 action.
# 2. Install Node.js dependencies using `npm ci`.
# 3. Create a `.env.test` file by copying from `.env.example`.
# 4. Set up environment variables in the `.env.test` file, including database connection details and other app-specific settings.
# 5. Run functional tests using the `node ace test functional --groups "ReferenceValidation"` command.
name: CI
run-name: Running tests for checkReferenceType branch
on:
push:
branches:
- feat/checkReferenceType
jobs:
container-job:
runs-on: ubuntu-latest
# Docker Hub image that `container-job` executes in
container: node:20-bullseye
services:
# Label used to access the service container
postgres:
image: postgres:latest
env:
POSTGRES_USER: alice
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
POSTGRES_DB: tethys_dev
# ports:
# - 5432:5432
options: |
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout repository
uses: actions/checkout@v3
# - name: Set up Node.js
# uses: actions/setup-node@v2
# with:
# node-version: '20'
- name: Install dependencies
run: npm ci
- name: Create .env.test file
run: cp .env.example .env.test
- name: Set up environment variables
run: |
echo "DB_CONNECTION=pg" >> .env.test
echo "PG_HOST=postgres" >> .env.test
echo "PG_PORT=5432" >> .env.test
echo "PG_USER=alice" >> .env.test
echo "PG_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env.test
echo "PG_DB_NAME=tethys_dev" >> .env.test
echo "NODE_ENV=test" >> .env.test
echo "ASSETS_DRIVER=fake" >> .env.test
echo "SESSION_DRIVER=memory" >> .env.test
echo "HASH_DRIVER=bcrypt" >> .env.test
echo "HOST=127.0.0.1" >> .env.test
echo "PORT=3333" >> .env.test
echo "APP_NAME=TethysCloud" >> .env.test
echo "APP_URL=http://${HOST}:${PORT}" >> .env.test
echo "CACHE_VIEWS=false" >> .env.test
echo "APP_KEY=pfi5N2ACN4tMJ5d8d8BPHfh3FEuvleej" >> .env.test
echo "DRIVE_DISK=local" >> .env.test
echo "OAI_LIST_SIZE=200" >> .env.test
echo "OPENSEARCH_HOST=${{ secrets.OPENSEARCH_HOST }}" >> .env.test
echo "OPENSEARCH_CORE=tethys-records" >> .env.test
- name: Run tests
run: node ace test functional --groups "ReferenceValidation"

12
.gitea/workflows/ci.yaml
View file

@ -4,13 +4,7 @@
name: CI Pipeline
run-name: ${{ github.actor }} is running CI pipeline
# trigger build when pushing, or when creating a pull request
on:
push:
branches:
- master
pull_request:
branches:
- master
on: [push, pull_request]
jobs:
# Label of the container job
@ -18,7 +12,7 @@ jobs:
# run build on latest ubuntu
runs-on: ubuntu-latest
container: node:20-bullseye
container: node:18-bullseye
services:
mydb:
@ -76,7 +70,6 @@ jobs:
&& echo "CACHE_VIEWS=false" >> .env.test
&& echo "APP_KEY=pfi5N2ACN4tMJ5d8d8BPHfh3FEuvleej" >> .env.test
&& echo "DRIVE_DISK=local" >> .env.test
&& echo "OAI_LIST_SIZE=200" >> .env.test
# finally run the tests
# - run: npm test
@ -102,4 +95,3 @@ jobs:
# uses: coverallsapp/github-action@master
# with:
# github-token: ${{ secrets.GITHUB_TOKEN }}

1
.gitignore vendored
View file

@ -7,4 +7,3 @@ coverage
tmp
docker-compose.yml
.env.test
public/assets

68
Dockerfile
View file

@ -1,63 +1,57 @@
################## First Stage - Creating base #########################
# Created a variable to hold our node base image
ARG NODE_IMAGE=node:22-trixie-slim
ARG NODE_IMAGE=node:20-bookworm-slim
FROM $NODE_IMAGE AS base
# Install dumb-init and ClamAV, and perform ClamAV database update
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
dumb-init \
clamav \
clamav-daemon \
clamdscan \
ca-certificates \
RUN apt update \
&& apt-get install -y dumb-init clamav clamav-daemon nano \
&& rm -rf /var/lib/apt/lists/* \
# Creating folders and changing ownerships
&& mkdir -p /home/node/app \
&& mkdir -p /var/lib/clamav \
&& mkdir -p /home/node/app && chown node:node /home/node/app \
&& mkdir -p /var/lib/clamav \
&& mkdir /usr/local/share/clamav \
&& chown -R node:clamav /var/lib/clamav /usr/local/share/clamav /etc/clamav \
# permissions
&& mkdir /var/run/clamav \
&& mkdir -p /var/log/clamav \
&& mkdir -p /tmp/clamav-logs \
# Set ownership and permissions
&& chown node:node /home/node/app \
# && chown -R node:clamav /var/lib/clamav /usr/local/share/clamav /etc/clamav /var/run/clamav \
&& chown -R node:clamav /var/lib/clamav /usr/local/share/clamav /etc/clamav /var/run/clamav /var/log/clamav \
&& chown -R node:clamav /etc/clamav \
&& chmod 755 /tmp/clamav-logs \
&& chmod 750 /var/run/clamav \
&& chmod 755 /var/lib/clamav \
&& chmod 755 /var/log/clamav \
# Add node user to clamav group and allow sudo for clamav commands
&& usermod -a -G clamav node
# && chmod 666 /var/run/clamav/clamd.socket
# Make directories group-writable so node (as member of clamav group) can access them
# && chmod 750 /var/run/clamav /var/lib/clamav /var/log/clamav /tmp/clamav-logs
&& chown node:clamav /var/run/clamav \
&& chmod 750 /var/run/clamav
# -----------------------------------------------
# --- ClamAV & FeshClam -------------------------
# -----------------------------------------------
# RUN \
# chmod 644 /etc/clamav/freshclam.conf && \
# freshclam && \
# mkdir /var/run/clamav && \
# chown -R clamav:root /var/run/clamav
# # initial update of av databases
# RUN freshclam
# Configure ClamAV - copy config files before switching user
# COPY --chown=node:clamav ./*.conf /etc/clamav/
# Configure Clam AV...
COPY --chown=node:clamav ./*.conf /etc/clamav/
# # permissions
# RUN mkdir /var/run/clamav && \
# chown node:clamav /var/run/clamav && \
# chmod 750 /var/run/clamav
# Setting the working directory
WORKDIR /home/node/app
# Changing the current active user to "node"
# Download initial ClamAV database as root before switching users
USER node
RUN freshclam --quiet || echo "Initial database download failed - will retry at runtime"
# Copy entrypoint script
# initial update of av databases
RUN freshclam
# VOLUME /var/lib/clamav
COPY --chown=node:clamav docker-entrypoint.sh /home/node/app/docker-entrypoint.sh
RUN chmod +x /home/node/app/docker-entrypoint.sh
ENV TZ="Europe/Vienna"
################## Second Stage - Installing dependencies ##########
# In this stage, we will start installing dependencies
FROM base AS dependencies
@ -76,6 +70,7 @@ ENV NODE_ENV=production
# We run "node ace build" to build the app (dist folder) for production
RUN node ace build --ignore-ts-errors
# RUN node ace build --production
# RUN node ace build --ignore-ts-errors
################## Final Stage - Production #########################
@ -93,7 +88,6 @@ RUN npm ci --omit=dev
# Copy files to the working directory from the build folder the user
COPY --chown=node:node --from=build /home/node/app/build .
# Expose port
# EXPOSE 3310
EXPOSE 3333
ENTRYPOINT ["/home/node/app/docker-entrypoint.sh"]
# Run the command to start the server using "dumb-init"

22
LICENSE
View file

@ -1,22 +0,0 @@
MIT License
Copyright (c) 2025 Tethys Research Repository
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

7
ace.js
View file

@ -15,11 +15,10 @@
/**
* Register hook to process TypeScript files using ts-node
*/
// import { register } from 'node:module';
// register('ts-node/esm', import.meta.url);
import 'ts-node-maintained/register/esm';
import { register } from 'node:module'
register('ts-node/esm', import.meta.url)
/**
* Import ace console entrypoint
*/
await import('./bin/console.js');
await import('./bin/console.js')

172
adonisrc.ts
View file

@ -1,7 +1,7 @@
import { defineConfig } from '@adonisjs/core/app';
import { defineConfig } from '@adonisjs/core/app'
export default defineConfig({
/*
/*
|--------------------------------------------------------------------------
| Commands
|--------------------------------------------------------------------------
@ -10,12 +10,12 @@ export default defineConfig({
| will be scanned automatically from the "./commands" directory.
*/
commands: [
() => import('@adonisjs/core/commands'),
() => import('@adonisjs/lucid/commands'),
() => import('@adonisjs/mail/commands')
],
/*
commands: [
() => import('@adonisjs/core/commands'),
() => import('@adonisjs/lucid/commands'),
() => import('@adonisjs/mail/commands')
],
/*
|--------------------------------------------------------------------------
| Preloads
|--------------------------------------------------------------------------
@ -23,23 +23,19 @@ export default defineConfig({
| List of modules to import before starting the application.
|
*/
preloads: [
() => import('./start/routes.js'),
() => import('./start/kernel.js'),
() => import('#start/validator'),
// () => import('#start/rules/unique'),
// () => import('#start/rules/translated_language'),
// () => import('#start/rules/unique_person'),
// // () => import('#start/rules/file_length'),
// // () => import('#start/rules/file_scan'),
// // () => import('#start/rules/allowed_extensions_mimetypes'),
// () => import('#start/rules/dependent_array_min_length'),
// () => import('#start/rules/referenceValidation'),
// () => import('#start/rules/valid_mimetype'),
// () => import('#start/rules/array_contains_types'),
// () => import('#start/rules/orcid'),
],
/*
preloads: [
() => import('./start/routes.js'),
() => import('./start/kernel.js'),
() => import('#start/validator'),
() => import('#start/rules/unique'),
() => import('#start/rules/translated_language'),
() => import('#start/rules/unique_person'),
() => import('#start/rules/file_length'),
() => import('#start/rules/file_scan'),
() => import('#start/rules/allowed_extensions_mimetypes'),
() => import('#start/rules/dependent_array_min_length')
],
/*
|--------------------------------------------------------------------------
| Service providers
|--------------------------------------------------------------------------
@ -48,50 +44,48 @@ export default defineConfig({
| application
|
*/
providers: [
// () => import('./providers/AppProvider.js'),
() => import('@adonisjs/core/providers/app_provider'),
() => import('@adonisjs/core/providers/hash_provider'),
{
file: () => import('@adonisjs/core/providers/repl_provider'),
environment: ['repl', 'test'],
},
() => import('@adonisjs/session/session_provider'),
() => import('@adonisjs/core/providers/edge_provider'),
() => import('@adonisjs/shield/shield_provider'),
// () => import('@eidellev/inertia-adonisjs'),
// () => import('@adonisjs/inertia/inertia_provider'),
() => import('#providers/app_provider'),
// () => import('#providers/inertia_provider'),
() => import('@adonisjs/inertia/inertia_provider'),
() => import('@adonisjs/lucid/database_provider'),
() => import('@adonisjs/auth/auth_provider'),
// () => import('@eidellev/adonis-stardust'),
() => import('@adonisjs/redis/redis_provider'),
// () => import('@adonisjs/encore/encore_provider'),
() => import('@adonisjs/static/static_provider'),
() => import('#providers/stardust_provider'),
() => import('#providers/query_builder_provider'),
() => import('#providers/token_worker_provider'),
() => import('#providers/rule_provider'),
// () => import('#providers/drive/provider/drive_provider'),
() => import('@adonisjs/drive/drive_provider'),
// () => import('@adonisjs/core/providers/vinejs_provider'),
() => import('#providers/vinejs_provider'),
() => import('@adonisjs/mail/mail_provider'),
() => import('@adonisjs/vite/vite_provider'),
],
metaFiles: [
{
pattern: 'public/**',
reloadServer: false,
},
{
pattern: 'resources/views/**/*.edge',
reloadServer: false,
},
],
/*
providers: [
// () => import('./providers/AppProvider.js'),
() => import('@adonisjs/core/providers/app_provider'),
() => import('@adonisjs/core/providers/hash_provider'),
{
file: () => import('@adonisjs/core/providers/repl_provider'),
environment: ['repl', 'test'],
},
() => import('@adonisjs/session/session_provider'),
() => import('@adonisjs/core/providers/edge_provider'),
() => import('@adonisjs/shield/shield_provider'),
// () => import('@eidellev/inertia-adonisjs'),
// () => import('@adonisjs/inertia/inertia_provider'),
() => import('#providers/app_provider'),
() => import('#providers/inertia_provider'),
() => import('@adonisjs/lucid/database_provider'),
() => import('@adonisjs/auth/auth_provider'),
// () => import('@eidellev/adonis-stardust'),
() => import('@adonisjs/redis/redis_provider'),
() => import('@adonisjs/encore/encore_provider'),
() => import('@adonisjs/static/static_provider'),
() => import('#providers/stardust_provider'),
() => import('#providers/query_builder_provider'),
() => import('#providers/token_worker_provider'),
// () => import('#providers/validator_provider'),
() => import('#providers/drive/provider/drive_provider'),
// () => import('@adonisjs/core/providers/vinejs_provider'),
() => import('#providers/vinejs_provider'),
() => import('@adonisjs/mail/mail_provider')
// () => import('#providers/mail_provider'),
],
metaFiles: [
{
pattern: 'public/**',
reloadServer: false,
},
{
pattern: 'resources/views/**/*.edge',
reloadServer: false,
},
],
/*
|--------------------------------------------------------------------------
| Tests
|--------------------------------------------------------------------------
@ -100,24 +94,22 @@ export default defineConfig({
| and add additional suites.
|
*/
tests: {
suites: [
{
files: ['tests/unit/**/*.spec(.ts|.js)'],
name: 'unit',
timeout: 2000,
},
{
files: ['tests/functional/**/*.spec(.ts|.js)'],
name: 'functional',
timeout: 30000,
},
],
forceExit: false,
},
assetsBundler: false,
hooks: {
onBuildStarting: [() => import('@adonisjs/vite/build_hook')],
},
// assetsBundler: false
});
tests: {
suites: [
{
files: ['tests/unit/**/*.spec(.ts|.js)'],
name: 'unit',
timeout: 2000,
},
{
files: ['tests/functional/**/*.spec(.ts|.js)'],
name: 'functional',
timeout: 30000,
},
],
forceExit: false,
},
})

14
app/Controllers/Http/Admin/AdminuserController.ts
View file

@ -85,9 +85,7 @@ export default class AdminuserController {
// return response.badRequest(error.messages);
throw error;
}
const input: Record<string, any> = request.only(['login', 'email', 'first_name', 'last_name']);
input.password = request.input('new_password');
const input = request.only(['login', 'email', 'password', 'first_name', 'last_name']);
const user = await User.create(input);
if (request.input('roles')) {
const roles: Array<number> = request.input('roles');
@ -97,6 +95,7 @@ export default class AdminuserController {
session.flash('message', 'User has been created successfully');
return response.redirect().toRoute('settings.user.index');
}
public async show({ request, inertia }: HttpContext) {
const id = request.param('id');
const user = await User.query().where('id', id).firstOrFail();
@ -140,11 +139,9 @@ export default class AdminuserController {
});
// password is optional
let input: Record<string, any>;
if (request.input('new_password')) {
input = request.only(['login', 'email', 'first_name', 'last_name']);
input.password = request.input('new_password');
let input;
if (request.input('password')) {
input = request.only(['login', 'email', 'password', 'first_name', 'last_name']);
} else {
input = request.only(['login', 'email', 'first_name', 'last_name']);
}
@ -159,6 +156,7 @@ export default class AdminuserController {
session.flash('message', 'User has been updated successfully');
return response.redirect().toRoute('settings.user.index');
}
public async destroy({ request, response, session }: HttpContext) {
const id = request.param('id');
const user = await User.findOrFail(id);

13
app/Controllers/Http/Admin/MimetypeController.ts
View file

@ -25,7 +25,6 @@ export default class MimetypeController {
const newDatasetSchema = vine.object({
name: vine.string().trim().isUnique({ table: 'mime_types', column: 'name' }),
file_extension: vine.array(vine.string()).minLength(1), // define at least one extension for the new mimetype
alternate_mimetype: vine.array(vine.string().isValidMimetype()).distinct().optional(), // define alias mimetypes
enabled: vine.boolean(),
});
// await request.validate({ schema: newDatasetSchema, messages: this.messages });
@ -34,21 +33,17 @@ export default class MimetypeController {
// await request.validate({ schema: newDatasetSchema, messages: this.messages });
const validator = vine.compile(newDatasetSchema);
validator.messagesProvider = new SimpleMessagesProvider(this.messages);
await request.validateUsing(validator, { messagesProvider: new SimpleMessagesProvider(this.messages) });
await request.validateUsing(validator);
} catch (error) {
// Step 3 - Handle errors
// return response.badRequest(error.messages);
throw error;
}
const input = request.only(['name', 'enabled', 'file_extension', 'alternate_mimetype']);
const input = request.only(['name', 'enabled', 'file_extension']);
// Concatenate the file_extensions array into a string with '|' as the separator
if (Array.isArray(input.file_extension)) {
input.file_extension = input.file_extension.join('|');
}
// Concatenate the alias_mimetype array into a string with '|' as the separator
if (Array.isArray(input.alternate_mimetype)) {
input.alternate_mimetype = input.alternate_mimetype.join('|');
}
await MimeType.create(input);
// if (request.input('roles')) {
// const roles: Array<number> = request.input('roles');
@ -64,7 +59,7 @@ export default class MimetypeController {
'maxLength': '{{ field }} must be less then {{ max }} characters long',
'isUnique': '{{ field }} must be unique, and this value is already taken',
'required': '{{ field }} is required',
'file_extension.array.minLength': 'at least {{ min }} mimetypes must be defined',
'file_extension.minLength': 'at least {{ min }} mimetypes must be defined',
'file_extension.*.string': 'Each file extension must be a valid string', // Adjusted to match the type
};
@ -168,7 +163,7 @@ export default class MimetypeController {
mimetype,
});
} catch (error) {
if (error.code === 'E_ROW_NOT_FOUND') {
if (error.code == 'E_ROW_NOT_FOUND') {
session.flash({ warning: 'Mimetype is not found in database' });
} else {
session.flash({ warning: 'general error occured, you cannot delete the mimetype' });

4
app/Controllers/Http/Admin/RoleController.ts
View file

@ -42,7 +42,7 @@ export default class RoleController {
can: {
create: await auth.user?.can(['user-create']),
edit: await auth.user?.can(['user-edit']),
delete: false, //await auth.user?.can(['user-delete']),
delete: await auth.user?.can(['user-delete']),
},
});
}
@ -124,7 +124,7 @@ export default class RoleController {
// password is optional
const input = request.only(['name', 'display_name', 'description']);
const input = request.only(['name', 'description']);
await role.merge(input).save();
// await user.save();

4
app/Controllers/Http/Admin/mailsettings_controller.ts
View file

@ -76,9 +76,9 @@ export default class MailSettingsController {
public async sendTestMail({ response, auth }: HttpContext) {
const user = auth.user!;
const userEmail = user.email;
// let mailManager = await app.container.make('mail.manager');
// let iwas = mailManager.use();
// let iwas = mailManager.use();
// let test = mail.config.mailers.smtp();
if (!userEmail) {
return response.badRequest({ message: 'User email is not set. Please update your profile.' });

37
app/Controllers/Http/Api/AuthorsController.ts
View file

@ -4,29 +4,17 @@ import Person from '#models/person';
// node ace make:controller Author
export default class AuthorsController {
public async index({}: HttpContext) {
public async index({}: HttpContext) {
// select * from gba.persons
// where exists (select * from gba.documents inner join gba.link_documents_persons on "documents"."id" = "link_documents_persons"."document_id"
// where ("link_documents_persons"."role" = 'author') and ("persons"."id" = "link_documents_persons"."person_id"));
const authors = await Person.query()
.select([
'id',
'academic_title',
'first_name',
'last_name',
'identifier_orcid',
'status',
'name_type',
'created_at'
// Note: 'email' is omitted
])
.preload('datasets')
.where('name_type', 'Personal')
.whereHas('datasets', (dQuery) => {
dQuery.wherePivot('role', 'author');
})
.withCount('datasets', (query) => {
query.as('datasets_count');
})
.orderBy('datasets_count', 'desc');
.whereHas('datasets', (dQuery) => {
dQuery.wherePivot('role', 'author');
})
.withCount('datasets', (query) => {
query.as('datasets_count');
});
return authors;
}
@ -37,10 +25,7 @@ export default class AuthorsController {
if (request.input('filter')) {
// users = users.whereRaw('name like %?%', [request.input('search')])
const searchTerm = request.input('filter');
authors.andWhere((query) => {
query.whereILike('first_name', `%${searchTerm}%`)
.orWhereILike('last_name', `%${searchTerm}%`);
});
authors.whereILike('first_name', `%${searchTerm}%`).orWhereILike('last_name', `%${searchTerm}%`);
// .orWhere('email', 'like', `%${searchTerm}%`);
}

231
app/Controllers/Http/Api/AvatarController.ts
View file

@ -1,212 +1,65 @@
import type { HttpContext } from '@adonisjs/core/http';
import { StatusCodes } from 'http-status-codes';
import redis from '@adonisjs/redis/services/main';
// import * as fs from 'fs';
// import * as path from 'path';
const PREFIXES = ['von', 'van', 'de', 'del', 'della', 'di', 'da', 'dos', 'du', 'le', 'la'];
const DEFAULT_SIZE = 50;
const MIN_SIZE = 16;
const MAX_SIZE = 512;
const FONT_SIZE_RATIO = 0.4;
const COLOR_LIGHTENING_PERCENT = 60;
const COLOR_DARKENING_FACTOR = 0.6;
const CACHE_TTL = 24 * 60 * 60; // 24 hours instead of 1 hour
const prefixes = ['von', 'van'];
// node ace make:controller Author
export default class AvatarController {
public async generateAvatar({ request, response }: HttpContext) {
try {
const { name, size = DEFAULT_SIZE } = request.only(['name', 'size']);
// Enhanced validation
if (!name || typeof name !== 'string' || name.trim().length === 0) {
return response.status(StatusCodes.BAD_REQUEST).json({
error: 'Name is required and must be a non-empty string',
});
}
const parsedSize = this.validateSize(size);
if (!parsedSize.isValid) {
return response.status(StatusCodes.BAD_REQUEST).json({
error: parsedSize.error,
});
}
// Build a unique cache key for the given name and size
const cacheKey = `avatar:${this.sanitizeName(name)}-${parsedSize.value}`;
// const cacheKey = `avatar:${name.trim().toLowerCase()}-${size}`;
try {
const cachedSvg = await redis.get(cacheKey);
if (cachedSvg) {
this.setResponseHeaders(response);
return response.send(cachedSvg);
}
} catch (redisError) {
// Log redis error but continue without cache
console.warn('Redis cache read failed:', redisError);
}
const { name, background, textColor, size } = request.only(['name', 'background', 'textColor', 'size']);
// Generate initials
// const initials = name
// .split(' ')
// .map((part) => part.charAt(0).toUpperCase())
// .join('');
const initials = this.getInitials(name);
const colors = this.generateColors(name);
const svgContent = this.createSvg(size, colors, initials);
// // Cache the generated avatar for future use, e.g. 1 hour expiry
try {
await redis.setex(cacheKey, CACHE_TTL, svgContent);
} catch (redisError) {
// Log but don't fail the request
console.warn('Redis cache write failed:', redisError);
}
// Define SVG content with dynamic values for initials, background color, text color, and size
const svgContent = `
<svg width="${size || 50}" height="${size || 50}" xmlns="http://www.w3.org/2000/svg">
<rect width="100%" height="100%" fill="#${background || '7F9CF5'}"/>
<text x="50%" y="50%" dominant-baseline="middle" text-anchor="middle" font-weight="bold" font-family="Arial, sans-serif" font-size="${
(size / 100) * 40 || 25
}" fill="#${textColor || 'ffffff'}">${initials}</text>
</svg>
`;
// Set response headers for SVG content
response.header('Content-type', 'image/svg+xml');
response.header('Cache-Control', 'no-cache');
response.header('Pragma', 'no-cache');
response.header('Expires', '0');
this.setResponseHeaders(response);
return response.send(svgContent);
} catch (error) {
console.error('Avatar generation error:', error);
return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
error: 'Failed to generate avatar',
});
return response.status(StatusCodes.OK).json({ error: error.message });
}
}
private validateSize(size: any): { isValid: boolean; value?: number; error?: string } {
const numSize = Number(size);
private getInitials(name: string) {
const parts = name.split(' ');
let initials = '';
if (isNaN(numSize)) {
return { isValid: false, error: 'Size must be a valid number' };
}
if (numSize < MIN_SIZE || numSize > MAX_SIZE) {
return {
isValid: false,
error: `Size must be between ${MIN_SIZE} and ${MAX_SIZE}`,
};
}
return { isValid: true, value: Math.floor(numSize) };
}
private sanitizeName(name: string): string {
return name
.trim()
.toLowerCase()
.replace(/[^a-z0-9\s-]/gi, '');
}
private getInitials(name: string): string {
const sanitized = name.trim().replace(/\s+/g, ' '); // normalize whitespace
const parts = sanitized
.split(' ')
.filter((part) => part.length > 0)
.map((part) => part.trim());
if (parts.length === 0) {
return 'NA';
}
if (parts.length === 1) {
// For single word, take first 2 characters or first char if only 1 char
return parts[0].substring(0, Math.min(2, parts[0].length)).toUpperCase();
}
return this.getMultiWordInitials(parts);
}
private getMultiWordInitials(parts: string[]): string {
// Filter out prefixes and short words
const significantParts = parts.filter((part) => !PREFIXES.includes(part.toLowerCase()) && part.length > 1);
if (significantParts.length === 0) {
// Fallback to first and last regardless of prefixes
if (parts.length >= 2) {
const firstName = parts[0];
const lastName = parts[parts.length - 1];
return (firstName.charAt(0) + lastName.charAt(0)).toUpperCase();
const firstInitial = firstName.charAt(0).toUpperCase();
const lastInitial = lastName.charAt(0).toUpperCase();
if (prefixes.includes(lastName.toLowerCase()) && lastName === lastName.toUpperCase()) {
initials = firstInitial + lastName.charAt(1).toUpperCase();
} else {
initials = firstInitial + lastInitial;
}
} else if (parts.length === 1) {
initials = parts[0].substring(0, 2).toUpperCase();
}
if (significantParts.length === 1) {
return significantParts[0].substring(0, 2).toUpperCase();
}
// Take first and last significant parts
const firstName = significantParts[0];
const lastName = significantParts[significantParts.length - 1];
return (firstName.charAt(0) + lastName.charAt(0)).toUpperCase();
}
private generateColors(name: string): { background: string; text: string } {
const baseColor = this.getColorFromName(name);
return {
background: this.lightenColor(baseColor, COLOR_LIGHTENING_PERCENT),
text: this.darkenColor(baseColor),
};
}
private createSvg(size: number, colors: { background: string; text: string }, initials: string): string {
const fontSize = Math.max(12, Math.floor(size * FONT_SIZE_RATIO)); // Ensure readable font size
// Escape any potential HTML/XML characters in initials
const escapedInitials = this.escapeXml(initials);
return `<svg width="${size}" height="${size}" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${size} ${size}">
<rect width="100%" height="100%" fill="#${colors.background}" rx="${size * 0.1}"/>
<text x="50%" y="50%" dominant-baseline="central" text-anchor="middle"
font-weight="600" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif"
font-size="${fontSize}" fill="#${colors.text}">${escapedInitials}</text>
</svg>`;
}
private escapeXml(text: string): string {
return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;').replace(/'/g, '&apos;');
}
private setResponseHeaders(response: HttpContext['response']): void {
response.header('Content-Type', 'image/svg+xml');
response.header('Cache-Control', 'public, max-age=86400'); // Cache for 1 day
response.header('ETag', `"${Date.now()}"`); // Simple ETag
}
private getColorFromName(name: string): string {
let hash = 0;
const normalizedName = name.toLowerCase().trim();
for (let i = 0; i < normalizedName.length; i++) {
hash = normalizedName.charCodeAt(i) + ((hash << 5) - hash);
hash = hash & hash; // Convert to 32-bit integer
}
// Ensure we get vibrant colors by constraining the color space
const colorParts = [];
for (let i = 0; i < 3; i++) {
let value = (hash >> (i * 8)) & 0xff;
// Ensure minimum color intensity for better contrast
value = Math.max(50, value);
colorParts.push(value.toString(16).padStart(2, '0'));
}
return colorParts.join('');
}
private lightenColor(hexColor: string, percent: number): string {
const r = parseInt(hexColor.substring(0, 2), 16);
const g = parseInt(hexColor.substring(2, 4), 16);
const b = parseInt(hexColor.substring(4, 6), 16);
const lightenValue = (value: number) => Math.min(255, Math.floor(value + (255 - value) * (percent / 100)));
const newR = lightenValue(r);
const newG = lightenValue(g);
const newB = lightenValue(b);
return ((newR << 16) | (newG << 8) | newB).toString(16).padStart(6, '0');
}
private darkenColor(hexColor: string): string {
const r = parseInt(hexColor.slice(0, 2), 16);
const g = parseInt(hexColor.slice(2, 4), 16);
const b = parseInt(hexColor.slice(4, 6), 16);
const darkenValue = (value: number) => Math.max(0, Math.floor(value * COLOR_DARKENING_FACTOR));
const darkerR = darkenValue(r);
const darkerG = darkenValue(g);
const darkerB = darkenValue(b);
return ((darkerR << 16) + (darkerG << 8) + darkerB).toString(16).padStart(6, '0');
return initials;
}
}

419
app/Controllers/Http/Api/DatasetController.ts
View file

@ -1,36 +1,19 @@
import type { HttpContext } from '@adonisjs/core/http';
// import Person from 'App/Models/Person';
import Dataset from '#models/dataset';
import { StatusCodes } from 'http-status-codes';
import DatasetReference from '#models/dataset_reference';
// node ace make:controller Author
export default class DatasetController {
/**
* GET /api/datasets
* Find all published datasets
*/
public async index({ response }: HttpContext) {
try {
const datasets = await Dataset.query()
.where(function (query) {
query.where('server_state', 'published').orWhere('server_state', 'deleted');
})
.preload('titles')
.preload('identifier')
.orderBy('server_date_published', 'desc');
public async index({}: HttpContext) {
// select * from gba.persons
// where exists (select * from gba.documents inner join gba.link_documents_persons on "documents"."id" = "link_documents_persons"."document_id"
// where ("link_documents_persons"."role" = 'author') and ("persons"."id" = "link_documents_persons"."person_id"));
const datasets = await Dataset.query().where('server_state', 'published').orWhere('server_state', 'deleted');
return response.status(StatusCodes.OK).json(datasets);
} catch (error) {
return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
message: error.message || 'Some error occurred while retrieving datasets.',
});
}
return datasets;
}
/**
* GET /api/dataset
* Find all published datasets
*/
public async findAll({ response }: HttpContext) {
try {
const datasets = await Dataset.query()
@ -46,368 +29,34 @@ export default class DatasetController {
}
}
/**
* GET /api/dataset/:publish_id
* Find one dataset by publish_id
*/
public async findOne({ response, params }: HttpContext) {
try {
const dataset = await Dataset.query()
.where('publish_id', params.publish_id)
.preload('titles')
.preload('descriptions') // Using 'descriptions' instead of 'abstracts'
.preload('user', (builder) => {
builder.select(['id', 'firstName', 'lastName', 'avatar', 'login']);
})
.preload('authors', (builder) => {
builder
.select(['id', 'academic_title', 'first_name', 'last_name', 'identifier_orcid', 'status', 'name_type'])
.withCount('datasets', (query) => {
query.as('datasets_count');
})
.pivotColumns(['role', 'sort_order'])
.orderBy('pivot_sort_order', 'asc');
})
.preload('contributors', (builder) => {
builder
.select(['id', 'academic_title', 'first_name', 'last_name', 'identifier_orcid', 'status', 'name_type'])
.withCount('datasets', (query) => {
query.as('datasets_count');
})
.pivotColumns(['role', 'sort_order', 'contributor_type'])
.orderBy('pivot_sort_order', 'asc');
})
.preload('subjects')
.preload('coverage')
.preload('licenses')
.preload('references')
.preload('project')
// .preload('referenced_by', (builder) => {
// builder.preload('dataset', (builder) => {
// builder.preload('identifier');
// });
// })
.preload('files', (builder) => {
builder.preload('hashvalues');
})
.preload('identifier')
.first(); // Use first() instead of firstOrFail() to handle not found gracefully
if (!dataset) {
return response.status(StatusCodes.NOT_FOUND).json({
message: `Cannot find Dataset with publish_id=${params.publish_id}.`,
public async findOne({ params }: HttpContext) {
const datasets = await Dataset.query()
.where('publish_id', params.publish_id)
.preload('titles')
.preload('descriptions')
.preload('user')
.preload('authors', (builder) => {
builder.orderBy('pivot_sort_order', 'asc');
})
.preload('contributors', (builder) => {
builder.orderBy('pivot_sort_order', 'asc');
})
.preload('subjects')
.preload('coverage')
.preload('licenses')
.preload('references')
.preload('project')
.preload('referenced_by', (builder) => {
builder.preload('dataset', (builder) => {
builder.preload('identifier');
});
}
})
.preload('files', (builder) => {
builder.preload('hashvalues');
})
.preload('identifier')
.firstOrFail();
// Build the version chain
const versionChain = await this.buildVersionChain(dataset);
// Add version chain to response
const responseData = {
...dataset.toJSON(),
versionChain: versionChain,
};
// return response.status(StatusCodes.OK).json(dataset);
return response.status(StatusCodes.OK).json(responseData);
} catch (error) {
return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
message: error.message || `Error retrieving Dataset with publish_id=${params.publish_id}.`,
});
}
}
/**
* GET /:prefix/:value
* Find dataset by identifier (e.g., https://doi.tethys.at/10.24341/tethys.99.2)
*/
public async findByIdentifier({ response, params }: HttpContext) {
const identifierValue = `${params.prefix}/${params.value}`;
// Optional: Validate DOI format
if (!identifierValue.match(/^10\.\d+\/[a-zA-Z0-9._-]+\.[0-9]+(?:\.[0-9]+)*$/)) {
return response.status(StatusCodes.BAD_REQUEST).json({
message: `Invalid DOI format: ${identifierValue}`,
});
}
try {
// Method 1: Using subquery with whereIn (most similar to your original)
const dataset = await Dataset.query()
// .whereIn('id', (subQuery) => {
// subQuery.select('dataset_id').from('dataset_identifiers').where('value', identifierValue);
// })
.whereHas('identifier', (builder) => {
builder.where('value', identifierValue);
})
.preload('titles')
.preload('descriptions') // Using 'descriptions' instead of 'abstracts'
.preload('user', (builder) => {
builder.select(['id', 'firstName', 'lastName', 'avatar', 'login']);
})
.preload('authors', (builder) => {
builder
.select(['id', 'academic_title', 'first_name', 'last_name', 'identifier_orcid', 'status', 'name_type'])
.withCount('datasets', (query) => {
query.as('datasets_count');
})
.pivotColumns(['role', 'sort_order'])
.wherePivot('role', 'author')
.orderBy('pivot_sort_order', 'asc');
})
.preload('contributors', (builder) => {
builder
.select(['id', 'academic_title', 'first_name', 'last_name', 'identifier_orcid', 'status', 'name_type'])
.withCount('datasets', (query) => {
query.as('datasets_count');
})
.pivotColumns(['role', 'sort_order', 'contributor_type'])
.wherePivot('role', 'contributor')
.orderBy('pivot_sort_order', 'asc');
})
.preload('subjects')
.preload('coverage')
.preload('licenses')
.preload('references')
.preload('project')
// .preload('referenced_by', (builder) => {
// builder.preload('dataset', (builder) => {
// builder.preload('identifier');
// });
// })
.preload('files', (builder) => {
builder.preload('hashvalues');
})
.preload('identifier')
.first();
if (!dataset) {
return response.status(StatusCodes.NOT_FOUND).json({
message: `Cannot find Dataset with identifier=${identifierValue}.`,
});
}
// Build the version chain
const versionChain = await this.buildVersionChain(dataset);
// Add version chain to response
const responseData = {
...dataset.toJSON(),
versionChain: versionChain,
};
// return response.status(StatusCodes.OK).json(dataset);
return response.status(StatusCodes.OK).json(responseData);
} catch (error) {
return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
message: error.message || `Error retrieving Dataset with identifier=${identifierValue}.`,
});
}
}
/**
* Build the complete version chain for a dataset
* Traverses both backwards (previous versions) and forwards (newer versions)
*/
private async buildVersionChain(dataset: Dataset) {
const versionChain = {
// current: {
// id: dataset.id,
// publish_id: dataset.publish_id,
// doi: dataset.identifier?.value || null,
// main_title: dataset.mainTitle || null,
// server_date_published: dataset.server_date_published,
// },
previousVersions: [] as any[],
newerVersions: [] as any[],
};
// Get all previous versions (going backwards in time)
versionChain.previousVersions = await this.getPreviousVersions(dataset.id);
// Get all newer versions (going forwards in time)
versionChain.newerVersions = await this.getNewerVersions(dataset.id);
return versionChain;
}
/**
* Recursively get all previous versions
*/
// private async getPreviousVersions(datasetId: number, visited: Set<number> = new Set()): Promise<any[]> {
// // Prevent infinite loops
// if (visited.has(datasetId)) {
// return [];
// }
// visited.add(datasetId);
// const previousVersions: any[] = [];
// // Find references where this dataset "IsNewVersionOf" another dataset
// const previousRefs = await DatasetReference.query()
// .where('document_id', datasetId)
// .where('relation', 'IsNewVersionOf')
// .whereNotNull('related_document_id');
// for (const ref of previousRefs) {
// if (!ref.related_document_id) continue;
// const previousDataset = await Dataset.query()
// .where('id', ref.related_document_id)
// .preload('identifier')
// .preload('titles')
// .first();
// if (previousDataset) {
// const versionInfo = {
// id: previousDataset.id,
// publish_id: previousDataset.publish_id,
// doi: previousDataset.identifier?.value || null,
// main_title: previousDataset.mainTitle || null,
// server_date_published: previousDataset.server_date_published,
// relation: 'IsPreviousVersionOf', // From perspective of current dataset
// };
// previousVersions.push(versionInfo);
// // Recursively get even older versions
// const olderVersions = await this.getPreviousVersions(previousDataset.id, visited);
// previousVersions.push(...olderVersions);
// }
// }
// return previousVersions;
// }
private async getPreviousVersions(datasetId: number, visited: Set<number> = new Set()): Promise<any[]> {
if (visited.has(datasetId)) return [];
visited.add(datasetId);
const result: any[] = [];
// A dataset points to its OLDER version via relation 'IsNewVersionOf'
const refs = await DatasetReference.query()
.where('document_id', datasetId)
.where('relation', 'IsNewVersionOf'); // ← removed .whereNotNull('related_document_id')
for (const ref of refs) {
const related = await this.resolveReferencedDataset(ref, datasetId);
if (!related) continue;
result.push({
id: related.id,
publish_id: related.publish_id,
doi: related.identifier?.value || null,
main_title: related.mainTitle || null,
server_date_published: related.server_date_published,
relation: 'IsPreviousVersionOf',
});
result.push(...(await this.getPreviousVersions(related.id, visited)));
}
return result;
}
/**
* Recursively get all newer versions
*/
// private async getNewerVersions(datasetId: number, visited: Set<number> = new Set()): Promise<any[]> {
// // Prevent infinite loops
// if (visited.has(datasetId)) {
// return [];
// }
// visited.add(datasetId);
// const newerVersions: any[] = [];
// // Find references where this dataset "IsPreviousVersionOf" another dataset
// const newerRefs = await DatasetReference.query()
// .where('document_id', datasetId)
// .where('relation', 'IsPreviousVersionOf')
// .whereNotNull('related_document_id');
// for (const ref of newerRefs) {
// if (!ref.related_document_id) continue;
// const newerDataset = await Dataset.query().where('id', ref.related_document_id).preload('identifier').preload('titles').first();
// if (newerDataset) {
// const versionInfo = {
// id: newerDataset.id,
// publish_id: newerDataset.publish_id,
// doi: newerDataset.identifier?.value || null,
// main_title: newerDataset.mainTitle || null,
// server_date_published: newerDataset.server_date_published,
// relation: 'IsNewVersionOf', // From perspective of current dataset
// };
// newerVersions.push(versionInfo);
// // Recursively get even newer versions
// const evenNewerVersions = await this.getNewerVersions(newerDataset.id, visited);
// newerVersions.push(...evenNewerVersions);
// }
// }
// return newerVersions;
// }
private async getNewerVersions(datasetId: number, visited: Set<number> = new Set()): Promise<any[]> {
if (visited.has(datasetId)) return [];
visited.add(datasetId);
const result: any[] = [];
// A dataset points to its NEWER version via relation 'IsPreviousVersionOf'
const refs = await DatasetReference.query()
.where('document_id', datasetId)
.where('relation', 'IsPreviousVersionOf'); // ← removed .whereNotNull(...)
for (const ref of refs) {
const related = await this.resolveReferencedDataset(ref, datasetId);
if (!related) continue;
result.push({
id: related.id,
publish_id: related.publish_id,
doi: related.identifier?.value || null,
main_title: related.mainTitle || null,
server_date_published: related.server_date_published,
relation: 'IsNewVersionOf',
});
result.push(...(await this.getNewerVersions(related.id, visited)));
}
return result;
}
private async resolveReferencedDataset(ref: DatasetReference, currentDatasetId: number) {
const doi = this.normalizeDoi(ref.value);
if (doi) {
const byDoi = await Dataset.query()
.whereHas('identifier', (q) => q.where('value', doi))
.preload('identifier')
.preload('titles') // needed so mainTitle computes
.first();
if (byDoi) return byDoi;
}
if (ref.related_document_id && ref.related_document_id !== currentDatasetId) {
return await Dataset.query()
.where('id', ref.related_document_id)
.preload('identifier')
.preload('titles')
.first();
}
return null;
}
private normalizeDoi(value: string | null): string | null {
if (!value) return null;
return value
.trim()
.replace(/^https?:\/\/(dx\.)?doi\.org\//i, '')
.replace(/^doi:/i, '');
return datasets;
}
}

126
app/Controllers/Http/Api/FileController.ts
View file

@ -2,103 +2,53 @@ import type { HttpContext } from '@adonisjs/core/http';
import File from '#models/file';
import { StatusCodes } from 'http-status-codes';
import * as fs from 'fs';
import { DateTime } from 'luxon';
import * as path from 'path';
// node ace make:controller Author
export default class FileController {
// @Get("download/:id")
public async findOne({ response, params }: HttpContext) {
const id = params.id;
// const file = await File.findOrFail(id);
// Load file with its related dataset to check embargo
const file = await File.query()
.where('id', id)
.preload('dataset') // or 'dataset' - whatever your relationship is named
.firstOrFail();
const file = await File.findOrFail(id);
// const file = await File.findOne({
// where: { id: id },
// });
if (file) {
const filePath = '/storage/app/public/' + file.pathName;
const ext = path.extname(filePath);
const fileName = file.label + ext;
try {
fs.accessSync(filePath, fs.constants.R_OK); //| fs.constants.W_OK);
// console.log("can read/write:", path);
response
.header('Cache-Control', 'no-cache private')
.header('Content-Description', 'File Transfer')
.header('Content-Type', file.mimeType)
.header('Content-Disposition', 'inline; filename=' + fileName)
.header('Content-Transfer-Encoding', 'binary')
.header('Access-Control-Allow-Origin', '*')
.header('Access-Control-Allow-Methods', 'GET,POST');
response.status(StatusCodes.OK).download(filePath);
} catch (err) {
// console.log("no access:", path);
response.status(StatusCodes.NOT_FOUND).send({
message: `File with id ${id} doesn't exist on file server`,
});
}
if (!file) {
return response.status(StatusCodes.NOT_FOUND).send({
// res.status(StatusCodes.OK).sendFile(filePath, (err) => {
// // res.setHeader("Content-Type", "application/json");
// // res.removeHeader("Content-Disposition");
// res.status(StatusCodes.NOT_FOUND).send({
// message: `File with id ${id} doesn't exist on file server`,
// });
// });
} else {
response.status(StatusCodes.NOT_FOUND).send({
message: `Cannot find File with id=${id}.`,
});
}
const dataset = file.dataset;
// Files from unpublished datasets are now blocked
if (dataset.server_state !== 'published') {
return response.status(StatusCodes.FORBIDDEN).send({
message: `File access denied: Dataset is not published.`,
});
}
if (dataset && this.isUnderEmbargo(dataset.embargo_date)) {
return response.status(StatusCodes.FORBIDDEN).send({
message: `File is under embargo until ${dataset.embargo_date?.toFormat('yyyy-MM-dd')}`,
});
}
// Proceed with file download
const filePath = '/storage/app/data/' + file.pathName;
const fileExt = file.filePath.split('.').pop() || '';
// const fileName = file.label + fileExt;
const fileName = file.label.toLowerCase().endsWith(`.${fileExt.toLowerCase()}`) ? file.label : `${file.label}.${fileExt}`;
// Determine if file can be previewed inline in browser
const canPreviewInline = (mimeType: string): boolean => {
const type = mimeType.toLowerCase();
return (
type === 'application/pdf' ||
type.startsWith('image/') ||
type.startsWith('text/') ||
type === 'application/json' ||
type === 'application/xml' ||
// Uncomment if you want video/audio inline
type.startsWith('video/') ||
type.startsWith('audio/')
);
};
const disposition = canPreviewInline(file.mimeType) ? 'inline' : 'attachment';
try {
fs.accessSync(filePath, fs.constants.R_OK); //| fs.constants.W_OK);
// console.log("can read/write:", filePath);
response
.header('Cache-Control', 'no-cache private')
.header('Content-Description', 'File Transfer')
.header('Content-Type', file.mimeType)
.header('Content-Disposition', `${disposition}; filename="${fileName}"`)
.header('Content-Transfer-Encoding', 'binary')
.header('Access-Control-Allow-Origin', '*')
.header('Access-Control-Allow-Methods', 'GET');
response.status(StatusCodes.OK).download(filePath);
} catch (err) {
// console.log("no access:", path);
response.status(StatusCodes.NOT_FOUND).send({
message: `File with id ${id} doesn't exist on file server`,
});
}
}
/**
* Check if the dataset is under embargo
* Compares only dates (ignoring time) for embargo check
* @param embargoDate - The embargo date from dataset
* @returns true if under embargo, false if embargo has passed or no embargo set
*/
private isUnderEmbargo(embargoDate: DateTime | null): boolean {
// No embargo date set - allow download
if (!embargoDate) {
return false;
}
// Get current date at start of day (00:00:00)
const today = DateTime.now().startOf('day');
// Get embargo date at start of day (00:00:00)
const embargoDateOnly = embargoDate.startOf('day');
// File is under embargo if embargo date is after today
// This means the embargo lifts at the start of the embargo date
return embargoDateOnly >= today;
}
}

24
app/Controllers/Http/Api/HomeController.ts
View file

@ -17,8 +17,7 @@ export default class HomeController {
// .preload('authors')
// .orderBy('server_date_published');
const datasets = await db
.from('documents as doc')
const datasets = await db.from('documents as doc')
.select(['publish_id', 'server_date_published', db.raw(`date_part('year', server_date_published) as pub_year`)])
.where('server_state', serverState)
.innerJoin('link_documents_persons as ba', 'doc.id', 'ba.document_id')
@ -60,6 +59,7 @@ export default class HomeController {
// const year = params.year;
// const from = parseInt(year);
try {
// const datasets = await Database.from('documents as doc')
// .select([Database.raw(`date_part('month', server_date_published) as pub_month`), Database.raw('COUNT(*) as count')])
// .where('server_state', serverState)
@ -68,12 +68,9 @@ export default class HomeController {
// .groupBy('pub_month');
// // .orderBy('server_date_published');
// Calculate the last 4 years including the current year
const currentYear = new Date().getFullYear();
const years = Array.from({ length: 4 }, (_, i) => currentYear - (i + 1)).reverse();
const years = [2021, 2022, 2023]; // Add the second year
const result = await db
.from('documents as doc')
const result = await db.from('documents as doc')
.select([
db.raw(`date_part('year', server_date_published) as pub_year`),
db.raw(`date_part('month', server_date_published) as pub_month`),
@ -86,7 +83,7 @@ export default class HomeController {
.groupBy('pub_year', 'pub_month')
.orderBy('pub_year', 'asc')
.orderBy('pub_month', 'asc');
const labels = Array.from({ length: 12 }, (_, i) => i + 1); // Assuming 12 months
const inputDatasets: Map<string, ChartDataset> = result.reduce((acc, item) => {
@ -103,15 +100,15 @@ export default class HomeController {
acc[pub_year].data[pub_month - 1] = parseInt(count);
return acc;
return acc ;
}, {});
const outputDatasets = Object.entries(inputDatasets).map(([year, data]) => ({
data: data.data,
label: year,
borderColor: data.borderColor,
fill: data.fill,
}));
fill: data.fill
}));
const data = {
labels: labels,
@ -129,11 +126,11 @@ export default class HomeController {
private getRandomHexColor() {
const letters = '0123456789ABCDEF';
let color = '#';
for (let i = 0; i < 6; i++) {
color += letters[Math.floor(Math.random() * 16)];
}
return color;
}
}
@ -142,4 +139,5 @@ interface ChartDataset {
label: string;
borderColor: string;
fill: boolean;
}

18
app/Controllers/Http/Api/UserController.ts
View file

@ -9,24 +9,6 @@ import BackupCode from '#models/backup_code';
// Here we are generating secret and recovery codes for the user that’s enabling 2FA and storing them to our database.
export default class UserController {
public async getSubmitters({ response }: HttpContext) {
try {
const submitters = await User.query()
.preload('roles', (query) => {
query.where('name', 'submitter')
})
.whereHas('roles', (query) => {
query.where('name', 'submitter')
})
.exec();
return submitters;
} catch (error) {
return response.status(StatusCodes.INTERNAL_SERVER_ERROR).json({
message: 'Invalid TOTP state',
});
}
}
public async enable({ auth, response, request }: HttpContext) {
const user = (await User.find(auth.user?.id)) as User;
// await user.load('totp_secret');

36
app/Controllers/Http/Api/collections_controller.ts
View file

@ -1,36 +0,0 @@
import type { HttpContext } from '@adonisjs/core/http';
import Collection from '#models/collection';
export default class CollectionsController {
public async show({ params, response }: HttpContext) {
// Get the collection id from route parameters
const collectionId = params.id;
// Find the selected collection by id
const collection = await Collection.find(collectionId);
if (!collection) {
return response.status(404).json({ message: 'Collection not found' });
}
// Query for narrower concepts: collections whose parent_id equals the selected collection's id
const narrowerCollections = await Collection.query().where('parent_id', collection.id) || [];
// For broader concept, if the selected collection has a parent_id fetch that record (otherwise null)
const broaderCollection: Collection[] | never[] | null = await (async () => {
if (collection.parent_id) {
// Try to fetch the parent...
const parent = await Collection.find(collection.parent_id)
// If found, return it wrapped in an array; if not found, return null (or empty array if you prefer)
return parent ? [parent] : null
}
return []
})()
// Return the selected collection along with its narrower and broader concepts in JSON format
return response.json({
selectedCollection: collection,
narrowerCollections,
broaderCollection,
});
}
}

152
app/Controllers/Http/Auth/AuthController.ts
View file

@ -1,68 +1,93 @@
import type { HttpContext } from '@adonisjs/core/http';
import User from '#models/user';
import BackupCode from '#models/backup_code';
// import Hash from '@ioc:Adonis/Core/Hash';
// import InvalidCredentialException from 'App/Exceptions/InvalidCredentialException';
import { authValidator } from '#validators/auth';
import hash from '@adonisjs/core/services/hash';
import db from '@adonisjs/lucid/services/db';
import TwoFactorAuthProvider from '#app/services/TwoFactorAuthProvider';
import ActivityLogger from '#services/activity_logger';
import logger from '@adonisjs/core/services/logger';
import TwoFactorAuthProvider from '#app/services/TwoFactorAuthProvider';
// import { Authenticator } from '@adonisjs/auth';
// import { LoginState } from 'Contracts/enums';
// import { StatusCodes } from 'http-status-codes';
// interface MyHttpsContext extends HttpContext {
// auth: Authenticator<User>
// }
export default class AuthController {
// login function{ request, auth, response }:HttpContext
public async login({ request, response, auth, session }: HttpContext) {
// console.log({
// registerBody: request.body(),
// });
// await request.validate(AuthValidator);
await request.validateUsing(authValidator);
// const plainPassword = await request.input('password');
// const email = await request.input('email');
// grab uid and password values off request body
const { email, password } = request.only(['email', 'password']);
try {
await db.connection().rawQuery('SELECT 1');
// // attempt to verify credential and login user
// await auth.use('web').attempt(email, plainPassword);
// const user = await auth.use('web').verifyCredentials(email, password);
const user = await User.verifyCredentials(email, password);
if (user.isTwoFactorEnabled) {
// Noch KEIN abgeschlossenes Login -> nicht loggen.
// session.put("login.id", user.id);
// return view.render("pages/two-factor-challenge");
session.flash('user_id', user.id);
return response.redirect().back();
// let state = LoginState.STATE_VALIDATED;
// return response.status(StatusCodes.OK).json({
// state: state,
// new_user_id: user.id,
// });
}
await auth.use('web').login(user);
this.recordAuthEvent('auth.login', { user, ip: request.ip() });
} catch (error: any) {
// DB nicht erreichbar -> kein fehlgeschlagener Login-Versuch, weiterwerfen
if (error.code === 'ECONNREFUSED') {
throw error;
}
// Echter Credential-Fehler -> als fehlgeschlagenen Versuch protokollieren
this.recordAuthEvent('auth.login_failed', { email, ip: request.ip() });
} catch (error) {
// if login fails, return vague form message and redirect back
session.flash('message', 'Your username, email, or password is incorrect');
return response.redirect().back();
}
return response.redirect('/apps/dashboard');
// otherwise, redirect todashboard
response.redirect('/apps/dashboard');
}
public async twoFactorChallenge({ request, session, auth, response }: HttpContext) {
const { code, backup_code, login_id } = request.only(['code', 'backup_code', 'login_id']);
const { code, backup_code, login_id } = request.only(['code', 'backup_code', 'login_id']);
const user = await User.query().where('id', login_id).firstOrFail();
if (code) {
const isValid = await TwoFactorAuthProvider.validate(user, code);
if (isValid) {
// login user and redirect to dashboard
await auth.use('web').login(user);
this.recordAuthEvent('auth.login', { user, email: user.email, ip: request.ip(), method: '2fa_totp' });
return response.redirect('/apps/dashboard');
}
session.flash('message', 'Your two-factor code is incorrect');
response.redirect('/apps/dashboard');
} else {
session.flash('message', 'Your two-factor code is incorrect');
return response.redirect().back();
}
if (backup_code) {
}
} else if (backup_code) {
const codes: BackupCode[] = await user.getBackupCodes();
// const verifiedBackupCodes = await Promise.all(
// codes.map(async (backupCode) => {
// let isVerified = await hash.verify(backupCode.code, backup_code);
// if (isVerified) {
// return backupCode;
// }
// }),
// );
// const backupCodeToDelete = verifiedBackupCodes.find(Boolean);
let backupCodeToDelete: BackupCode | null = null;
let backupCodeToDelete = null;
for (const backupCode of codes) {
const isVerified = await hash.verify(backupCode.code, backup_code);
if (isVerified) {
@ -71,68 +96,29 @@ export default class AuthController {
}
}
if (!backupCodeToDelete) {
if (backupCodeToDelete) {
if (backupCodeToDelete.used === false) {
backupCodeToDelete.used = true;
await backupCodeToDelete.save();
console.log(`BackupCode with id ${backupCodeToDelete.id} has been marked as used.`);
await auth.use('web').login(user);
response.redirect('/apps/dashboard');
} else {
session.flash('message', 'BackupCode already used');
return response.redirect().back();
}
} else {
session.flash('message', 'BackupCode not found');
return response.redirect().back();
}
if (backupCodeToDelete.used) {
session.flash('message', 'BackupCode already used');
return response.redirect().back();
}
backupCodeToDelete.used = true;
await backupCodeToDelete.save();
await auth.use('web').login(user);
this.recordAuthEvent('auth.login', { user, email: user.email, ip: request.ip(), method: '2fa_backup_code' });
return response.redirect('/apps/dashboard');
}
}
// Weder code noch backup_code übergeben
session.flash('message', 'No verification code provided');
return response.redirect().back();
}
public async logout({ auth, request, response }: HttpContext) {
// Session auflösen -> füllt auth.user, falls eingeloggt
await auth.use('web').check();
const user = auth.use('web').user;
// logout function
public async logout({ auth, response }: HttpContext) {
// await auth.logout();
await auth.use('web').logout();
if (user) {
this.recordAuthEvent('auth.logout', { user, email: user.email, ip: request.ip() });
}
return response.redirect('/app/login');
}
/**
* Zentraler Audit-Logger für Auth-Events.
* Fire-and-forget: ein Fehler beim Schreiben darf Login/Logout nie blockieren.
*/
private recordAuthEvent(
type: 'auth.login' | 'auth.logout' | 'auth.login_failed',
opts: { user?: User; email?: string; ip: string; method?: string },
) {
const { user, email, ip, method } = opts;
const description =
type === 'auth.login'
? `${user!.firstName} ${user!.lastName} signed in`
: type === 'auth.logout'
? `${user!.firstName} ${user!.lastName} signed out`
: `Failed login attempt for ${email ?? 'unknown'}`;
void ActivityLogger.log({
type,
description,
userId: user?.id ?? null,
subjectType: user ? 'User' : null,
subjectId: user?.id ?? null,
properties: { ip, ...(method ? { method } : {}) },
}).catch((err) => logger.error({ err }, `failed to record ${type} activity`));
// return response.status(200);
}
}

184
app/Controllers/Http/Auth/UserController.ts
View file

@ -6,11 +6,6 @@ import hash from '@adonisjs/core/services/hash';
// import { schema, rules } from '@adonisjs/validator';
import vine from '@vinejs/vine';
import BackupCodeStorage, { SecureRandom } from '#services/backup_code_storage';
import path from 'path';
import crypto from 'crypto';
// import drive from '#services/drive';
import drive from '@adonisjs/drive/services/main';
import logger from '@adonisjs/core/services/logger';
// Here we are generating secret and recovery codes for the user that’s enabling 2FA and storing them to our database.
export default class UserController {
@ -33,7 +28,7 @@ export default class UserController {
user: user,
twoFactorEnabled: user.isTwoFactorEnabled,
// code: await TwoFactorAuthProvider.generateQrCode(user),
backupState: backupState,
backupState: backupState,
});
}
@ -45,8 +40,10 @@ export default class UserController {
// });
const passwordSchema = vine.object({
// first step
old_password: vine.string().trim(),
// .regex(/^[a-zA-Z0-9]+$/),
old_password: vine
.string()
.trim()
.regex(/^[a-zA-Z0-9]+$/),
new_password: vine.string().confirmed({ confirmationField: 'confirm_password' }).trim().minLength(8).maxLength(255),
});
try {
@ -57,9 +54,9 @@ export default class UserController {
// return response.badRequest(error.messages);
throw error;
}
try {
const user = (await auth.user) as User;
const user = await auth.user as User;
const { old_password, new_password } = request.only(['old_password', 'new_password']);
// if (!(old_password && new_password && confirm_password)) {
@ -85,171 +82,6 @@ export default class UserController {
}
}
public async profile({ inertia, auth }: HttpContext) {
const user = await User.find(auth.user?.id);
// let test = await drive.use().getUrl(user?.avatar);
// user?.preload('roles');
const avatarFullPathUrl = user?.avatar ? await drive.use('public').getUrl(user.avatar) : null;
return inertia.render('profile/show', {
user: user,
defaultUrl: avatarFullPathUrl,
});
}
/**
* Update the user's profile information.
*
* @param {HttpContext} ctx - The HTTP context object.
* @returns {Promise<void>}
*/
public async profileUpdate({ auth, request, response, session }: HttpContext) {
if (!auth.user) {
session.flash('error', 'You must be logged in to update your profile.');
return response.redirect().toRoute('login');
}
const updateProfileValidator = vine.withMetaData<{ userId: number }>().compile(
vine.object({
first_name: vine.string().trim().minLength(4).maxLength(255),
last_name: vine.string().trim().minLength(4).maxLength(255),
login: vine.string().trim().minLength(4).maxLength(255),
email: vine
.string()
.trim()
.maxLength(255)
.email()
.normalizeEmail()
.isUnique({ table: 'accounts', column: 'email', whereNot: (field) => field.meta.userId }),
avatar: vine
.myfile({
size: '2mb',
extnames: ['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'],
})
// .allowedMimetypeExtensions({
// allowedExtensions: ['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'],
// })
.optional(),
}),
);
const user = await User.find(auth.user.id);
if (!user) {
session.flash('error', 'User not found.');
return response.redirect().toRoute('login');
}
try {
// validate update form
await request.validateUsing(updateProfileValidator, {
meta: {
userId: user.id,
},
});
const { login, email, first_name, last_name } = request.only(['login', 'email', 'first_name', 'last_name']);
const sanitizedData: { [key: string]: any } = {
login: login?.trim(),
email: email?.toLowerCase().trim(),
first_name: first_name?.trim(),
last_name: last_name?.trim(),
// avatar: "",
};
const toCamelCase = (str: string) => str.replace(/_([a-z])/g, (g) => g[1].toUpperCase());
const hasInputChanges = Object.keys(sanitizedData).some((key) => {
const camelKey = toCamelCase(key);
return sanitizedData[key] !== (user.$attributes as { [key: string]: any })[camelKey];
});
let hasAvatarChanged = false;
const avatar = request.file('avatar');
if (avatar) {
const fileHash = crypto
.createHash('sha256')
.update(avatar.clientName + avatar.size)
.digest('hex');
const fileName = `avatar-${fileHash}.${avatar.extname}`;
const avatarFullPath = path.join('/uploads', `${user.login}`, fileName);
if (user.avatar != avatarFullPath) {
if (user.avatar) {
await drive.use('public').delete(user.avatar);
}
hasAvatarChanged = user.avatar !== avatarFullPath;
await avatar.moveToDisk(avatarFullPath, 'public', {
name: fileName,
overwrite: true, // overwrite in case of conflict
disk: 'public',
});
sanitizedData.avatar = avatarFullPath;
}
}
if (!hasInputChanges && !hasAvatarChanged) {
session.flash('message', 'No changes were made.');
return response.redirect().back();
}
await user.merge(sanitizedData).save();
session.flash('message', 'User has been updated successfully');
return response.redirect().toRoute('settings.profile.edit');
} catch (error) {
logger.error('Profile update failed:', error);
// session.flash('errors', 'Profile update failed. Please try again.');
// return response.redirect().back();
throw error;
}
}
public async passwordUpdate({ auth, request, response, session }: HttpContext) {
// const passwordSchema = schema.create({
// old_password: schema.string({ trim: true }, [rules.required()]),
// new_password: schema.string({ trim: true }, [rules.minLength(8), rules.maxLength(255), rules.confirmed('confirm_password')]),
// confirm_password: schema.string({ trim: true }, [rules.required()]),
// });
const passwordSchema = vine.object({
// first step
old_password: vine.string().trim(),
// .regex(/^[a-zA-Z0-9]+$/),
new_password: vine.string().confirmed({ confirmationField: 'confirm_password' }).trim().minLength(8).maxLength(255),
});
try {
// await request.validate({ schema: passwordSchema });
const validator = vine.compile(passwordSchema);
await request.validateUsing(validator);
} catch (error) {
// return response.badRequest(error.messages);
throw error;
}
try {
const user = (await auth.user) as User;
const { old_password, new_password } = request.only(['old_password', 'new_password']);
// if (!(old_password && new_password && confirm_password)) {
// return response.status(400).send({ warning: 'Old password and new password are required.' });
// }
// Verify if the provided old password matches the user's current password
const isSame = await hash.verify(user.password, old_password);
if (!isSame) {
session.flash('warning', 'Old password is incorrect.');
return response.redirect().back();
// return response.flash('warning', 'Old password is incorrect.').redirect().back();
}
// Hash the new password before updating the user's password
user.password = new_password;
await user.save();
// return response.status(200).send({ message: 'Password updated successfully.' });
session.flash({ message: 'Password updated successfully.' });
return response.redirect().toRoute('settings.profile.edit');
} catch (error) {
// return response.status(500).send({ message: 'Internal server error.' });
return response.flash('warning', `Invalid server state. Internal server error.`).redirect().back();
}
}
public async enableTwoFactorAuthentication({ auth, response, session }: HttpContext): Promise<void> {
// const user: User | undefined = auth?.user;
const user = (await User.find(auth.user?.id)) as User;
@ -283,7 +115,7 @@ export default class UserController {
} else {
session.flash('error', 'User not found.');
}
return response.redirect().back();
// return inertia.render('Auth/AccountInfo', {
// // status: {

869
app/Controllers/Http/Editor/DatasetController.ts
View file

File diff suppressed because it is too large Load diff

481
app/Controllers/Http/Oai/OaiController.ts
View file

@ -1,4 +1,5 @@
import type { HttpContext } from '@adonisjs/core/http';
// import { RequestContract } from '@ioc:Adonis/Core/Request';
import { Request } from '@adonisjs/core/http';
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
import { create } from 'xmlbuilder2';
@ -14,14 +15,18 @@ import { OaiModelException, BadOaiModelException } from '#app/exceptions/OaiMode
import Dataset from '#models/dataset';
import Collection from '#models/collection';
import { getDomain, preg_match } from '#app/utils/utility-functions';
import DatasetXmlSerializer from '#app/Library/DatasetXmlSerializer';
import XmlModel from '#app/Library/XmlModel';
import logger from '@adonisjs/core/services/logger';
import ResumptionToken from '#app/Library/Oai/ResumptionToken';
import config from '@adonisjs/core/services/config';
import { inject } from '@adonisjs/core';
// import Config from '@ioc:Adonis/Core/Config';
import config from '@adonisjs/core/services/config'
// import { inject } from '@adonisjs/fold';
import { inject } from '@adonisjs/core'
// import { TokenWorkerContract } from "MyApp/Models/TokenWorker";
import TokenWorkerContract from '#library/Oai/TokenWorkerContract';
import { ModelQueryBuilderContract } from '@adonisjs/lucid/types/model';
interface XslTParameter {
[key: string]: any;
}
@ -30,14 +35,12 @@ interface Dictionary {
[index: string]: string;
}
interface PagingParameter {
interface ListParameter {
cursor: number;
totalLength: number;
totalIds: number;
start: number;
nextDocIds: number[];
activeWorkIds: number[];
reldocIds: (number | null)[];
metadataPrefix: string;
queryParams: Object;
}
@inject()
@ -46,7 +49,6 @@ export default class OaiController {
private sampleRegEx = /^[A-Za-zäüÄÜß0-9\-_.!~]+$/;
private xsltParameter: XslTParameter;
private firstPublishedDataset: Dataset | null;
/**
* Holds xml representation of document information to be processed.
*
@ -55,6 +57,7 @@ export default class OaiController {
private xml: XMLBuilder;
private proc;
constructor(public tokenWorker: TokenWorkerContract) {
// Load the XSLT file
this.proc = readFileSync('public/assets2/datasetxml2oai.sef.json');
@ -79,13 +82,13 @@ export default class OaiController {
xsltParameter['oai_error_message'] = 'Only POST and GET methods are allowed for OAI-PMH.';
}
let earliestDateFromDb;
// const oaiRequest: OaiParameter = request.body;
try {
this.firstPublishedDataset = await Dataset.earliestPublicationDate();
// Pflichtfeld laut OAI-PMH: auch bei leerem Repository einen validen
// UTCdatetime liefern, sonst entsteht ein ungültiges leeres Element.
this.xsltParameter['earliestDatestamp'] =
this.firstPublishedDataset?.server_date_published.toFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") ?? '1970-01-01T00:00:00Z';
const firstPublishedDataset: Dataset | null = await Dataset.earliestPublicationDate();
firstPublishedDataset != null &&
(earliestDateFromDb = firstPublishedDataset.server_date_published.toFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"));
this.xsltParameter['earliestDatestamp'] = earliestDateFromDb;
// start the request
await this.handleRequest(oaiRequest, request);
} catch (error) {
@ -118,7 +121,7 @@ export default class OaiController {
// logLevel: 10,
});
xmlOutput = result.principalResult;
} catch (error: any) {
} catch (error) {
return response.status(500).json({
message: 'An error occurred while creating the user',
error: error.message,
@ -153,25 +156,28 @@ export default class OaiController {
const verb = oaiRequest['verb'];
this.xsltParameter['oai_verb'] = verb;
if (verb === 'Identify') {
this.handleIdentify(oaiRequest);
this.handleIdentify();
} else if (verb === 'ListMetadataFormats') {
this.handleListMetadataFormats();
} else if (verb == 'GetRecord') {
await this.handleGetRecord(oaiRequest);
} else if (verb == 'ListRecords') {
// Get browser fingerprint from the request:
const browserFingerprint = this.getBrowserFingerprint(request);
await this.handleListRecords(oaiRequest, browserFingerprint);
await this.handleListRecords(oaiRequest);
} else if (verb == 'ListIdentifiers') {
// Get browser fingerprint from the request:
const browserFingerprint = this.getBrowserFingerprint(request);
await this.handleListIdentifiers(oaiRequest, browserFingerprint);
await this.handleListIdentifiers(oaiRequest);
} else if (verb == 'ListSets') {
await this.handleListSets();
} else {
this.handleIllegalVerb();
}
} else {
// // try {
// // console.log("Async code example.")
// const err = new PageNotFoundException("verb not found");
// throw err;
// // } catch (error) { // manually catching
// // next(error); // passing to default middleware error handler
// // }
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'The verb provided in the request is illegal.',
@ -180,15 +186,12 @@ export default class OaiController {
}
}
protected handleIdentify(oaiRequest: Dictionary) {
// OAI-PMH: Identify akzeptiert außer `verb` keine Argumente.
this.assertOnlyVerb(oaiRequest);
protected handleIdentify() {
const email = process.env.OAI_EMAIL || 'repository@geosphere.at';
const repositoryName = 'Tethys RDR';
const repIdentifier = 'tethys.at';
const sampleIdentifier = 'oai:' + repIdentifier + ':1'; //$this->_configuration->getSampleIdentifier();
// Get configuration values from environment or a dedicated configuration service
const email = process.env.OAI_EMAIL ?? 'repository@geosphere.at';
const repositoryName = process.env.OAI_REPOSITORY_NAME ?? 'Tethys RDR';
const repIdentifier = process.env.OAI_REP_IDENTIFIER ?? 'tethys.at';
const sampleIdentifier = `oai:${repIdentifier}:1`;
// Dataset::earliestPublicationDate()->server_date_published->format('Y-m-d\TH:i:s\Z') : null;
// earliestDateFromDb!= null && (this.xsltParameter['earliestDatestamp'] = earliestDateFromDb?.server_date_published);
@ -202,21 +205,6 @@ export default class OaiController {
this.xml.root().ele('Datasets');
}
/**
* Wirft badArgument, wenn der Request andere Parameter als `verb` enthält.
* Für Verben ohne zusätzliche Argumente (Identify, ListSets, ListMetadataFormats).
*/
private assertOnlyVerb(oaiRequest: Dictionary) {
const illegalKeys = Object.keys(oaiRequest).filter((key) => key !== 'verb');
if (illegalKeys.length > 0) {
throw new OaiModelException(
StatusCodes.BAD_REQUEST,
`The request includes illegal arguments: ${illegalKeys.join(', ')}.`,
OaiErrorCodes.BADARGUMENT,
);
}
}
protected handleListMetadataFormats() {
this.xml.root().ele('Datasets');
}
@ -228,7 +216,7 @@ export default class OaiController {
const sets: { [key: string]: string } = {
'open_access': 'Set for open access licenses',
'openaire_data': 'OpenAIRE',
'openaire_data': "OpenAIRE",
'doc-type:ResearchData': 'Set for document type ResearchData',
...(await this.getSetsForDatasetTypes()),
...(await this.getSetsForCollections()),
@ -246,15 +234,7 @@ export default class OaiController {
const repIdentifier = 'tethys.at';
this.xsltParameter['repIdentifier'] = repIdentifier;
// Validate that required parameter exists early
if (!('identifier' in oaiRequest)) {
throw new BadOaiModelException('The prefix of the identifier argument is unknown.');
}
// Validate and extract the dataset identifier from the request
const dataId = this.validateAndGetIdentifier(oaiRequest);
// Retrieve dataset with associated XML cache and collection roles
const dataset = await Dataset.query()
.where('publish_id', dataId)
.preload('xmlCache')
@ -271,61 +251,59 @@ export default class OaiController {
);
}
// Validate and set the metadata prefix parameter
const metadataPrefix = this.validateAndGetMetadataPrefix(oaiRequest);
this.xsltParameter['oai_metadataPrefix'] = metadataPrefix;
// Ensure that the dataset is in an exportable state
// do not deliver datasets which are restricted by document state defined in deliveringStates
this.validateDatasetState(dataset);
// Build the XML for the dataset record and add it to the root node
// add xml elements
const datasetNode = this.xml.root().ele('Datasets');
await this.createXmlRecord(dataset, datasetNode);
}
protected async handleListIdentifiers(oaiRequest: Dictionary, browserFingerprint: string) {
if (!this.tokenWorker.isConnected) {
await this.tokenWorker.connect();
}
protected async handleListIdentifiers(oaiRequest: Dictionary) {
!this.tokenWorker.isConnected && (await this.tokenWorker.connect());
const maxIdentifier: number = config.get('oai.max.listidentifiers', 100);
await this.handleLists(oaiRequest, maxIdentifier, browserFingerprint);
await this.handleLists(oaiRequest, maxIdentifier);
}
protected async handleListRecords(oaiRequest: Dictionary, browserFingerprint: string) {
if (!this.tokenWorker.isConnected) {
await this.tokenWorker.connect();
}
protected async handleListRecords(oaiRequest: Dictionary) {
!this.tokenWorker.isConnected && (await this.tokenWorker.connect());
const maxRecords: number = config.get('oai.max.listrecords', 100);
await this.handleLists(oaiRequest, maxRecords, browserFingerprint);
await this.handleLists(oaiRequest, maxRecords);
}
private async handleLists(oaiRequest: Dictionary, maxRecords: number, browserFingerprint: string) {
private async handleLists(oaiRequest: Dictionary, maxRecords: number) {
maxRecords = maxRecords || 100;
const repIdentifier = 'tethys.at';
this.xsltParameter['repIdentifier'] = repIdentifier;
const datasetNode = this.xml.root().ele('Datasets');
const paginationParams: PagingParameter = {
// list initialisation
const numWrapper: ListParameter = {
cursor: 0,
totalLength: 0,
totalIds: 0,
start: maxRecords + 1,
nextDocIds: [],
activeWorkIds: [],
reldocIds: [],
metadataPrefix: '',
queryParams: {},
};
// resumptionToken is defined
if ('resumptionToken' in oaiRequest) {
await this.handleResumptionToken(oaiRequest, maxRecords, paginationParams);
await this.handleResumptionToken(oaiRequest, maxRecords, numWrapper);
} else {
await this.handleNoResumptionToken(oaiRequest, paginationParams, maxRecords);
// no resumptionToken is given
await this.handleNoResumptionToken(oaiRequest, numWrapper);
}
const nextIds: number[] = paginationParams.nextDocIds;
const workIds: number[] = paginationParams.activeWorkIds;
// handling of document ids
const restIds = numWrapper.reldocIds as number[];
const workIds = restIds.splice(0, maxRecords) as number[]; // array_splice(restIds, 0, maxRecords);
if (workIds.length === 0) {
// no records returned
if (workIds.length == 0) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'The combination of the given values results in an empty list.',
@ -333,222 +311,169 @@ export default class OaiController {
);
}
const datasets = await Dataset.query()
const datasets: Dataset[] = await Dataset.query()
.whereIn('publish_id', workIds)
.preload('xmlCache')
.preload('collections', (builder) => {
builder.preload('collectionRole');
})
.orderBy('publish_id');
for (const dataset of datasets) {
await this.createXmlRecord(dataset, datasetNode);
}
await this.setResumptionToken(nextIds, paginationParams, browserFingerprint);
// store the further Ids in a resumption-file
const countRestIds = restIds.length; //84
if (countRestIds > 0) {
const token = new ResumptionToken();
token.startPosition = numWrapper.start; //101
token.totalIds = numWrapper.totalIds; //184
token.documentIds = restIds; //101 -184
token.metadataPrefix = numWrapper.metadataPrefix;
// $tokenWorker->storeResumptionToken($token);
const res: string = await this.tokenWorker.set(token);
// set parameters for the resumptionToken-node
// const res = token.ResumptionId;
this.setParamResumption(res, numWrapper.cursor, numWrapper.totalIds);
}
}
private async handleNoResumptionToken(oaiRequest: Dictionary, paginationParams: PagingParameter, maxRecords: number) {
this.validateMetadataPrefix(oaiRequest, paginationParams);
const finder: ModelQueryBuilderContract<typeof Dataset, Dataset> = Dataset.query().whereIn(
'server_state',
this.deliveringDocumentStates,
);
this.applySetFilter(finder, oaiRequest);
this.applyDateFilters(finder, oaiRequest);
await this.fetchAndSetResults(finder, paginationParams, oaiRequest, maxRecords);
}
private async fetchAndSetResults(
finder: ModelQueryBuilderContract<typeof Dataset, Dataset>,
paginationParams: PagingParameter,
oaiRequest: Dictionary,
maxRecords: number,
) {
const totalResult = await finder
.clone()
.count('* as total')
.first()
.then((res) => res?.$extras.total);
paginationParams.totalLength = Number(totalResult);
const combinedRecords: Dataset[] = await finder
.select('publish_id')
.orderBy('publish_id')
.offset(0)
.limit(maxRecords * 2);
paginationParams.activeWorkIds = combinedRecords.slice(0, 100).map((dat) => Number(dat.publish_id));
paginationParams.nextDocIds = combinedRecords.slice(100).map((dat) => Number(dat.publish_id));
// No resumption token was used – set queryParams from the current oaiRequest
paginationParams.queryParams = {
...oaiRequest,
deliveringStates: this.deliveringDocumentStates,
};
// paginationParams.totalLength = 230;
}
private async handleResumptionToken(oaiRequest: Dictionary, maxRecords: number, paginationParams: PagingParameter) {
const resParam = oaiRequest['resumptionToken'];
private async handleResumptionToken(oaiRequest: Dictionary, maxRecords: number, numWrapper: ListParameter) {
const resParam = oaiRequest['resumptionToken']; //e.g. "158886496600000"
const token = await this.tokenWorker.get(resParam);
if (!token) {
throw new OaiModelException(StatusCodes.INTERNAL_SERVER_ERROR, 'cache is outdated.', OaiErrorCodes.BADRESUMPTIONTOKEN);
}
// this.setResumptionParameters(token, maxRecords, paginationParams);
paginationParams.cursor = token.startPosition - 1;
paginationParams.start = token.startPosition + maxRecords;
paginationParams.totalLength = token.totalIds;
paginationParams.activeWorkIds = token.documentIds;
paginationParams.metadataPrefix = token.metadataPrefix;
paginationParams.queryParams = token.queryParams;
this.xsltParameter['oai_metadataPrefix'] = token.metadataPrefix;
numWrapper.cursor = token.startPosition - 1; //startet dann bei Index 10
numWrapper.start = token.startPosition + maxRecords;
numWrapper.totalIds = token.totalIds;
numWrapper.reldocIds = token.documentIds;
numWrapper.metadataPrefix = token.metadataPrefix;
const finder = this.buildDatasetQueryViaToken(token);
const nextRecords: Dataset[] = await this.fetchNextRecords(finder, token, maxRecords);
paginationParams.nextDocIds = nextRecords.map((dat) => Number(dat.publish_id));
this.xsltParameter['oai_metadataPrefix'] = numWrapper.metadataPrefix;
}
private async setResumptionToken(nextIds: number[], paginationParams: PagingParameter, browserFingerprint: string) {
const countRestIds = nextIds.length;
if (countRestIds > 0) {
// const token = this.createResumptionToken(paginationParams, nextIds);
const token = new ResumptionToken();
token.startPosition = paginationParams.start;
token.totalIds = paginationParams.totalLength;
token.documentIds = nextIds;
token.metadataPrefix = paginationParams.metadataPrefix;
token.queryParams = paginationParams.queryParams;
const res: string = await this.tokenWorker.set(token, browserFingerprint);
this.setParamResumption(res, paginationParams.cursor, paginationParams.totalLength);
}
}
private buildDatasetQueryViaToken(token: ResumptionToken) {
const finder = Dataset.query();
const originalQuery = token.queryParams || {};
const deliveringStates = originalQuery.deliveringStates || this.deliveringDocumentStates;
finder.whereIn('server_state', deliveringStates);
this.applySetFilter(finder, originalQuery);
this.applyDateFilters(finder, originalQuery);
return finder;
}
private async fetchNextRecords(finder: ModelQueryBuilderContract<typeof Dataset, Dataset>, token: ResumptionToken, maxRecords: number) {
return finder
.select('publish_id')
.orderBy('publish_id')
.offset(token.startPosition - 1 + maxRecords)
.limit(100);
}
private validateMetadataPrefix(oaiRequest: Dictionary, paginationParams: PagingParameter) {
if (!('metadataPrefix' in oaiRequest)) {
private async handleNoResumptionToken(oaiRequest: Dictionary, numWrapper: ListParameter) {
// no resumptionToken is given
if ('metadataPrefix' in oaiRequest) {
numWrapper.metadataPrefix = oaiRequest['metadataPrefix'];
} else {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'The prefix of the metadata argument is unknown.',
OaiErrorCodes.BADARGUMENT,
);
}
paginationParams.metadataPrefix = oaiRequest['metadataPrefix'];
this.xsltParameter['oai_metadataPrefix'] = paginationParams.metadataPrefix;
}
this.xsltParameter['oai_metadataPrefix'] = numWrapper.metadataPrefix;
private applySetFilter(finder: ModelQueryBuilderContract<typeof Dataset, Dataset>, queryParams: any) {
if ('set' in queryParams) {
const [setType, setValue] = queryParams['set'].split(':');
let finder: ModelQueryBuilderContract<typeof Dataset, Dataset> = Dataset.query();
// add server state restrictions
finder.whereIn('server_state', this.deliveringDocumentStates);
if ('set' in oaiRequest) {
const set = oaiRequest['set'] as string;
const setArray = set.split(':');
switch (setType) {
case 'data-type':
setValue && finder.where('type', setValue);
break;
case 'open_access':
finder.andWhereHas('licenses', (query) => {
query.whereIn('name', ['CC-BY-4.0', 'CC-BY-SA-4.0']);
if (setArray[0] == 'data-type') {
if (setArray.length == 2 && setArray[1]) {
finder.where('type', setArray[1]);
}
} else if (setArray[0] == 'open_access') {
const openAccessLicences = ['CC-BY-4.0', 'CC-BY-SA-4.0'];
finder.andWhereHas('licenses', (query) => {
query.whereIn('name', openAccessLicences);
});
} else if (setArray[0] == 'ddc') {
if (setArray.length == 2 && setArray[1] != '') {
finder.andWhereHas('collections', (query) => {
query.where('number', setArray[1]);
});
break;
case 'ddc':
setValue &&
finder.andWhereHas('collections', (query) => {
query.where('number', setValue);
});
break;
}
}
}
}
private applyDateFilters(finder: ModelQueryBuilderContract<typeof Dataset, Dataset>, queryParams: any) {
const { from, until } = queryParams;
// const timeZone = "Europe/Vienna"; // Canonical time zone name
// &from=2020-09-03&until2020-09-03
// &from=2020-09-11&until=2021-05-11
if ('from' in oaiRequest && 'until' in oaiRequest) {
const from = oaiRequest['from'] as string;
let fromDate = dayjs(from); //.tz(timeZone);
const until = oaiRequest['until'] as string;
let untilDate = dayjs(until); //.tz(timeZone);
if (!fromDate.isValid() || !untilDate.isValid()) {
throw new OaiModelException(StatusCodes.INTERNAL_SERVER_ERROR, 'Date Parameter is not valid.', OaiErrorCodes.BADARGUMENT);
}
fromDate = dayjs.tz(from, 'Europe/Vienna');
untilDate = dayjs.tz(until, 'Europe/Vienna');
if (from && until) {
this.handleFromUntilFilter(finder, from, until);
} else if (from) {
this.handleFromFilter(finder, from);
} else if (until) {
this.handleUntilFilter(finder, until);
}
}
if (from.length != until.length) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'The request has different granularities for the from and until parameters.',
OaiErrorCodes.BADARGUMENT,
);
}
fromDate.hour() == 0 && (fromDate = fromDate.startOf('day'));
untilDate.hour() == 0 && (untilDate = untilDate.endOf('day'));
private handleFromUntilFilter(finder: ModelQueryBuilderContract<typeof Dataset, Dataset>, from: string, until: string) {
const fromDate = this.parseDateWithValidation(from, 'From');
const untilDate = this.parseDateWithValidation(until, 'Until');
finder.whereBetween('server_date_published', [fromDate.format('YYYY-MM-DD HH:mm:ss'), untilDate.format('YYYY-MM-DD HH:mm:ss')]);
} else if ('from' in oaiRequest && !('until' in oaiRequest)) {
const from = oaiRequest['from'] as string;
let fromDate = dayjs(from);
if (!fromDate.isValid()) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'From date parameter is not valid.',
OaiErrorCodes.BADARGUMENT,
);
}
fromDate = dayjs.tz(from, 'Europe/Vienna');
fromDate.hour() == 0 && (fromDate = fromDate.startOf('day'));
if (from.length !== until.length) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'The request has different granularities for the from and until parameters.',
OaiErrorCodes.BADARGUMENT,
);
const now = dayjs();
if (fromDate.isAfter(now)) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'Given from date is greater than now. The given values results in an empty list.',
OaiErrorCodes.NORECORDSMATCH,
);
} else {
finder.andWhere('server_date_published', '>=', fromDate.format('YYYY-MM-DD HH:mm:ss'));
}
} else if (!('from' in oaiRequest) && 'until' in oaiRequest) {
const until = oaiRequest['until'] as string;
let untilDate = dayjs(until);
if (!untilDate.isValid()) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'Until date parameter is not valid.',
OaiErrorCodes.BADARGUMENT,
);
}
untilDate = dayjs.tz(until, 'Europe/Vienna');
untilDate.hour() == 0 && (untilDate = untilDate.endOf('day'));
const firstPublishedDataset: Dataset = (await Dataset.earliestPublicationDate()) as Dataset;
const earliestPublicationDate = dayjs(firstPublishedDataset.server_date_published.toISO()); //format("YYYY-MM-DDThh:mm:ss[Z]"));
if (earliestPublicationDate.isAfter(untilDate)) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
`earliestDatestamp is greater than given until date.
The given values results in an empty list.`,
OaiErrorCodes.NORECORDSMATCH,
);
} else {
finder.andWhere('server_date_published', '<=', untilDate.format('YYYY-MM-DD HH:mm:ss'));
}
}
finder.whereBetween('server_date_published', [fromDate.format('YYYY-MM-DD HH:mm:ss'), untilDate.format('YYYY-MM-DD HH:mm:ss')]);
}
private handleFromFilter(finder: ModelQueryBuilderContract<typeof Dataset, Dataset>, from: string) {
const fromDate = this.parseDateWithValidation(from, 'From');
const now = dayjs();
if (fromDate.isAfter(now)) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'Given from date is greater than now. The given values results in an empty list.',
OaiErrorCodes.NORECORDSMATCH,
);
}
finder.andWhere('server_date_published', '>=', fromDate.format('YYYY-MM-DD HH:mm:ss'));
}
private handleUntilFilter(finder: ModelQueryBuilderContract<typeof Dataset, Dataset>, until: string) {
const untilDate = this.parseDateWithValidation(until, 'Until');
const earliestPublicationDate = dayjs(this.firstPublishedDataset?.server_date_published.toISO());
if (earliestPublicationDate.isAfter(untilDate)) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'earliestDatestamp is greater than given until date. The given values results in an empty list.',
OaiErrorCodes.NORECORDSMATCH,
);
}
finder.andWhere('server_date_published', '<=', untilDate.format('YYYY-MM-DD HH:mm:ss'));
}
private parseDateWithValidation(dateStr: string, label: string) {
let date = dayjs(dateStr);
if (!date.isValid()) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
`${label} date parameter is not valid.`,
OaiErrorCodes.BADARGUMENT,
);
}
date = dayjs.tz(dateStr, 'Europe/Vienna');
return date.hour() === 0 ? (label === 'From' ? date.startOf('day') : date.endOf('day')) : date;
let reldocIdsDocs = await finder.select('publish_id').orderBy('publish_id');
numWrapper.reldocIds = reldocIdsDocs.map((dat) => dat.publish_id);
numWrapper.totalIds = numWrapper.reldocIds.length; //212
}
private setParamResumption(res: string, cursor: number, totalIds: number) {
@ -620,17 +545,19 @@ export default class OaiController {
}
private async getDatasetXmlDomNode(dataset: Dataset) {
const serializer = new DatasetXmlSerializer(dataset).enableCaching().excludeEmptyFields();
const xmlModel = new XmlModel(dataset);
// xmlModel.setModel(dataset);
xmlModel.excludeEmptyFields();
xmlModel.caching = true;
// const cache = dataset.xmlCache ? dataset.xmlCache : null;
// dataset.load('xmlCache');
if (dataset.xmlCache) {
serializer.setCache(dataset.xmlCache);
xmlModel.xmlCache = dataset.xmlCache;
}
// return cache.toXmlDocument();
const xmlDocument: XMLBuilder | null = await serializer.toXmlDocument();
return xmlDocument;
// return cache.getDomDocument();
const domDocument: XMLBuilder | null = await xmlModel.getDomDocument();
return domDocument;
}
private addSpecInformation(domNode: XMLBuilder, information: string) {
@ -714,30 +641,4 @@ export default class OaiController {
this.xsltParameter['oai_error_code'] = 'badVerb';
this.xsltParameter['oai_error_message'] = 'The verb provided in the request is illegal.';
}
/**
* Helper method to build a browser fingerprint by combining:
* - User-Agent header,
* - the IP address,
* - Accept-Language header,
* - current timestamp rounded to the hour.
*
* Every new hour, this will return a different fingerprint.
*/
private getBrowserFingerprint(request: Request): string {
const userAgent = request.header('user-agent') || 'unknown';
// Check for X-Forwarded-For header to use the client IP from the proxy if available.
const xForwardedFor = request.header('x-forwarded-for');
let ip = request.ip();
// console.log(ip);
if (xForwardedFor) {
// X-Forwarded-For may contain a comma-separated list of IPs; the first one is the client IP.
ip = xForwardedFor.split(',')[0].trim();
// console.log('xforwardedfor ip' + ip);
}
const locale = request.header('accept-language') || 'default';
// Round the current time to the start of the hour.
const timestampHour = dayjs().startOf('hour').format('YYYY-MM-DDTHH');
return `${userAgent}-${ip}-${locale}-${timestampHour}`;
}
}

120
app/Controllers/Http/Reviewer/DatasetController.ts
View file

@ -9,7 +9,6 @@ import vine from '@vinejs/vine';
import mail from '@adonisjs/mail/services/main';
import logger from '@adonisjs/core/services/logger';
import { validate } from 'deep-email-validator';
import File from '#models/file';
interface Dictionary {
[index: string]: string;
@ -39,21 +38,13 @@ export default class DatasetsController {
}
datasets.orderBy(attribute, sortOrder);
} else {
// datasets.orderBy('id', 'asc');
// Custom ordering to prioritize rejected_editor state
datasets.orderByRaw(`
CASE
WHEN server_state = 'rejected_to_reviewer' THEN 0
ELSE 1
END ASC,
id ASC
`);
// users.orderBy('created_at', 'desc');
datasets.orderBy('id', 'asc');
}
// const users = await User.query().orderBy('login').paginate(page, limit);
const myDatasets = await datasets
// .where('server_state', 'approved')
.whereIn('server_state', ['approved', 'rejected_to_reviewer'])
.where('server_state', 'approved')
.where('reviewer_id', user.id)
.preload('titles')
@ -71,51 +62,7 @@ export default class DatasetsController {
});
}
public async review({ request, inertia, response, auth }: HttpContext) {
const id = request.param('id');
const datasetQuery = Dataset.query().where('id', id);
datasetQuery
.preload('titles', (query) => query.orderBy('id', 'asc'))
.preload('descriptions', (query) => query.orderBy('id', 'asc'))
.preload('coverage')
.preload('licenses')
.preload('authors', (query) => query.orderBy('pivot_sort_order', 'asc'))
.preload('contributors', (query) => query.orderBy('pivot_sort_order', 'asc'))
// .preload('subjects')
.preload('subjects', (builder) => {
builder.orderBy('id', 'asc').withCount('datasets');
})
.preload('references')
.preload('project')
.preload('files', (query) => {
query.orderBy('sort_order', 'asc'); // Sort by sort_order column
});
const dataset = await datasetQuery.firstOrFail();
const validStates = ['approved', 'rejected_to_reviewer'];
if (!validStates.includes(dataset.server_state)) {
// session.flash('errors', 'Invalid server state!');
return response
.flash(
'warning',
`Invalid server state. Dataset with id ${id} cannot be reviewed. Datset has server state ${dataset.server_state}.`,
)
.redirect()
.toRoute('reviewer.dataset.list');
}
return inertia.render('Reviewer/Dataset/Review', {
dataset,
can: {
review: await auth.user?.can(['dataset-review']),
reject: await auth.user?.can(['dataset-review-reject']),
},
});
}
public async review_old({ request, inertia, response, auth }: HttpContext) {
public async review({ request, inertia, response }: HttpContext) {
const id = request.param('id');
const dataset = await Dataset.query()
.where('id', id)
@ -211,10 +158,6 @@ export default class DatasetsController {
return inertia.render('Reviewer/Dataset/Review', {
dataset,
fields: fields,
can: {
review: await auth.user?.can(['dataset-review']),
reject: await auth.user?.can(['dataset-review-reject']),
},
});
}
@ -223,7 +166,7 @@ export default class DatasetsController {
// const { id } = params;
const dataset = await Dataset.findOrFail(id);
const validStates = ['approved', 'rejected_to_reviewer'];
const validStates = ['approved'];
if (!validStates.includes(dataset.server_state)) {
// throw new Error('Invalid server state!');
// return response.flash('warning', 'Invalid server state. Dataset cannot be released to editor').redirect().back();
@ -237,10 +180,6 @@ export default class DatasetsController {
}
dataset.server_state = 'reviewed';
// if editor has rejected to reviewer:
if (dataset.reject_editor_note != null) {
dataset.reject_editor_note = null;
}
try {
// await dataset.related('editor').associate(user); // speichert schon ab
@ -264,7 +203,7 @@ export default class DatasetsController {
})
.firstOrFail();
const validStates = ['approved', 'rejected_to_reviewer'];
const validStates = ['approved'];
if (!validStates.includes(dataset.server_state)) {
// session.flash('errors', 'Invalid server state!');
return response
@ -311,12 +250,12 @@ export default class DatasetsController {
throw error;
}
const validStates = ['approved', 'rejected_to_reviewer'];
const validStates = ['approved'];
if (!validStates.includes(dataset.server_state)) {
// throw new Error('Invalid server state!');
// return response.flash('warning', 'Invalid server state. Dataset cannot be released to editor').redirect().back();
return response
.flash(
.flash(
`Invalid server state. Dataset with id ${id} cannot be rejected. Datset has server state ${dataset.server_state}.`,
'warning',
)
@ -337,7 +276,7 @@ export default class DatasetsController {
validateSMTP: false,
});
const validRecipientEmail: boolean = validationResult.valid;
// let emailStatusMessage = '';
let emailStatusMessage = '';
if (sendMail == true) {
if (dataset.editor.email && validRecipientEmail) {
@ -350,7 +289,7 @@ export default class DatasetsController {
<p>Best regards,<br>Your Tethys reviewer: ${authUser.login}</p>
`);
});
// emailStatusMessage = ` A rejection email was successfully sent to ${dataset.editor.email}.`;
emailStatusMessage = ` A rejection email was successfully sent to ${dataset.editor.email}.`;
} catch (error) {
logger.error(error);
return response
@ -358,7 +297,7 @@ export default class DatasetsController {
.toRoute('reviewer.dataset.list');
}
} else {
// emailStatusMessage = ` However, the email could not be sent because the editor's email address (${dataset.editor.email}) is not valid.`;
emailStatusMessage = ` However, the email could not be sent because the editor's email address (${dataset.editor.email}) is not valid.`;
}
}
@ -368,41 +307,4 @@ export default class DatasetsController {
.toRoute('reviewer.dataset.list')
.flash(`You have rejected dataset ${dataset.id}! to editor ${dataset.editor.login}`, 'message');
}
// public async download({ params, response }: HttpContext) {
// const id = params.id;
// // Find the file by ID
// const file = await File.findOrFail(id);
// // const filePath = await drive.use('local').getUrl('/'+ file.filePath)
// const filePath = file.filePath;
// const fileExt = file.filePath.split('.').pop() || '';
// // Set the response headers and download the file
// response.header('Content-Type', file.mime_type || 'application/octet-stream');
// response.attachment(`${file.label}.${fileExt}`);
// return response.download(filePath);
// }
public async download({ params, response }: HttpContext) {
const id = params.id;
// Find the file by ID
const file = await File.findOrFail(id);
// const filePath = await drive.use('local').getUrl('/'+ file.filePath)
const filePath = file.filePath;
const fileExt = file.filePath.split('.').pop() || '';
// Check if label already includes the extension
const fileName = file.label.toLowerCase().endsWith(`.${fileExt.toLowerCase()}`) ? file.label : `${file.label}.${fileExt}`;
// Set the response headers and download the file
response
.header('Cache-Control', 'no-cache private')
.header('Content-Description', 'File Transfer')
.header('Content-Type', file.mime_type || 'application/octet-stream')
// .header('Content-Disposition', 'inline; filename=' + fileName)
.header('Content-Transfer-Encoding', 'binary')
.header('Access-Control-Allow-Origin', '*')
.header('Access-Control-Allow-Methods', 'GET');
response.attachment(fileName);
return response.download(filePath);
}
}

730
app/Controllers/Http/Submitter/DatasetController.ts
View file

File diff suppressed because it is too large Load diff

231
app/Library/DatasetXmlSerializer.ts
View file

@ -1,231 +0,0 @@
import DocumentXmlCache from '#models/DocumentXmlCache';
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
import Dataset from '#models/dataset';
import Strategy from './Strategy.js';
import { builder } from 'xmlbuilder2';
import logger from '@adonisjs/core/services/logger';
/**
* Configuration for XML serialization
*
* @interface XmlSerializationConfig
*/
export interface XmlSerializationConfig {
/** The dataset model to serialize */
model: Dataset;
/** DOM representation (if available) */
dom?: XMLBuilder;
/** Fields to exclude from serialization */
excludeFields: Array<string>;
/** Whether to exclude empty fields */
excludeEmpty: boolean;
/** Base URI for xlink:ref elements */
baseUri: string;
}
/**
* Options for controlling serialization behavior
*/
export interface SerializationOptions {
/** Enable XML caching */
enableCaching?: boolean;
/** Exclude empty fields from output */
excludeEmptyFields?: boolean;
/** Custom base URI */
baseUri?: string;
/** Fields to exclude */
excludeFields?: string[];
}
/**
* DatasetXmlSerializer
*
* Handles XML serialization of Dataset models with intelligent caching.
* Generates XML representations and manages cache lifecycle to optimize performance.
*
* @example
* ```typescript
* const serializer = new DatasetXmlSerializer(dataset);
* serializer.enableCaching();
* serializer.excludeEmptyFields();
*
* const xmlDocument = await serializer.toXmlDocument();
* ```
*/
export default class DatasetXmlSerializer {
private readonly config: XmlSerializationConfig;
private readonly strategy: Strategy;
private cache: DocumentXmlCache | null = null;
private cachingEnabled = false;
constructor(dataset: Dataset, options: SerializationOptions = {}) {
this.config = {
model: dataset,
excludeEmpty: options.excludeEmptyFields ?? false,
baseUri: options.baseUri ?? '',
excludeFields: options.excludeFields ?? [],
};
this.strategy = new Strategy({
excludeEmpty: options.excludeEmptyFields ?? false,
baseUri: options.baseUri ?? '',
excludeFields: options.excludeFields ?? [],
model: dataset,
});
if (options.enableCaching) {
this.cachingEnabled = true;
}
}
/**
* Enable caching for XML generation
* When enabled, generated XML is stored in database for faster retrieval
*/
public enableCaching(): this {
this.cachingEnabled = true;
return this;
}
/**
* Disable caching for XML generation
*/
public disableCaching(): this {
this.cachingEnabled = false;
return this;
}
set model(model: Dataset) {
this.config.model = model;
}
/**
* Configure to exclude empty fields from XML output
*/
public excludeEmptyFields(): this {
this.config.excludeEmpty = true;
return this;
}
/**
* Set the cache instance directly (useful when preloading)
* @param cache - The DocumentXmlCache instance
*/
public setCache(cache: DocumentXmlCache): this {
this.cache = cache;
return this;
}
/**
* Get the current cache instance
*/
public getCache(): DocumentXmlCache | null {
return this.cache;
}
/**
* Get DOM document with intelligent caching
* Returns cached version if valid, otherwise generates new document
*/
public async toXmlDocument(): Promise<XMLBuilder | null> {
const dataset = this.config.model;
// Try to get from cache first
let cachedDocument: XMLBuilder | null = await this.retrieveFromCache();
if (cachedDocument) {
logger.debug(`Using cached XML for dataset ${dataset.id}`);
return cachedDocument;
}
// Generate fresh document
logger.debug(`[DatasetXmlSerializer] Cache miss - generating fresh XML for dataset ${dataset.id}`);
const freshDocument = await this.strategy.createDomDocument();
if (!freshDocument) {
logger.error(`[DatasetXmlSerializer] Failed to generate XML for dataset ${dataset.id}`);
return null;
}
// Cache if caching is enabled
if (this.cachingEnabled) {
await this.persistToCache(freshDocument, dataset);
}
// Extract the dataset-specific node
return this.extractDatasetNode(freshDocument);
}
/**
* Generate XML string representation
* Convenience method that converts XMLBuilder to string
*/
public async toXmlString(): Promise<string | null> {
const document = await this.toXmlDocument();
return document ? document.end({ prettyPrint: false }) : null;
}
/**
* Persist generated XML document to cache
* Non-blocking - failures are logged but don't interrupt the flow
*/
private async persistToCache(domDocument: XMLBuilder, dataset: Dataset): Promise<void> {
try {
this.cache = this.cache || new DocumentXmlCache();
this.cache.document_id = dataset.id;
this.cache.xml_version = 1;
this.cache.server_date_modified = dataset.server_date_modified.toFormat('yyyy-MM-dd HH:mm:ss');
this.cache.xml_data = domDocument.end();
await this.cache.save();
logger.debug(`Cached XML for dataset ${dataset.id}`);
} catch (error) {
logger.error(`Failed to cache XML for dataset ${dataset.id}: ${error.message}`);
// Don't throw - caching failure shouldn't break the flow
}
}
/**
* Extract the Rdr_Dataset node from full document
*/
private extractDatasetNode(domDocument: XMLBuilder): XMLBuilder | null {
const node = domDocument.find((n) => n.node.nodeName === 'Rdr_Dataset', false, true)?.node;
if (node) {
return builder({ version: '1.0', encoding: 'UTF-8', standalone: true }, node);
}
return domDocument;
}
/**
* Attempt to retrieve valid cached XML document
* Returns null if cache doesn't exist or is stale
*/
private async retrieveFromCache(): Promise<XMLBuilder | null> {
const dataset: Dataset = this.config.model;
if (!this.cache) {
return null;
}
// Check if cache is still valid
const actuallyCached = await DocumentXmlCache.hasValidEntry(dataset.id, dataset.server_date_modified);
if (!actuallyCached) {
logger.debug(`Cache invalid for dataset ${dataset.id}`);
return null;
}
//cache is actual return cached document
try {
if (this.cache) {
return this.cache.getDomDocument();
} else {
return null;
}
} catch (error) {
logger.error(`Failed to retrieve cached document for dataset ${dataset.id}: ${error.message}`);
return null;
}
}
}

249
app/Library/Doi/DoiClient.ts
View file

@ -1,22 +1,25 @@
// import { Client } from 'guzzle';
// import { Log } from '@adonisjs/core/build/standalone';
// import { DoiInterface } from './interfaces/DoiInterface';
import DoiClientContract from '#app/Library/Doi/DoiClientContract';
import DoiClientException from '#app/exceptions/DoiClientException';
import { StatusCodes } from 'http-status-codes';
import logger from '@adonisjs/core/services/logger';
import { AxiosResponse } from 'axios';
import { default as axios } from 'axios';
import axios from 'axios';
export class DoiClient implements DoiClientContract {
public username: string;
public password: string;
public serviceUrl: string;
public apiUrl: string;
constructor() {
// const datacite_environment = process.env.DATACITE_ENVIRONMENT || 'debug';
this.username = process.env.DATACITE_USERNAME || '';
this.password = process.env.DATACITE_PASSWORD || '';
this.serviceUrl = process.env.DATACITE_SERVICE_URL || '';
this.apiUrl = process.env.DATACITE_API_URL || 'https://api.datacite.org';
// this.prefix = process.env.DATACITE_PREFIX || '';
// this.base_domain = process.env.BASE_DOMAIN || '';
if (this.username === '' || this.password === '' || this.serviceUrl === '') {
const message = 'issing configuration settings to properly initialize DOI client';
@ -47,7 +50,7 @@ export class DoiClient implements DoiClientContract {
'Content-Type': 'application/xml;charset=UTF-8',
};
try {
const metadataResponse = await axios.put(`${this.serviceUrl}/metadata/${doiValue}`, xmlMeta, { auth, headers });
const metadataResponse = await axios.default.put(`${this.serviceUrl}/metadata/${doiValue}`, xmlMeta, { auth, headers });
// Response Codes
// 201 Created: operation successful
@ -62,7 +65,7 @@ export class DoiClient implements DoiClientContract {
throw new DoiClientException(metadataResponse.status, message);
}
const doiResponse = await axios.put(`${this.serviceUrl}/doi/${doiValue}`, `doi=${doiValue}\nurl=${landingPageUrl}`, {
const doiResponse = await axios.default.put(`${this.serviceUrl}/doi/${doiValue}`, `doi=${doiValue}\nurl=${landingPageUrl}`, {
auth,
headers,
});
@ -87,240 +90,4 @@ export class DoiClient implements DoiClientContract {
throw new DoiClientException(error.response.status, error.response.data);
}
}
/**
* Retrieves DOI information from DataCite REST API
*
* @param doiValue The DOI identifier e.g. '10.5072/tethys.999'
* @returns Promise with DOI information or null if not found
*/
public async getDoiInfo(doiValue: string): Promise<any | null> {
try {
// Use configurable DataCite REST API URL
const dataciteApiUrl = `${this.apiUrl}/dois/${doiValue}`;
const response = await axios.get(dataciteApiUrl, {
headers: {
Accept: 'application/vnd.api+json',
},
});
if (response.status === 200 && response.data.data) {
return {
created: response.data.data.attributes.created,
registered: response.data.data.attributes.registered,
updated: response.data.data.attributes.updated,
published: response.data.data.attributes.published,
state: response.data.data.attributes.state,
url: response.data.data.attributes.url,
metadata: response.data.data.attributes,
};
}
} catch (error) {
if (error.response?.status === 404) {
logger.debug(`DOI ${doiValue} not found in DataCite`);
return null;
}
logger.debug(`DataCite REST API failed for ${doiValue}: ${error.message}`);
// Fallback to MDS API
return await this.getDoiInfoFromMds(doiValue);
}
return null;
}
/**
* Fallback method to get DOI info from MDS API
*
* @param doiValue The DOI identifier
* @returns Promise with basic DOI information or null
*/
private async getDoiInfoFromMds(doiValue: string): Promise<any | null> {
try {
const auth = {
username: this.username,
password: this.password,
};
// Get DOI URL
const doiResponse = await axios.get(`${this.serviceUrl}/doi/${doiValue}`, { auth });
if (doiResponse.status === 200) {
// Get metadata if available
try {
const metadataResponse = await axios.get(`${this.serviceUrl}/metadata/${doiValue}`, {
auth,
headers: {
Accept: 'application/xml',
},
});
return {
url: doiResponse.data.trim(),
metadata: metadataResponse.data,
created: new Date().toISOString(), // MDS doesn't provide creation dates
registered: new Date().toISOString(), // Use current time as fallback
source: 'mds',
};
} catch (metadataError) {
// Return basic info even if metadata fetch fails
return {
url: doiResponse.data.trim(),
created: new Date().toISOString(),
registered: new Date().toISOString(),
source: 'mds',
};
}
}
} catch (error) {
if (error.response?.status === 404) {
logger.debug(`DOI ${doiValue} not found in DataCite MDS`);
return null;
}
logger.debug(`DataCite MDS API failed for ${doiValue}: ${error.message}`);
}
return null;
}
/**
* Checks if a DOI exists in DataCite
*
* @param doiValue The DOI identifier
* @returns Promise<boolean> True if DOI exists
*/
public async doiExists(doiValue: string): Promise<boolean> {
const doiInfo = await this.getDoiInfo(doiValue);
return doiInfo !== null;
}
/**
* Gets the last modification date of a DOI
*
* @param doiValue The DOI identifier
* @returns Promise<Date | null> Last modification date or creation date if never updated, null if not found
*/
public async getDoiLastModified(doiValue: string): Promise<Date | null> {
const doiInfo = await this.getDoiInfo(doiValue);
if (doiInfo) {
// Use updated date if available, otherwise fall back to created/registered date
const dateToUse = doiInfo.updated || doiInfo.registered || doiInfo.created;
if (dateToUse) {
logger.debug(
`DOI ${doiValue}: Using ${doiInfo.updated ? 'updated' : doiInfo.registered ? 'registered' : 'created'} date: ${dateToUse}`,
);
return new Date(dateToUse);
}
}
return null;
}
/**
* Makes a DOI unfindable (registered but not discoverable)
* Note: DOIs cannot be deleted, only made unfindable
* await doiClient.makeDoiUnfindable('10.21388/tethys.231');
*
* @param doiValue The DOI identifier e.g. '10.5072/tethys.999'
* @returns Promise<AxiosResponse<any>> The http response
*/
public async makeDoiUnfindable(doiValue: string): Promise<AxiosResponse<any>> {
const auth = {
username: this.username,
password: this.password,
};
try {
// First, check if DOI exists
const exists = await this.doiExists(doiValue);
if (!exists) {
throw new DoiClientException(404, `DOI ${doiValue} not found`);
}
// Delete the DOI URL mapping to make it unfindable
// This removes the URL but keeps the metadata registered
const response = await axios.delete(`${this.serviceUrl}/doi/${doiValue}`, { auth });
// Response Codes for DELETE /doi/{doi}
// 200 OK: operation successful
// 401 Unauthorized: no login
// 403 Forbidden: login problem, quota exceeded
// 404 Not Found: DOI does not exist
if (response.status !== 200) {
const message = `Unexpected DataCite MDS response code ${response.status}`;
logger.error(message);
throw new DoiClientException(response.status, message);
}
logger.info(`DOI ${doiValue} successfully made unfindable`);
return response;
} catch (error) {
logger.error(`Failed to make DOI ${doiValue} unfindable: ${error.message}`);
if (error instanceof DoiClientException) {
throw error;
}
throw new DoiClientException(error.response?.status || 500, error.response?.data || error.message);
}
}
/**
* Makes a DOI findable again by re-registering the URL
* await doiClient.makeDoiFindable(
* '10.21388/tethys.231',
* 'https://doi.dev.tethys.at/10.21388/tethys.231'
* );
*
* @param doiValue The DOI identifier e.g. '10.5072/tethys.999'
* @param landingPageUrl The landing page URL
* @returns Promise<AxiosResponse<any>> The http response
*/
public async makeDoiFindable(doiValue: string, landingPageUrl: string): Promise<AxiosResponse<any>> {
const auth = {
username: this.username,
password: this.password,
};
try {
// Re-register the DOI with its URL to make it findable again
const response = await axios.put(`${this.serviceUrl}/doi/${doiValue}`, `doi=${doiValue}\nurl=${landingPageUrl}`, { auth });
// Response Codes for PUT /doi/{doi}
// 201 Created: operation successful
// 400 Bad Request: request body must be exactly two lines: DOI and URL
// 401 Unauthorized: no login
// 403 Forbidden: login problem, quota exceeded
// 412 Precondition failed: metadata must be uploaded first
if (response.status !== 201) {
const message = `Unexpected DataCite MDS response code ${response.status}`;
logger.error(message);
throw new DoiClientException(response.status, message);
}
logger.info(`DOI ${doiValue} successfully made findable again`);
return response;
} catch (error) {
logger.error(`Failed to make DOI ${doiValue} findable: ${error.message}`);
if (error instanceof DoiClientException) {
throw error;
}
throw new DoiClientException(error.response?.status || 500, error.response?.data || error.message);
}
}
/**
* Gets the current state of a DOI (draft, registered, findable)
* const state = await doiClient.getDoiState('10.21388/tethys.231');
* console.log(`Current state: ${state}`); // 'findable'
*
* @param doiValue The DOI identifier
* @returns Promise<string | null> The DOI state or null if not found
*/
public async getDoiState(doiValue: string): Promise<string | null> {
const doiInfo = await this.getDoiInfo(doiValue);
return doiInfo?.state || null;
}
}

9
app/Library/Oai/ResumptionToken.ts
View file

@ -4,7 +4,6 @@ export default class ResumptionToken {
private _resumptionId = '';
private _startPosition = 0;
private _totalIds = 0;
private _queryParams: Record<string, any> = {};
get key(): string {
return this.metadataPrefix + this.startPosition + this.totalIds;
@ -49,12 +48,4 @@ export default class ResumptionToken {
set totalIds(totalIds: number) {
this._totalIds = totalIds;
}
get queryParams(): Record<string, any> {
return this._queryParams;
}
set queryParams(params: Record<string, any>) {
this._queryParams = params;
}
}

2
app/Library/Oai/TokenWorkerContract.ts
View file

@ -6,6 +6,6 @@ export default abstract class TokenWorkerContract {
abstract connect(): void;
abstract close(): void;
abstract get(key: string): Promise<ResumptionToken | null>;
abstract set(token: ResumptionToken, browserFingerprint: string): Promise<string>;
abstract set(token: ResumptionToken): Promise<string>;
}

90
app/Library/Oai/TokenWorkerSerice.ts
View file

@ -1,8 +1,8 @@
import ResumptionToken from './ResumptionToken.js';
import { createClient, RedisClientType } from 'redis';
import InternalServerErrorException from '#app/exceptions/InternalServerException';
// import { sprintf } from 'sprintf-js';
// import dayjs from 'dayjs';
import { sprintf } from 'sprintf-js';
import dayjs from 'dayjs';
import TokenWorkerContract from './TokenWorkerContract.js';
export default class TokenWorkerService implements TokenWorkerContract {
@ -40,80 +40,30 @@ export default class TokenWorkerService implements TokenWorkerContract {
return result !== undefined && result !== null;
}
/**
* Simplified set method that stores the token using a browser fingerprint key.
* If the token for that fingerprint already exists and its documentIds match the new token,
* then the fingerprint key is simply returned.
*/
public async set(token: ResumptionToken, browserFingerprint: string): Promise<string> {
// Generate a 15-digit unique number string based on the fingerprint
const uniqueNumberKey = this.createUniqueNumberFromFingerprint(browserFingerprint, token.documentIds, token.totalIds);
// Optionally, you could prefix it if desired, e.g. 'rs_' + uniqueNumberKey
const fingerprintKey = uniqueNumberKey;
// const fingerprintKey = `rs_fp_${browserFingerprint}`;
const existingTokenString = await this.cache.get(fingerprintKey);
if (existingTokenString) {
const existingToken = this.parseToken(existingTokenString);
if (this.arraysAreEqual(existingToken.documentIds, token.documentIds)) {
return fingerprintKey;
}
}
public async set(token: ResumptionToken): Promise<string> {
const uniqueName = await this.generateUniqueName();
const serialToken = JSON.stringify(token);
await this.cache.setEx(fingerprintKey, this.ttl, serialToken);
return fingerprintKey;
await this.cache.setEx(uniqueName, this.ttl, serialToken);
return uniqueName;
}
// Updated helper method to generate a unique key based on fingerprint and documentIds
private createUniqueNumberFromFingerprint(browserFingerprint: string, documentIds: number[], totalIds: number): string {
// Combine the fingerprint, document IDs and totalIds to produce the input string
const combined = browserFingerprint + ':' + documentIds.join('-') + ':' + totalIds;
// Simple hash algorithm
let hash = 0;
for (let i = 0; i < combined.length; i++) {
hash = (hash << 5) - hash + combined.charCodeAt(i);
hash |= 0; // Convert to 32-bit integer
}
// Ensure positive number and limit it to at most 15 digits
const positiveHash = Math.abs(hash) % 1000000000000000;
// Pad with trailing zeros to ensure a 15-digit string
return positiveHash.toString().padEnd(15, '0');
private async generateUniqueName(): Promise<string> {
let fc = 0;
const uniqueId = dayjs().unix().toString();
let uniqueName: string;
let cacheKeyExists: boolean;
do {
// format values
// %s - String
// %d - Signed decimal number (negative, zero or positive)
// [0-9] (Specifies the minimum width held of to the variable value)
uniqueName = sprintf('%s%05d', uniqueId, fc++);
cacheKeyExists = await this.has(uniqueName);
} while (cacheKeyExists);
return uniqueName;
}
// Add a helper function to compare two arrays of numbers with identical order
private arraysAreEqual(arr1: number[], arr2: number[]): boolean {
if (arr1.length !== arr2.length) {
return false;
}
return arr1.every((num, index) => num === arr2[index]);
}
// public async set(token: ResumptionToken): Promise<string> {
// const uniqueName = await this.generateUniqueName();
// const serialToken = JSON.stringify(token);
// await this.cache.setEx(uniqueName, this.ttl, serialToken);
// return uniqueName;
// }
// private async generateUniqueName(): Promise<string> {
// let fc = 0;
// const uniqueId = dayjs().unix().toString();
// let uniqueName: string;
// let cacheKeyExists: boolean;
// do {
// // format values
// // %s - String
// // %d - Signed decimal number (negative, zero or positive)
// // [0-9] (Specifies the minimum width held of to the variable value)
// uniqueName = sprintf('%s%05d', uniqueId, fc++);
// cacheKeyExists = await this.has(uniqueName);
// } while (cacheKeyExists);
// return uniqueName;
// }
public async get(key: string): Promise<ResumptionToken | null> {
if (!this.cache) {
throw new InternalServerErrorException('Dataset is not available for OAI export!');

138
app/Library/Utils/Index.ts
View file

@ -2,7 +2,7 @@ import Dataset from '#models/dataset';
import { Client } from '@opensearch-project/opensearch';
import { create } from 'xmlbuilder2';
import SaxonJS from 'saxon-js';
import DatasetXmlSerializer from '#app/Library/DatasetXmlSerializer';
import XmlModel from '#app/Library/XmlModel';
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
import logger from '@adonisjs/core/services/logger';
import { readFileSync } from 'fs';
@ -17,7 +17,7 @@ interface XslTParameter {
}
export default {
// opensearchNode: process.env.OPENSEARCH_HOST || 'localhost',
client: new Client({ node: `${process.env.OPENSEARCH_HOST || 'localhost'}` }), // replace with your OpenSearch endpoint
client: new Client({ node: `http://${process.env.OPENSEARCH_HOST || 'localhost'}` }), // replace with your OpenSearch endpoint
async getDoiRegisterString(dataset: Dataset): Promise<string | undefined> {
try {
@ -72,42 +72,31 @@ export default {
}
},
/**
* Index a dataset document to OpenSearch/Elasticsearch
*/
async indexDocument(dataset: Dataset, index_name: string): Promise<void> {
try {
// Load XSLT transformation file
const xsltProc = readFileSync('public/assets2/solr.sef.json');
const proc = readFileSync('public/assets2/solr.sef.json');
const doc: string = await this.getTransformedString(dataset, proc);
// Transform dataset to JSON document
const jsonDoc: string = await this.getTransformedString(dataset, xsltProc);
const document = JSON.parse(jsonDoc);
// Index document to OpenSearch with doument json body
let document = JSON.parse(doc);
await this.client.index({
id: dataset.publish_id?.toString(),
index: index_name,
body: document,
refresh: true, // make immediately searchable
refresh: true,
});
logger.info(`Dataset ${dataset.publish_id} successfully indexed to ${index_name}`);
logger.info(`dataset with publish_id ${dataset.publish_id} successfully indexed`);
} catch (error) {
logger.error(`Failed to index dataset ${dataset.publish_id}: ${error.message}`);
throw error; // Re-throw to allow caller to handle
logger.error(`An error occurred while indexing datsaet with publish_id ${dataset.publish_id}.`);
}
},
/**
* Transform dataset XML to JSON using XSLT
*/
async getTransformedString(dataset: Dataset, proc: Buffer): Promise<string> {
// Generate XML string from dataset
const xmlString = await this.generateDatasetXml(dataset);
let xml = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
const datasetNode = xml.root().ele('Dataset');
await createXmlRecord(dataset, datasetNode);
const xmlString = xml.end({ prettyPrint: false });
try {
// Apply XSLT transformation
const result = await SaxonJS.transform({
stylesheetText: proc,
destination: 'serialized',
@ -119,18 +108,6 @@ export default {
return '';
}
},
/**
* Generate XML string from dataset model
*/
async generateDatasetXml(dataset: Dataset): Promise<string> {
const xml = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
const datasetNode = xml.root().ele('Dataset');
await createXmlRecord(dataset, datasetNode);
return xml.end({ prettyPrint: false });
},
};
/**
* Return the default global focus trap stack
@ -138,49 +115,74 @@ export default {
* @return {import('focus-trap').FocusTrap[]}
*/
/**
* Create complete XML record for dataset
* Handles caching and metadata enrichment
*/
// export const indexDocument = async (dataset: Dataset, index_name: string, proc: Buffer): Promise<void> => {
// try {
// const doc = await getJsonString(dataset, proc);
// let document = JSON.parse(doc);
// await client.index({
// id: dataset.publish_id?.toString(),
// index: index_name,
// body: document,
// refresh: true,
// });
// Logger.info(`dataset with publish_id ${dataset.publish_id} successfully indexed`);
// } catch (error) {
// Logger.error(`An error occurred while indexing datsaet with publish_id ${dataset.publish_id}.`);
// }
// };
// const getJsonString = async (dataset, proc): Promise<string> => {
// let xml = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
// const datasetNode = xml.root().ele('Dataset');
// await createXmlRecord(dataset, datasetNode);
// const xmlString = xml.end({ prettyPrint: false });
// try {
// const result = await transform({
// stylesheetText: proc,
// destination: 'serialized',
// sourceText: xmlString,
// });
// return result.principalResult;
// } catch (error) {
// Logger.error(`An error occurred while creating the user, error: ${error.message},`);
// return '';
// }
// };
const createXmlRecord = async (dataset: Dataset, datasetNode: XMLBuilder): Promise<void> => {
const domNode = await getDatasetXmlDomNode(dataset);
if (!domNode) {
throw new Error(`Failed to generate XML DOM node for dataset ${dataset.id}`);
}
// Enrich with landing page URL
if (dataset.publish_id) {
addLandingPageAttribute(domNode, dataset.publish_id.toString());
}
// Add data type specification
addSpecInformation(domNode, `data-type:${dataset.type}`);
// Add collection information
if (dataset.collections) {
for (const coll of dataset.collections) {
const collRole = coll.collectionRole;
addSpecInformation(domNode, `${collRole.oai_name}:${coll.number}`);
if (domNode) {
// add frontdoor url and data-type
dataset.publish_id && addLandingPageAttribute(domNode, dataset.publish_id.toString());
addSpecInformation(domNode, 'data-type:' + dataset.type);
if (dataset.collections) {
for (const coll of dataset.collections) {
const collRole = coll.collectionRole;
addSpecInformation(domNode, collRole.oai_name + ':' + coll.number);
}
}
}
datasetNode.import(domNode);
datasetNode.import(domNode);
}
};
const getDatasetXmlDomNode = async (dataset: Dataset): Promise<XMLBuilder | null> => {
const serializer = new DatasetXmlSerializer(dataset).enableCaching().excludeEmptyFields();
const xmlModel = new XmlModel(dataset);
// xmlModel.setModel(dataset);
// Load cache relationship if not already loaded
xmlModel.excludeEmptyFields();
xmlModel.caching = true;
// const cache = dataset.xmlCache ? dataset.xmlCache : null;
// dataset.load('xmlCache');
await dataset.load('xmlCache');
if (dataset.xmlCache) {
serializer.setCache(dataset.xmlCache);
xmlModel.xmlCache = dataset.xmlCache;
}
// Generate or retrieve cached DOM document
const xmlDocument: XMLBuilder | null = await serializer.toXmlDocument();
return xmlDocument;
// return cache.getDomDocument();
const domDocument: XMLBuilder | null = await xmlModel.getDomDocument();
return domDocument;
};
const addLandingPageAttribute = (domNode: XMLBuilder, dataid: string) => {
@ -190,6 +192,6 @@ const addLandingPageAttribute = (domNode: XMLBuilder, dataid: string) => {
domNode.att('landingpage', url);
};
const addSpecInformation = (domNode: XMLBuilder, information: string) => {
const addSpecInformation= (domNode: XMLBuilder, information: string) => {
domNode.ele('SetSpec').att('Value', information);
};
};

129
app/Library/XmlModel.ts Normal file
View file

@ -0,0 +1,129 @@
import DocumentXmlCache from '#models/DocumentXmlCache';
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
import Dataset from '#models/dataset';
import Strategy from './Strategy.js';
import { DateTime } from 'luxon';
import { builder } from 'xmlbuilder2';
/**
* This is the description of the interface
*
* @interface Conf
* @member {Model} model holds the current dataset model
* @member {XMLBuilder} dom holds the current DOM representation
* @member {Array<string>} excludeFields List of fields to skip on serialization.
* @member {boolean} excludeEmpty True, if empty fields get excluded from serialization.
* @member {string} baseUri Base URI for xlink:ref elements
*/
export interface Conf {
model: Dataset;
dom?: XMLBuilder;
excludeFields: Array<string>;
excludeEmpty: boolean;
baseUri: string;
}
export default class XmlModel {
private config: Conf;
// private strategy = null;
private cache: DocumentXmlCache | null = null;
private _caching = false;
private strategy: Strategy;
constructor(dataset: Dataset) {
// $this->strategy = new Strategy();// Opus_Model_Xml_Version1;
// $this->config = new Conf();
// $this->strategy->setup($this->config);
this.config = {
excludeEmpty: false,
baseUri: '',
excludeFields: [],
model: dataset,
};
this.strategy = new Strategy({
excludeEmpty: true,
baseUri: '',
excludeFields: [],
model: dataset,
});
}
set model(model: Dataset) {
this.config.model = model;
}
public excludeEmptyFields(): void {
this.config.excludeEmpty = true;
}
get xmlCache(): DocumentXmlCache | null {
return this.cache;
}
set xmlCache(cache: DocumentXmlCache) {
this.cache = cache;
}
get caching(): boolean {
return this._caching;
}
set caching(caching: boolean) {
this._caching = caching;
}
public async getDomDocument(): Promise<XMLBuilder | null> {
const dataset = this.config.model;
let domDocument: XMLBuilder | null = await this.getDomDocumentFromXmlCache();
if (domDocument == null) {
domDocument = await this.strategy.createDomDocument();
// domDocument = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
if (this._caching) {
// caching is desired:
this.cache = this.cache || new DocumentXmlCache();
this.cache.document_id = dataset.id;
this.cache.xml_version = 1; // (int)$this->strategy->getVersion();
this.cache.server_date_modified = dataset.server_date_modified.toFormat('yyyy-MM-dd HH:mm:ss');
this.cache.xml_data = domDocument.end();
await this.cache.save();
}
const node = domDocument.find(
(n) => {
const test = n.node.nodeName == 'Rdr_Dataset';
return test;
},
false,
true,
)?.node;
if (node != undefined) {
domDocument = builder({ version: '1.0', encoding: 'UTF-8', standalone: true }, node);
}
}
return domDocument;
}
private async getDomDocumentFromXmlCache(): Promise<XMLBuilder | null> {
const dataset: Dataset = this.config.model;
if (!this.cache) {
return null;
}
//.toFormat('YYYY-MM-DD HH:mm:ss');
let date: DateTime = dataset.server_date_modified;
const actuallyCached: boolean = await DocumentXmlCache.hasValidEntry(dataset.id, date);
if (!actuallyCached) {
return null;
}
//cache is actual return it for oai:
try {
if (this.cache) {
return this.cache.getDomDocument();
} else {
return null;
}
} catch (error) {
return null;
}
}
}

43
app/controllers/activities_controller.ts
View file

@ -1,43 +0,0 @@
// app/controllers/activities_controller.ts
import type { HttpContext } from '@adonisjs/core/http';
import Activity from '#models/activity';
export default class ActivitiesController {
async index({ response }: HttpContext) {
// const activities = await Activity.query()
// .preload('user', (q) => q.select('id', 'login'))
// .orderBy('created_at', 'desc')
// .limit(10);
// return response.json(
// activities.map((a) => ({
// id: a.id,
// type: a.type,
// description: a.description,
// user: a.user?.login ?? null,
// created_at: a.createdAt.toISO(), // relativeTime() expects ISO
// })),
// );
try {
const activities = await Activity.query()
.preload('user', (q) => q.select('id', 'login'))
.orderBy('created_at', 'desc')
.limit(10);
return response.json(
activities.map((a) => ({
id: a.id,
type: a.type,
description: a.description,
user: a.user?.login ?? null,
created_at: a.createdAt.toISO(), // relativeTime() expects ISO
})),
);
} catch (error) {
console.error('Error fetching activities:', error);
return response.status(500).json({ error: 'Internal Server Error' });
}
}
}

54
app/controllers/projects_controller.ts
View file

@ -1,54 +0,0 @@
// app/controllers/projects_controller.ts
import Project from '#models/project';
import type { HttpContext } from '@adonisjs/core/http';
import { createProjectValidator, updateProjectValidator } from '#validators/project';
export default class ProjectsController {
// GET /settings/projects
public async index({ inertia, auth }: HttpContext) {
const projects = await Project.all();
// return inertia.render('Admin/Project/Index', { projects });
return inertia.render('Admin/Project/Index', {
projects: projects,
can: {
edit: await auth.user?.can(['settings']),
create: await auth.user?.can(['settings']),
},
});
}
// GET /settings/projects/create
public async create({ inertia }: HttpContext) {
return inertia.render('Admin/Project/Create');
}
// POST /settings/projects
public async store({ request, response, session }: HttpContext) {
// Validate the request data
const data = await request.validateUsing(createProjectValidator);
await Project.create(data);
session.flash('success', 'Project created successfully');
return response.redirect().toRoute('settings.project.index');
}
// GET /settings/projects/:id/edit
public async edit({ params, inertia }: HttpContext) {
const project = await Project.findOrFail(params.id);
return inertia.render('Admin/Project/Edit', { project });
}
// PUT /settings/projects/:id
public async update({ params, request, response, session }: HttpContext) {
const project = await Project.findOrFail(params.id);
// Validate the request data
const data = await request.validateUsing(updateProjectValidator);
await project.merge(data).save();
session.flash('success', 'Project updated successfully');
return response.redirect().toRoute('settings.project.index');
}
}

43
app/exceptions/db_handler.ts
View file

@ -1,43 +0,0 @@
// import { Exception } from '@adonisjs/core/exceptions'
import { HttpContext, ExceptionHandler } from '@adonisjs/core/http';
export default class DbHandlerException extends ExceptionHandler {
// constructor() {
// super(Logger)
// }
async handle(error: any, ctx: HttpContext) {
// Check for AggregateError type
if (error.type === 'AggregateError' && error.aggregateErrors) {
const dbErrors = error.aggregateErrors.some((err: any) => err.code === 'ECONNREFUSED' && err.port === 5432);
if (dbErrors) {
return ctx.response.status(503).json({
status: 'error',
message: 'PostgreSQL database connection failed. Please ensure the database service is running.',
details: {
code: error.code,
type: error.type,
ports: error.aggregateErrors.map((err: any) => ({
port: err.port,
address: err.address,
})),
},
});
}
}
// Handle simple ECONNREFUSED errors
if (error.code === 'ECONNREFUSED') {
return ctx.response.status(503).json({
status: 'error',
message: 'Database connection failed. Please ensure PostgreSQL is running.',
code: error.code,
});
}
return super.handle(error, ctx);
}
static status = 500;
}

168
app/exceptions/handler.ts
View file

@ -1,53 +1,125 @@
import app from '@adonisjs/core/services/app'
import { HttpContext, ExceptionHandler } from '@adonisjs/core/http'
import type { StatusPageRange, StatusPageRenderer } from '@adonisjs/core/types/http'
/*
|--------------------------------------------------------------------------
| Http Exception Handler
|--------------------------------------------------------------------------
|
| AdonisJs will forward all exceptions occurred during an HTTP request to
| the following class. You can learn more about exception handling by
| reading docs.
|
| The exception handler extends a base `HttpExceptionHandler` which is not
| mandatory, however it can do lot of heavy lifting to handle the errors
| properly.
|
*/
import app from '@adonisjs/core/services/app';
import { HttpContext, ExceptionHandler } from '@adonisjs/core/http';
// import logger from '@adonisjs/core/services/logger';
import type { StatusPageRange, StatusPageRenderer } from '@adonisjs/core/types/http';
export default class HttpExceptionHandler extends ExceptionHandler {
protected debug = !app.inProduction
protected renderStatusPages = true
protected statusPages: Record<StatusPageRange, StatusPageRenderer> = {
'404': (error, ctx) =>
ctx.inertia
? ctx.inertia.render('Errors/ServerError', { error: error.message, code: error.status })
: ctx.response.status(error.status).send(error.message),
'401..403': (error, ctx) => {
if (ctx.inertia) {
return ctx.inertia.render('Errors/ServerError', { error: error.message, code: error.status });
}
return ctx.response.status(error.status).send(error.message);
},
'500..599': (error, ctx) => {
const isDbError =
error.code === 'ECONNREFUSED' &&
(error.errors?.some((e: any) => e.port === 5432) ?? error.message?.includes('5432'));
if (isDbError && ctx.inertia) {
return ctx.inertia.render('Errors/postgres_error', {
status: 'error',
message: 'PostgreSQL database connection failed.',
details: {
code: error.code,
type: error.status
// Entferne das .map() auf error.errors, da es oft undefined ist
}
});
}
if (ctx.inertia) {
return ctx.inertia.render('Errors/ServerError', { error: error.message, code: 500 });
}
return ctx.response.status(500).send(error.message);
}
};
public async handle(error: any, ctx: HttpContext) {
/**
* WICHTIG: Validierungsfehler (422) NICHT manuell abfangen!
* AdonisJS 6 + VineJS + Inertia machen das automatisch.
* Wenn du es hier manuell machst, überschreibst du den Standard-Flow.
* In debug mode, the exception handler will display verbose errors
* with pretty printed stack traces.
*/
protected debug = !app.inProduction;
return super.handle(error, ctx)
}
}
/**
* Status pages are used to display a custom HTML pages for certain error
* codes. You might want to enable them in production only, but feel
* free to enable them in development as well.
*/
protected renderStatusPages = true; //app.inProduction;
/**
* Status pages is a collection of error code range and a callback
* to return the HTML contents to send as a response.
*/
// protected statusPages: Record<StatusPageRange, StatusPageRenderer> = {
// '401..403': (error, { view }) => {
// return view.render('./errors/unauthorized', { error });
// },
// '404': (error, { view }) => {
// return view.render('./errors/not-found', { error });
// },
// '500..599': (error, { view }) => {
// return view.render('./errors/server-error', { error });
// },
// };
protected statusPages: Record<StatusPageRange, StatusPageRenderer> = {
'404': (error, { inertia }) => {
return inertia.render('Errors/ServerError', {
error: error.message,
code: error.status,
});
},
'401..403': async (error, { inertia }) => {
// session.flash('errors', error.message);
return inertia.render('Errors/ServerError', {
error: error.message,
code: error.status,
});
},
'500..599': (error, { inertia }) => inertia.render('Errors/ServerError', { error: error.message, code: error.status }),
};
// constructor() {
// super(logger);
// }
public async handle(error: any, ctx: HttpContext) {
const { response, request, session } = ctx;
/**
* Handle failed authentication attempt
*/
// if (['E_INVALID_AUTH_PASSWORD', 'E_INVALID_AUTH_UID'].includes(error.code)) {
// session.flash('errors', { login: error.message });
// return response.redirect('/login');
// }
// if ([401].includes(error.status)) {
// session.flash('errors', { login: error.message });
// return response.redirect('/dashboard');
// }
// https://github.com/inertiajs/inertia-laravel/issues/56
// let test = response.getStatus(); //200
// let header = request.header('X-Inertia'); // true
// if (request.header('X-Inertia') && [500, 503, 404, 403, 401, 200].includes(response.getStatus())) {
if (request.header('X-Inertia') && [422].includes(error.status)) {
// session.flash('errors', error.messages.errors);
session.flash('errors', error.messages);
return response.redirect().back();
// return inertia.render('errors/server_error', {
// return inertia.render('errors/server_error', {
// // status: response.getStatus(),
// error: error,
// });
// ->toResponse($request)
// ->setStatusCode($response->status());
}
// Dynamically change the error templates based on the absence of X-Inertia header
// if (!ctx.request.header('X-Inertia')) {
// this.statusPages = {
// '401..403': (error, { view }) => view.render('./errors/unauthorized', { error }),
// '404': (error, { view }) => view.render('./errors/not-found', { error }),
// '500..599': (error, { view }) => view.render('./errors/server-error', { error }),
// };
// }
/**
* Forward rest of the exceptions to the parent class
*/
return super.handle(error, ctx);
}
/**
* The method is used to report error to the logging service or
* the a third party error monitoring service.
*
* @note You should not attempt to send a response from this method.
*/
async report(error: unknown, ctx: HttpContext) {
return super.report(error, ctx);
}
}

50
app/models/DocumentXmlCache.ts
View file

@ -4,8 +4,7 @@ import { builder, create } from 'xmlbuilder2';
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
import db from '@adonisjs/lucid/services/db';
import { DateTime } from 'luxon';
import type { BelongsTo } from '@adonisjs/lucid/types/relations';
import logger from '@adonisjs/core/services/logger';
import type { BelongsTo } from "@adonisjs/lucid/types/relations";
export default class DocumentXmlCache extends BaseModel {
public static namingStrategy = new SnakeCaseNamingStrategy();
@ -67,38 +66,33 @@ export default class DocumentXmlCache extends BaseModel {
}
/**
* Check if a valid (non-stale) cache entry exists
* Cache is valid only if it was created AFTER the dataset's last modification
* Check if a dataset in a specific xml version is already cached or not.
*
* @param datasetId - The dataset ID to check
* @param datasetServerDateModified - The dataset's last modification timestamp
* @returns true if valid cache exists, false otherwise
* @param mixed datasetId
* @param mixed serverDateModified
* @returns {Promise<boolean>} Returns true on cached hit else false.
*/
// public static async hasValidEntry(datasetId: number, datasetServerDateModified: DateTime): Promise<boolean> {
// // const formattedDate = dayjs(datasetServerDateModified).format('YYYY-MM-DD HH:mm:ss');
// const query = Database.from(this.table)
// .where('document_id', datasetId)
// .where('server_date_modified', '2023-08-17 16:51:03')
// .first();
// const row = await query;
// return !!row;
// }
// Assuming 'DocumentXmlCache' has a table with a 'server_date_modified' column in your database
public static async hasValidEntry(datasetId: number, datasetServerDateModified: DateTime): Promise<boolean> {
const serverDateModifiedString: string = datasetServerDateModified.toFormat('yyyy-MM-dd HH:mm:ss'); // Convert DateTime to ISO string
const row = await db
.from(this.table)
const query = db.from(this.table)
.where('document_id', datasetId)
.where('server_date_modified', '>', serverDateModifiedString) // Check if server_date_modified is newer or equal
.where('server_date_modified', '>=', serverDateModifiedString) // Check if server_date_modified is newer or equal
.first();
const isValid = !!row;
if (isValid) {
logger.debug(`Valid cache found for dataset ${datasetId}`);
} else {
logger.debug(`No valid cache for dataset ${datasetId} (dataset modified: ${serverDateModifiedString})`);
}
return isValid;
}
/**
* Invalidate (delete) cache entry
*/
public async invalidate(): Promise<void> {
await this.delete();
logger.debug(`Invalidated cache for document ${this.document_id}`);
const row = await query;
return !!row;
}
}

57
app/models/activity.ts
View file

@ -1,57 +0,0 @@
// app/models/activity.ts
import { DateTime } from 'luxon';
import { belongsTo, column } from '@adonisjs/lucid/orm';
import BaseModel from './base_model.js';
import type { BelongsTo } from '@adonisjs/lucid/types/relations';
import User from '#models/user';
import { SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm';
export default class Activity extends BaseModel {
public static namingStrategy = new SnakeCaseNamingStrategy();
public static primaryKey = 'id';
public static table = 'activities';
@column({ isPrimary: true })
declare id: number;
@column()
declare type: string;
@column()
declare userId: number | null;
@column()
declare subjectType: string | null;
@column()
declare subjectId: number | null;
@column()
declare description: string;
// Manual JSON (de)serialization keeps this working on SQLite/MySQL.
// On Postgres json/jsonb the driver already parses — drop the `consume`
// JSON.parse there to avoid double-handling.
// @column({
// prepare: (value: Record<string, any> | null) => (value ? JSON.stringify(value) : null),
// consume: (value: string | null) => (value ? JSON.parse(value) : null),
// })
// declare properties: Record<string, any> | null;
@column()
declare properties: Record<string, any> | null;
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime;
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime;
// @belongsTo(() => User)
// declare user: BelongsTo<typeof User>;
@belongsTo(() => User, {
foreignKey: 'userId',
})
declare user: BelongsTo<typeof User>;
}

65
app/models/dataset.ts
View file

@ -5,10 +5,7 @@ import {
belongsTo,
hasMany,
computed,
hasOne,
afterCreate,
beforeUpdate,
afterUpdate,
hasOne
} from '@adonisjs/lucid/orm';
import { DateTime } from 'luxon';
import dayjs from 'dayjs';
@ -26,11 +23,10 @@ import DatasetIdentifier from './dataset_identifier.js';
import Project from './project.js';
import DocumentXmlCache from './DocumentXmlCache.js';
import DatasetExtension from '#models/traits/dataset_extension';
import type { ManyToMany } from '@adonisjs/lucid/types/relations';
import type { BelongsTo } from '@adonisjs/lucid/types/relations';
import type { HasMany } from '@adonisjs/lucid/types/relations';
import type { HasOne } from '@adonisjs/lucid/types/relations';
import ActivityLogger from '#services/activity_logger';
import type { ManyToMany } from "@adonisjs/lucid/types/relations";
import type { BelongsTo } from "@adonisjs/lucid/types/relations";
import type { HasMany } from "@adonisjs/lucid/types/relations";
import type { HasOne } from "@adonisjs/lucid/types/relations";
export default class Dataset extends DatasetExtension {
public static namingStrategy = new SnakeCaseNamingStrategy();
@ -50,7 +46,7 @@ export default class Dataset extends DatasetExtension {
@column({ columnName: 'creating_corporation' })
public creating_corporation: string;
@column.dateTime({
@column.dateTime({
columnName: 'embargo_date',
serialize: (value: Date | null) => {
return value ? dayjs(value).format('YYYY-MM-DD') : value;
@ -64,7 +60,7 @@ export default class Dataset extends DatasetExtension {
@column({})
public language: string;
@column({ columnName: 'publish_id' })
@column({columnName: 'publish_id'})
public publish_id: number | null = null;
@column({})
@ -213,15 +209,6 @@ export default class Dataset extends DatasetExtension {
return mainTitle ? mainTitle.value : null;
}
@computed({
serializeAs: 'doi_identifier',
})
public get doiIdentifier() {
// return `${this.firstName} ${this.lastName}`;
const identifier: DatasetIdentifier = this.identifier;
return identifier ? identifier.value : null;
}
@manyToMany(() => Person, {
pivotForeignKey: 'document_id',
pivotRelatedForeignKey: 'person_id',
@ -270,12 +257,10 @@ export default class Dataset extends DatasetExtension {
return model || null;
}
static async getMax(column: string) {
let dataset = await this.query()
.max(column + ' as max_publish_id')
.firstOrFail();
static async getMax (column: string) {
let dataset = await this.query().max(column + ' as max_publish_id').firstOrFail();
return dataset.$extras.max_publish_id;
}
}
@computed({
serializeAs: 'remaining_time',
@ -290,34 +275,4 @@ export default class Dataset extends DatasetExtension {
return 0;
}
}
// @afterCreate()
// static async logUploaded(dataset: Dataset) {
// await dataset.preload('titles');
// await ActivityLogger.log({
// type: 'dataset.uploaded',
// description: `New publication uploaded: ${dataset.mainTitle ?? 'Untitled'}`,
// subjectType: 'Dataset',
// subjectId: dataset.id,
// });
// }
@beforeUpdate()
static capturePublish(dataset: Dataset) {
// $dirty is populated here, before persistence
(dataset as any).$becamePublished = dataset.$dirty.status !== undefined && dataset.status === 'published';
}
@afterUpdate()
static async logPublished(dataset: Dataset) {
if ((dataset as any).$becamePublished) {
await ActivityLogger.log({
type: 'dataset.published',
description: `Publication published: ${dataset.mainTitle}`,
subjectType: 'Dataset',
subjectId: dataset.id,
});
}
}
}

9
app/models/file.ts
View file

@ -3,12 +3,12 @@ import { column, hasMany, belongsTo, SnakeCaseNamingStrategy, computed } from '@
import HashValue from './hash_value.js';
import Dataset from './dataset.js';
import BaseModel from './base_model.js';
// import { Buffer } from 'buffer';
import * as fs from 'fs';
import crypto from 'crypto';
// import Drive from '@ioc:Adonis/Core/Drive';
// import Drive from '@adonisjs/drive';
// import drive from '#services/drive';
import drive from '@adonisjs/drive/services/main';
import drive from '#services/drive';
import type { HasMany } from "@adonisjs/lucid/types/relations";
import type { BelongsTo } from "@adonisjs/lucid/types/relations";
@ -88,8 +88,7 @@ export default class File extends BaseModel {
serializeAs: 'filePath',
})
public get filePath() {
// return `/storage/app/public/${this.pathName}`;
return `/storage/app/data/${this.pathName}`;
return `/storage/app/public/${this.pathName}`;
// const mainTitle = this.titles?.find((title) => title.type === 'Main');
// return mainTitle ? mainTitle.value : null;
}
@ -166,7 +165,7 @@ export default class File extends BaseModel {
public async delete() {
if (this.pathName) {
// Delete file from additional storage
await drive.use('local').delete(this.pathName);
await drive.delete(this.pathName);
}
// Call the original delete method of the BaseModel to remove the record from the database

5
app/models/mime_type.ts
View file

@ -16,14 +16,9 @@ export default class MimeType extends BaseModel {
@column({})
public name: string;
// 1 : n file_extensions are separated by '|' in the database
@column({})
public file_extension: string;
// 1 : n alternate_mimetype are separated by '|' in the database
@column({})
public alternate_mimetype: string;
@column({})
public enabled: boolean;

52
app/models/person.ts
View file

@ -3,7 +3,7 @@ import { DateTime } from 'luxon';
import dayjs from 'dayjs';
import Dataset from './dataset.js';
import BaseModel from './base_model.js';
import type { ManyToMany } from '@adonisjs/lucid/types/relations';
import type { ManyToMany } from "@adonisjs/lucid/types/relations";
export default class Person extends BaseModel {
public static namingStrategy = new SnakeCaseNamingStrategy();
@ -30,7 +30,7 @@ export default class Person extends BaseModel {
@column({})
public lastName: string;
@column({ columnName: 'identifier_orcid' })
@column({})
public identifierOrcid: string;
@column({})
@ -51,7 +51,7 @@ export default class Person extends BaseModel {
serializeAs: 'name',
})
public get fullName() {
return [this.firstName, this.lastName].filter(Boolean).join(' ');
return `${this.firstName} ${this.lastName}`;
}
// @computed()
@ -64,12 +64,10 @@ export default class Person extends BaseModel {
// return '2023-03-21 08:45:00';
// }
@computed({
serializeAs: 'dataset_count',
})
@computed()
public get datasetCount() {
const stock = this.$extras.datasets_count; //my pivot column name was "stock"
return Number(stock);
return stock;
}
@computed()
@ -78,16 +76,6 @@ export default class Person extends BaseModel {
return contributor_type;
}
@computed({ serializeAs: 'allow_email_contact' })
public get allowEmailContact() {
// If the datasets relation is missing or empty, return false instead of null.
if (!this.datasets || this.datasets.length === 0) {
return false;
}
// Otherwise return the pivot attribute from the first related dataset.
return this.datasets[0].$extras?.pivot_allow_email_contact;
}
@manyToMany(() => Dataset, {
pivotForeignKey: 'person_id',
pivotRelatedForeignKey: 'document_id',
@ -95,34 +83,4 @@ export default class Person extends BaseModel {
pivotColumns: ['role', 'sort_order', 'allow_email_contact'],
})
public datasets: ManyToMany<typeof Dataset>;
// public toJSON() {
// const json = super.toJSON();
// // Check if this person is loaded through a pivot relationship with sensitive roles
// const pivotRole = this.$extras?.pivot_role;
// if (pivotRole === 'author' || pivotRole === 'contributor') {
// // Remove sensitive information for public-facing roles
// delete json.email;
// // delete json.identifierOrcid;
// }
// return json;
// }
// @afterFind()
// public static async afterFindHook(person: Person) {
// if (person.$extras?.pivot_role === 'author' || person.$extras?.pivot_role === 'contributor') {
// person.email = undefined as any;
// }
// }
// @afterFetch()
// public static async afterFetchHook(persons: Person[]) {
// persons.forEach(person => {
// if (person.$extras?.pivot_role === 'author' || person.$extras?.pivot_role === 'contributor') {
// person.email = undefined as any;
// }
// });
// }
}

57
app/models/types.ts
View file

@ -1,57 +0,0 @@
/**
* Qs module config
*/
type QueryStringConfig = {
depth?: number
allowPrototypes?: boolean
plainObjects?: boolean
parameterLimit?: number
arrayLimit?: number
ignoreQueryPrefix?: boolean
delimiter?: RegExp | string
allowDots?: boolean
charset?: 'utf-8' | 'iso-8859-1' | undefined
charsetSentinel?: boolean
interpretNumericEntities?: boolean
parseArrays?: boolean
comma?: boolean
}
/**
* Base config used by all types
*/
type BodyParserBaseConfig = {
encoding: string
limit: string | number
types: string[]
}
/**
* Body parser config for parsing JSON requests
*/
export type BodyParserJSONConfig = BodyParserBaseConfig & {
strict: boolean
convertEmptyStringsToNull: boolean
}
/**
* Parser config for parsing form data
*/
export type BodyParserFormConfig = BodyParserBaseConfig & {
queryString: QueryStringConfig
convertEmptyStringsToNull: boolean
}
/**
* Parser config for parsing raw body (untouched)
*/
export type BodyParserRawConfig = BodyParserBaseConfig
/**
* Body parser config for all supported form types
*/
export type BodyParserConfig = {
allowedMethods: string[]
json: BodyParserJSONConfig
form: BodyParserFormConfig
raw: BodyParserRawConfig
multipart: BodyParserMultipartConfig
}

45
app/models/user.ts
View file

@ -1,6 +1,6 @@
import { DateTime } from 'luxon';
import { withAuthFinder } from '@adonisjs/auth/mixins/lucid';
import { column, manyToMany, hasMany, SnakeCaseNamingStrategy, computed, beforeFetch, beforeFind } from '@adonisjs/lucid/orm';
import { column, manyToMany, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm';
import hash from '@adonisjs/core/services/hash';
import Role from './role.js';
import db from '@adonisjs/lucid/services/db';
@ -14,7 +14,6 @@ import type { ManyToMany } from '@adonisjs/lucid/types/relations';
import type { HasMany } from '@adonisjs/lucid/types/relations';
import { compose } from '@adonisjs/core/helpers';
import BackupCode from './backup_code.js';
import Activity from './activity.js';
const AuthFinder = withAuthFinder(() => hash.use('laravel'), {
uids: ['email'],
@ -50,6 +49,7 @@ export default class User extends compose(BaseModel, AuthFinder) {
@column()
public login: string;
@column()
public firstName: string;
@ -87,8 +87,17 @@ export default class User extends compose(BaseModel, AuthFinder) {
@column({})
public state: number;
@column({})
public avatar: string;
// @hasOne(() => TotpSecret, {
// foreignKey: 'user_id',
// })
// public totp_secret: HasOne<typeof TotpSecret>;
// @beforeSave()
// public static async hashPassword(user: User) {
// if (user.$dirty.password) {
// user.password = await hash.use('laravel').make(user.password);
// }
// }
public get isTwoFactorEnabled(): boolean {
return Boolean(this?.twoFactorSecret && this.state == TotpState.STATE_ENABLED);
@ -112,34 +121,6 @@ export default class User extends compose(BaseModel, AuthFinder) {
})
public backupcodes: HasMany<typeof BackupCode>;
@hasMany(() => Activity, {
foreignKey: 'user_id',
})
public activities: HasMany<typeof Activity>;
@computed({
serializeAs: 'is_admin',
})
public get isAdmin(): boolean {
const roles = this.roles;
const isAdmin = roles?.map((role: Role) => role.name).includes('administrator');
return isAdmin;
}
// public toJSON() {
// return {
// ...super.toJSON(),
// roles: []
// };
// }
@beforeFind()
@beforeFetch()
public static preloadRoles(user: User) {
user.preload('roles', (builder) => {
builder.select(['id', 'name', 'display_name', 'description']);
});
}
public async getBackupCodes(this: User): Promise<BackupCode[]> {
const test = await this.related('backupcodes').query();
// return test.map((role) => role.code);

27
app/services/activity_logger.ts
View file

@ -1,27 +0,0 @@
// app/services/activity_logger.ts
import Activity from '#models/activity'
interface LogOptions {
type: string
description: string
userId?: number | null
subjectType?: string | null
subjectId?: number | string | null
properties?: Record<string, any>
}
export default class ActivityLogger {
static async log(options: LogOptions): Promise<void> {
await Activity.create({
type: options.type,
description: options.description,
userId: options.userId ?? null,
subjectType: options.subjectType ?? null,
subjectId: options.subjectId != null ? Number(options.subjectId) : null,
properties: options.properties ?? null,
})
// Invalidate the cache if you add one (see Redis section).
// await redis.del('activities:recent')
}
}

54
app/services/validation_service.ts
View file

@ -1,54 +0,0 @@
import app from '@adonisjs/core/services/app'
import { errors } from '@vinejs/vine'
/**
* The ValidationService handles manual construction of validation errors
* that are compatible with the VanillaErrorReporter and AdonisJS Session.
*/
export class ValidationService {
/**
* Builds a validation error in the array-of-objects format without throwing it.
* Use this when you need the error object itself, e.g. multipart.abort(error).
*/
make(field: string, message: string, rule: string = 'manual') {
return new errors.E_VALIDATION_ERROR([
{
field,
message,
rule,
},
])
}
/**
* Throws a manual validation error in the array-of-objects format
* which prevents the ".reduce is not a function" error in the session.
*/
throw(field: string, message: string, rule: string = 'manual') {
throw this.make(field, message, rule)
}
/**
* Throws multiple manual validation errors at once.
*/
throwMany(errorObjects: Array<{ field: string; message: string; rule?: string }>) {
throw new errors.E_VALIDATION_ERROR(
errorObjects.map((err) => ({
field: err.field,
message: err.message,
rule: err.rule || 'manual',
}))
)
}
}
/**
* Initialize and export the singleton instance
*/
let validation: ValidationService
await app.booted(async () => {
validation = await app.container.make(ValidationService)
})
export { validation as default }

103
app/utils/utility-functions.ts
View file

@ -1,16 +1,3 @@
import { join, isAbsolute } from 'node:path';
import type { BodyParserConfig } from '#models/types';
import { createId } from '@paralleldrive/cuid2';
import { tmpdir } from 'node:os';
import config from '@adonisjs/core/services/config';
import Dataset from '#models/dataset';
import { TransactionClientContract } from '@adonisjs/lucid/types/database';
import Person from '#models/person';
interface Dictionary {
[index: string]: string;
}
export function sum(a: number, b: number): number {
return a + b;
}
@ -37,93 +24,3 @@ export function preg_match(regex: RegExp, str: string) {
const result: boolean = regex.test(str);
return result;
}
/**
* Returns the tmp path for storing the files temporarly
*/
export function getTmpPath(config: BodyParserConfig['multipart']): string {
if (typeof config.tmpFileName === 'function') {
const tmpPath = config.tmpFileName();
return isAbsolute(tmpPath) ? tmpPath : join(tmpdir(), tmpPath);
}
return join(tmpdir(), createId());
}
/**
* Returns config for a given type
*/
export function getConfigFor<K extends keyof BodyParserConfig>(type: K): BodyParserConfig[K] {
const bodyParserConfig: BodyParserConfig = config.get('bodyparser');
const configType = bodyParserConfig[type];
return configType;
}
export function parseBytesSize(size: string): number {
const units: Record<string, number> = {
kb: 1024,
mb: 1024 * 1024,
gb: 1024 * 1024 * 1024,
tb: 1024 * 1024 * 1024 * 1024,
};
const match = size.match(/^(\d+)(kb|mb|gb|tb)$/i); // Regex to match size format
if (!match) {
throw new Error('Invalid size format');
}
const [, value, unit] = match;
return parseInt(value) * units[unit.toLowerCase()];
}
// Helper function to format bytes as human-readable text
export function formatBytes(bytes: number): string {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
export async function savePersons(dataset: Dataset, persons: any[], role: string, trx: TransactionClientContract) {
for (const [key, person] of persons.entries()) {
const pivotData = {
role: role,
sort_order: key + 1,
allow_email_contact: false,
...extractPivotAttributes(person), // Merge pivot attributes here
};
if (person.id !== undefined) {
await dataset
.useTransaction(trx)
.related('persons')
.attach({
[person.id]: pivotData,
});
} else {
const dataPerson = new Person();
dataPerson.fill(person);
await dataset.useTransaction(trx).related('persons').save(dataPerson, false, pivotData);
}
}
}
// Helper function to extract pivot attributes from a person object
function extractPivotAttributes(person: any) {
const pivotAttributes: Dictionary = {};
for (const key in person) {
if (key.startsWith('pivot_')) {
// pivotAttributes[key] = person[key];
const cleanKey = key.replace('pivot_', ''); // Remove 'pivot_' prefix
pivotAttributes[cleanKey] = person[key];
}
}
return pivotAttributes;
}
// in #app/utils/utility-functions
export function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}

182
app/validators/dataset.ts
View file

@ -1,7 +1,6 @@
import vine, { SimpleMessagesProvider } from '@vinejs/vine';
import { TitleTypes, DescriptionTypes, ContributorTypes, ReferenceIdentifierTypes, RelationTypes } from '#contracts/enums';
import dayjs from 'dayjs';
// import MimeType from '#models/mime_type';
// const enabledExtensions = await MimeType.query().select('file_extension').where('enabled', true).exec();
@ -40,8 +39,7 @@ export const createDatasetValidator = vine.compile(
.translatedLanguage({ mainLanguageField: 'language', typeField: 'type' }),
}),
)
// .minLength(2)
.arrayContainsTypes({ typeA: 'main', typeB: 'translated' }),
.minLength(1),
descriptions: vine
.array(
vine.object({
@ -55,8 +53,7 @@ export const createDatasetValidator = vine.compile(
.translatedLanguage({ mainLanguageField: 'language', typeField: 'type' }),
}),
)
// .minLength(1),
.arrayContainsTypes({ typeA: 'abstract', typeB: 'translated' }),
.minLength(1),
authors: vine
.array(
vine.object({
@ -67,9 +64,8 @@ export const createDatasetValidator = vine.compile(
.email()
.normalizeEmail()
.isUniquePerson({ table: 'persons', column: 'email', idField: 'id' }),
first_name: vine.string().trim().minLength(3).maxLength(255).optional().requiredWhen('name_type', '=', 'Personal'),
first_name: vine.string().trim().minLength(3).maxLength(255),
last_name: vine.string().trim().minLength(3).maxLength(255),
identifier_orcid: vine.string().trim().maxLength(255).orcid().optional(),
}),
)
.minLength(1)
@ -84,10 +80,9 @@ export const createDatasetValidator = vine.compile(
.email()
.normalizeEmail()
.isUniquePerson({ table: 'persons', column: 'email', idField: 'id' }),
first_name: vine.string().trim().minLength(3).maxLength(255).optional().requiredWhen('name_type', '=', 'Personal'),
first_name: vine.string().trim().minLength(3).maxLength(255),
last_name: vine.string().trim().minLength(3).maxLength(255),
pivot_contributor_type: vine.enum(Object.keys(ContributorTypes)),
identifier_orcid: vine.string().trim().maxLength(255).orcid().optional(),
}),
)
.distinct('email')
@ -130,7 +125,7 @@ export const createDatasetValidator = vine.compile(
references: vine
.array(
vine.object({
value: vine.string().trim().minLength(3).maxLength(255).validateReference({ typeField: 'type' }),
value: vine.string().trim().minLength(3).maxLength(255),
type: vine.enum(Object.values(ReferenceIdentifierTypes)),
relation: vine.enum(Object.values(RelationTypes)),
label: vine.string().trim().minLength(2).maxLength(255),
@ -191,8 +186,7 @@ export const updateDatasetValidator = vine.compile(
.translatedLanguage({ mainLanguageField: 'language', typeField: 'type' }),
}),
)
// .minLength(2)
.arrayContainsTypes({ typeA: 'main', typeB: 'translated' }),
.minLength(1),
descriptions: vine
.array(
vine.object({
@ -206,7 +200,7 @@ export const updateDatasetValidator = vine.compile(
.translatedLanguage({ mainLanguageField: 'language', typeField: 'type' }),
}),
)
.arrayContainsTypes({ typeA: 'abstract', typeB: 'translated' }),
.minLength(1),
authors: vine
.array(
vine.object({
@ -217,9 +211,8 @@ export const updateDatasetValidator = vine.compile(
.email()
.normalizeEmail()
.isUniquePerson({ table: 'persons', column: 'email', idField: 'id' }),
first_name: vine.string().trim().minLength(3).maxLength(255).optional().requiredWhen('name_type', '=', 'Personal'),
first_name: vine.string().trim().minLength(3).maxLength(255),
last_name: vine.string().trim().minLength(3).maxLength(255),
identifier_orcid: vine.string().trim().maxLength(255).orcid().optional(),
}),
)
.minLength(1)
@ -234,9 +227,8 @@ export const updateDatasetValidator = vine.compile(
.email()
.normalizeEmail()
.isUniquePerson({ table: 'persons', column: 'email', idField: 'id' }),
first_name: vine.string().trim().minLength(3).maxLength(255).optional().requiredWhen('name_type', '=', 'Personal'),
first_name: vine.string().trim().minLength(3).maxLength(255),
last_name: vine.string().trim().minLength(3).maxLength(255),
identifier_orcid: vine.string().trim().maxLength(255).orcid().optional(),
pivot_contributor_type: vine.enum(Object.keys(ContributorTypes)),
}),
)
@ -280,7 +272,7 @@ export const updateDatasetValidator = vine.compile(
references: vine
.array(
vine.object({
value: vine.string().trim().minLength(3).maxLength(255).validateReference({ typeField: 'type' }),
value: vine.string().trim().minLength(3).maxLength(255),
type: vine.enum(Object.values(ReferenceIdentifierTypes)),
relation: vine.enum(Object.values(RelationTypes)),
label: vine.string().trim().minLength(2).maxLength(255),
@ -310,149 +302,21 @@ export const updateDatasetValidator = vine.compile(
.fileScan({ removeInfected: true }),
)
.dependentArrayMinLength({ dependentArray: 'fileInputs', min: 1 }),
fileInputs: vine
.array(
vine.object({
label: vine.string().trim().maxLength(100),
}),
)
.optional(),
fileInputs: vine.array(
vine.object({
label: vine.string().trim().maxLength(100),
//extnames: extensions,
}),
),
}),
);
export const updateEditorDatasetValidator = vine.compile(
vine.object({
// first step
language: vine
.string()
.trim()
.regex(/^[a-zA-Z0-9]+$/),
licenses: vine.array(vine.number()).minLength(1), // define at least one license for the new dataset
rights: vine.string().in(['true']),
// second step
type: vine.string().trim().minLength(3).maxLength(255),
creating_corporation: vine.string().trim().minLength(3).maxLength(255),
titles: vine
.array(
vine.object({
value: vine.string().trim().minLength(3).maxLength(255),
type: vine.enum(Object.values(TitleTypes)),
language: vine
.string()
.trim()
.minLength(2)
.maxLength(255)
.translatedLanguage({ mainLanguageField: 'language', typeField: 'type' }),
}),
)
// .minLength(2)
.arrayContainsTypes({ typeA: 'main', typeB: 'translated' }),
descriptions: vine
.array(
vine.object({
value: vine.string().trim().minLength(3).maxLength(2500),
type: vine.enum(Object.values(DescriptionTypes)),
language: vine
.string()
.trim()
.minLength(2)
.maxLength(255)
.translatedLanguage({ mainLanguageField: 'language', typeField: 'type' }),
}),
)
.arrayContainsTypes({ typeA: 'abstract', typeB: 'translated' }),
authors: vine
.array(
vine.object({
email: vine
.string()
.trim()
.maxLength(255)
.email()
.normalizeEmail()
.isUniquePerson({ table: 'persons', column: 'email', idField: 'id' }),
first_name: vine.string().trim().minLength(3).maxLength(255).optional().requiredWhen('name_type', '=', 'Personal'),
last_name: vine.string().trim().minLength(3).maxLength(255),
identifier_orcid: vine.string().trim().maxLength(255).orcid().optional(),
}),
)
.minLength(1)
.distinct('email'),
contributors: vine
.array(
vine.object({
email: vine
.string()
.trim()
.maxLength(255)
.email()
.normalizeEmail()
.isUniquePerson({ table: 'persons', column: 'email', idField: 'id' }),
first_name: vine.string().trim().minLength(3).maxLength(255).optional().requiredWhen('name_type', '=', 'Personal'),
last_name: vine.string().trim().minLength(3).maxLength(255),
identifier_orcid: vine.string().trim().maxLength(255).orcid().optional(),
pivot_contributor_type: vine.enum(Object.keys(ContributorTypes)),
}),
)
.distinct('email')
.optional(),
// third step
project_id: vine.number().optional(),
// embargo_date: schema.date.optional({ format: 'yyyy-MM-dd' }, [rules.after(10, 'days')]),
embargo_date: vine
.date({
formats: ['YYYY-MM-DD'],
})
.afterOrEqual((_field) => {
return dayjs().add(10, 'day').format('YYYY-MM-DD');
})
.optional(),
coverage: vine.object({
x_min: vine.number(),
x_max: vine.number(),
y_min: vine.number(),
y_max: vine.number(),
elevation_absolut: vine.number().positive().optional(),
elevation_min: vine.number().positive().optional().requiredIfExists('elevation_max'),
elevation_max: vine.number().positive().optional().requiredIfExists('elevation_min'),
// type: vine.enum(Object.values(DescriptionTypes)),
depth_absolut: vine.number().negative().optional(),
depth_min: vine.number().negative().optional().requiredIfExists('depth_max'),
depth_max: vine.number().negative().optional().requiredIfExists('depth_min'),
time_abolute: vine.date({ formats: { utc: true } }).optional(),
time_min: vine
.date({ formats: { utc: true } })
.beforeField('time_max')
.optional()
.requiredIfExists('time_max'),
time_max: vine
.date({ formats: { utc: true } })
.afterField('time_min')
.optional()
.requiredIfExists('time_min'),
}),
references: vine
.array(
vine.object({
value: vine.string().trim().minLength(3).maxLength(255).validateReference({ typeField: 'type' }),
type: vine.enum(Object.values(ReferenceIdentifierTypes)),
relation: vine.enum(Object.values(RelationTypes)),
label: vine.string().trim().minLength(2).maxLength(255),
}),
)
.optional(),
subjects: vine
.array(
vine.object({
value: vine.string().trim().minLength(3).maxLength(255),
// pivot_contributor_type: vine.enum(Object.keys(ContributorTypes)),
language: vine.string().trim().minLength(2).maxLength(255),
}),
)
.minLength(3)
.distinct('value'),
}),
);
// files: schema.array([rules.minLength(1)]).members(
// schema.file({
// size: '512mb',
// extnames: ['jpg', 'gif', 'png', 'tif', 'pdf', 'zip', 'fgb', 'nc', 'qml', 'ovr', 'gpkg', 'gml', 'gpx', 'kml', 'kmz', 'json'],
// }),
// ),
let messagesProvider = new SimpleMessagesProvider({
'minLength': '{{ field }} must be at least {{ min }} characters long',
@ -504,10 +368,8 @@ let messagesProvider = new SimpleMessagesProvider({
'files.array.minLength': 'At least {{ min }} file upload is required.',
'files.*.size': 'file size is to big',
'files.*.extnames': 'file extension is not supported',
'embargo_date.date.afterOrEqual': `Embargo date must be on or after ${dayjs().add(10, 'day').format('DD.MM.YYYY')}`,
});
createDatasetValidator.messagesProvider = messagesProvider;
updateDatasetValidator.messagesProvider = messagesProvider;
updateEditorDatasetValidator.messagesProvider = messagesProvider;
// export default createDatasetValidator;

28
app/validators/project.ts
View file

@ -1,28 +0,0 @@
// app/validators/project.ts
import vine from '@vinejs/vine';
export const createProjectValidator = vine.compile(
vine.object({
label: vine.string().trim().minLength(1).maxLength(50) .regex(/^[a-z0-9-]+$/),
name: vine
.string()
.trim()
.minLength(3)
.maxLength(255)
.regex(/^[a-zA-Z0-9äöüßÄÖÜ\s-]+$/),
description: vine.string().trim().maxLength(255).minLength(5).optional(),
}),
);
export const updateProjectValidator = vine.compile(
vine.object({
// label is NOT included since it's readonly
name: vine
.string()
.trim()
.minLength(3)
.maxLength(255)
.regex(/^[a-zA-Z0-9äöüßÄÖÜ\s-]+$/),
description: vine.string().trim().maxLength(255).minLength(5).optional(),
}),
);

29
app/validators/role.ts
View file

@ -8,20 +8,20 @@ export const createRoleValidator = vine.compile(
vine.object({
name: vine
.string()
.isUnique({ table: 'roles', column: 'name' })
.trim()
.minLength(3)
.maxLength(255)
.isUnique({ table: 'roles', column: 'name' })
.regex(/^[a-zA-Z0-9]+$/), // Must be alphanumeric
.regex(/^[a-zA-Z0-9]+$/), //Must be alphanumeric with hyphens or underscores
display_name: vine
.string()
.isUnique({ table: 'roles', column: 'display_name' })
.trim()
.minLength(3)
.maxLength(255)
.isUnique({ table: 'roles', column: 'display_name' })
.regex(/^[a-zA-Z0-9]+$/),
description: vine.string().trim().escape().minLength(3).maxLength(255).optional(),
permissions: vine.array(vine.number()).minLength(1), // At least one permission required
permissions: vine.array(vine.number()).minLength(1), // define at least one permission for the new role
}),
);
@ -29,28 +29,21 @@ export const updateRoleValidator = vine.withMetaData<{ roleId: number }>().compi
vine.object({
name: vine
.string()
.trim()
.minLength(3)
.maxLength(255)
// .unique(async (db, value, field) => {
// const result = await db.from('roles').select('id').whereNot('id', field.meta.roleId).where('name', value).first();
// return result.length ? false : true;
// })
.isUnique({
table: 'roles',
column: 'name',
whereNot: (field) => field.meta.roleId,
})
.regex(/^[a-zA-Z0-9]+$/),
display_name: vine
.string()
.trim()
.minLength(3)
.maxLength(255)
.isUnique({
table: 'roles',
column: 'display_name',
whereNot: (field) => field.meta.roleId,
})
.regex(/^[a-zA-Z0-9]+$/),
.maxLength(255),
description: vine.string().trim().escape().minLength(3).maxLength(255).optional(),
permissions: vine.array(vine.number()).minLength(1), // At least one permission required
permissions: vine.array(vine.number()).minLength(1), // define at least one permission for the new role
}),
);

4
app/validators/user.ts
View file

@ -16,7 +16,7 @@ export const createUserValidator = vine.compile(
first_name: vine.string().trim().minLength(3).maxLength(255),
last_name: vine.string().trim().minLength(3).maxLength(255),
email: vine.string().maxLength(255).email().normalizeEmail().isUnique({ table: 'accounts', column: 'email' }),
new_password: vine.string().confirmed({ confirmationField: 'password_confirmation' }).trim().minLength(3).maxLength(60),
password: vine.string().confirmed().trim().minLength(3).maxLength(60),
roles: vine.array(vine.number()).minLength(1), // define at least one role for the new user
}),
);
@ -42,7 +42,7 @@ export const updateUserValidator = vine.withMetaData<{ objId: number }>().compil
.email()
.normalizeEmail()
.isUnique({ table: 'accounts', column: 'email', whereNot: (field) => field.meta.objId }),
new_password: vine.string().confirmed({ confirmationField: 'password_confirmation' }).trim().minLength(3).maxLength(60).optional(),
password: vine.string().confirmed().trim().minLength(3).maxLength(60).optional(),
roles: vine.array(vine.number()).minLength(1), // define at least one role for the new user
}),
);

239
app/validators/vanilla_error_reporter.ts
View file

@ -1,50 +1,203 @@
import { errors } from '@vinejs/vine'
import type { ErrorReporterContract, FieldContext } from '@vinejs/vine/types'
// import { ValidationError } from '../errors/validation_error.js';
import { errors } from '@vinejs/vine';
import type { ErrorReporterContract, FieldContext } from '@vinejs/vine/types';
import string from '@poppinss/utils/string';
/**
* Der VanillaErrorReporter sammelt Validierungsfehler im Standardformat,
* damit die AdonisJS Session-Middleware sie korrekt verarbeiten (reducen) kann.
* Shape of the Vanilla error node
*/
export type VanillaErrorNode = {
[field: string]: string[];
};
export interface MessagesBagContract {
get(pointer: string, rule: string, message: string, arrayExpressionPointer?: string, args?: any): string;
}
/**
* Message bag exposes the API to pull the most appropriate message for a
* given validation failure.
*/
export class MessagesBag implements MessagesBagContract {
messages: Message;
wildCardCallback;
constructor(messages: string[]) {
this.messages = messages;
this.wildCardCallback = typeof this.messages['*'] === 'function' ? this.messages['*'] : undefined;
}
/**
* Transform message by replace placeholders with runtime values
*/
transform(message: any, rule: string, pointer: string, args: any) {
/**
* No interpolation required
*/
if (!message.includes('{{')) {
return message;
}
return string.interpolate(message, { rule, field: pointer, options: args || {} });
}
/**
* Returns the most appropriate message for the validation failure.
*/
get(pointer: string, rule: string, message: string, arrayExpressionPointer: string, args: any) {
let validationMessage = this.messages[`${pointer}.${rule}`];
/**
* Fetch message for the array expression pointer if it exists
*/
if (!validationMessage && arrayExpressionPointer) {
validationMessage = this.messages[`${arrayExpressionPointer}.${rule}`];
}
/**
* Fallback to the message for the rule
*/
if (!validationMessage) {
validationMessage = this.messages[rule];
}
/**
* Transform and return message. The wildcard callback is invoked when custom message
* is not defined
*/
return validationMessage
? this.transform(validationMessage, rule, pointer, args)
: this.wildCardCallback
? this.wildCardCallback(pointer, rule, arrayExpressionPointer, args)
: message;
}
}
/**
* Shape of the error message collected by the SimpleErrorReporter
*/
type SimpleError = {
message: string;
field: string;
rule: string;
index?: number;
meta?: Record<string, any>;
};
export interface Message {
[key: string]: any;
}
/**
* Simple error reporter collects error messages as an array of object.
* Each object has following properties.
*
* - message: string
* - field: string
* - rule: string
* - index?: number (in case of an array member)
* - args?: Record<string, any>
*/
export class VanillaErrorReporter implements ErrorReporterContract {
/**
* Boolean, um zu prüfen, ob Fehler vorliegen
*/
hasErrors: boolean = false
/**
* Sammlung der Fehler als Array (erforderlich für AdonisJS 6 Session)
*/
errors: any[] = []
/**
* Diese Methode wird von VineJS für jeden Validierungsfehler aufgerufen
*/
report(
message: string,
rule: string,
field: FieldContext,
meta?: Record<string, any>
): void {
this.hasErrors = true
// private messages;
// private bail;
/**
* Wir pushen das Objekt in das Array.
* Das Feld 'field' erhält den vollständigen Pfad (z.B. "user.email").
* Boolean to know one or more errors have been reported
*/
hasErrors: boolean = false;
/**
* Collection of errors
*/
// errors: SimpleError[] = [];
errors: Message = {};
/**
* Report an error.
*/
this.errors.push({
message,
rule,
field: field.getFieldPath(),
...meta,
});
}
/**
* Erstellt die eigentliche Exception.
* Da 'this.errors' nun ein Array ist, funktioniert .reduce()
* in der Session-Middleware reibungslos.
*/
createError() {
return new errors.E_VALIDATION_ERROR(this.errors);
}
}
// constructor(messages: MessagesBagContract) {
// this.messages = messages;
// }
report(message: string, rule: string, field: FieldContext, meta?: Record<string, any> | undefined): void {
// const error: SimpleError = {
// message,
// rule,
// field: field.getFieldPath()
// };
// if (meta) {
// error.meta = meta;
// }
// if (field.isArrayMember) {
// error.index = field.name as number;
// }
// this.errors.push(error);
this.hasErrors = true;
// if (this.errors[field.getFieldPath()]) {
// this.errors[field.getFieldPath()]?.push(message);
// } else {
// this.errors[field.getFieldPath()] = [message];
// }
const error: SimpleError = {
message,
rule,
field: field.getFieldPath(), // ?field.wildCardPath.split('.')[0] : field.getFieldPath(),
};
// field: 'titles.0.value'
// message: 'Main Title is required'
// rule: 'required' "required"
if (meta) {
error.meta = meta;
}
// if (field.isArrayMember) {
// error.index = field.name;
// }
this.hasErrors = true;
var test = field.getFieldPath();
// this.errors.push(error);
// if (this.errors[error.field]) {
// this.errors[error.field]?.push(message);
// }
if (field.isArrayMember) {
// Check if the field has wildCardPath and if the error field already exists
if (this.errors[error.field]) {
// Do nothing, as we don't want to push further messages
} else {
// If the error field already exists, push the message
if (this.errors[error.field]) {
this.errors[error.field].push(message);
} else {
this.errors[error.field] = [message];
}
}
} else {
if (this.errors[error.field]) {
this.errors[error.field]?.push(message);
} else {
this.errors[error.field] = [message];
}
}
// } else {
// // normal field
// this.errors[field.field] = [message];
// }
/**
* Collecting errors as per the JSONAPI spec
*/
// this.errors.push({
// code: rule,
// detail: message,
// source: {
// pointer: field.wildCardPath,
// },
// ...(meta ? { meta } : {}),
// });
// let pointer: string = field.wildCardPath as string; //'display_name'
// // if (field.isArrayMember) {
// // this.errors[pointer] = field.name;
// // }
// this.errors[pointer] = this.errors[pointer] || [];
// // this.errors[pointer].push(message);
// this.errors[pointer].push(this.messages.get(pointer, rule, message, arrayExpressionPointer, args));
}
/**
* Returns an instance of the validation error
*/
createError() {
return new errors.E_VALIDATION_ERROR(this.errors);
}
}
export {};

20
clamd.conf
View file

@ -5,23 +5,7 @@ LogSyslog no
LogVerbose yes
DatabaseDirectory /var/lib/clamav
LocalSocket /var/run/clamav/clamd.socket
# LocalSocketMode 666
# Optional: allow multiple threads
MaxThreads 20
# Disable TCP socket
# TCPSocket 0
# TCP port address.
# Default: no
# TCPSocket 3310
# TCP address.
# By default we bind to INADDR_ANY, probably not wise.
# Enable the following to provide some degree of protection
# from the outside world.
# Default: no
# TCPAddr 127.0.0.1
Foreground no
PidFile /var/run/clamav/clamd.pid
# LocalSocketGroup node # Changed from 'clamav'
# User node # Changed from 'clamav' - clamd runs as clamav user
LocalSocketGroup node
User node

482
commands/fix_dataset_cross_references.ts
View file

@ -1,482 +0,0 @@
/*
|--------------------------------------------------------------------------
| node ace make:command fix-dataset-cross-references
| DONE: create commands/fix_dataset_cross_references.ts
|--------------------------------------------------------------------------
*/
import { BaseCommand, flags } from '@adonisjs/core/ace';
import type { CommandOptions } from '@adonisjs/core/types/ace';
import { DateTime } from 'luxon';
import Dataset from '#models/dataset';
import DatasetReference from '#models/dataset_reference';
import AppConfig from '#models/appconfig';
// import env from '#start/env';
interface MissingCrossReference {
sourceDatasetId: number;
targetDatasetId: number;
sourcePublishId: number | null;
targetPublishId: number | null;
sourceDoi: string | null;
targetDoi: string | null;
referenceType: string;
relation: string;
doi: string | null;
reverseRelation: string;
sourceReferenceLabel: string | null;
}
export default class DetectMissingCrossReferences extends BaseCommand {
static commandName = 'detect:missing-cross-references';
static description = 'Detect missing bidirectional cross-references between versioned datasets';
public static needsApplication = true;
@flags.boolean({ alias: 'f', description: 'Fix missing cross-references automatically' })
public fix: boolean = false;
@flags.boolean({ alias: 'v', description: 'Verbose output' })
public verbose: boolean = false;
@flags.number({ alias: 'p', description: 'Filter by specific publish_id (source or target dataset)' })
public publish_id?: number;
// example: node ace detect:missing-cross-references --verbose -p 227 //if you want to filter by specific publish_id with details
// example: node ace detect:missing-cross-references --verbose
// example: node ace detect:missing-cross-references --fix -p 227 //if you want to filter by specific publish_id and fix it
// example: node ace detect:missing-cross-references
public static options: CommandOptions = {
startApp: true,
staysAlive: false,
};
// Define the allowed relations that we want to process
private readonly ALLOWED_RELATIONS = [
'IsNewVersionOf',
'IsPreviousVersionOf',
'IsVariantFormOf',
'IsOriginalFormOf',
'Continues',
'IsContinuedBy',
'HasPart',
'IsPartOf',
];
// private readonly ALLOWED_RELATIONS = ['IsPreviousVersionOf', 'IsOriginalFormOf'];
async run() {
this.logger.info('🔍 Detecting missing cross-references...');
this.logger.info(`📋 Processing only these relations: ${this.ALLOWED_RELATIONS.join(', ')}`);
if (this.publish_id) {
this.logger.info(`Filtering by publish_id: ${this.publish_id}`);
}
try {
const missingReferences = await this.findMissingCrossReferences();
// Store count in AppConfig if not fixing and count >= 1
if (!this.fix && missingReferences.length >= 1) {
await this.storeMissingCrossReferencesCount(missingReferences.length);
}
if (missingReferences.length === 0) {
const filterMsg = this.publish_id ? ` for publish_id ${this.publish_id}` : '';
this.logger.success(`All cross-references are properly linked for the specified relations${filterMsg}!`);
// Clear the count if no missing references
if (!this.fix) {
await this.storeMissingCrossReferencesCount(0);
}
return;
}
const filterMsg = this.publish_id ? ` (filtered by publish_id ${this.publish_id})` : '';
this.logger.warning(`Found ${missingReferences.length} missing cross-reference(s)${filterMsg}:`);
// Show brief list if not verbose mode
if (!this.verbose) {
for (const missing of missingReferences) {
const sourceDoi = missing.sourceDoi ? ` DOI: ${missing.sourceDoi}` : '';
const targetDoi = missing.targetDoi ? ` DOI: ${missing.targetDoi}` : '';
this.logger.info(
`Dataset ${missing.sourceDatasetId} (Publish ID: ${missing.sourcePublishId}${sourceDoi}) ${missing.relation} Dataset ${missing.targetDatasetId} (Publish ID: ${missing.targetPublishId}${targetDoi}) → missing reverse: ${missing.reverseRelation}`,
);
}
} else {
// Verbose mode - show detailed info
for (const missing of missingReferences) {
this.logger.info(
`Dataset ${missing.sourceDatasetId} references ${missing.targetDatasetId}, but reverse reference is missing`,
);
this.logger.info(` - Reference type: ${missing.referenceType}`);
this.logger.info(` - Relation: ${missing.relation}`);
this.logger.info(` - DOI: ${missing.doi}`);
}
}
if (this.fix) {
await this.fixMissingReferences(missingReferences);
// Clear the count after fixing
await this.storeMissingCrossReferencesCount(0);
this.logger.success('All missing cross-references have been fixed!');
} else {
if (this.verbose) {
this.printMissingReferencesList(missingReferences);
}
this.logger.info('💡 Run with --fix flag to automatically create missing cross-references');
if (this.publish_id) {
this.logger.info(`🎯 Currently filtering by publish_id: ${this.publish_id}`);
}
}
} catch (error) {
this.logger.error('Error detecting missing cross-references:', error);
process.exit(1);
}
}
private async storeMissingCrossReferencesCount(count: number): Promise<void> {
try {
await AppConfig.updateOrCreate(
{
appid: 'commands',
configkey: 'missing_cross_references_count',
},
{
configvalue: count.toString(),
},
);
this.logger.info(`📊 Stored missing cross-references count in database: ${count}`);
} catch (error) {
this.logger.error('Failed to store missing cross-references count:', error);
}
}
private async findMissingCrossReferences(): Promise<MissingCrossReference[]> {
const missingReferences: {
sourceDatasetId: number;
targetDatasetId: number;
sourcePublishId: number | null;
targetPublishId: number | null;
sourceDoi: string | null;
targetDoi: string | null;
referenceType: string;
relation: string;
doi: string | null;
reverseRelation: string;
sourceReferenceLabel: string | null;
}[] = [];
this.logger.info('📊 Querying dataset references...');
// Find all references that point to Tethys datasets (DOI or URL containing tethys DOI)
// Only from datasets that are published AND only for allowed relations
const tethysReferencesQuery = DatasetReference.query()
.whereIn('type', ['DOI', 'URL'])
.whereIn('relation', this.ALLOWED_RELATIONS) // Only process allowed relations
.where((query) => {
query.where('value', 'like', '%doi.org/10.24341/tethys.%').orWhere('value', 'like', '%tethys.at/dataset/%');
})
.preload('dataset', (datasetQuery) => {
datasetQuery.preload('identifier');
})
.whereHas('dataset', (datasetQuery) => {
datasetQuery.where('server_state', 'published');
});
if (typeof this.publish_id === 'number') {
tethysReferencesQuery.whereHas('dataset', (datasetQuery) => {
datasetQuery.where('publish_id', this.publish_id as number);
});
}
const tethysReferences = await tethysReferencesQuery.exec();
this.logger.info(`🔗 Found ${tethysReferences.length} Tethys references from published datasets (allowed relations only)`);
let processedCount = 0;
let skippedCount = 0;
for (const reference of tethysReferences) {
processedCount++;
// if (this.verbose && processedCount % 10 === 0) {
// this.logger.info(`📈 Processed ${processedCount}/${tethysReferences.length} references...`);
// }
// Double-check that this relation is in our allowed list (safety check)
if (!this.ALLOWED_RELATIONS.includes(reference.relation)) {
skippedCount++;
if (this.verbose) {
this.logger.info(`⏭️ Skipping relation "${reference.relation}" - not in allowed list`);
}
continue;
}
// Extract dataset publish_id from DOI or URL
// const targetDatasetPublish = this.extractDatasetPublishIdFromReference(reference.value);
// Extract DOI from reference URL
const doi = this.extractDoiFromReference(reference.value);
// if (!targetDatasetPublish) {
// if (this.verbose) {
// this.logger.warning(`Could not extract publish ID from: ${reference.value}`);
// }
// continue;
// }
if (!doi) {
if (this.verbose) {
this.logger.warning(`Could not extract DOI from: ${reference.value}`);
}
continue;
}
// // Check if target dataset exists and is published
// const targetDataset = await Dataset.query()
// .where('publish_id', targetDatasetPublish)
// .where('server_state', 'published')
// .preload('identifier')
// .first();
// Check if target dataset exists and is published by querying via identifier
const targetDataset = await Dataset.query()
.where('server_state', 'published')
.whereHas('identifier', (query) => {
query.where('value', doi);
})
.preload('identifier')
.first();
if (!targetDataset) {
if (this.verbose) {
this.logger.warning(`⚠️ Target dataset with publish_id ${doi} not found or not published`);
}
continue;
}
// Ensure we have a valid source dataset with proper preloading
if (!reference.dataset) {
this.logger.warning(`⚠️ Source dataset ${reference.document_id} not properly loaded, skipping...`);
continue;
}
// Check if reverse reference exists
const reverseReferenceExists = await this.checkReverseReferenceExists(
targetDataset.id,
reference.document_id,
reference.relation,
reference.dataset.identifier.value
);
if (!reverseReferenceExists) {
const reverseRelation = this.getReverseRelation(reference.relation);
if (reverseRelation) {
// Only add if we have a valid reverse relation
missingReferences.push({
sourceDatasetId: reference.document_id,
targetDatasetId: targetDataset.id,
sourcePublishId: reference.dataset.publish_id || null,
targetPublishId: targetDataset.publish_id || null,
referenceType: reference.type,
relation: reference.relation,
doi: reference.value,
reverseRelation: reverseRelation,
sourceDoi: reference.dataset.identifier ? reference.dataset.identifier.value : null,
targetDoi: targetDataset.identifier ? targetDataset.identifier.value : null,
sourceReferenceLabel: reference.label || null,
});
}
}
}
this.logger.info(`✅ Processed ${processedCount} references (${skippedCount} skipped due to relation filtering)`);
return missingReferences;
}
private extractDoiFromReference(reference: string): string | null {
// Match DOI pattern, with or without URL prefix
const doiPattern = /(?:https?:\/\/)?(?:doi\.org\/)?(.+)/i;
const match = reference.match(doiPattern);
if (match && match[1]) {
return match[1]; // Returns just "10.24341/tethys.99.2"
}
return null;
}
private extractDatasetPublishIdFromReference(value: string): number | null {
// Extract from DOI: https://doi.org/10.24341/tethys.107 -> 107
const doiMatch = value.match(/10\.24341\/tethys\.(\d+)/);
if (doiMatch) {
return parseInt(doiMatch[1]);
}
// Extract from URL: https://tethys.at/dataset/107 -> 107
const urlMatch = value.match(/tethys\.at\/dataset\/(\d+)/);
if (urlMatch) {
return parseInt(urlMatch[1]);
}
return null;
}
private async checkReverseReferenceExists(
targetDatasetId: number,
sourceDatasetId: number,
originalRelation: string,
sourceDatasetIdentifier: string | null,
): Promise<boolean> {
const reverseRelation = this.getReverseRelation(originalRelation);
if (!reverseRelation) {
return true; // If no reverse relation is defined, consider it as "exists" to skip processing
}
// Only check for reverse references where the source dataset is also published
const reverseReference = await DatasetReference.query()
// We don't filter by source document_id here to find any incoming reference from any published dataset
.where('document_id', targetDatasetId)
// .where('related_document_id', sourceDatasetId) // Ensure it's an incoming reference
.where('relation', reverseRelation)
.where('value', 'like', `%${sourceDatasetIdentifier}`) // Basic check to ensure it points back to source dataset
.first();
return !!reverseReference;
}
private getReverseRelation(relation: string): string | null {
const relationMap: Record<string, string> = {
IsNewVersionOf: 'IsPreviousVersionOf',
IsPreviousVersionOf: 'IsNewVersionOf',
IsVariantFormOf: 'IsOriginalFormOf',
IsOriginalFormOf: 'IsVariantFormOf',
Continues: 'IsContinuedBy',
IsContinuedBy: 'Continues',
HasPart: 'IsPartOf',
IsPartOf: 'HasPart',
};
// Only return reverse relation if it exists in our map, otherwise return null
return relationMap[relation] || null;
}
private printMissingReferencesList(missingReferences: MissingCrossReference[]) {
console.log('┌─────────────────────────────────────────────────────────────────────────────────┐');
console.log('│ MISSING CROSS-REFERENCES REPORT │');
console.log('│ (Published Datasets Only - Filtered Relations) │');
console.log('└─────────────────────────────────────────────────────────────────────────────────┘');
console.log();
missingReferences.forEach((missing, index) => {
console.log(
`${index + 1}. Dataset ${missing.sourceDatasetId} (Publish ID: ${missing.sourcePublishId} Identifier: ${missing.sourceDoi})
${missing.relation} Dataset ${missing.targetDatasetId} (Publish ID: ${missing.targetPublishId} Identifier: ${missing.targetDoi})`,
);
console.log(` ├─ Current relation: "${missing.relation}"`);
console.log(` ├─ Missing reverse relation: "${missing.reverseRelation}"`);
console.log(` ├─ Reference type: ${missing.referenceType}`);
console.log(` └─ DOI/URL: ${missing.doi}`);
console.log();
});
console.log('┌─────────────────────────────────────────────────────────────────────────────────┐');
console.log(`│ SUMMARY: ${missingReferences.length} missing reverse reference(s) detected │`);
console.log(`│ Processed relations: ${this.ALLOWED_RELATIONS.join(', ')} │`);
console.log('└─────────────────────────────────────────────────────────────────────────────────┘');
}
private async fixMissingReferences(missingReferences: MissingCrossReference[]) {
this.logger.info('🔧 Creating missing cross-references in database...');
let fixedCount = 0;
let errorCount = 0;
for (const [index, missing] of missingReferences.entries()) {
try {
// Get both source and target datasets
const sourceDataset = await Dataset.query()
.where('id', missing.sourceDatasetId)
.where('server_state', 'published')
.preload('identifier')
.preload('titles') // Preload titles to get mainTitle
.first();
const targetDataset = await Dataset.query().where('id', missing.targetDatasetId).where('server_state', 'published').first();
if (!sourceDataset) {
this.logger.warning(`⚠️ Source dataset ${missing.sourceDatasetId} not found or not published, skipping...`);
errorCount++;
continue;
}
if (!targetDataset) {
this.logger.warning(`⚠️ Target dataset ${missing.targetDatasetId} not found or not published, skipping...`);
errorCount++;
continue;
}
// **NEW: Update the original reference if related_document_id is missing**
const originalReference = await DatasetReference.query()
.where('document_id', missing.sourceDatasetId)
.where('relation', missing.relation)
.where('value', 'like', `%${missing.targetDoi}%`)
.first();
if (originalReference && !originalReference.related_document_id) {
originalReference.related_document_id = missing.targetDatasetId;
await originalReference.save();
if (this.verbose) {
this.logger.info(`🔗 Updated original reference with related_document_id: ${missing.targetDatasetId}`);
}
}
// Create the reverse reference using the referenced_by relationship
// Example: If Dataset 297 IsNewVersionOf Dataset 144
// We create an incoming reference for Dataset 144 that shows Dataset 297 IsPreviousVersionOf it
const reverseReference = new DatasetReference();
// Don't set document_id - this creates an incoming reference via related_document_id
reverseReference.document_id = missing.targetDatasetId; //
reverseReference.related_document_id = missing.sourceDatasetId;
reverseReference.type = 'DOI';
reverseReference.relation = missing.reverseRelation;
// Use the source dataset's DOI for the value (what's being referenced)
if (sourceDataset.identifier?.value) {
reverseReference.value = `https://doi.org/${sourceDataset.identifier.value}`;
} else {
// Fallback to dataset URL if no DOI
reverseReference.value = `https://tethys.at/dataset/${sourceDataset.publish_id || missing.sourceDatasetId}`;
}
// Use the source dataset's main title for the label
//reverseReference.label = sourceDataset.mainTitle || `Dataset ${missing.sourceDatasetId}`;
// get label of forward reference
reverseReference.label = missing.sourceReferenceLabel || sourceDataset.mainTitle || `Dataset ${missing.sourceDatasetId}`;
// reverseReference.notes = `Auto-created by detect:missing-cross-references command on ${DateTime.now().toISO()} to fix missing bidirectional reference.`;
// Save the new reverse reference
// Also save 'server_date_modified' on target dataset to trigger any downstream updates (e.g. search index)
targetDataset.server_date_modified = DateTime.now();
await targetDataset.save();
await reverseReference.save();
fixedCount++;
if (this.verbose) {
this.logger.info(
`✅ [${index + 1}/${missingReferences.length}] Created reverse reference: Dataset ${missing.sourceDatasetId} -> ${missing.targetDatasetId} (${missing.reverseRelation})`,
);
} else if ((index + 1) % 10 === 0) {
this.logger.info(`📈 Fixed ${fixedCount}/${missingReferences.length} references...`);
}
} catch (error) {
this.logger.error(
`❌ Error creating reverse reference for datasets ${missing.targetDatasetId} -> ${missing.sourceDatasetId}:`,
error,
);
errorCount++;
}
}
this.logger.info(`📊 Fix completed: ${fixedCount} created, ${errorCount} errors`);
}
}

189
commands/fix_version_related_ids.ts
View file

@ -1,189 +0,0 @@
/*
|--------------------------------------------------------------------------
| node ace make:command fix-version-related-ids
| DONE: create commands/fix_version_related_ids.ts
|--------------------------------------------------------------------------
| Repairs the `related_document_id` foreign key on version references
| (IsNewVersionOf / IsPreviousVersionOf, both directions).
|
| The DOI stored in `value` is the reliable link; `related_document_id`
| is frequently NULL or self-referential. This command resolves the target
| dataset via its DOI and sets `related_document_id` accordingly, correcting
| both NULL and wrong-but-non-null values.
|
| Examples:
| node ace fix:version-related-ids // dry run, all datasets
| node ace fix:version-related-ids --verbose // dry run with per-row detail
| node ace fix:version-related-ids --fix // apply changes
| node ace fix:version-related-ids --fix -p 226 // apply, only refs owned by publish_id 226
*/
import { BaseCommand, flags } from '@adonisjs/core/ace';
import type { CommandOptions } from '@adonisjs/core/types/ace';
import Dataset from '#models/dataset';
import DatasetReference from '#models/dataset_reference';
export default class FixVersionRelatedIds extends BaseCommand {
static commandName = 'fix:version-related-ids';
static description =
'Backfill/repair related_document_id on IsNewVersionOf / IsPreviousVersionOf references by resolving the target dataset via its DOI';
public static needsApplication = true;
@flags.boolean({ alias: 'f', description: 'Apply changes. Without this flag the command runs as a dry run.' })
public fix: boolean = false;
@flags.boolean({ alias: 'v', description: 'Verbose output (per-reference detail)' })
public verbose: boolean = false;
@flags.number({ alias: 'p', description: 'Only process references owned by this publish_id' })
public publish_id?: number;
public static options: CommandOptions = {
startApp: true,
staysAlive: false,
};
// Only the version relations, both directions.
private readonly VERSION_RELATIONS = ['IsNewVersionOf', 'IsPreviousVersionOf'];
async run() {
this.logger.info(`🔍 Scanning ${this.VERSION_RELATIONS.join(' / ')} references...`);
this.logger.info(this.fix ? '✏️ Mode: APPLY (changes will be written)' : '👀 Mode: DRY RUN (no changes written)');
if (typeof this.publish_id === 'number') {
this.logger.info(`🎯 Filtering by owning publish_id: ${this.publish_id}`);
}
try {
const query = DatasetReference.query()
.whereIn('relation', this.VERSION_RELATIONS)
.whereIn('type', ['DOI', 'URL'])
.where((q) => {
q.where('value', 'like', '%doi.org/10.24341/tethys.%').orWhere('value', 'like', '%tethys.at/dataset/%');
});
// Restrict to references owned by a specific dataset (by publish_id), if requested.
if (typeof this.publish_id === 'number') {
query.whereHas('dataset', (d) => d.where('publish_id', this.publish_id as number));
}
const refs = await query.exec();
this.logger.info(`🔗 Found ${refs.length} version reference(s) to inspect`);
let alreadyCorrect = 0;
let filledFromNull = 0;
let correctedWrong = 0;
let unresolved = 0;
for (const ref of refs) {
const target = await this.resolveTarget(ref);
if (!target) {
unresolved++;
if (this.verbose) {
this.logger.warning(`⚠️ Reference ${ref.id}: could not resolve target (value: ${ref.value})`);
}
continue;
}
// Never let a reference point at its own owning document.
if (target.id === ref.document_id) {
unresolved++;
if (this.verbose) {
this.logger.warning(
`⚠️ Reference ${ref.id}: target resolves to its own document (${ref.document_id}); skipping self-link`,
);
}
continue;
}
if (ref.related_document_id === target.id) {
alreadyCorrect++;
continue;
}
const previous = ref.related_document_id;
const wasNull = previous === null || previous === undefined;
if (this.fix) {
ref.related_document_id = target.id;
await ref.save();
}
if (wasNull) {
filledFromNull++;
} else {
correctedWrong++;
}
if (this.verbose) {
const action = this.fix ? 'Updated' : '📝 Would update';
this.logger.info(
`${action} reference ${ref.id} (doc ${ref.document_id}, ${ref.relation}): ` +
`related_document_id ${previous ?? 'NULL'} → ${target.id} (publish_id ${target.publish_id})`,
);
}
}
this.logger.info('────────────────────────────────────────');
this.logger.info(`✔️ Already correct: ${alreadyCorrect}`);
this.logger.info(`➕ Filled from NULL: ${filledFromNull}`);
this.logger.info(`🔧 Corrected wrong value: ${correctedWrong}`);
this.logger.info(`⚠️ Unresolved/skipped: ${unresolved}`);
this.logger.info('────────────────────────────────────────');
const changes = filledFromNull + correctedWrong;
if (!this.fix && changes > 0) {
this.logger.info(`💡 Dry run only. Re-run with --fix to write ${changes} change(s).`);
} else if (this.fix) {
this.logger.success(`Done. ${changes} reference(s) updated.`);
} else {
this.logger.success('Nothing to change — all version references already linked correctly.');
}
} catch (error) {
this.logger.error('Error fixing version related_document_id values:', error);
process.exit(1);
}
}
/**
* Resolve the dataset a version reference points to.
* Prefers the DOI in `value` (reliable); falls back to a tethys publish_id URL.
*/
private async resolveTarget(ref: DatasetReference): Promise<Dataset | null> {
const doi = this.normalizeDoi(ref.value);
if (doi) {
const byDoi = await Dataset.query()
.whereHas('identifier', (q) => q.where('value', doi))
.first();
if (byDoi) return byDoi;
}
const publishId = this.extractPublishId(ref.value);
if (publishId) {
const byPublishId = await Dataset.query().where('publish_id', publishId).first();
if (byPublishId) return byPublishId;
}
return null;
}
/**
* Strip the resolver prefix so a reference value like
* "https://doi.org/10.24341/tethys.108.2" matches the identifier
* table value "10.24341/tethys.108.2". Returns null if it isn't a DOI.
*/
private normalizeDoi(value: string | null): string | null {
if (!value) return null;
const cleaned = value
.trim()
.replace(/^https?:\/\/(dx\.)?doi\.org\//i, '')
.replace(/^doi:/i, '');
return /^10\.\d{4,}\//.test(cleaned) ? cleaned : null;
}
private extractPublishId(value: string | null): number | null {
if (!value) return null;
const urlMatch = value.match(/tethys\.at\/dataset\/(\d+)/);
return urlMatch ? parseInt(urlMatch[1], 10) : null;
}
}

79
commands/index_datasets.ts
View file

@ -4,7 +4,7 @@
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
import { create } from 'xmlbuilder2';
import Dataset from '#models/dataset';
import XmlModel from '#app/Library/DatasetXmlSerializer';
import XmlModel from '#app/Library/XmlModel';
import { readFileSync } from 'fs';
import SaxonJS from 'saxon-js';
import { Client } from '@opensearch-project/opensearch';
@ -12,8 +12,10 @@ import { getDomain } from '#app/utils/utility-functions';
import { BaseCommand, flags } from '@adonisjs/core/ace';
import { CommandOptions } from '@adonisjs/core/types/ace';
import env from '#start/env';
// import db from '@adonisjs/lucid/services/db';
// import { default as Dataset } from '#models/dataset';
import logger from '@adonisjs/core/services/logger';
import { DateTime } from 'luxon';
const opensearchNode = env.get('OPENSEARCH_HOST', 'localhost');
const client = new Client({ node: `${opensearchNode}` }); // replace with your OpenSearch endpoint
@ -28,10 +30,11 @@ export default class IndexDatasets extends BaseCommand {
public publish_id: number;
public static options: CommandOptions = {
startApp: true, // Ensures the IoC container is ready to use
staysAlive: false, // Command exits after running
startApp: true,
staysAlive: false,
};
async run() {
logger.debug('Hello world!');
// const { default: Dataset } = await import('#models/dataset');
@ -41,12 +44,10 @@ export default class IndexDatasets extends BaseCommand {
const index_name = 'tethys-records';
for (var dataset of datasets) {
const shouldUpdate = await this.shouldUpdateDataset(dataset, index_name);
if (shouldUpdate) {
await this.indexDocument(dataset, index_name, proc);
} else {
logger.info(`Dataset with publish_id ${dataset.publish_id} is up to date, skipping indexing`);
}
// Logger.info(`File publish_id ${dataset.publish_id}`);
// const jsonString = await this.getJsonString(dataset, proc);
// console.log(jsonString);
await this.indexDocument(dataset, index_name, proc);
}
}
@ -64,46 +65,6 @@ export default class IndexDatasets extends BaseCommand {
return await query.exec();
}
private async shouldUpdateDataset(dataset: Dataset, index_name: string): Promise<boolean> {
try {
// Check if publish_id exists before proceeding
if (!dataset.publish_id) {
// Return true to update since document doesn't exist in OpenSearch yet
return true;
}
// Get the existing document from OpenSearch
const response = await client.get({
index: index_name,
id: dataset.publish_id?.toString(),
});
const existingDoc = response.body._source;
// Compare server_date_modified
if (existingDoc && existingDoc.server_date_modified) {
// Convert Unix timestamp (seconds) to milliseconds for DateTime.fromMillis()
const existingModified = DateTime.fromMillis(Number(existingDoc.server_date_modified) * 1000);
const currentModified = dataset.server_date_modified;
// Only update if the dataset has been modified more recently
if (currentModified <= existingModified) {
return false;
}
}
return true;
} catch (error) {
// If document doesn't exist or other error, we should index it
if (error.statusCode === 404) {
logger.info(`Dataset with publish_id ${dataset.publish_id} not found in index, will create new document`);
return true;
}
logger.warn(`Error checking existing document for publish_id ${dataset.publish_id}: ${error.message}`);
return true; // Index anyway if we can't determine the status
}
}
private async indexDocument(dataset: Dataset, index_name: string, proc: Buffer): Promise<void> {
try {
const doc = await this.getJsonString(dataset, proc);
@ -117,8 +78,7 @@ export default class IndexDatasets extends BaseCommand {
});
logger.info(`dataset with publish_id ${dataset.publish_id} successfully indexed`);
} catch (error) {
logger.error(`An error occurred while indexing dataset with publish_id ${dataset.publish_id}.
Error: ${error.message}`);
logger.error(`An error occurred while indexing dataset with publish_id ${dataset.publish_id}.`);
}
}
@ -151,16 +111,19 @@ export default class IndexDatasets extends BaseCommand {
}
private async getDatasetXmlDomNode(dataset: Dataset): Promise<XMLBuilder | null> {
const serializer = new XmlModel(dataset).enableCaching().excludeEmptyFields();
const xmlModel = new XmlModel(dataset);
// xmlModel.setModel(dataset);
xmlModel.excludeEmptyFields();
xmlModel.caching = true;
// const cache = dataset.xmlCache ? dataset.xmlCache : null;
// dataset.load('xmlCache');
if (dataset.xmlCache) {
serializer.setCache(dataset.xmlCache);
xmlModel.xmlCache = dataset.xmlCache;
}
// return cache.toXmlDocument();
const xmlDocument: XMLBuilder | null = await serializer.toXmlDocument();
return xmlDocument;
// return cache.getDomDocument();
const domDocument: XMLBuilder | null = await xmlModel.getDomDocument();
return domDocument;
}
private addSpecInformation(domNode: XMLBuilder, information: string) {

346
commands/list_updatable_datacite.ts
View file

@ -1,346 +0,0 @@
/*
|--------------------------------------------------------------------------
| node ace make:command list-updateable-datacite
| DONE: create commands/list_updeatable_datacite.ts
|--------------------------------------------------------------------------
*/
import { BaseCommand, flags } from '@adonisjs/core/ace';
import { CommandOptions } from '@adonisjs/core/types/ace';
import Dataset from '#models/dataset';
import { DoiClient } from '#app/Library/Doi/DoiClient';
import env from '#start/env';
import logger from '@adonisjs/core/services/logger';
import { DateTime } from 'luxon';
import pLimit from 'p-limit';
export default class ListUpdateableDatacite extends BaseCommand {
static commandName = 'list:updateable-datacite';
static description = 'List all datasets that need DataCite DOI updates';
public static needsApplication = true;
// private chunkSize = 100; // Set chunk size for pagination
@flags.boolean({ alias: 'v', description: 'Verbose output showing detailed information' })
public verbose: boolean = false;
@flags.boolean({ alias: 'c', description: 'Show only count of updatable datasets' })
public countOnly: boolean = false;
@flags.boolean({ alias: 'i', description: 'Show only publish IDs (useful for scripting)' })
public idsOnly: boolean = false;
@flags.number({ description: 'Chunk size for processing datasets (default: 50)' })
public chunkSize: number = 50;
//example: node ace list:updateable-datacite
//example: node ace list:updateable-datacite --verbose
//example: node ace list:updateable-datacite --count-only
//example: node ace list:updateable-datacite --ids-only
//example: node ace list:updateable-datacite --chunk-size 50
public static options: CommandOptions = {
startApp: true,
stayAlive: false,
};
async run() {
const prefix = env.get('DATACITE_PREFIX', '');
const base_domain = env.get('BASE_DOMAIN', '');
if (!prefix || !base_domain) {
logger.error('Missing DATACITE_PREFIX or BASE_DOMAIN environment variables');
return;
}
// Prevent conflicting flags
if ((this.verbose && this.countOnly) || (this.verbose && this.idsOnly)) {
logger.error('Flags --verbose cannot be combined with --count-only or --ids-only');
return;
}
const chunkSize = this.chunkSize || 50;
let page = 1;
let hasMoreDatasets = true;
let totalProcessed = 0;
const updatableDatasets: Dataset[] = [];
if (!this.countOnly && !this.idsOnly) {
logger.info(`Processing datasets in chunks of ${chunkSize}...`);
}
while (hasMoreDatasets) {
const datasets = await this.getDatasets(page, chunkSize);
if (datasets.length === 0) {
hasMoreDatasets = false;
break;
}
if (!this.countOnly && !this.idsOnly) {
logger.info(`Processing chunk ${page} (${datasets.length} datasets)...`);
}
const chunkUpdatableDatasets = await this.processChunk(datasets);
updatableDatasets.push(...chunkUpdatableDatasets);
totalProcessed += datasets.length;
page += 1;
if (datasets.length < chunkSize) {
hasMoreDatasets = false;
}
}
if (!this.countOnly && !this.idsOnly) {
logger.info(`Processed ${totalProcessed} datasets total, found ${updatableDatasets.length} that need updates`);
}
if (this.countOnly) {
console.log(updatableDatasets.length);
} else if (this.idsOnly) {
updatableDatasets.forEach((dataset) => console.log(dataset.publish_id));
} else if (this.verbose) {
await this.showVerboseOutput(updatableDatasets);
} else {
this.showSimpleOutput(updatableDatasets);
}
}
/**
* Processes a chunk of datasets to determine which ones need DataCite updates
*
* This method handles parallel processing of datasets within a chunk, providing
* efficient error handling and filtering of results.
*
* @param datasets - Array of Dataset objects to process
* @returns Promise<Dataset[]> - Array of datasets that need updates
*/
// private async processChunk(datasets: Dataset[]): Promise<Dataset[]> {
// // Process datasets in parallel using Promise.allSettled for better error handling
// //
// // Why Promise.allSettled vs Promise.all?
// // - Promise.all fails fast: if ANY promise rejects, the entire operation fails
// // - Promise.allSettled waits for ALL promises: some can fail, others succeed
// // - This is crucial for batch processing where we don't want one bad dataset
// // to stop processing of the entire chunk
// const results = await Promise.allSettled(
// datasets.map(async (dataset) => {
// try {
// // Check if this specific dataset needs a DataCite update
// const needsUpdate = await this.shouldUpdateDataset(dataset);
// // Return the dataset if it needs update, null if it doesn't
// // This creates a sparse array that we'll filter later
// return needsUpdate ? dataset : null;
// } catch (error) {
// // Error handling for individual dataset checks
// //
// // Log warnings only if we're not in silent modes (count-only or ids-only)
// // This prevents log spam when running automated scripts
// if (!this.countOnly && !this.idsOnly) {
// logger.warn(`Error checking dataset ${dataset.publish_id}: ${error.message}`);
// }
// // IMPORTANT DECISION: Return the dataset anyway if we can't determine status
// //
// // Why? It's safer to include a dataset that might not need updating
// // than to miss one that actually does need updating. This follows the
// // "fail-safe" principle - if we're unsure, err on the side of caution
// return dataset;
// }
// }),
// );
// // Filter and extract results from Promise.allSettled response
// //
// // Promise.allSettled returns an array of objects with this structure:
// // - { status: 'fulfilled', value: T } for successful promises
// // - { status: 'rejected', reason: Error } for failed promises
// //
// // We need to:
// // 1. Only get fulfilled results (rejected ones are already handled above)
// // 2. Filter out null values (datasets that don't need updates)
// // 3. Extract the actual Dataset objects from the wrapper
// return results
// .filter(
// (result): result is PromiseFulfilledResult<Dataset | null> =>
// // Type guard: only include fulfilled results that have actual values
// // This filters out:
// // - Rejected promises (shouldn't happen due to try/catch, but safety first)
// // - Fulfilled promises that returned null (datasets that don't need updates)
// result.status === 'fulfilled' && result.value !== null,
// )
// .map((result) => result.value!); // Extract the Dataset from the wrapper
// // The ! is safe here because we filtered out null values above
// }
private async processChunk(datasets: Dataset[]): Promise<Dataset[]> {
// Limit concurrency to avoid API flooding (e.g., max 5 at once)
const limit = pLimit(5);
const tasks = datasets.map((dataset) =>
limit(async () => {
try {
const needsUpdate = await this.shouldUpdateDataset(dataset);
return needsUpdate ? dataset : null;
} catch (error) {
if (!this.countOnly && !this.idsOnly) {
logger.warn(
`Error checking dataset ${dataset.publish_id}: ${
error instanceof Error ? error.message : JSON.stringify(error)
}`,
);
}
// Fail-safe: include dataset if uncertain
return dataset;
}
}),
);
const results = await Promise.allSettled(tasks);
return results
.filter((result): result is PromiseFulfilledResult<Dataset | null> => result.status === 'fulfilled' && result.value !== null)
.map((result) => result.value!);
}
private async getDatasets(page: number, chunkSize: number): Promise<Dataset[]> {
return await Dataset.query()
.orderBy('publish_id', 'asc')
.preload('identifier')
.preload('xmlCache')
.preload('titles')
.where('server_state', 'published')
.whereHas('identifier', (identifierQuery) => {
identifierQuery.where('type', 'doi');
})
.forPage(page, chunkSize); // Get files for the current page
}
private async shouldUpdateDataset(dataset: Dataset): Promise<boolean> {
try {
let doiIdentifier = dataset.identifier;
if (!doiIdentifier) {
await dataset.load('identifier');
doiIdentifier = dataset.identifier;
}
if (!doiIdentifier || doiIdentifier.type !== 'doi') {
return false;
}
const datasetModified =
dataset.server_date_modified instanceof DateTime
? dataset.server_date_modified
: DateTime.fromJSDate(dataset.server_date_modified);
if (!datasetModified) {
return true;
}
if (datasetModified > DateTime.now()) {
return false;
}
const doiClient = new DoiClient();
const DOI_CHECK_TIMEOUT = 300; // ms
const doiLastModified = await Promise.race([
doiClient.getDoiLastModified(doiIdentifier.value),
this.createTimeoutPromise(DOI_CHECK_TIMEOUT),
]).catch(() => null);
if (!doiLastModified) {
// If uncertain, better include dataset for update
return true;
}
const doiModified = DateTime.fromJSDate(doiLastModified);
if (datasetModified > doiModified) {
const diffInSeconds = Math.abs(datasetModified.diff(doiModified, 'seconds').seconds);
const toleranceSeconds = 600;
return diffInSeconds > toleranceSeconds;
}
return false;
} catch (error) {
return true; // safer: include dataset if unsure
}
}
/**
* Create a timeout promise for API calls
*/
private createTimeoutPromise(timeoutMs: number): Promise<never> {
return new Promise((_, reject) => {
setTimeout(() => reject(new Error(`API call timeout after ${timeoutMs}ms`)), timeoutMs);
});
}
private showSimpleOutput(updatableDatasets: Dataset[]): void {
if (updatableDatasets.length === 0) {
console.log('No datasets need DataCite updates.');
return;
}
console.log(`\nFound ${updatableDatasets.length} dataset(s) that need DataCite updates:\n`);
updatableDatasets.forEach((dataset) => {
console.log(`publish_id ${dataset.publish_id} needs update - ${dataset.mainTitle || 'Untitled'}`);
});
console.log(`\nTo update these datasets, run:`);
console.log(` node ace update:datacite`);
console.log(`\nOr update specific datasets:`);
console.log(` node ace update:datacite -p <publish_id>`);
}
private async showVerboseOutput(updatableDatasets: Dataset[]): Promise<void> {
if (updatableDatasets.length === 0) {
console.log('No datasets need DataCite updates.');
return;
}
console.log(`\nFound ${updatableDatasets.length} dataset(s) that need DataCite updates:\n`);
for (const dataset of updatableDatasets) {
await this.showDatasetDetails(dataset);
}
console.log(`\nSummary: ${updatableDatasets.length} datasets need updates`);
}
private async showDatasetDetails(dataset: Dataset): Promise<void> {
try {
let doiIdentifier = dataset.identifier;
if (!doiIdentifier) {
await dataset.load('identifier');
doiIdentifier = dataset.identifier;
}
const doiValue = doiIdentifier?.value || 'N/A';
const datasetModified = dataset.server_date_modified;
// Get DOI info from DataCite
const doiClient = new DoiClient();
const doiLastModified = await doiClient.getDoiLastModified(doiValue);
const doiState = await doiClient.getDoiState(doiValue);
console.log(`┌─ Dataset ${dataset.publish_id} ───────────────────────────────────────────────────────────────`);
console.log(`│ Title: ${dataset.mainTitle || 'Untitled'}`);
console.log(`│ DOI: ${doiValue}`);
console.log(`│ DOI State: ${doiState || 'Unknown'}`);
console.log(`│ Dataset Modified: ${datasetModified ? datasetModified.toISO() : 'N/A'}`);
console.log(`│ DOI Modified: ${doiLastModified ? DateTime.fromJSDate(doiLastModified).toISO() : 'N/A'}`);
console.log(`│ Status: NEEDS UPDATE`);
console.log(`└─────────────────────────────────────────────────────────────────────────────────────────────\n`);
} catch (error) {
console.log(`┌─ Dataset ${dataset.publish_id} ───────────────────────────────────────────────────────────────`);
console.log(`│ Title: ${dataset.mainTitle || 'Untitled'}`);
console.log(`│ DOI: ${dataset.identifier?.value || 'N/A'}`);
console.log(`│ Error: ${error.message}`);
console.log(`│ Status: NEEDS UPDATE (Error checking)`);
console.log(`└─────────────────────────────────────────────────────────────────────────────────────────────\n`);
}
}
}

266
commands/update_datacite.ts
View file

@ -1,266 +0,0 @@
/*
|--------------------------------------------------------------------------
| node ace make:command update-datacite
| DONE: create commands/update_datacite.ts
|--------------------------------------------------------------------------
*/
import { BaseCommand, flags } from '@adonisjs/core/ace';
import { CommandOptions } from '@adonisjs/core/types/ace';
import Dataset from '#models/dataset';
import { DoiClient } from '#app/Library/Doi/DoiClient';
import DoiClientException from '#app/exceptions/DoiClientException';
import Index from '#app/Library/Utils/Index';
import env from '#start/env';
import logger from '@adonisjs/core/services/logger';
import { DateTime } from 'luxon';
import { getDomain } from '#app/utils/utility-functions';
export default class UpdateDatacite extends BaseCommand {
static commandName = 'update:datacite';
static description = 'Update DataCite DOI records for published datasets';
public static needsApplication = true;
@flags.number({ alias: 'p', description: 'Specific publish_id to update' })
public publish_id: number;
@flags.boolean({ alias: 'f', description: 'Force update all records regardless of modification date' })
public force: boolean = false;
@flags.boolean({ alias: 'd', description: 'Dry run - show what would be updated without making changes' })
public dryRun: boolean = false;
@flags.boolean({ alias: 's', description: 'Show detailed stats for each dataset that needs updating' })
public stats: boolean = false;
//example: node ace update:datacite -p 123 --force --dry-run
public static options: CommandOptions = {
startApp: true, // Whether to boot the application before running the command
stayAlive: false, // Whether to keep the process alive after the command has executed
};
async run() {
logger.info('Starting DataCite update process...');
const prefix = env.get('DATACITE_PREFIX', '');
const base_domain = env.get('BASE_DOMAIN', '');
const apiUrl = env.get('DATACITE_API_URL', 'https://api.datacite.org');
if (!prefix || !base_domain) {
logger.error('Missing DATACITE_PREFIX or BASE_DOMAIN environment variables');
return;
}
logger.info(`Using DataCite API: ${apiUrl}`);
const datasets = await this.getDatasets();
logger.info(`Found ${datasets.length} datasets to process`);
let updated = 0;
let skipped = 0;
let errors = 0;
for (const dataset of datasets) {
try {
const shouldUpdate = this.force || (await this.shouldUpdateDataset(dataset));
if (this.stats) {
// Stats mode: show detailed information for datasets that need updating
if (shouldUpdate) {
await this.showDatasetStats(dataset);
updated++;
} else {
skipped++;
}
continue;
}
if (!shouldUpdate) {
logger.info(`Dataset ${dataset.publish_id}: Up to date, skipping`);
skipped++;
continue;
}
if (this.dryRun) {
logger.info(`Dataset ${dataset.publish_id}: Would update DataCite record (dry run)`);
updated++;
continue;
}
await this.updateDataciteRecord(dataset, prefix, base_domain);
logger.info(`Dataset ${dataset.publish_id}: Successfully updated DataCite record`);
updated++;
} catch (error) {
logger.error(`Dataset ${dataset.publish_id}: Failed to update - ${error.message}`);
errors++;
}
}
if (this.stats) {
logger.info(`\nDataCite Stats Summary: ${updated} datasets need updating, ${skipped} are up to date`);
} else {
logger.info(`DataCite update completed. Updated: ${updated}, Skipped: ${skipped}, Errors: ${errors}`);
}
}
private async getDatasets(): Promise<Dataset[]> {
const query = Dataset.query()
.preload('identifier')
.preload('xmlCache')
.where('server_state', 'published')
.whereHas('identifier', (identifierQuery) => {
identifierQuery.where('type', 'doi');
});
if (this.publish_id) {
query.where('publish_id', this.publish_id);
}
return await query.exec();
}
private async shouldUpdateDataset(dataset: Dataset): Promise<boolean> {
try {
let doiIdentifier = dataset.identifier;
if (!doiIdentifier) {
await dataset.load('identifier');
doiIdentifier = dataset.identifier;
}
if (!doiIdentifier || doiIdentifier.type !== 'doi') {
return false;
}
const datasetModified = dataset.server_date_modified;
const now = DateTime.now();
if (!datasetModified) {
return true; // Update if modification date is missing
}
if (datasetModified > now) {
return false; // Skip invalid future dates
}
// Check DataCite DOI modification date
const doiClient = new DoiClient();
const doiLastModified = await doiClient.getDoiLastModified(doiIdentifier.value);
if (!doiLastModified) {
return false; // not Update if we can't get DOI info
}
const doiModified = DateTime.fromJSDate(doiLastModified);
if (datasetModified > doiModified) {
// if dataset was modified after DOI creation
// Calculate the difference in seconds
const diffInSeconds = Math.abs(datasetModified.diff(doiModified, 'seconds').seconds);
// Define tolerance threshold (60 seconds = 1 minute)
const toleranceSeconds = 60;
// Only update if the difference is greater than the tolerance
// This prevents unnecessary updates for minor timestamp differences
return diffInSeconds > toleranceSeconds;
} else {
return false; // No update needed
}
} catch (error) {
return false; // not update if we can't determine status or other error
}
}
private async updateDataciteRecord(dataset: Dataset, prefix: string, base_domain: string): Promise<void> {
try {
// Get the DOI identifier (HasOne relationship)
let doiIdentifier = dataset.identifier;
if (!doiIdentifier) {
await dataset.load('identifier');
doiIdentifier = dataset.identifier;
}
if (!doiIdentifier || doiIdentifier.type !== 'doi') {
throw new Error('No DOI identifier found for dataset');
}
// Generate XML metadata
const xmlMeta = (await Index.getDoiRegisterString(dataset)) as string;
if (!xmlMeta) {
throw new Error('Failed to generate XML metadata');
}
// Construct DOI value and landing page URL
const doiValue = doiIdentifier.value; // Use existing DOI value
const landingPageUrl = `https://doi.${getDomain(base_domain)}/${doiValue}`;
// Update DataCite record
const doiClient = new DoiClient();
const dataciteResponse = await doiClient.registerDoi(doiValue, xmlMeta, landingPageUrl);
if (dataciteResponse?.status === 201) {
// // Update dataset modification date
// dataset.server_date_modified = DateTime.now();
// await dataset.save();
// // Update search index
// const index_name = 'tethys-records';
// await Index.indexDocument(dataset, index_name);
logger.debug(`Dataset ${dataset.publish_id}: DataCite record and search index updated successfully`);
} else {
throw new DoiClientException(
dataciteResponse?.status || 500,
`Unexpected DataCite response code: ${dataciteResponse?.status}`,
);
}
} catch (error) {
if (error instanceof DoiClientException) {
throw error;
}
throw new Error(`Failed to update DataCite record: ${error.message}`);
}
}
/**
* Shows detailed statistics for a dataset that needs updating
*/
private async showDatasetStats(dataset: Dataset): Promise<void> {
try {
let doiIdentifier = dataset.identifier;
if (!doiIdentifier) {
await dataset.load('identifier');
doiIdentifier = dataset.identifier;
}
const doiValue = doiIdentifier?.value || 'N/A';
const doiStatus = doiIdentifier?.status || 'N/A';
const datasetModified = dataset.server_date_modified;
// Get DOI info from DataCite
const doiClient = new DoiClient();
const doiLastModified = await doiClient.getDoiLastModified(doiValue);
const doiState = await doiClient.getDoiState(doiValue);
console.log(`
┌─ Dataset ${dataset.publish_id} ───────────────────────────────────────────────────────────────
│ DOI Value: ${doiValue}
│ DOI Status (DB): ${doiStatus}
│ DOI State (DataCite): ${doiState || 'Unknown'}
│ Dataset Modified: ${datasetModified ? datasetModified.toISO() : 'N/A'}
│ DOI Modified: ${doiLastModified ? DateTime.fromJSDate(doiLastModified).toISO() : 'N/A'}
│ Needs Update: YES - Dataset newer than DOI
└─────────────────────────────────────────────────────────────────────────────────────────────`);
} catch (error) {
console.log(`
┌─ Dataset ${dataset.publish_id} ───────────────────────────────────────────────────────────────
│ DOI Value: ${dataset.identifier?.value || 'N/A'}
│ Error: ${error.message}
│ Needs Update: YES - Error checking status
└─────────────────────────────────────────────────────────────────────────────────────────────`);
}
}
}

2
commands/validate_checksum.ts
View file

@ -88,7 +88,7 @@ export default class ValidateChecksum extends BaseCommand {
);
// Construct the file path
const filePath = '/storage/app/data/' + file.pathName;
const filePath = '/storage/app/public/' + file.pathName;
try {
// Calculate the MD5 checksum of the file

18
components.d.ts vendored
View file

@ -11,21 +11,3 @@ declare module '@vue/runtime-core' {
NInput: (typeof import('naive-ui'))['NInput'];
}
}
// types/leaflet-src-dom-DomEvent.d.ts
declare module 'leaflet/src/dom/DomEvent' {
export type DomEventHandler = (e?: any) => void;
// Attach event listeners. `obj` can be any DOM node or object with event handling.
export function on(obj: any, types: string, fn: DomEventHandler, context?: any): void;
// Detach event listeners.
export function off(obj: any, types: string, fn?: DomEventHandler, context?: any): void;
// Prevent default on native events
export function preventDefault(ev?: Event | undefined): void;
// Optional: other helpers you might need later
export function stopPropagation(ev?: Event | undefined): void;
export function stop(ev?: Event | undefined): void;
}

3
config/app.ts
View file

@ -80,8 +80,7 @@ export const http = defineConfig({
| headers.
|
*/
// trustProxy: proxyAddr.compile('loopback'),
trustProxy: proxyAddr.compile(['127.0.0.1', '::1/128']),
trustProxy: proxyAddr.compile('loopback'),
/*
|--------------------------------------------------------------------------

6
config/bodyparser.ts
View file

@ -128,7 +128,7 @@ allowedMethods: ['POST', 'PUT', 'PATCH', 'DELETE'],
| projects/:id/file
| ```
*/
processManually: ['/submitter/dataset/submit', '/submitter/dataset/:id/update'],
processManually: [],
/*
|--------------------------------------------------------------------------
@ -185,8 +185,8 @@ allowedMethods: ['POST', 'PUT', 'PATCH', 'DELETE'],
| and fields data.
|
*/
limit: '513mb',
//limit: env.get('UPLOAD_LIMIT', '513mb'),
// limit: '20mb',
limit: env.get('UPLOAD_LIMIT', '513mb'),
/*
|--------------------------------------------------------------------------

2
config/database.ts
View file

@ -47,7 +47,7 @@ const databaseConfig = defineConfig({
migrations: {
naturalSort: true,
},
// healthCheck: false,
healthCheck: false,
debug: false,
pool: { min: 1, max: 100 },
},

190
config/drive.ts
View file

@ -1,45 +1,151 @@
// import env from '#start/env'
// import app from '@adonisjs/core/services/app'
import { defineConfig, services } from '@adonisjs/drive'
/**
* Config source: https://git.io/JBt3o
*
* Feel free to let us know via PR, if you find something broken in this config
* file.
*/
import { defineConfig } from '#providers/drive/src/types/define_config';
import env from '#start/env';
// import { driveConfig } from '@adonisjs/core/build/config';
// import { driveConfig } from "@adonisjs/drive/build/config.js";
// import Application from '@ioc:Adonis/Core/Application';
const driveConfig = defineConfig({
default: 'public',
/*
|--------------------------------------------------------------------------
| Drive Config
|--------------------------------------------------------------------------
|
| The `DriveConfig` relies on the `DisksList` interface which is
| defined inside the `contracts` directory.
|
*/
export default defineConfig({
/*
|--------------------------------------------------------------------------
| Default disk
|--------------------------------------------------------------------------
|
| The default disk to use for managing file uploads. The value is driven by
| the `DRIVE_DISK` environment variable.
|
*/
disk: env.get('DRIVE_DISK', 'local'),
services: {
/**
* Persist files on the local filesystem
*/
public: services.fs({
location: '/storage/app/public/',
serveFiles: true,
routeBasePath: '/public',
visibility: 'public',
}),
local: services.fs({
location: '/storage/app/data/',
serveFiles: true,
routeBasePath: '/data',
visibility: 'public',
}),
/**
* Persist files on Digital Ocean spaces
*/
// spaces: services.s3({
// credentials: {
// accessKeyId: env.get('SPACES_KEY'),
// secretAccessKey: env.get('SPACES_SECRET'),
// },
// region: env.get('SPACES_REGION'),
// bucket: env.get('SPACES_BUCKET'),
// endpoint: env.get('SPACES_ENDPOINT'),
// visibility: 'public',
// }),
disks: {
/*
|--------------------------------------------------------------------------
| Local
|--------------------------------------------------------------------------
|
| Uses the local file system to manage files. Make sure to turn off serving
| files when not using this disk.
|
*/
local: {
driver: 'local',
visibility: 'public',
/*
|--------------------------------------------------------------------------
| Storage root - Local driver only
|--------------------------------------------------------------------------
|
| Define an absolute path to the storage directory from where to read the
| files.
|
*/
// root: Application.tmpPath('uploads'),
root: '/storage/app/public',
/*
|--------------------------------------------------------------------------
| Serve files - Local driver only
|--------------------------------------------------------------------------
|
| When this is set to true, AdonisJS will configure a files server to serve
| files from the disk root. This is done to mimic the behavior of cloud
| storage services that has inbuilt capabilities to serve files.
|
*/
serveFiles: true,
/*
|--------------------------------------------------------------------------
| Base path - Local driver only
|--------------------------------------------------------------------------
|
| Base path is always required when "serveFiles = true". Also make sure
| the `basePath` is unique across all the disks using "local" driver and
| you are not registering routes with this prefix.
|
*/
basePath: '/uploads',
},
/*
|--------------------------------------------------------------------------
| S3 Driver
|--------------------------------------------------------------------------
|
| Uses the S3 cloud storage to manage files. Make sure to install the s3
| drive separately when using it.
|
|**************************************************************************
| npm i @adonisjs/drive-s3
|**************************************************************************
|
*/
// s3: {
// driver: 's3',
// visibility: 'public',
// key: Env.get('S3_KEY'),
// secret: Env.get('S3_SECRET'),
// region: Env.get('S3_REGION'),
// bucket: Env.get('S3_BUCKET'),
// endpoint: Env.get('S3_ENDPOINT'),
//
// // For minio to work
// // forcePathStyle: true,
// },
/*
|--------------------------------------------------------------------------
| GCS Driver
|--------------------------------------------------------------------------
|
| Uses the Google cloud storage to manage files. Make sure to install the GCS
| drive separately when using it.
|
|**************************************************************************
| npm i @adonisjs/drive-gcs
|**************************************************************************
|
*/
// gcs: {
// driver: 'gcs',
// visibility: 'public',
// keyFilename: Env.get('GCS_KEY_FILENAME'),
// bucket: Env.get('GCS_BUCKET'),
/*
|--------------------------------------------------------------------------
| Uniform ACL - Google cloud storage only
|--------------------------------------------------------------------------
|
| When using the Uniform ACL on the bucket, the "visibility" option is
| ignored. Since, the files ACL is managed by the google bucket policies
| directly.
|
|**************************************************************************
| Learn more: https://cloud.google.com/storage/docs/uniform-bucket-level-access
|**************************************************************************
|
| The following option just informs drive whether your bucket is using uniform
| ACL or not. The actual setting needs to be toggled within the Google cloud
| console.
|
*/
// usingUniformAcl: false,
// },
},
})
export default driveConfig
});

233
config/drive_self.ts
View file

@ -1,233 +0,0 @@
/**
* Config source: https://git.io/JBt3o
*
* Feel free to let us know via PR, if you find something broken in this config
* file.
*/
import { defineConfig } from '#providers/drive/src/types/define_config';
import env from '#start/env';
// import { driveConfig } from '@adonisjs/core/build/config';
// import { driveConfig } from "@adonisjs/drive/build/config.js";
// import Application from '@ioc:Adonis/Core/Application';
/*
|--------------------------------------------------------------------------
| Drive Config
|--------------------------------------------------------------------------
|
| The `DriveConfig` relies on the `DisksList` interface which is
| defined inside the `contracts` directory.
|
*/
export default defineConfig({
/*
|--------------------------------------------------------------------------
| Default disk
|--------------------------------------------------------------------------
|
| The default disk to use for managing file uploads. The value is driven by
| the `DRIVE_DISK` environment variable.
|
*/
disk: env.get('DRIVE_DISK', 'local'),
disks: {
/*
|--------------------------------------------------------------------------
| Local
|--------------------------------------------------------------------------
|
| Uses the local file system to manage files. Make sure to turn off serving
| files when not using this disk.
|
*/
local: {
driver: 'local',
visibility: 'public',
/*
|--------------------------------------------------------------------------
| Storage root - Local driver only
|--------------------------------------------------------------------------
|
| Define an absolute path to the storage directory from where to read the
| files.
|
*/
// root: Application.tmpPath('uploads'),
root: '/storage/app/data',
/*
|--------------------------------------------------------------------------
| Serve files - Local driver only
|--------------------------------------------------------------------------
|
| When this is set to true, AdonisJS will configure a files server to serve
| files from the disk root. This is done to mimic the behavior of cloud
| storage services that has inbuilt capabilities to serve files.
|
*/
serveFiles: true,
/*
|--------------------------------------------------------------------------
| Base path - Local driver only
|--------------------------------------------------------------------------
|
| Base path is always required when "serveFiles = true". Also make sure
| the `basePath` is unique across all the disks using "local" driver and
| you are not registering routes with this prefix.
|
*/
basePath: '/files',
},
local: {
driver: 'local',
visibility: 'public',
/*
|--------------------------------------------------------------------------
| Storage root - Local driver only
|--------------------------------------------------------------------------
|
| Define an absolute path to the storage directory from where to read the
| files.
|
*/
// root: Application.tmpPath('uploads'),
root: '/storage/app/data',
/*
|--------------------------------------------------------------------------
| Serve files - Local driver only
|--------------------------------------------------------------------------
|
| When this is set to true, AdonisJS will configure a files server to serve
| files from the disk root. This is done to mimic the behavior of cloud
| storage services that has inbuilt capabilities to serve files.
|
*/
serveFiles: true,
/*
|--------------------------------------------------------------------------
| Base path - Local driver only
|--------------------------------------------------------------------------
|
| Base path is always required when "serveFiles = true". Also make sure
| the `basePath` is unique across all the disks using "local" driver and
| you are not registering routes with this prefix.
|
*/
basePath: '/files',
},
fs: {
driver: 'local',
visibility: 'public',
/*
|--------------------------------------------------------------------------
| Storage root - Local driver only
|--------------------------------------------------------------------------
|
| Define an absolute path to the storage directory from where to read the
| files.
|
*/
// root: Application.tmpPath('uploads'),
root: '/storage/app/public',
/*
|--------------------------------------------------------------------------
| Serve files - Local driver only
|--------------------------------------------------------------------------
|
| When this is set to true, AdonisJS will configure a files server to serve
| files from the disk root. This is done to mimic the behavior of cloud
| storage services that has inbuilt capabilities to serve files.
|
*/
serveFiles: true,
/*
|--------------------------------------------------------------------------
| Base path - Local driver only
|--------------------------------------------------------------------------
|
| Base path is always required when "serveFiles = true". Also make sure
| the `basePath` is unique across all the disks using "local" driver and
| you are not registering routes with this prefix.
|
*/
basePath: '/public',
},
/*
|--------------------------------------------------------------------------
| S3 Driver
|--------------------------------------------------------------------------
|
| Uses the S3 cloud storage to manage files. Make sure to install the s3
| drive separately when using it.
|
|**************************************************************************
| npm i @adonisjs/drive-s3
|**************************************************************************
|
*/
// s3: {
// driver: 's3',
// visibility: 'public',
// key: Env.get('S3_KEY'),
// secret: Env.get('S3_SECRET'),
// region: Env.get('S3_REGION'),
// bucket: Env.get('S3_BUCKET'),
// endpoint: Env.get('S3_ENDPOINT'),
//
// // For minio to work
// // forcePathStyle: true,
// },
/*
|--------------------------------------------------------------------------
| GCS Driver
|--------------------------------------------------------------------------
|
| Uses the Google cloud storage to manage files. Make sure to install the GCS
| drive separately when using it.
|
|**************************************************************************
| npm i @adonisjs/drive-gcs
|**************************************************************************
|
*/
// gcs: {
// driver: 'gcs',
// visibility: 'public',
// keyFilename: Env.get('GCS_KEY_FILENAME'),
// bucket: Env.get('GCS_BUCKET'),
/*
|--------------------------------------------------------------------------
| Uniform ACL - Google cloud storage only
|--------------------------------------------------------------------------
|
| When using the Uniform ACL on the bucket, the "visibility" option is
| ignored. Since, the files ACL is managed by the google bucket policies
| directly.
|
|**************************************************************************
| Learn more: https://cloud.google.com/storage/docs/uniform-bucket-level-access
|**************************************************************************
|
| The following option just informs drive whether your bucket is using uniform
| ACL or not. The actual setting needs to be toggled within the Google cloud
| console.
|
*/
// usingUniformAcl: false,
// },
},
});

30
config/inertia.ts
View file

@ -1,9 +1,7 @@
import { defineConfig } from '@adonisjs/inertia';
import type { HttpContext } from '@adonisjs/core/http';
import type { InferSharedProps } from '@adonisjs/inertia/types';
import env from '#start/env';
const inertiaConfig = defineConfig({
export default defineConfig({
/**
* Path to the Edge view that will be used as the root view for Inertia responses
*/
@ -22,8 +20,6 @@ const inertiaConfig = defineConfig({
return ctx.session?.flashMessages.get('user_id');
},
opensearch_host: env.get('OPENSEARCH_HOST'),
flash: (ctx) => {
return {
message: ctx.session?.flashMessages.get('message'),
@ -34,15 +30,15 @@ const inertiaConfig = defineConfig({
// params: ({ params }) => params,
authUser: async ({ auth }: HttpContext) => {
if (!auth?.user) return null
await auth.user.load('roles') // sicherstellen, dass geladen ist
return {
id: auth.user.id,
login: auth.user.login,
email: auth.user.email,
first_name: auth.user.first_name,
last_name: auth.user.last_name,
roles: auth.user.roles.map((role) => role.name),
if (auth?.user) {
await auth.user.load('roles');
return auth.user;
// {
// 'id': auth.user.id,
// 'login': auth.user.login,
// };
} else {
return null;
}
},
},
@ -56,12 +52,6 @@ const inertiaConfig = defineConfig({
},
});
export default inertiaConfig
declare module '@adonisjs/inertia/types' {
export interface SharedProps extends InferSharedProps<typeof inertiaConfig> { }
}
// import { InertiaConfig } from '@ioc:EidelLev/Inertia';
// /*

12
config/mail.ts
View file

@ -12,11 +12,11 @@ const mailConfig = defineConfig({
mailers: {
smtp: transports.smtp({
// socketTimeout: 5000,// Overall timeout (5 seconds)
socketTimeout: 5000,// Overall timeout (5 seconds)
host: env.get('SMTP_HOST', ''),
port: env.get('SMTP_PORT'),
secure: false,
ignoreTLS: true,
// ignoreTLS: true,
requireTLS: false,
/**
@ -30,10 +30,10 @@ const mailConfig = defineConfig({
}, */
}),
// resend: transports.resend({
// key: env.get('RESEND_API_KEY'),
// baseUrl: 'https://api.resend.com',
// }),
resend: transports.resend({
key: env.get('RESEND_API_KEY'),
baseUrl: 'https://api.resend.com',
}),
},
});

7
config/session.ts
View file

@ -6,7 +6,7 @@
*/
import env from '#start/env';
// import app from '@adonisjs/core/services/app';
import app from '@adonisjs/core/services/app';
import { defineConfig, stores } from '@adonisjs/session';
const sessionConfig = defineConfig({
@ -96,7 +96,7 @@ const sessionConfig = defineConfig({
* variable in order to infer the store name without any
* errors.
*/
store: 'file', //env.get('SESSION_DRIVER'),
store: env.get('SESSION_DRIVER'),
/*
|--------------------------------------------------------------------------
| Configuration for the file driver
@ -122,9 +122,6 @@ const sessionConfig = defineConfig({
// redisConnection: 'local',
stores: {
cookie: stores.cookie(),
file: stores.file({
location: './tmp/sessions', // Where the data will live
}),
},
});
export default sessionConfig;

32
config/vite.ts
View file

@ -1,32 +0,0 @@
import { defineConfig } from '@adonisjs/vite';
const viteBackendConfig = defineConfig({
/**
* The output of vite will be written inside this
* directory. The path should be relative from
* the application root.
*/
buildDirectory: 'public/assets',
/**
* The path to the manifest file generated by the
* "vite build" command.
*/
manifestFile: 'public/assets/.vite/manifest.json',
/**
* Feel free to change the value of the "assetsUrl" to
* point to a CDN in production.
*/
assetsUrl: '/assets',
/**
* Add defer attribute to scripts for better performance.
*/
scriptAttributes: {
defer: true,
},
});
export default viteBackendConfig;

1
contracts/enums.ts
View file

@ -21,7 +21,6 @@ export enum ServerStates {
rejected_reviewer = 'rejected_reviewer',
rejected_editor = 'rejected_editor',
reviewed = 'reviewed',
rejected_to_reviewer = 'rejected_to_reviewer',
}
// for table dataset_titles

1472
database/factories/create_collections_data_ccs.sql
View file

File diff suppressed because it is too large Load diff

1044
database/factories/create_collections_data_ddc.sql
View file

File diff suppressed because it is too large Load diff

0
database/migrations/config_2_backupcodes_table.ts → database/migrations/1721999420193_create_backupcodes_table.ts
View file

0
database/migrations/config_1_appconfigs_table.ts → database/migrations/1723813974183_create_appconfigs_table.ts
View file

25
database/migrations/1782294801884_create_activities_table.ts
View file

@ -1,25 +0,0 @@
import { BaseSchema } from '@adonisjs/lucid/schema';
export default class extends BaseSchema {
protected tableName = 'activities';
async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id');
table.string('type').notNullable().index(); // 'dataset.uploaded', 'auth.login'
table.integer('user_id').unsigned().nullable().references('id').inTable('accounts').onDelete('SET NULL');
table.string('subject_type').nullable(); // manual morph: model name
table.bigInteger('subject_id').unsigned().nullable();
table.string('description').notNullable();
table.json('properties').nullable();
table.timestamp('created_at');
table.timestamp('updated_at');
table.index(['subject_type', 'subject_id'])
table.index('created_at')
});
}
async down() {
this.schema.dropTable(this.tableName);
}
}

5
database/migrations/acl_4_accounts.ts
View file

@ -18,7 +18,6 @@ export default class Accounts extends BaseSchema {
table.text("two_factor_recovery_codes").nullable();
table.smallint('state').nullable();
table.bigint('last_counter').nullable();
table.string('avatar').nullable();
});
}
@ -44,7 +43,6 @@ export default class Accounts extends BaseSchema {
// two_factor_recovery_codes text COLLATE pg_catalog."default",
// state smallint,
// last_counter bigint,
// avatar character varying(255),
// )
// ALTER TABLE gba.accounts
@ -87,6 +85,3 @@ export default class Accounts extends BaseSchema {
// GRANT ALL ON SEQUENCE gba.totp_secrets_id_seq TO tethys_admin;
// ALTER TABLE gba.totp_secrets ALTER COLUMN id SET DEFAULT nextval('gba.totp_secrets_id_seq');
// ALTER TABLE "accounts" ADD COLUMN "avatar" VARCHAR(255) NULL

19
database/migrations/dataset_2_documents.ts
View file

@ -86,22 +86,3 @@ export default class Documents extends BaseSchema {
// CONSTRAINT documents_server_state_check CHECK (server_state::text = ANY (ARRAY['deleted'::character varying::text, 'inprogress'::character varying::text, 'published'::character varying::text, 'released'::character varying::text, 'editor_accepted'::character varying::text, 'approved'::character varying::text, 'rejected_reviewer'::character varying::text, 'rejected_editor'::character varying::text, 'reviewed'::character varying::text])),
// CONSTRAINT documents_type_check CHECK (type::text = ANY (ARRAY['analysisdata'::character varying::text, 'measurementdata'::character varying::text, 'monitoring'::character varying::text, 'remotesensing'::character varying::text, 'gis'::character varying::text, 'models'::character varying::text, 'mixedtype'::character varying::text]))
// )
// ALTER TABLE documents DROP CONSTRAINT documents_server_state_check;
// ALTER TABLE documents
// ADD CONSTRAINT documents_server_state_check CHECK (
// server_state::text = ANY (ARRAY[
// 'deleted',
// 'inprogress',
// 'published',
// 'released',
// 'editor_accepted',
// 'approved',
// 'rejected_reviewer',
// 'rejected_editor',
// 'reviewed',
// 'rejected_to_reviewer' -- new value added
// ]::text[])
// );

18
database/migrations/dataset_6_collection_roles.ts
View file

@ -32,21 +32,3 @@ export default class CollectionsRoles extends BaseSchema {
// visible_oai boolean NOT NULL DEFAULT true,
// CONSTRAINT collections_roles_pkey PRIMARY KEY (id)
// )
// change to normal intzeger:
// ALTER TABLE collections_roles ALTER COLUMN id DROP DEFAULT;
// DROP SEQUENCE IF EXISTS collections_roles_id_seq;
// -- Step 1: Temporarily change one ID to a value not currently used
// UPDATE collections_roles SET id = 99 WHERE name = 'ccs';
// -- Step 2: Change 'ddc' ID to 2 (the old 'ccs' ID)
// UPDATE collections_roles SET id = 2 WHERE name = 'ddc';
// -- Step 3: Change the temporary ID (99) to 3 (the old 'ddc' ID)
// UPDATE collections_roles SET id = 3 WHERE name = 'ccs';
// UPDATE collections_roles SET id = 99 WHERE name = 'bk';
// UPDATE collections_roles SET id = 1 WHERE name = 'institutes';
// UPDATE collections_roles SET id = 4 WHERE name = 'pacs';
// UPDATE collections_roles SET id = 7 WHERE name = 'bk';

32
database/migrations/dataset_7_collections.ts
View file

@ -5,7 +5,7 @@ export default class Collections extends BaseSchema {
public async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id');//.defaultTo("nextval('collections_id_seq')");
table.increments('id').defaultTo("nextval('collections_id_seq')");
table.integer('role_id').unsigned();
table
.foreign('role_id', 'collections_role_id_foreign')
@ -25,8 +25,6 @@ export default class Collections extends BaseSchema {
.onUpdate('CASCADE');
table.boolean('visible').notNullable().defaultTo(true);
table.boolean('visible_publish').notNullable().defaultTo(true);
table.integer('left_id').unsigned();
table.integer('right_id').unsigned();
});
}
@ -56,31 +54,3 @@ export default class Collections extends BaseSchema {
// ON UPDATE CASCADE
// ON DELETE CASCADE
// )
// change to normal intzeger:
// ALTER TABLE collections ALTER COLUMN id DROP DEFAULT;
// DROP SEQUENCE IF EXISTS collections_id_seq;
// ALTER TABLE collections
// ADD COLUMN left_id INTEGER;
// COMMENT ON COLUMN collections.left_id IS 'comment';
// ALTER TABLE collections
// ADD COLUMN right_id INTEGER;
// COMMENT ON COLUMN collections.right_id IS 'comment';
// -- Step 1: Drop the existing default
// ALTER TABLE collections
// ALTER COLUMN visible DROP DEFAULT,
// ALTER COLUMN visible_publish DROP DEFAULT;
// -- Step 2: Change column types with proper casting
// ALTER TABLE collections
// ALTER COLUMN visible TYPE smallint USING CASE WHEN visible THEN 1 ELSE 0 END,
// ALTER COLUMN visible_publish TYPE smallint USING CASE WHEN visible_publish THEN 1 ELSE 0 END;
// -- Step 3: Set new defaults as smallint
// ALTER TABLE collections
// ALTER COLUMN visible SET DEFAULT 1,
// ALTER COLUMN visible_publish SET DEFAULT 1;

18
database/migrations/update_1_to_mime_types.ts
View file

@ -1,18 +0,0 @@
import { BaseSchema } from "@adonisjs/lucid/schema";
export default class AddAlternateMimetypeToMimeTypes extends BaseSchema {
protected tableName = 'mime_types';
public async up () {
this.schema.alterTable(this.tableName, (table) => {
table.string('alternate_mimetype').nullable();
});
}
public async down () {
this.schema.alterTable(this.tableName, (table) => {
table.dropColumn('alternate_mimetype');
});
}
}
// ALTER TABLE "mime_types" ADD COLUMN "alternate_mimetype" VARCHAR(255) NULL

101
docker-entrypoint.sh
View file

@ -1,74 +1,47 @@
#!/bin/bash
# # Run freshclam to update virus definitions
# freshclam
# # Sleep for a few seconds to give ClamAV time to start
# sleep 5
# # Start the ClamAV daemon
# /etc/init.d/clamav-daemon start
# bootstrap clam av service and clam av database updater
set -m
echo "Starting ClamAV services..."
function process_file() {
if [[ ! -z "$1" ]]; then
local SETTING_LIST=$(echo "$1" | tr ',' '\n' | grep "^[A-Za-z][A-Za-z]*=.*$")
local SETTING
for SETTING in ${SETTING_LIST}; do
# Remove any existing copies of this setting. We do this here so that
# settings with multiple values (e.g. ExtraDatabase) can still be added
# multiple times below
local KEY=${SETTING%%=*}
sed -i $2 -e "/^${KEY} /d"
done
for SETTING in ${SETTING_LIST}; do
# Split on first '='
local KEY=${SETTING%%=*}
local VALUE=${SETTING#*=}
echo "${KEY} ${VALUE}" >> "$2"
done
fi
}
# Try to download database if missing
# if [ ! "$(ls -A /var/lib/clamav 2>/dev/null)" ]; then
# echo "Downloading ClamAV database (this may take a while)..."
# # Simple freshclam run without complex config
# if freshclam --datadir=/var/lib/clamav --quiet; then
# echo "✓ Database downloaded successfully"
# else
# echo "⚠ Database download failed - creating minimal setup"
# # Create a dummy file so clamd doesn't immediately fail
# touch /var/lib/clamav/.dummy
# fi
# fi
# Start freshclam daemon for automatic updates
echo "Starting freshclam daemon for automatic updates..."
# sg clamav -c "freshclam -d" &
# Added --daemon-notify to freshclam - This notifies clamd when the database updates
freshclam -d --daemon-notify=/etc/clamav/clamd.conf &
#freshclam -d &
# process_file "${CLAMD_SETTINGS_CSV}" /etc/clamav/clamd.conf
# process_file "${FRESHCLAM_SETTINGS_CSV}" /etc/clamav/freshclam.conf
# start in background
freshclam -d &
# /etc/init.d/clamav-freshclam start &
# Start clamd in background
# Start clamd in foreground (so dumb-init can supervise it)
clamd
# /etc/init.d/clamav-daemon start &
# Give freshclam a moment to start
sleep 2
# Start clamd daemon in background using sg
echo "Starting ClamAV daemon..."
# sg clamav -c "clamd" &
# Use sg to run clamd with proper group permissions
# sg clamav -c "clamd" &
# clamd --config-file=/etc/clamav/clamd.conf &
clamd &
# Give services time to start
echo "Waiting for services to initialize..."
sleep 8
# simple check
if pgrep clamd > /dev/null; then
echo "✓ ClamAV daemon is running"
else
echo "⚠ ClamAV daemon status uncertain, but continuing..."
fi
# Check if freshclam daemon is running
if pgrep freshclam > /dev/null; then
echo "✓ Freshclam daemon is running"
else
echo "⚠ Freshclam daemon status uncertain, but continuing..."
fi
# # Optional: Test socket connectivity
# if [ -S /var/run/clamav/clamd.socket ]; then
# echo "✓ ClamAV socket exists"
# else
# echo "⚠ WARNING: ClamAV socket not found - services may still be starting"
# fi
# # change back to CMD of dockerfile
echo "✓ ClamAV setup complete"
echo "Starting main application..."
# exec dumb-init -- "$@"
# change back to CMD of dockerfile
exec "$@"

278
docs/commands/index-datasets.md
View file

@ -1,278 +0,0 @@
# Dataset Indexing Command
AdonisJS Ace command for indexing and synchronizing published datasets with OpenSearch for search functionality.
## Overview
The `index:datasets` command processes published datasets and creates/updates corresponding search index documents in OpenSearch. It intelligently compares modification timestamps to only re-index datasets when necessary, optimizing performance while maintaining search index accuracy.
## Command Syntax
```bash
node ace index:datasets [options]
```
## Options
| Flag | Alias | Description |
|------|-------|-------------|
| `--publish_id <number>` | `-p` | Index a specific dataset by publish_id |
## Usage Examples
### Basic Operations
```bash
# Index all published datasets that have been modified since last indexing
node ace index:datasets
# Index a specific dataset by publish_id
node ace index:datasets --publish_id 231
node ace index:datasets -p 231
```
## How It Works
### 1. **Dataset Selection**
The command processes datasets that meet these criteria:
- `server_state = 'published'` - Only published datasets
- Has preloaded `xmlCache` relationship for metadata transformation
- Optionally filtered by specific `publish_id`
### 2. **Smart Update Detection**
For each dataset, the command:
- Checks if the dataset exists in the OpenSearch index
- Compares `server_date_modified` timestamps
- Only re-indexes if the dataset is newer than the indexed version
### 3. **Document Processing**
The indexing process involves:
1. **XML Generation**: Creates structured XML from dataset metadata
2. **XSLT Transformation**: Converts XML to JSON using Saxon-JS processor
3. **Index Update**: Updates or creates the document in OpenSearch
4. **Logging**: Records success/failure for each operation
## Index Structure
### Index Configuration
- **Index Name**: `tethys-records`
- **Document ID**: Dataset `publish_id`
- **Refresh**: `true` (immediate availability)
### Document Fields
The indexed documents contain:
- **Metadata Fields**: Title, description, authors, keywords
- **Identifiers**: DOI, publish_id, and other identifiers
- **Temporal Data**: Publication dates, coverage periods
- **Geographic Data**: Spatial coverage information
- **Technical Details**: Data formats, access information
- **Timestamps**: Creation and modification dates
## Example Output
### Successful Run
```bash
node ace index:datasets
```
```
Found 150 published datasets to process
Dataset with publish_id 231 successfully indexed
Dataset with publish_id 245 is up to date, skipping indexing
Dataset with publish_id 267 successfully indexed
An error occurred while indexing dataset with publish_id 289. Error: Invalid XML metadata
Processing completed: 148 indexed, 1 skipped, 1 error
```
### Specific Dataset
```bash
node ace index:datasets --publish_id 231
```
```
Found 1 published dataset to process
Dataset with publish_id 231 successfully indexed
Processing completed: 1 indexed, 0 skipped, 0 errors
```
## Update Logic
The command uses intelligent indexing to avoid unnecessary processing:
| Condition | Action | Reason |
|-----------|--------|--------|
| Dataset not in index | ✅ Index | New dataset needs indexing |
| Dataset newer than indexed version | ✅ Re-index | Dataset has been updated |
| Dataset same/older than indexed version | ❌ Skip | Already up to date |
| OpenSearch document check fails | ✅ Index | Better safe than sorry |
| Invalid XML metadata | ❌ Skip + Log Error | Cannot process invalid data |
### Timestamp Comparison
```typescript
// Example comparison logic
const existingModified = DateTime.fromMillis(Number(existingDoc.server_date_modified) * 1000);
const currentModified = dataset.server_date_modified;
if (currentModified <= existingModified) {
// Skip - already up to date
return false;
}
// Proceed with indexing
```
## XML Transformation Process
### 1. **XML Generation**
```xml
<?xml version="1.0" encoding="UTF-8" standalone="true"?>
<root>
<Dataset>
<!-- Dataset metadata fields -->
<title>Research Dataset Title</title>
<description>Dataset description...</description>
<!-- Additional metadata -->
</Dataset>
</root>
```
### 2. **XSLT Processing**
The command uses Saxon-JS with a compiled stylesheet (`solr.sef.json`) to transform XML to JSON:
```javascript
const result = await SaxonJS.transform({
stylesheetText: proc,
destination: 'serialized',
sourceText: xmlString,
});
```
### 3. **Final JSON Document**
```json
{
"id": "231",
"title": "Research Dataset Title",
"description": "Dataset description...",
"authors": ["Author Name"],
"server_date_modified": 1634567890,
"publish_id": 231
}
```
## Configuration Requirements
### Environment Variables
```bash
# OpenSearch Configuration
OPENSEARCH_HOST=localhost:9200
# For production:
# OPENSEARCH_HOST=your-opensearch-cluster:9200
```
### Required Files
- **XSLT Stylesheet**: `public/assets2/solr.sef.json` - Compiled Saxon-JS stylesheet for XML transformation
### Database Relationships
The command expects these model relationships:
```typescript
// Dataset model must have:
@hasOne(() => XmlCache, { foreignKey: 'dataset_id' })
public xmlCache: HasOne<typeof XmlCache>
```
## Error Handling
The command handles various error scenarios gracefully:
### Common Errors and Solutions
| Error | Cause | Solution |
|-------|-------|----------|
| `XSLT transformation failed` | Invalid XML or missing stylesheet | Check XML structure and stylesheet path |
| `OpenSearch connection error` | Service unavailable | Verify OpenSearch is running and accessible |
| `JSON parse error` | Malformed transformation result | Check XSLT stylesheet output format |
| `Missing xmlCache relationship` | Data integrity issue | Ensure xmlCache exists for dataset |
### Error Logging
```bash
# Typical error log entry
An error occurred while indexing dataset with publish_id 231.
Error: XSLT transformation failed: Invalid XML structure at line 15
```
## Performance Considerations
### Batch Processing
- Processes datasets sequentially to avoid overwhelming OpenSearch
- Each dataset is committed individually for reliability
- Failed indexing of one dataset doesn't stop processing others
### Resource Usage
- **Memory**: XML/JSON transformations require temporary memory
- **Network**: OpenSearch API calls for each dataset
- **CPU**: XSLT transformations are CPU-intensive
### Optimization Tips
```bash
# Index only recently modified datasets (run regularly)
node ace index:datasets
# Index specific datasets when needed
node ace index:datasets --publish_id 231
# Consider running during off-peak hours for large batches
```
## Integration with Other Systems
### Search Functionality
The indexed documents power:
- **Dataset Search**: Full-text search across metadata
- **Faceted Browsing**: Filter by authors, keywords, dates
- **Geographic Search**: Spatial query capabilities
- **Auto-complete**: Suggest dataset titles and keywords
### Related Commands
- [`update:datacite`](update-datacite.md) - Often run after indexing to sync DOI metadata
- **Database migrations** - May require re-indexing after schema changes
### API Integration
The indexed data is consumed by:
- **Search API**: `/api/search` endpoints
- **Browse API**: `/api/datasets` with filtering
- **Recommendations**: Related dataset suggestions
## Monitoring and Maintenance
### Regular Tasks
```bash
# Daily indexing (recommended cron job)
0 2 * * * cd /path/to/project && node ace index:datasets
# Weekly full re-index (if needed)
0 3 * * 0 cd /path/to/project && node ace index:datasets --force
```
### Health Checks
- Monitor OpenSearch cluster health
- Check for failed indexing operations in logs
- Verify search functionality is working
- Compare dataset counts between database and index
### Troubleshooting
```bash
# Check specific dataset indexing
node ace index:datasets --publish_id 231
# Verify OpenSearch connectivity
curl -X GET "localhost:9200/_cluster/health"
# Check index statistics
curl -X GET "localhost:9200/tethys-records/_stats"
```
## Best Practices
1. **Regular Scheduling**: Run the command regularly (daily) to keep the search index current
2. **Monitor Logs**: Watch for transformation errors or OpenSearch issues
3. **Backup Strategy**: Include OpenSearch indices in backup procedures
4. **Resource Management**: Monitor OpenSearch cluster resources during bulk operations
5. **Testing**: Verify search functionality after major indexing operations
6. **Coordination**: Run indexing before DataCite updates when both are needed

216
docs/commands/update-datacite.md
View file

@ -1,216 +0,0 @@
# DataCite Update Command
AdonisJS Ace command for updating DataCite DOI records for published datasets.
## Overview
The `update:datacite` command synchronizes your local dataset metadata with DataCite DOI records. It intelligently compares modification dates to only update records when necessary, reducing unnecessary API calls and maintaining data consistency.
## Command Syntax
```bash
node ace update:datacite [options]
```
## Options
| Flag | Alias | Description |
|------|-------|-------------|
| `--publish_id <number>` | `-p` | Update a specific dataset by publish_id |
| `--force` | `-f` | Force update all records regardless of modification date |
| `--dry-run` | `-d` | Preview what would be updated without making changes |
| `--stats` | `-s` | Show detailed statistics for datasets that need updating |
## Usage Examples
### Basic Operations
```bash
# Update all datasets that have been modified since their DOI was last updated
node ace update:datacite
# Update a specific dataset
node ace update:datacite --publish_id 231
node ace update:datacite -p 231
# Force update all datasets with DOIs (ignores modification dates)
node ace update:datacite --force
```
### Preview and Analysis
```bash
# Preview what would be updated (dry run)
node ace update:datacite --dry-run
# Show detailed statistics for datasets that need updating
node ace update:datacite --stats
# Show stats for a specific dataset
node ace update:datacite --stats --publish_id 231
```
### Combined Options
```bash
# Dry run for a specific dataset
node ace update:datacite --dry-run --publish_id 231
# Show stats for all datasets (including up-to-date ones)
node ace update:datacite --stats --force
```
## Command Modes
### 1. **Normal Mode** (Default)
Updates DataCite records for datasets that have been modified since their DOI was last updated.
**Example Output:**
```
Using DataCite API: https://api.test.datacite.org
Found 50 datasets to process
Dataset 231: Successfully updated DataCite record
Dataset 245: Up to date, skipping
Dataset 267: Successfully updated DataCite record
DataCite update completed. Updated: 15, Skipped: 35, Errors: 0
```
### 2. **Dry Run Mode** (`--dry-run`)
Shows what would be updated without making any changes to DataCite.
**Use Case:** Preview updates before running the actual command.
**Example Output:**
```
Dataset 231: Would update DataCite record (dry run)
Dataset 267: Would update DataCite record (dry run)
Dataset 245: Up to date, skipping
DataCite update completed. Updated: 2, Skipped: 1, Errors: 0
```
### 3. **Stats Mode** (`--stats`)
Shows detailed information for each dataset that needs updating, including why it needs updating.
**Use Case:** Debug synchronization issues, monitor dataset/DOI status, generate reports.
**Example Output:**
```
┌─ Dataset 231 ─────────────────────────────────────────────────────────
│ DOI Value: 10.21388/tethys.231
│ DOI Status (DB): findable
│ DOI State (DataCite): findable
│ Dataset Modified: 2024-09-15T10:30:00.000Z
│ DOI Modified: 2024-09-10T08:15:00.000Z
│ Needs Update: YES - Dataset newer than DOI
└───────────────────────────────────────────────────────────────────────
┌─ Dataset 267 ─────────────────────────────────────────────────────────
│ DOI Value: 10.21388/tethys.267
│ DOI Status (DB): findable
│ DOI State (DataCite): findable
│ Dataset Modified: 2024-09-18T14:20:00.000Z
│ DOI Modified: 2024-09-16T12:45:00.000Z
│ Needs Update: YES - Dataset newer than DOI
└───────────────────────────────────────────────────────────────────────
DataCite Stats Summary: 2 datasets need updating, 48 are up to date
```
## Update Logic
The command uses intelligent update detection:
1. **Compares modification dates**: Dataset `server_date_modified` vs DOI last modification date from DataCite
2. **Validates data integrity**: Checks for missing or future dates
3. **Handles API failures gracefully**: Updates anyway if DataCite info can't be retrieved
4. **Uses dual API approach**: DataCite REST API (primary) with MDS API fallback
### When Updates Happen
| Condition | Action | Reason |
|-----------|--------|--------|
| Dataset modified > DOI modified | ✅ Update | Dataset has newer changes |
| Dataset modified ≤ DOI modified | ❌ Skip | DOI is up to date |
| Dataset date in future | ❌ Skip | Invalid data, needs investigation |
| Dataset date missing | ✅ Update | Can't determine staleness |
| DataCite API error | ✅ Update | Better safe than sorry |
| `--force` flag used | ✅ Update | Override all logic |
## Environment Configuration
Required environment variables:
```bash
# DataCite Credentials
DATACITE_USERNAME=your_username
DATACITE_PASSWORD=your_password
# API Endpoints (environment-specific)
DATACITE_API_URL=https://api.test.datacite.org # Test environment
DATACITE_SERVICE_URL=https://mds.test.datacite.org # Test MDS
DATACITE_API_URL=https://api.datacite.org # Production
DATACITE_SERVICE_URL=https://mds.datacite.org # Production MDS
# Project Configuration
DATACITE_PREFIX=10.21388 # Your DOI prefix
BASE_DOMAIN=tethys.at # Your domain
```
## Error Handling
The command handles various error scenarios:
- **Invalid modification dates**: Logs errors but continues processing other datasets
- **DataCite API failures**: Falls back to MDS API, then to safe update
- **Missing DOI identifiers**: Skips datasets without DOI identifiers
- **Network issues**: Continues with next dataset after logging error
## Integration
The command integrates with:
- **Dataset Model**: Uses `server_date_modified` for change detection
- **DatasetIdentifier Model**: Reads DOI values and status
- **OpenSearch Index**: Updates search index after DataCite update
- **DoiClient**: Handles all DataCite API interactions
## Common Workflows
### Daily Maintenance
```bash
# Update any datasets modified today
node ace update:datacite
```
### Pre-Deployment Check
```bash
# Check what would be updated before deployment
node ace update:datacite --dry-run
```
### Debugging Sync Issues
```bash
# Investigate why specific dataset isn't syncing
node ace update:datacite --stats --publish_id 231
```
### Full Resync
```bash
# Force update all DOI records (use with caution)
node ace update:datacite --force
```
### Monitoring Report
```bash
# Generate sync status report
node ace update:datacite --stats > datacite-sync-report.txt
```
## Best Practices
1. **Regular Updates**: Run daily or after bulk dataset modifications
2. **Test First**: Use `--dry-run` or `--stats` before bulk operations
3. **Monitor Logs**: Check for data integrity warnings
4. **Environment Separation**: Use correct API URLs for test vs production
5. **Rate Limiting**: The command handles DataCite rate limits automatically

218
freshclam.conf
View file

@ -1,47 +1,229 @@
##
## Container-optimized freshclam configuration
## Example config file for freshclam
## Please read the freshclam.conf(5) manual before editing this file.
##
# Database directory
# Comment or remove the line below.
# Path to the database directory.
# WARNING: It must match clamd.conf's directive!
# Default: hardcoded (depends on installation options)
DatabaseDirectory /var/lib/clamav
# Log to stdout for container logging
# Path to the log file (make sure it has proper permissions)
# Default: disabled
# UpdateLogFile /dev/stdout
# Basic logging settings
# Maximum size of the log file.
# Value of 0 disables the limit.
# You may use 'M' or 'm' for megabytes (1M = 1m = 1048576 bytes)
# and 'K' or 'k' for kilobytes (1K = 1k = 1024 bytes).
# in bytes just don't use modifiers. If LogFileMaxSize is enabled,
# log rotation (the LogRotate option) will always be enabled.
# Default: 1M
#LogFileMaxSize 2M
# Log time with each message.
# Default: no
LogTime yes
# Enable verbose logging.
# Default: no
LogVerbose yes
# Use system logger (can work together with UpdateLogFile).
# Default: no
LogSyslog no
# PID file location
# Specify the type of syslog messages - please refer to 'man syslog'
# for facility names.
# Default: LOG_LOCAL6
#LogFacility LOG_MAIL
# Enable log rotation. Always enabled when LogFileMaxSize is enabled.
# Default: no
#LogRotate yes
# This option allows you to save the process identifier of the daemon
# Default: disabled
#PidFile /var/run/freshclam.pid
PidFile /var/run/clamav/freshclam.pid
# Database owner
# By default when started freshclam drops privileges and switches to the
# "clamav" user. This directive allows you to change the database owner.
# Default: clamav (may depend on installation options)
DatabaseOwner node
# Mirror settings for Austria
# Use DNS to verify virus database version. Freshclam uses DNS TXT records
# to verify database and software versions. With this directive you can change
# the database verification domain.
# WARNING: Do not touch it unless you're configuring freshclam to use your
# own database verification domain.
# Default: current.cvd.clamav.net
#DNSDatabaseInfo current.cvd.clamav.net
# Uncomment the following line and replace XY with your country
# code. See http://www.iana.org/cctld/cctld-whois.htm for the full list.
# You can use db.XY.ipv6.clamav.net for IPv6 connections.
DatabaseMirror db.at.clamav.net
# database.clamav.net is a round-robin record which points to our most
# reliable mirrors. It's used as a fall back in case db.XY.clamav.net is
# not working. DO NOT TOUCH the following line unless you know what you
# are doing.
DatabaseMirror database.clamav.net
# How many attempts to make before giving up.
# Default: 3 (per mirror)
#MaxAttempts 5
# With this option you can control scripted updates. It's highly recommended
# to keep it enabled.
# Default: yes
# Update settings
ScriptedUpdates yes
#ScriptedUpdates yes
# By default freshclam will keep the local databases (.cld) uncompressed to
# make their handling faster. With this option you can enable the compression;
# the change will take effect with the next database update.
# Default: no
#CompressLocalDatabase no
# With this option you can provide custom sources (http:// or file://) for
# database files. This option can be used multiple times.
# Default: no custom URLs
#DatabaseCustomURL http://myserver.com/mysigs.ndb
#DatabaseCustomURL file:///mnt/nfs/local.hdb
# This option allows you to easily point freshclam to private mirrors.
# If PrivateMirror is set, freshclam does not attempt to use DNS
# to determine whether its databases are out-of-date, instead it will
# use the If-Modified-Since request or directly check the headers of the
# remote database files. For each database, freshclam first attempts
# to download the CLD file. If that fails, it tries to download the
# CVD file. This option overrides DatabaseMirror, DNSDatabaseInfo
# and ScriptedUpdates. It can be used multiple times to provide
# fall-back mirrors.
# Default: disabled
#PrivateMirror mirror1.mynetwork.com
#PrivateMirror mirror2.mynetwork.com
# Number of database checks per day.
# Default: 12 (every two hours)
Checks 12
#Checks 24
# Don't fork (good for containers)
# Proxy settings
# Default: disabled
#HTTPProxyServer myproxy.com
#HTTPProxyPort 1234
#HTTPProxyUsername myusername
#HTTPProxyPassword mypass
# If your servers are behind a firewall/proxy which applies User-Agent
# filtering you can use this option to force the use of a different
# User-Agent header.
# Default: clamav/version_number
#HTTPUserAgent SomeUserAgentIdString
# Use aaa.bbb.ccc.ddd as client address for downloading databases. Useful for
# multi-homed systems.
# Default: Use OS'es default outgoing IP address.
#LocalIPAddress aaa.bbb.ccc.ddd
# Send the RELOAD command to clamd.
# Default: no
#NotifyClamd /path/to/clamd.conf
# Run command after successful database update.
# Default: disabled
#OnUpdateExecute command
# Run command when database update process fails.
# Default: disabled
#OnErrorExecute command
# Run command when freshclam reports outdated version.
# In the command string %v will be replaced by the new version number.
# Default: disabled
#OnOutdatedExecute command
# Don't fork into background.
# Default: no
Foreground no
# Connection timeouts
ConnectTimeout 60
ReceiveTimeout 60
# Enable debug messages in libclamav.
# Default: no
#Debug yes
# Test databases before using them
TestDatabases yes
# Timeout in seconds when connecting to database server.
# Default: 30
#ConnectTimeout 60
# Enable bytecode signatures
Bytecode yes
# Timeout in seconds when reading from database server.
# Default: 30
#ReceiveTimeout 60
# With this option enabled, freshclam will attempt to load new
# databases into memory to make sure they are properly handled
# by libclamav before replacing the old ones.
# Default: yes
#TestDatabases yes
# When enabled freshclam will submit statistics to the ClamAV Project about
# the latest virus detections in your environment. The ClamAV maintainers
# will then use this data to determine what types of malware are the most
# detected in the field and in what geographic area they are.
# Freshclam will connect to clamd in order to get recent statistics.
# Default: no
#SubmitDetectionStats /path/to/clamd.conf
# Country of origin of malware/detection statistics (for statistical
# purposes only). The statistics collector at ClamAV.net will look up
# your IP address to determine the geographical origin of the malware
# reported by your installation. If this installation is mainly used to
# scan data which comes from a different location, please enable this
# option and enter a two-letter code (see http://www.iana.org/domains/root/db/)
# of the country of origin.
# Default: disabled
#DetectionStatsCountry country-code
# This option enables support for our "Personal Statistics" service.
# When this option is enabled, the information on malware detected by
# your clamd installation is made available to you through our website.
# To get your HostID, log on http://www.stats.clamav.net and add a new
# host to your host list. Once you have the HostID, uncomment this option
# and paste the HostID here. As soon as your freshclam starts submitting
# information to our stats collecting service, you will be able to view
# the statistics of this clamd installation by logging into
# http://www.stats.clamav.net with the same credentials you used to
# generate the HostID. For more information refer to:
# http://www.clamav.net/documentation.html#cctts
# This feature requires SubmitDetectionStats to be enabled.
# Default: disabled
#DetectionStatsHostID unique-id
# This option enables support for Google Safe Browsing. When activated for
# the first time, freshclam will download a new database file (safebrowsing.cvd)
# which will be automatically loaded by clamd and clamscan during the next
# reload, provided that the heuristic phishing detection is turned on. This
# database includes information about websites that may be phishing sites or
# possible sources of malware. When using this option, it's mandatory to run
# freshclam at least every 30 minutes.
# Freshclam uses the ClamAV's mirror infrastructure to distribute the
# database and its updates but all the contents are provided under Google's
# terms of use. See http://www.google.com/transparencyreport/safebrowsing
# and http://www.clamav.net/documentation.html#safebrowsing
# for more information.
# Default: disabled
#SafeBrowsing yes
# This option enables downloading of bytecode.cvd, which includes additional
# detection mechanisms and improvements to the ClamAV engine.
# Default: enabled
#Bytecode yes
# Download an additional 3rd party signature database distributed through
# the ClamAV mirrors.
# This option can be used multiple times.
#ExtraDatabase dbname1
#ExtraDatabase dbname2

6
index.d.ts vendored
View file

@ -183,9 +183,3 @@ declare module 'saxon-js' {
export function transform(options: ITransformOptions): Promise<ITransformOutput> | ITransformOutput;
}
declare global {
interface File {
sort_order?: number;
}
}

14826
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

88
package.json
View file

@ -4,8 +4,7 @@
"private": true,
"scripts": {
"type-check": "tsc --noEmit",
"dev": "node ace serve",
"devInspect": "node ace serve --watch --node-args='--inspect'",
"dev": "node ace serve --watch",
"compress:xslt": "./node_modules/xslt3/xslt3.js -xsl:public/assets2/datasetxml2oai-pmh.xslt -export:public/assets2/datasetxml2oai.sef.json -t -nogo '-ns:##html5'",
"compress:solr": "./node_modules/xslt3/xslt3.js -xsl:public/assets2/solr.xslt -export:public/assets2/solr.sef.json -t -nogo '-ns:##html5'",
"compress:doi": "./node_modules/xslt3/xslt3.js -xsl:public/assets2/doi_datacite.xslt -export:public/assets2/doi_datacite.sef.json -t -nogo '-ns:##html5'",
@ -16,90 +15,90 @@
"format-check": "prettier --check ./**/*.{ts,js}",
"test": "node ace test"
},
"eslintConfig": {
"ignorePatterns": [
"build"
]
},
"eslintIgnore": [
"build"
],
"alias": {
"vue": "./node_modules/vue/dist/vue.esm-bundler.js"
},
"devDependencies": {
"@adonisjs/assembler": "^7.1.1",
"@adonisjs/tsconfig": "^1.4.0",
"@headlessui/vue": "^1.7.23",
"@japa/assert": "^4.0.1",
"@japa/plugin-adonisjs": "^4.0.0",
"@japa/runner": "^4.2.0",
"@adonisjs/tsconfig": "^1.2.1",
"@babel/core": "^7.20.12",
"@babel/plugin-proposal-class-properties": "^7.18.6",
"@babel/plugin-proposal-decorators": "^7.20.13",
"@babel/plugin-transform-runtime": "^7.19.6",
"@babel/preset-env": "^7.20.2",
"@babel/preset-typescript": "^7.18.6",
"@japa/api-client": "^2.0.3",
"@japa/assert": "^3.0.0",
"@japa/plugin-adonisjs": "^3.0.0",
"@japa/runner": "^3.1.1",
"@mdi/js": "^7.1.96",
"@poppinss/utils": "^6.7.2",
"@swc/wasm": "^1.10.14",
"@swc/core": "^1.4.2",
"@symfony/webpack-encore": "^5.0.1",
"@tailwindcss/forms": "^0.5.2",
"@types/bcryptjs": "^2.4.6",
"@types/clamscan": "^2.0.4",
"@types/escape-html": "^1.0.4",
"@types/fs-extra": "^11.0.4",
"@types/leaflet": "^1.9.21",
"@types/leaflet": "^1.9.3",
"@types/luxon": "^3.4.2",
"@types/node": "^22.10.2",
"@types/node": "^22.5.5",
"@types/proxy-addr": "^2.0.0",
"@types/qrcode": "^1.5.5",
"@types/source-map-support": "^0.5.6",
"@types/sprintf-js": "^1.1.4",
"@types/supertest": "^6.0.2",
"@vitejs/plugin-vue": "^5.2.1",
"autoprefixer": "^10.4.13",
"babel-preset-typescript-vue3": "^2.0.17",
"chart.js": "^4.2.0",
"dotenv-webpack": "^8.0.1",
"eslint": "^8.57.1",
"eslint-config-prettier": "^10.0.1",
"eslint-config-prettier": "^9.0.0",
"eslint-plugin-adonis": "^2.1.1",
"eslint-plugin-prettier": "^5.0.0-alpha.2",
"numeral": "^2.0.6",
"pinia": "^3.0.2",
"pino-pretty": "^13.0.0",
"pinia": "^2.0.30",
"pino-pretty": "^11.2.2",
"postcss-loader": "^8.1.1",
"prettier": "^3.4.2",
"prettier": "^3.0.0",
"supertest": "^6.3.3",
"tailwindcss": "^3.4.17",
"tailwindcss": "^3.2.4",
"ts-loader": "^9.4.2",
"ts-node-maintained": "^10.9.5",
"typescript": "^5.9.3",
"vite": "^6.0.11",
"ts-node": "^10.9.2",
"typescript": "^5.1.3",
"vue": "^3.4.26",
"vue-facing-decorator": "^4.0.1",
"vue-facing-decorator": "^3.0.0",
"vue-loader": "^17.0.1",
"webpack-dev-server": "^5.1.0",
"xslt3": "^2.5.0"
},
"dependencies": {
"@adonisjs/auth": "^9.2.4",
"@adonisjs/bodyparser": "^10.0.1",
"@adonisjs/core": "^6.21.0",
"@adonisjs/auth": "^9.1.1",
"@adonisjs/core": "^6.3.1",
"@adonisjs/cors": "^2.2.1",
"@adonisjs/drive": "^3.2.0",
"@adonisjs/inertia": "^3.1.1",
"@adonisjs/lucid": "^21.5.1",
"@adonisjs/drive": "^2.3.0",
"@adonisjs/encore": "^1.0.0",
"@adonisjs/inertia": "^1.0.0-7",
"@adonisjs/lucid": "^21.1.0",
"@adonisjs/mail": "^9.2.2",
"@adonisjs/redis": "^9.1.0",
"@adonisjs/session": "^7.5.0",
"@adonisjs/session": "^7.1.1",
"@adonisjs/shield": "^8.1.1",
"@adonisjs/static": "^1.1.1",
"@adonisjs/vite": "^4.0.0",
"@eidellev/adonis-stardust": "^3.0.0",
"@fontsource/archivo-black": "^5.0.1",
"@fontsource/inter": "^5.0.1",
"@inertiajs/inertia": "^0.11.1",
"@inertiajs/vue3": "^2.3.25",
"@opensearch-project/opensearch": "^3.2.0",
"@inertiajs/vue3": "^1.0.0",
"@opensearch-project/opensearch": "^2.4.0",
"@phc/format": "^1.0.0",
"@poppinss/manager": "^5.0.2",
"@vinejs/vine": "^3.0.0",
"axios": "^1.7.9",
"@vinejs/vine": "^2.0.0",
"bcrypt": "^5.1.1",
"bcryptjs": "^2.4.3",
"clamscan": "^2.1.2",
"crypto": "^1.0.1",
"dayjs": "^1.11.7",
"deep-email-validator": "^0.1.21",
"edge.js": "^6.0.1",
@ -114,20 +113,13 @@
"node-exceptions": "^4.0.1",
"notiwind": "^2.0.0",
"pg": "^8.9.0",
"phc-argon2": "^1.1.4",
"qrcode": "^1.5.3",
"redis": "^6.0.0",
"redis": "^4.6.10",
"reflect-metadata": "^0.2.1",
"saxon-js": "^2.5.0",
"toastify-js": "^1.12.0",
"vuedraggable": "^4.1.0",
"xmlbuilder2": "^4.0.3"
},
"hotHook": {
"boundaries": [
"./app/Controllers/**/*.ts",
"./app/middleware/*.ts"
]
"xmlbuilder2": "^3.1.1"
},
"type": "module",
"imports": {

5
postcss.config.cjs
View file

@ -1,10 +1,7 @@
module.exports = {
plugins: {
// 'postcss-import': {},
// 'postcss-nesting': {},
'tailwindcss/nesting': {},
// "@tailwindcss/postcss": {},
// tailwindcss: {},
'tailwindcss/nesting': {},
tailwindcss: {},
autoprefixer: {},
},

3
providers/drive/drivers/local.ts
View file

@ -74,8 +74,7 @@ export class LocalDriver implements LocalDriverContract {
*/
public async exists(location: string): Promise<boolean> {
try {
let path_temp = this.makePath(location); //'/storage/app/files/421'
return await this.adapter.pathExists(path_temp);
return await this.adapter.pathExists(this.makePath(location));
} catch (error) {
throw CannotGetMetaDataException.invoke(location, 'exists', error);
}

2
providers/mail_provider.ts
View file

@ -69,7 +69,7 @@ export default class MailProvider {
const mailConfigProvider = this.app.config.get('mail');
const config = await configProvider.resolve<any>(this.app, mailConfigProvider);
await config.mailers.smtp();
const iwas = await config.mailers.smtp();
// iwas.config.host = 'hhhost';
// this.app.config.set('mail.mailers.smtp.host', 'xhost');
// const iwas = await config.mailers.smtp();

16
providers/query_builder_provider.ts
View file

@ -63,15 +63,6 @@ export default class QueryBuilderProvider {
public register() {
// Register your own bindings
// const ModelQueryBuilder = this.app.container.bind('@adonisjs/lucid/orm/ModelQueryBuilder');
// ModelQueryBuilder.macro('whereTrue', function (columnName: string) {
// return this.where(columnName, true);
// });
// ModelQueryBuilder.macro('whereFalse', function (columnName: string) {
// return this.where(columnName, false);
// });
}
public async boot() {
@ -82,14 +73,15 @@ export default class QueryBuilderProvider {
// let rolesPluck = {};
let rolesPluck: { [key: number]: any } = {};
const result = await this.exec();
result.forEach((user: { [key: string]: any }, index: number) => {
let idc: number;
result.forEach((user, index) => {
let idc;
if (!id) {
idc = index;
} else {
idc = user[id];
}
const value: any = user[valueColumn];
const value = user[valueColumn];
// rolesPluck[idc] = user.name;
rolesPluck[idc] = value;
});
return rolesPluck;

34
providers/rule_provider.ts
View file

@ -1,34 +0,0 @@
import { ApplicationService } from '@adonisjs/core/types';
export default class RuleProvider {
constructor(protected app: ApplicationService) {}
public register() {
// Register your own bindings
}
public async boot() {
// IoC container is ready
// await import("../src/rules/index.js");
await import('#start/rules/unique');
await import('#start/rules/translated_language');
await import('#start/rules/unique_person');
// () => import('#start/rules/file_length'),
// () => import('#start/rules/file_scan'),
// () => import('#start/rules/allowed_extensions_mimetypes'),
await import('#start/rules/dependent_array_min_length');
await import('#start/rules/referenceValidation');
await import('#start/rules/valid_mimetype');
await import('#start/rules/array_contains_types');
await import('#start/rules/orcid');
}
public async ready() {
// App is ready
}
public async shutdown() {
// Cleanup, since app is going down
}
}

181
providers/vinejs_provider.ts
View file

@ -4,8 +4,9 @@
|--------------------------------------------------------------------------
|*/
import type { ApplicationService } from '@adonisjs/core/types';
import vine, { symbols, BaseLiteralType, Vine } from '@vinejs/vine';
import type { FieldContext, FieldOptions } from '@vinejs/vine/types';
import vine, { BaseLiteralType, Vine } from '@vinejs/vine';
import type { Validation, FieldContext, FieldOptions } from '@vinejs/vine/types';
// import type { MultipartFile, FileValidationOptions } from '@adonisjs/bodyparser/types';
import type { MultipartFile } from '@adonisjs/core/bodyparser';
import type { FileValidationOptions } from '@adonisjs/core/types/bodyparser';
import { Request, RequestValidator } from '@adonisjs/core/http';
@ -15,7 +16,6 @@ import MimeType from '#models/mime_type';
* Validation options accepted by the "file" rule
*/
export type FileRuleValidationOptions = Partial<FileValidationOptions> | ((field: FieldContext) => Partial<FileValidationOptions>);
/**
* Extend VineJS
*/
@ -24,66 +24,31 @@ declare module '@vinejs/vine' {
myfile(options?: FileRuleValidationOptions): VineMultipartFile;
}
}
/**
* Extend HTTP request class
*/
declare module '@adonisjs/core/http' {
interface Request extends RequestValidator {}
interface Request extends RequestValidator {
}
}
/**
* Checks if the value is an instance of multipart file
* from bodyparser.
*/
export function isBodyParserFile(file: MultipartFile | unknown): file is MultipartFile {
export function isBodyParserFile(file: MultipartFile | unknown): boolean {
return !!(file && typeof file === 'object' && 'isMultipartFile' in file);
}
export async function getEnabledExtensions() {
const enabledExtensions = await MimeType.query().select('file_extension').where('enabled', true).exec();
const extensions = enabledExtensions
.map((extension) => {
return extension.file_extension.split('|');
})
.flat();
/**
* Cache for enabled extensions to reduce database queries
*/
let extensionsCache: string[] | null = null;
let cacheTimestamp = 0;
const CACHE_DURATION = 5 * 60 * 1000; // 5 minutes
/**
* Get enabled extensions with caching
*/
export async function getEnabledExtensions(): Promise<string[]> {
const now = Date.now();
if (extensionsCache && now - cacheTimestamp < CACHE_DURATION) {
return extensionsCache;
}
try {
const enabledExtensions = await MimeType.query().select('file_extension').where('enabled', true).exec();
const extensions = enabledExtensions
.map((extension) => extension.file_extension.split('|'))
.flat()
.map((ext) => ext.toLowerCase().trim())
.filter((ext) => ext.length > 0);
extensionsCache = [...new Set(extensions)]; // Remove duplicates
cacheTimestamp = now;
return extensionsCache;
} catch (error) {
console.error('Error fetching enabled extensions:', error);
return extensionsCache || [];
}
}
/**
* Clear extensions cache
*/
export function clearExtensionsCache(): void {
extensionsCache = null;
cacheTimestamp = 0;
}
return extensions;
};
/**
* VineJS validation rule that validates the file to be an
* instance of BodyParser MultipartFile class.
@ -100,7 +65,6 @@ const isMultipartFile = vine.createRule(async (file: MultipartFile | unknown, op
// At this point, you can use type assertion to explicitly tell TypeScript that file is of type MultipartFile
const validatedFile = file as MultipartFile;
const validationOptions = typeof options === 'function' ? options(field) : options;
/**
* Set size when it's defined in the options and missing
* on the file instance
@ -108,29 +72,28 @@ const isMultipartFile = vine.createRule(async (file: MultipartFile | unknown, op
if (validatedFile.sizeLimit === undefined && validationOptions.size) {
validatedFile.sizeLimit = validationOptions.size;
}
/**
* Set extensions when it's defined in the options and missing
* on the file instance
*/
if (validatedFile.allowedExtensions === undefined) {
if (validationOptions.extnames !== undefined) {
validatedFile.allowedExtensions = validationOptions.extnames;
} else {
validatedFile.allowedExtensions = await getEnabledExtensions();
}
// if (validatedFile.allowedExtensions === undefined && validationOptions.extnames) {
// validatedFile.allowedExtensions = validationOptions.extnames;
// }
if (validatedFile.allowedExtensions === undefined && validationOptions.extnames) {
validatedFile.allowedExtensions = await getEnabledExtensions();
}
/**
* wieder löschen
* Set extensions when it's defined in the options and missing
* on the file instance
*/
// if (file.clientNameSizeLimit === undefined && validationOptions.clientNameSizeLimit) {
// file.clientNameSizeLimit = validationOptions.clientNameSizeLimit;
// }
/**
* Validate file
*/
try {
validatedFile.validate();
} catch (error) {
field.report(`File validation failed: ${error.message}`, 'file.validation_error', field, validationOptions);
return;
}
validatedFile.validate();
/**
* Report errors
*/
@ -139,40 +102,24 @@ const isMultipartFile = vine.createRule(async (file: MultipartFile | unknown, op
});
});
const MULTIPART_FILE: typeof symbols.SUBTYPE = symbols.SUBTYPE;
export class VineMultipartFile extends BaseLiteralType<MultipartFile, MultipartFile, MultipartFile> {
[MULTIPART_FILE]: string;
public validationOptions?: FileRuleValidationOptions;
// #private;
// constructor(validationOptions?: FileRuleValidationOptions, options?: FieldOptions, validations?: Validation<any>[]);
// clone(): this;
public validationOptions;
// extnames: (18) ['gpkg', 'htm', 'html', 'csv', 'txt', 'asc', 'c', 'cc', 'h', 'srt', 'tiff', 'pdf', 'png', 'zip', 'jpg', 'jpeg', 'jpe', 'xlsx']
// size: '512mb'
public constructor(validationOptions?: FileRuleValidationOptions, options?: FieldOptions) {
public constructor(validationOptions?: FileRuleValidationOptions, options?: FieldOptions, validations?: Validation<any>[]) {
// super(options, validations);
super(options, [isMultipartFile(validationOptions || {})]);
this.validationOptions = validationOptions;
}
public clone(): any {
return new VineMultipartFile(this.validationOptions, this.cloneOptions());
return new VineMultipartFile(this.validationOptions, this.cloneOptions(), this.cloneValidations());
}
/**
* Set maximum file size
*/
public maxSize(size: string | number): this {
const newOptions = { ...this.validationOptions, size };
return new VineMultipartFile(newOptions, this.cloneOptions()) as this;
}
/**
* Set allowed extensions
*/
public extensions(extnames: string[]): this {
const newOptions = { ...this.validationOptions, extnames };
return new VineMultipartFile(newOptions, this.cloneOptions()) as this;
}
}
export default class VinejsProvider {
@ -191,8 +138,13 @@ export default class VinejsProvider {
/**
* The container bindings have booted
*/
boot(): void {
Vine.macro('myfile', function (this: Vine, options?: FileRuleValidationOptions) {
// VineString.macro('translatedLanguage', function (this: VineString, options: Options) {
// return this.use(translatedLanguageRule(options));
// });
Vine.macro('myfile', function (this: Vine, options) {
return new VineMultipartFile(options);
});
@ -200,47 +152,10 @@ export default class VinejsProvider {
* The validate method can be used to validate the request
* data for the current request using VineJS validators
*/
Request.macro('validateUsing', function (this: Request, ...args) {
if (!this.ctx) {
throw new Error('HttpContext is not available');
}
return new RequestValidator(this.ctx).validateUsing(...args);
});
Request.macro('validateUsing', function (...args) {
return new RequestValidator(this.ctx).validateUsing(...args);
});
// Ensure MIME validation macros are loaded
this.loadMimeValidationMacros();
this.loadFileScanMacros();
this.loadFileLengthMacros();
}
/**
* Load MIME validation macros - called during boot to ensure they're available
*/
private async loadMimeValidationMacros(): Promise<void> {
try {
// Dynamically import the MIME validation rule to ensure macros are registered
await import('#start/rules/allowed_extensions_mimetypes');
} catch (error) {
console.warn('Could not load MIME validation macros:', error);
}
}
private async loadFileScanMacros(): Promise<void> {
try {
// Dynamically import the MIME validation rule to ensure macros are registered
await import('#start/rules/file_scan');
} catch (error) {
console.warn('Could not load MIME validation macros:', error);
}
}
private async loadFileLengthMacros(): Promise<void> {
try {
// Dynamically import the MIME validation rule to ensure macros are registered
await import('#start/rules/file_length');
} catch (error) {
console.warn('Could not load MIME validation macros:', error);
}
}
/**
@ -256,7 +171,5 @@ export default class VinejsProvider {
/**
* Preparing to shutdown the app
*/
async shutdown() {
clearExtensionsCache();
}
async shutdown() {}
}

BIN
public/android-chrome-192x192.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 11 KiB

BIN
public/android-chrome-512x512.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 37 KiB

BIN
public/apple-touch-icon.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 9.8 KiB

12
public/assets/entrypoints.json Normal file
View file

@ -0,0 +1,12 @@
{
"entrypoints": {
"app": {
"css": [
"http://localhost:8080/assets/app.css"
],
"js": [
"http://localhost:8080/assets/app.js"
]
}
}
}

103
public/assets/manifest.json Normal file
View file

@ -0,0 +1,103 @@
{
"assets/app.css": "http://localhost:8080/assets/app.css",
"assets/app.js": "http://localhost:8080/assets/app.js",
"assets/resources_js_apps_settings_l18n_de_js.js": "http://localhost:8080/assets/resources_js_apps_settings_l18n_de_js.js",
"assets/resources_js_apps_settings_l18n_en_js.js": "http://localhost:8080/assets/resources_js_apps_settings_l18n_en_js.js",
"assets/resources_js_Pages_Admin_License_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_License_Index_vue.js",
"assets/resources_js_Pages_Admin_Mimetype_Create_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Mimetype_Create_vue.js",
"assets/resources_js_Pages_Admin_Mimetype_Delete_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Mimetype_Delete_vue.js",
"assets/resources_js_Pages_Admin_Mimetype_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Mimetype_Index_vue.js",
"assets/resources_js_Pages_Admin_Permission_Create_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Permission_Create_vue.js",
"assets/resources_js_Pages_Admin_Permission_Edit_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Permission_Edit_vue.js",
"assets/resources_js_Pages_Admin_Permission_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Permission_Index_vue.js",
"assets/resources_js_Pages_Admin_Permission_Show_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Permission_Show_vue.js",
"assets/resources_js_Pages_Admin_Role_Create_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Role_Create_vue.js",
"assets/resources_js_Pages_Admin_Role_Edit_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Role_Edit_vue.js",
"assets/resources_js_Pages_Admin_Role_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Role_Index_vue.js",
"assets/resources_js_Pages_Admin_Role_Show_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Role_Show_vue.js",
"assets/resources_js_Pages_Admin_Settings_vue-resources_js_utils_toast_css.css": "http://localhost:8080/assets/resources_js_Pages_Admin_Settings_vue-resources_js_utils_toast_css.css",
"assets/resources_js_Pages_Admin_Settings_vue-resources_js_utils_toast_css.js": "http://localhost:8080/assets/resources_js_Pages_Admin_Settings_vue-resources_js_utils_toast_css.js",
"assets/resources_js_Pages_Admin_User_Create_vue-resources_js_Components_SimplePasswordMeter_password-f3312a.css": "http://localhost:8080/assets/resources_js_Pages_Admin_User_Create_vue-resources_js_Components_SimplePasswordMeter_password-f3312a.css",
"assets/resources_js_Pages_Admin_User_Create_vue-resources_js_Components_SimplePasswordMeter_password-f3312a.js": "http://localhost:8080/assets/resources_js_Pages_Admin_User_Create_vue-resources_js_Components_SimplePasswordMeter_password-f3312a.js",
"assets/resources_js_Pages_Admin_User_Edit_vue-resources_js_Components_SimplePasswordMeter_password-m-6dc207.css": "http://localhost:8080/assets/resources_js_Pages_Admin_User_Edit_vue-resources_js_Components_SimplePasswordMeter_password-m-6dc207.css",
"assets/resources_js_Pages_Admin_User_Edit_vue-resources_js_Components_SimplePasswordMeter_password-m-6dc207.js": "http://localhost:8080/assets/resources_js_Pages_Admin_User_Edit_vue-resources_js_Components_SimplePasswordMeter_password-m-6dc207.js",
"assets/resources_js_Pages_Admin_User_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_User_Index_vue.js",
"assets/resources_js_Pages_Admin_User_Show_vue.js": "http://localhost:8080/assets/resources_js_Pages_Admin_User_Show_vue.js",
"assets/resources_js_Pages_App_vue.js": "http://localhost:8080/assets/resources_js_Pages_App_vue.js",
"assets/resources_js_Pages_Auth_AccountInfo_vue-resources_js_utils_toast_css-resources_js_Components_-06c7b5.css": "http://localhost:8080/assets/resources_js_Pages_Auth_AccountInfo_vue-resources_js_utils_toast_css-resources_js_Components_-06c7b5.css",
"assets/resources_js_Pages_Auth_AccountInfo_vue-resources_js_utils_toast_css-resources_js_Components_-06c7b5.js": "http://localhost:8080/assets/resources_js_Pages_Auth_AccountInfo_vue-resources_js_utils_toast_css-resources_js_Components_-06c7b5.js",
"assets/resources_js_Pages_Auth_Login_vue.js": "http://localhost:8080/assets/resources_js_Pages_Auth_Login_vue.js",
"assets/resources_js_Pages_Auth_Register_vue.js": "http://localhost:8080/assets/resources_js_Pages_Auth_Register_vue.js",
"assets/resources_js_Pages_Dashboard_vue.js": "http://localhost:8080/assets/resources_js_Pages_Dashboard_vue.js",
"assets/resources_js_Pages_Editor_Dataset_Approve_vue.js": "http://localhost:8080/assets/resources_js_Pages_Editor_Dataset_Approve_vue.js",
"assets/resources_js_Pages_Editor_Dataset_Doi_vue.js": "http://localhost:8080/assets/resources_js_Pages_Editor_Dataset_Doi_vue.js",
"assets/resources_js_Pages_Editor_Dataset_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Editor_Dataset_Index_vue.js",
"assets/resources_js_Pages_Editor_Dataset_Publish_vue.js": "http://localhost:8080/assets/resources_js_Pages_Editor_Dataset_Publish_vue.js",
"assets/resources_js_Pages_Editor_Dataset_Receive_vue.js": "http://localhost:8080/assets/resources_js_Pages_Editor_Dataset_Receive_vue.js",
"assets/resources_js_Pages_Editor_Dataset_Reject_vue.js": "http://localhost:8080/assets/resources_js_Pages_Editor_Dataset_Reject_vue.js",
"assets/resources_js_Pages_Error_vue.js": "http://localhost:8080/assets/resources_js_Pages_Error_vue.js",
"assets/resources_js_Pages_Errors_ServerError_vue.js": "http://localhost:8080/assets/resources_js_Pages_Errors_ServerError_vue.js",
"assets/resources_js_Pages_Errors_not_found_vue.js": "http://localhost:8080/assets/resources_js_Pages_Errors_not_found_vue.js",
"assets/resources_js_Pages_Map_vue-resources_js_Components_Map_draw_component_vue-resources_js_Compon-b0925c.css": "http://localhost:8080/assets/resources_js_Pages_Map_vue-resources_js_Components_Map_draw_component_vue-resources_js_Compon-b0925c.css",
"assets/resources_js_Pages_Map_vue-resources_js_Components_Map_draw_component_vue-resources_js_Compon-b0925c.js": "http://localhost:8080/assets/resources_js_Pages_Map_vue-resources_js_Components_Map_draw_component_vue-resources_js_Compon-b0925c.js",
"assets/resources_js_Pages_ProfileView_vue.js": "http://localhost:8080/assets/resources_js_Pages_ProfileView_vue.js",
"assets/resources_js_Pages_Reviewer_Dataset_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Reviewer_Dataset_Index_vue.js",
"assets/resources_js_Pages_Reviewer_Dataset_Reject_vue.js": "http://localhost:8080/assets/resources_js_Pages_Reviewer_Dataset_Reject_vue.js",
"assets/resources_js_Pages_Reviewer_Dataset_Review_vue.js": "http://localhost:8080/assets/resources_js_Pages_Reviewer_Dataset_Review_vue.js",
"assets/resources_js_Pages_Submitter_Dataset_Category_vue.css": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Category_vue.css",
"assets/resources_js_Pages_Submitter_Dataset_Category_vue.js": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Category_vue.js",
"assets/resources_js_Pages_Submitter_Dataset_Create_vue-resources_js_utils_toast_css-resources_js_Com-03a898.css": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Create_vue-resources_js_utils_toast_css-resources_js_Com-03a898.css",
"assets/resources_js_Pages_Submitter_Dataset_Create_vue-resources_js_utils_toast_css-resources_js_Com-03a898.js": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Create_vue-resources_js_utils_toast_css-resources_js_Com-03a898.js",
"assets/resources_js_Pages_Submitter_Dataset_Delete_vue.js": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Delete_vue.js",
"assets/resources_js_Pages_Submitter_Dataset_Edit_vue-resources_js_utils_toast_css-resources_js_Compo-a37b65.css": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Edit_vue-resources_js_utils_toast_css-resources_js_Compo-a37b65.css",
"assets/resources_js_Pages_Submitter_Dataset_Edit_vue-resources_js_utils_toast_css-resources_js_Compo-a37b65.js": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Edit_vue-resources_js_utils_toast_css-resources_js_Compo-a37b65.js",
"assets/resources_js_Pages_Submitter_Dataset_Index_vue.css": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Index_vue.css",
"assets/resources_js_Pages_Submitter_Dataset_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Index_vue.js",
"assets/resources_js_Pages_Submitter_Dataset_Release_vue.js": "http://localhost:8080/assets/resources_js_Pages_Submitter_Dataset_Release_vue.js",
"assets/resources_js_Pages_Submitter_Person_Index_vue.js": "http://localhost:8080/assets/resources_js_Pages_Submitter_Person_Index_vue.js",
"assets/resources_js_Pages_register-view_register-view-component_vue.js": "http://localhost:8080/assets/resources_js_Pages_register-view_register-view-component_vue.js",
"assets/vendors-node_modules_mdi_js_mdi_js-node_modules_vue-loader_dist_exportHelper_js.js": "http://localhost:8080/assets/vendors-node_modules_mdi_js_mdi_js-node_modules_vue-loader_dist_exportHelper_js.js",
"assets/vendors-node_modules_focus-trap_dist_focus-trap_esm_js-node_modules_notiwind_dist_index_esm_js.js": "http://localhost:8080/assets/vendors-node_modules_focus-trap_dist_focus-trap_esm_js-node_modules_notiwind_dist_index_esm_js.js",
"assets/vendors-node_modules_vue-facing-decorator_dist_esm_utils_js.js": "http://localhost:8080/assets/vendors-node_modules_vue-facing-decorator_dist_esm_utils_js.js",
"assets/vendors-node_modules_toastify-js_src_toastify_js.js": "http://localhost:8080/assets/vendors-node_modules_toastify-js_src_toastify_js.js",
"assets/vendors-node_modules_leaflet_dist_leaflet-src_js-node_modules_leaflet_src_control_Control_Att-adabdc.js": "http://localhost:8080/assets/vendors-node_modules_leaflet_dist_leaflet-src_js-node_modules_leaflet_src_control_Control_Att-adabdc.js",
"assets/vendors-node_modules_buffer_index_js-node_modules_vuedraggable_dist_vuedraggable_umd_js.js": "http://localhost:8080/assets/vendors-node_modules_buffer_index_js-node_modules_vuedraggable_dist_vuedraggable_umd_js.js",
"assets/vendors-node_modules_mime_dist_src_index_js.js": "http://localhost:8080/assets/vendors-node_modules_mime_dist_src_index_js.js",
"assets/vendors-node_modules_numeral_numeral_js-node_modules_chart_js_dist_chart_js.js": "http://localhost:8080/assets/vendors-node_modules_numeral_numeral_js-node_modules_chart_js_dist_chart_js.js",
"assets/resources_js_Components_BaseButton_vue.js": "http://localhost:8080/assets/resources_js_Components_BaseButton_vue.js",
"assets/resources_js_Stores_main_ts-resources_js_Components_BaseDivider_vue-resources_js_Components_C-b45805.js": "http://localhost:8080/assets/resources_js_Stores_main_ts-resources_js_Components_BaseDivider_vue-resources_js_Components_C-b45805.js",
"assets/resources_js_Layouts_LayoutAuthenticated_vue.css": "http://localhost:8080/assets/resources_js_Layouts_LayoutAuthenticated_vue.css",
"assets/resources_js_Layouts_LayoutAuthenticated_vue.js": "http://localhost:8080/assets/resources_js_Layouts_LayoutAuthenticated_vue.js",
"assets/resources_js_Components_BaseButtons_vue-resources_js_Components_FormControl_vue-resources_js_-d830d6.js": "http://localhost:8080/assets/resources_js_Components_BaseButtons_vue-resources_js_Components_FormControl_vue-resources_js_-d830d6.js",
"assets/resources_js_Components_Admin_Pagination_vue-resources_js_Components_BaseButtons_vue-resource-6f3a70.js": "http://localhost:8080/assets/resources_js_Components_Admin_Pagination_vue-resources_js_Components_BaseButtons_vue-resource-6f3a70.js",
"assets/resources_js_utils_toast_ts-resources_js_Components_NotificationBar_vue.js": "http://localhost:8080/assets/resources_js_utils_toast_ts-resources_js_Components_NotificationBar_vue.js",
"assets/resources_js_Components_Map_draw_component_vue-resources_js_Components_Map_zoom_component_vue-058bcc.js": "http://localhost:8080/assets/resources_js_Components_Map_draw_component_vue-resources_js_Components_Map_zoom_component_vue-058bcc.js",
"assets/resources_js_Components_SectionMain_vue-resources_js_Components_SectionTitleLineWithButton_vu-764dfe.js": "http://localhost:8080/assets/resources_js_Components_SectionMain_vue-resources_js_Components_SectionTitleLineWithButton_vu-764dfe.js",
"assets/resources_js_Components_BaseButtons_vue-resources_js_Components_NotificationBar_vue-resources-7e06d8.js": "http://localhost:8080/assets/resources_js_Components_BaseButtons_vue-resources_js_Components_NotificationBar_vue-resources-7e06d8.js",
"assets/resources_js_Components_Admin_Sort_vue-resources_js_Components_SectionTitleLineWithButton_vue.js": "http://localhost:8080/assets/resources_js_Components_Admin_Sort_vue-resources_js_Components_SectionTitleLineWithButton_vue.js",
"assets/resources_js_Components_CardBoxModal_vue.js": "http://localhost:8080/assets/resources_js_Components_CardBoxModal_vue.js",
"assets/resources_js_Components_FileUpload_vue-resources_js_Components_FormCheckRadioGroup_vue-resour-25e686.js": "http://localhost:8080/assets/resources_js_Components_FileUpload_vue-resources_js_Components_FormCheckRadioGroup_vue-resour-25e686.js",
"assets/fonts/inter-latin-ext-400-normal.woff": "http://localhost:8080/assets/fonts/inter-latin-ext-400-normal.1c20f7dc.woff",
"assets/fonts/inter-latin-400-normal.woff": "http://localhost:8080/assets/fonts/inter-latin-400-normal.b0c8fe9d.woff",
"assets/fonts/inter-latin-ext-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-latin-ext-400-normal.3d10c85f.woff2",
"assets/fonts/inter-latin-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-latin-400-normal.9698cc7d.woff2",
"assets/fonts/archivo-black-latin-400-normal.woff2": "http://localhost:8080/assets/fonts/archivo-black-latin-400-normal.fc847a1f.woff2",
"assets/fonts/archivo-black-latin-ext-400-normal.woff2": "http://localhost:8080/assets/fonts/archivo-black-latin-ext-400-normal.21761451.woff2",
"assets/fonts/inter-cyrillic-ext-400-normal.woff": "http://localhost:8080/assets/fonts/inter-cyrillic-ext-400-normal.e8945162.woff",
"assets/fonts/archivo-black-latin-400-normal.woff": "http://localhost:8080/assets/fonts/archivo-black-latin-400-normal.58a301a6.woff",
"assets/fonts/inter-cyrillic-ext-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-cyrillic-ext-400-normal.fd1478dc.woff2",
"assets/fonts/inter-cyrillic-400-normal.woff": "http://localhost:8080/assets/fonts/inter-cyrillic-400-normal.e2841352.woff",
"assets/fonts/inter-greek-400-normal.woff": "http://localhost:8080/assets/fonts/inter-greek-400-normal.a42da273.woff",
"assets/fonts/archivo-black-latin-ext-400-normal.woff": "http://localhost:8080/assets/fonts/archivo-black-latin-ext-400-normal.5ab5ba92.woff",
"assets/fonts/inter-greek-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-greek-400-normal.a8de720a.woff2",
"assets/fonts/inter-cyrillic-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-cyrillic-400-normal.cb04b2ee.woff2",
"assets/fonts/inter-greek-ext-400-normal.woff": "http://localhost:8080/assets/fonts/inter-greek-ext-400-normal.b9e1e894.woff",
"assets/fonts/inter-vietnamese-400-normal.woff": "http://localhost:8080/assets/fonts/inter-vietnamese-400-normal.96f8adc7.woff",
"assets/fonts/inter-greek-ext-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-greek-ext-400-normal.f2fa0d9e.woff2",
"assets/fonts/inter-vietnamese-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-vietnamese-400-normal.44c9df13.woff2",
"assets/images/marker-icon.png": "http://localhost:8080/assets/images/marker-icon.2b3e1faf.png",
"assets/images/layers-2x.png": "http://localhost:8080/assets/images/layers-2x.8f2c4d11.png",
"assets/images/layers.png": "http://localhost:8080/assets/images/layers.416d9136.png",
"assets/images/Close.svg": "http://localhost:8080/assets/images/Close.e4887675.svg",
"assets/vendors-node_modules_vue-facing-decorator_dist_esm_index_js-node_modules_vue-facing-decorator-818045.js": "http://localhost:8080/assets/vendors-node_modules_vue-facing-decorator_dist_esm_index_js-node_modules_vue-facing-decorator-818045.js"
}

2
public/assets2/doi_datacite.sef.json
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more