- first commit
This commit is contained in:
commit
407717d4b5
57 changed files with 5510 additions and 0 deletions
184
src/library/XmlModel.ts
Normal file
184
src/library/XmlModel.ts
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
import DocumentXmlCache from "../models/DocumentXmlCache";
|
||||
// import { XMLDocument } from "xmlbuilder";
|
||||
import { XMLBuilder } from "xmlbuilder2/lib/interfaces";
|
||||
import Dataset from "../models/Dataset";
|
||||
import Logger from "jet-logger";
|
||||
import { create } from "xmlbuilder2";
|
||||
import dayjs from "dayjs";
|
||||
|
||||
/**
|
||||
* This is the description of the interface
|
||||
*
|
||||
* @interface Conf
|
||||
* @member {Model} model holds the current dataset model
|
||||
* @member {XMLBuilder} dom holds the current DOM representation
|
||||
*/
|
||||
export interface Conf {
|
||||
/**
|
||||
* Holds the current model either directly set or deserialized from XML.
|
||||
*/
|
||||
model: Dataset;
|
||||
|
||||
/**
|
||||
* Holds the current DOM representation.
|
||||
*/
|
||||
dom?: XMLBuilder;
|
||||
|
||||
/**
|
||||
* List of fields to skip on serialization.
|
||||
*/
|
||||
excludeFields: Array<string>;
|
||||
|
||||
/**
|
||||
* True, if empty fields get excluded from serialization.
|
||||
*/
|
||||
excludeEmpty: boolean;
|
||||
|
||||
/**
|
||||
* Base URI for xlink:ref elements
|
||||
*/
|
||||
baseUri: string;
|
||||
}
|
||||
|
||||
export default class XmlModel {
|
||||
// private config: { [key: string]: any } = {};
|
||||
private config: Conf; // = { excludeEmpty: false, baseUri: "" };
|
||||
// private strategy = null;
|
||||
private cache: DocumentXmlCache;
|
||||
private _caching = false;
|
||||
|
||||
constructor(dataset: Dataset) {
|
||||
// $this->strategy = new Strategy();// Opus_Model_Xml_Version1;
|
||||
// $this->config = new Conf();
|
||||
// $this->strategy->setup($this->config);
|
||||
|
||||
// this.strategy = new Strategy();
|
||||
this.config = {
|
||||
excludeEmpty: false,
|
||||
baseUri: "",
|
||||
excludeFields: [],
|
||||
model: dataset,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the Model for XML generation.
|
||||
*
|
||||
* @param \App\Models\Dataset model Model to serialize.
|
||||
*
|
||||
*/
|
||||
set setModel(model: Dataset) {
|
||||
this.config.model = model;
|
||||
// return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Define that empty fields (value===null) shall be excluded.
|
||||
*
|
||||
*/
|
||||
public excludeEmptyFields(): void {
|
||||
this.config.excludeEmpty = true;
|
||||
// return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return cache table.
|
||||
*
|
||||
* @returns {DocumentXmlCache}
|
||||
*/
|
||||
get getXmlCache(): DocumentXmlCache {
|
||||
return this.cache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a new XML version with current configuration up.
|
||||
*
|
||||
* @param { DocumentXmlCache } cache table
|
||||
*/
|
||||
set setXmlCache(cache: DocumentXmlCache) {
|
||||
this.cache = cache;
|
||||
}
|
||||
|
||||
get caching(): boolean {
|
||||
return this._caching;
|
||||
}
|
||||
set caching(caching: boolean) {
|
||||
this._caching = caching;
|
||||
}
|
||||
|
||||
public async getDomDocument() {
|
||||
const dataset = this.config.model;
|
||||
|
||||
let domDocument: XMLBuilder | null = await this.getDomDocumentFromXmlCache();
|
||||
if (domDocument) {
|
||||
return domDocument;
|
||||
} else {
|
||||
//create domDocument during runtime
|
||||
// domDocument = $this->strategy->getDomDocument();
|
||||
domDocument = create({ version: "1.0", encoding: "UTF-8", standalone: true }, "<root></root>");
|
||||
}
|
||||
|
||||
//if caching isn't wanted return only dom Document
|
||||
if (this._caching != true) {
|
||||
return domDocument;
|
||||
//otherwise caching is desired:
|
||||
// save xml cache to db and return domDocument
|
||||
} else {
|
||||
// save new DocumentXmlCache
|
||||
if (!this.cache) {
|
||||
this.cache = new DocumentXmlCache();
|
||||
this.cache.document_id = dataset.id;
|
||||
}
|
||||
// if (!this.cache.document_id) {
|
||||
// this.cache.document_id = dataset.id;
|
||||
// }
|
||||
this.cache.xml_version = 1; // (int)$this->strategy->getVersion();
|
||||
this.cache.server_date_modified = dayjs(dataset.server_date_modified).format("YYYY-MM-DD HH:mm:ss");
|
||||
this.cache.xml_data = domDocument.end();
|
||||
|
||||
//save xml cache
|
||||
this.cache.save();
|
||||
// return newly created xml cache
|
||||
return domDocument;
|
||||
}
|
||||
}
|
||||
|
||||
private async getDomDocumentFromXmlCache(): Promise<XMLBuilder | null> {
|
||||
const dataset: Dataset = this.config.model;
|
||||
if (null == this.cache) {
|
||||
//$logger->debug(__METHOD__ . ' skipping cache for ' . get_class($model));
|
||||
// Log::debug(__METHOD__ . ' skipping cache for ' . get_class($dataset));
|
||||
Logger.warn(`__METHOD__ . skipping cache for ${dataset}`);
|
||||
return null;
|
||||
}
|
||||
// const dataset: Dataset = this.config.model;
|
||||
const actuallyCached: boolean = await DocumentXmlCache.hasValidEntry(dataset.id, dataset.server_date_modified);
|
||||
|
||||
//no cache or no actual cache
|
||||
if (true != actuallyCached) {
|
||||
Logger.warn(" cache missing for " + "#" + dataset.id);
|
||||
return null;
|
||||
}
|
||||
|
||||
//cache is actual return it for oai:
|
||||
// Log::debug(__METHOD__ . ' cache hit for ' . get_class($dataset) . '#' . $dataset->id);
|
||||
try {
|
||||
// return $this->_cache->get($model->getId(), (int) $this->_strategy->getVersion());
|
||||
// const cache = await DocumentXmlCache.findOne({
|
||||
// where: { document_id: dataset.id },
|
||||
// });
|
||||
if (this.cache) {
|
||||
return this.cache.getDomDocument();
|
||||
} else {
|
||||
Logger.warn(" Access to XML cache failed on " + dataset + "#" + dataset.id + ". Trying to recover.");
|
||||
return null;
|
||||
}
|
||||
// return this.cache.getDomDocument();
|
||||
} catch (error) {
|
||||
Logger.warn(" Access to XML cache failed on " + dataset + "#" + dataset.id + ". Trying to recover.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// export default XmlModel;
|
||||
71
src/library/oai/OaiConfiguration.ts
Normal file
71
src/library/oai/OaiConfiguration.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import config from "../../config/oai.config";
|
||||
|
||||
export default class Configuration {
|
||||
/**
|
||||
* Hold path where to store temporary resumption token files.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
private _pathTokens = "";
|
||||
|
||||
private _maxListIds = 15;
|
||||
|
||||
/**
|
||||
* Return maximum number of listable identifiers per request.
|
||||
*
|
||||
* @return {number} Maximum number of listable identifiers per request.
|
||||
*/
|
||||
public get maxListIds(): number {
|
||||
return this._maxListIds;
|
||||
}
|
||||
public set maxListIds(value: number) {
|
||||
this._maxListIds = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Holds maximum number of records to list per request.
|
||||
*
|
||||
* @var number
|
||||
*/
|
||||
private _maxListRecs = 15;
|
||||
|
||||
/**
|
||||
* Return maximum number of listable records per request.
|
||||
*
|
||||
* @return {number} Maximum number of listable records per request.
|
||||
*/
|
||||
public get maxListRecs() {
|
||||
return this._maxListRecs;
|
||||
}
|
||||
public set maxListRecs(value) {
|
||||
this._maxListRecs = value;
|
||||
}
|
||||
|
||||
constructor() {
|
||||
this._maxListIds = config.max.listidentifiers as number;
|
||||
this._maxListRecs = config.max.listrecords as number;
|
||||
// $this->maxListIds = config('oai.max.listidentifiers');
|
||||
// $this->maxListRecs = config('oai.max.listrecords');
|
||||
// $this->pathTokens = config('app.workspacePath')
|
||||
// . DIRECTORY_SEPARATOR .'tmp'
|
||||
// . DIRECTORY_SEPARATOR . 'resumption';
|
||||
}
|
||||
|
||||
/**
|
||||
* Return temporary path for resumption tokens.
|
||||
*
|
||||
* @returns {string} token path.
|
||||
*/
|
||||
get getResumptionTokenPath(): string {
|
||||
return this._pathTokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return maximum number of listable records per request.
|
||||
*
|
||||
* @return {number} Maximum number of listable records per request.
|
||||
*/
|
||||
// get getMaxListRecords(): number {
|
||||
// return this._maxListRecs;
|
||||
// }
|
||||
}
|
||||
107
src/library/oai/ResumptionToken.ts
Normal file
107
src/library/oai/ResumptionToken.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
export default class ResumptionToken {
|
||||
/**
|
||||
* Holds dcoument ids
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
private _documentIds: number[] = [];
|
||||
|
||||
/**
|
||||
* Holds metadata prefix information
|
||||
*
|
||||
* @var {string}
|
||||
*/
|
||||
private _metadataPrefix = "";
|
||||
|
||||
/**
|
||||
* Holds resumption id (only if token is stored)
|
||||
*
|
||||
* @var {string}
|
||||
*/
|
||||
private _resumptionId = "";
|
||||
|
||||
/**
|
||||
* Holds start postion
|
||||
*
|
||||
* @var {number}
|
||||
*/
|
||||
private _startPosition = 0;
|
||||
|
||||
/**
|
||||
* Holds total amount of document ids
|
||||
*
|
||||
* @var {number}
|
||||
*/
|
||||
private _totalIds = 0;
|
||||
|
||||
//#region properties
|
||||
|
||||
get Key(): string{
|
||||
return this.MetadataPrefix + this.StartPosition + this.TotalIds;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns current holded document ids.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public get DocumentIds(): number[] {
|
||||
return this._documentIds;
|
||||
}
|
||||
public set DocumentIds(idsToStore: number | number[]) {
|
||||
if (!Array.isArray(idsToStore)) {
|
||||
idsToStore = new Array(idsToStore);
|
||||
}
|
||||
this._documentIds = idsToStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns metadata prefix information.
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public get MetadataPrefix(): string {
|
||||
return this._metadataPrefix;
|
||||
}
|
||||
public set MetadataPrefix(value) {
|
||||
this._metadataPrefix = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return setted resumption id after successful storing of resumption token.
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public get ResumptionId() {
|
||||
return this._resumptionId;
|
||||
}
|
||||
public set ResumptionId(resumptionId) {
|
||||
this._resumptionId = resumptionId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns start position.
|
||||
*
|
||||
* @return in
|
||||
*/
|
||||
public get StartPosition() {
|
||||
return this._startPosition;
|
||||
}
|
||||
public set StartPosition(startPosition) {
|
||||
this._startPosition = startPosition;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns total number of document ids for this request
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public get TotalIds() {
|
||||
return this._totalIds;
|
||||
}
|
||||
public set TotalIds(totalIds) {
|
||||
this._totalIds = totalIds;
|
||||
}
|
||||
|
||||
//#endregion properties
|
||||
}
|
||||
194
src/library/oai/TokenWorker.ts
Normal file
194
src/library/oai/TokenWorker.ts
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
import ResumptionToken from "./ResumptionToken";
|
||||
import { realpathSync } from "fs";
|
||||
import { createClient, RedisClientType } from "redis";
|
||||
import InternalServerErrorException from "../../exceptions/InternalServerError";
|
||||
import { sprintf } from "sprintf-js";
|
||||
import dayjs from "dayjs";
|
||||
import * as crypto from "crypto";
|
||||
|
||||
export default class TokenWorker {
|
||||
private resumptionPath = "";
|
||||
// private resumptionId = null;
|
||||
protected filePrefix = "rs_";
|
||||
protected fileExtension = "txt";
|
||||
|
||||
private cache: RedisClientType;
|
||||
private ttl: number;
|
||||
private url: string;
|
||||
private connected: boolean = false;
|
||||
|
||||
constructor(ttl: number) {
|
||||
// if (resPath) {
|
||||
// this.setResumptionPath(resPath);
|
||||
// }
|
||||
// [1] define ttl and create redis connection
|
||||
this.ttl = ttl;
|
||||
this.url = process.env.REDIS_URL || "redis://127.0.0.1:6379";
|
||||
|
||||
// this.cache.on("connect", () => {
|
||||
// console.log(`Redis connection established`);
|
||||
// });
|
||||
|
||||
// this.cache.on("error", (error: string) => {
|
||||
// console.error(`Redis error, service degraded: ${error}`);
|
||||
// });
|
||||
// The Redis client must be created in an async closure
|
||||
// (async () => {
|
||||
// this.cache = createClient({
|
||||
// url,
|
||||
// });
|
||||
// this.cache.on("error", (err) => console.log("[Redis] Redis Client Error: ", err));
|
||||
// await this.cache.connect();
|
||||
// console.log("[Redis]: Successfully connected to the Redis server");
|
||||
// })();
|
||||
}
|
||||
public async connect() {
|
||||
const url = process.env.REDIS_URL || "redis://localhost:6379";
|
||||
this.cache = createClient({
|
||||
url,
|
||||
});
|
||||
this.cache.on("error", (err) => {
|
||||
this.connected = false;
|
||||
console.log("[Redis] Redis Client Error: ", err);
|
||||
});
|
||||
this.cache.on("connect", () => {
|
||||
this.connected = true;
|
||||
// console.log(`Redis connection established`);
|
||||
});
|
||||
await this.cache.connect();
|
||||
}
|
||||
|
||||
public get Connected(): boolean {
|
||||
return this.connected;
|
||||
}
|
||||
|
||||
public async has(key: string): Promise<boolean> {
|
||||
const result = await this.cache.get(key);
|
||||
return result !== undefined && result !== null;
|
||||
}
|
||||
|
||||
public async set(token: ResumptionToken) {
|
||||
let fc = 0;
|
||||
const uniqueId = dayjs().unix().toString(); // 1548381600;
|
||||
let uniqueName: string;
|
||||
let cacheKeyExists = true;
|
||||
do {
|
||||
// format values
|
||||
// %s - String
|
||||
// %d - Signed decimal number (negative, zero or positive)
|
||||
// [0-9] (Specifies the minimum width held of to the variable value)
|
||||
uniqueName = sprintf("%s%05d", uniqueId, fc++);
|
||||
// let file = uniqueName;
|
||||
cacheKeyExists = await this.has(uniqueName);
|
||||
} while (cacheKeyExists);
|
||||
// uniqueName = this.checksum(token.Key);
|
||||
|
||||
const serialToken = JSON.stringify(token);
|
||||
await this.cache.setEx(uniqueName, this.ttl, serialToken);
|
||||
return uniqueName;
|
||||
|
||||
// token.ResumptionId = uniqueName;
|
||||
}
|
||||
|
||||
// public connected(): boolean {
|
||||
// return this.cache.connected;
|
||||
// }
|
||||
|
||||
public async get(key: string): Promise<ResumptionToken | null> {
|
||||
if (!this.cache) {
|
||||
throw new InternalServerErrorException("Dataset is not available for OAI export!");
|
||||
}
|
||||
|
||||
const result = await this.cache.get(key);
|
||||
if (result) {
|
||||
const rToken: ResumptionToken = new ResumptionToken();
|
||||
const parsed = JSON.parse(result);
|
||||
Object.assign(rToken, parsed);
|
||||
return rToken;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public del(key: string) {
|
||||
this.cache.del(key);
|
||||
}
|
||||
|
||||
public flush() {
|
||||
this.cache.flushAll();
|
||||
}
|
||||
public async close() {
|
||||
await this.cache.disconnect();
|
||||
this.connected = false;
|
||||
}
|
||||
|
||||
private checksum(str: string, algorithm?: string, encoding?: string): string {
|
||||
/**
|
||||
* @type {BinaryToTextEncoding}
|
||||
*/
|
||||
const ENCODING_OUT = "hex"; // Initializer type string is not assignable to variable type BinaryToTextEncoding
|
||||
|
||||
return crypto
|
||||
.createHash(algorithm || 'md5')
|
||||
.update(str, 'utf8')
|
||||
.digest(ENCODING_OUT)
|
||||
}
|
||||
|
||||
/**
|
||||
* Set resumption path where the resumption token files are stored.
|
||||
*
|
||||
* @throws Oai_Model_ResumptionTokenException Thrown if directory operations failed.
|
||||
* @return void
|
||||
*/
|
||||
public setResumptionPath(resPath: string): void {
|
||||
// expanding all symbolic links and resolving references
|
||||
const realPath = realpathSync(resPath);
|
||||
|
||||
// if (empty($realPath) or false === is_dir($realPath)) {
|
||||
// throw new Oai_Model_ResumptionTokenException(
|
||||
// 'Given resumption path "' . $resPath . '" (real path: "' . $realPath . '") is not a directory.'
|
||||
// );
|
||||
// }
|
||||
|
||||
// if (false === is_writable($realPath)) {
|
||||
// throw new Oai_Model_ResumptionTokenException(
|
||||
// 'Given resumption path "' . $resPath . '" (real path: "' . $realPath . '") is not writeable.'
|
||||
// );
|
||||
// }
|
||||
this.resumptionPath = realPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a resumption token
|
||||
*
|
||||
* @param Oai_Model_Resumptiontoken $token Token to store.
|
||||
* @throws Oai_Model_ResumptionTokenException Thrown on file operation error.
|
||||
* @return void
|
||||
*/
|
||||
public storeResumptionToken(token: ResumptionToken): void {
|
||||
// $fileName = $this->generateResumptionName();
|
||||
const uniqueName = "100";
|
||||
const serialToken = JSON.stringify(token);
|
||||
|
||||
// Cache::put($uniqueName, $serialToken, now()->addMinutes(60));
|
||||
this.cache.setEx(uniqueName, 86400, serialToken);
|
||||
// $token->setResumptionId($this->resumptionId);
|
||||
}
|
||||
|
||||
// private async get(key: string) {
|
||||
// return await this.redisClient.get(key);
|
||||
// }
|
||||
|
||||
// public async getResumptionToken(resId: string): Promise<ResumptionToken | null> {
|
||||
// let token: ResumptionToken | null = null;
|
||||
|
||||
// var data = await this.get(resId);
|
||||
// if (data) {
|
||||
// token = JSON.parse(data);
|
||||
// if (token instanceof ResumptionToken) {
|
||||
// return token;
|
||||
// }
|
||||
// }
|
||||
// return token;
|
||||
// }
|
||||
}
|
||||
Loading…
Add table
editor.link_modal.header
Reference in a new issue