/* global nhashFile */
const BlobStoreCustom = require('./blobStoreCustom')
const path = require('path')
const fs = require('fs')
/**
* Random number <= max
* @param {number} max
* @returns {number}
*/
function getRandomInt (max) {
return Math.floor(Math.random() * Math.floor(max))
}
const STORE_SUBFOLDER_COUNT = 400
const MAX_COUNTER = Math.pow(2, 31)
/**
* @classdesc
* Blob store implementation for storing content inside folder, defined in application.blobStores.path
*
* Singleton
*/
class FileSystemBlobStore extends BlobStoreCustom {
/**
* @param {object} storeConfig
* @param {ServerApp} appInstance
* @param {UBSession} sessionInstance
* @param {object} [options]
* @param {boolean} [options.checkStorePath=true]
*/
constructor (storeConfig, appInstance, sessionInstance, options) {
if (storeConfig.historyDepth && storeConfig.keepOriginalFileNames) {
console.warn(`BLOB store "${storeConfig.name}": "historyDepth" is ignored for store with "keepOriginalFileNames"!`)
delete storeConfig.historyDepth
}
super(storeConfig, appInstance, sessionInstance, options)
/**
* Store options
* @type {{checkStorePath: boolean}}
*/
this.options = Object.assign({ checkStorePath: true }, options)
const storePath = this.config.path // already normalized inside argv
if (this.options.checkStorePath) {
if (!fs.existsSync(storePath)) throw new Error(`BLOB store "${this.name}" path "${storePath}" not exists`)
const fStat = fs.statSync(storePath)
if (!fStat.isDirectory()) {
throw new Error(`BLOB store "${this.name}" path "${storePath}" is not a folder`)
}
}
/**
* Normalized path to the store root
*/
this.fullStorePath = storePath
/**
* Logical Unit Count for store. If > 0 then files are stored inside `Logical Unit` sub-folders `/LU01`, `/LU02` etc
* Write operations works with last LU folder
*/
this.LUCount = this.config.LUCount || 0
const tmpFolder = this.tempFolder // already normalized inside argv
if (!fs.existsSync(tmpFolder)) {
throw new Error(`Temp folder "${tmpFolder}" for BLOB store "${this.name}" doesn't exist. Check a "tempPath" store config parameter`)
} else {
this.tempFolder = tmpFolder
}
this.keepOriginalFileNames = (this.config.keepOriginalFileNames === true)
this.storeSize = this.config.storeSize || 'Simple'
this._folderCounter = 0
this.SIZES = {
Simple: 'Simple', Medium: 'Medium', Large: 'Large', Monthly: 'Monthly', Daily: 'Daily', Hourly: 'Hourly'
}
if (!this.SIZES[this.storeSize]) {
throw new Error(`Invalid storeSize "${this.storeSize}" for BLOB store "${this.name}"`)
}
if (this.storeSize === this.SIZES.Medium) {
this._folderCounter = getRandomInt(STORE_SUBFOLDER_COUNT) + 100
} else if (this.storeSize === this.SIZES.Large) {
this._folderCounter = (getRandomInt(STORE_SUBFOLDER_COUNT) + 1) * (getRandomInt(STORE_SUBFOLDER_COUNT) + 1)
}
if ((this.storeSize === this.SIZES.Simple) && (this.LUCount > 0)) {
throw new Error(`BLOB Store '${this.name}': LUCount can not be set for 'Simple' store`)
}
}
/**
* Returns full path to the file with BLOB content
* @param {BlobStoreRequest} request
* @param {BlobStoreItem} blobInfo JSON retrieved from a DB
* @returns {string}
*/
getContentFilePath (request, blobInfo) {
return request.isDirty ? this.getTempFileName(request) : this.getPermanentFileName(blobInfo, request)
}
/**
* Retrieve BLOB content from blob store.
* @param {BlobStoreRequest} request
* @param {BlobStoreItem} blobInfo JSON retrieved from a DB.
* @param {object} [options]
* @param {string|null} [options.encoding] Possible values:
* 'bin' 'ascii' 'binary' 'hex' ucs2/ucs-2/utf16le/utf-16le utf8/utf-8 base64
* if `null` will return {@link Buffer}, if `bin` - ArrayBuffer
* @returns {string|Buffer|ArrayBuffer|null}
*/
getContent (request, blobInfo, options) {
const filePath = this.getContentFilePath(request, blobInfo)
return filePath ? fs.readFileSync(filePath, options) : undefined
}
/**
* Fill HTTP response for getDocument request
* @param {BlobStoreRequest} requestParams
* @param {BlobStoreItem} blobInfo Document metadata. Not used for dirty requests
* @param {THTTPRequest} req
* @param {THTTPResponse} resp
* @param {boolean} [preventChangeRespOnError=false] If `true` - prevents sets resp status code - just returns false on error
* @returns {boolean}
*/
fillResponse (requestParams, blobInfo, req, resp, preventChangeRespOnError) {
let filePath, ct
if (requestParams.isDirty) {
filePath = this.getTempFileName(requestParams)
if (requestParams.fileName) {
ct = this.getMimeType(requestParams.fileName)
}
} else {
filePath = this.getPermanentFileName(blobInfo, requestParams)
ct = blobInfo.ct
}
if (!ct) ct = 'application/octet-stream'
if (filePath) {
resp.statusCode = 200
if (this.PROXY_SEND_FILE_HEADER) {
const storeRelPath = path.relative(this.fullStorePath, filePath)
let head = `${this.PROXY_SEND_FILE_HEADER}: /${this.PROXY_SEND_FILE_LOCATION_ROOT}/${this.name}/${storeRelPath}`
head += `\r\nContent-Type: ${ct}\r\nx-query-params: ${req.parameters}`
// to download file UI uses <a href="..." download="origFileName">,
// so Content-Disposition not required anymore
// moreover - if it passed, then PDF viewer do not open file from passed direct link, but tries to save it
// if (blobInfo && blobInfo.origName) {
// head += `\r\nContent-Disposition: attachment;filename="${blobInfo.origName}"`
// }
console.debug('<- ', head)
resp.writeHead(head)
resp.writeEnd('')
} else {
// if (blobInfo && blobInfo.origName) {
// resp.writeHead(`Content-Type: !STATICFILE\r\nContent-Type: ${ct}\r\nContent-Disposition: attachment;filename="${blobInfo.origName}"`)
// } else {
resp.writeHead(`Content-Type: !STATICFILE\r\nContent-Type: ${ct}`)
// }
resp.writeEnd(filePath)
}
return true
} else {
return preventChangeRespOnError
? false
: resp.notFound(`File path for BLOB item ${requestParams.entity}_${requestParams.attribute}_${requestParams.ID} is empty`)
}
}
/**
* Validate temp file exists, return full path to temp file, null in case of dirty item deletion or throw an error
*
* @param {UBEntityAttribute} attribute
* @param {number} ID
* @param {BlobStoreItem} dirtyItem
* @return {string}
*/
checkTempFileBeforePersist (attribute, ID, dirtyItem) {
if (dirtyItem.deleting) {
const tempPath = this.getTempFileName({
entity: attribute.entity.name,
attribute: attribute.name,
ID
})
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath)
return null
}
const tempPath = this.getTempFileName({
entity: attribute.entity.name,
attribute: attribute.name,
ID
})
if (!fs.existsSync(tempPath)) {
throw new Error(`BLOB store persist: temp file '${tempPath}' does not exist`)
}
const tmpFileStat = fs.statSync(tempPath)
if (tmpFileStat.size <= 0) {
console.error(`BLOB store persist: temp file '${tempPath}' size ${tmpFileStat.size} is invalid`)
throw new global.UB.UBAbort('<<<invalidFileSize>>>')
}
return tempPath
}
/**
* Move content defined by `dirtyItem` from temporary to permanent store.
* TIPS: in v0 (UB<5) if file updated then implementation takes a store from old item.
* This raise a problem - old store may be in archive state (readonly)
* So in UB5 we change implementation to use a store defined in the attribute for new items
*
* Return a new attribute content which describe a place of the BLOB in permanent store
*
* @param {UBEntityAttribute} attribute
* @param {number} ID
* @param {BlobStoreItem} dirtyItem
* @param {number} newRevision
* @returns {BlobStoreItem|null}
*/
persist (attribute, ID, dirtyItem, newRevision) {
const tempPath = this.checkTempFileBeforePersist(attribute, ID, dirtyItem)
if (tempPath === null) { // deleted
return null
}
const newPlacement = this.genNewPlacement(attribute, dirtyItem, ID)
fs.renameSync(tempPath, newPlacement.fullFn)
const newMD5 = nhashFile(newPlacement.fullFn, 'MD5')
const ct = this.getMimeType(newPlacement.ext, true)
const stat = fs.statSync(newPlacement.fullFn)
const resp = {
v: 1,
store: this.name,
fName: newPlacement.fn,
origName: dirtyItem.origName,
relPath: newPlacement.relPath,
ct,
size: stat.size,
md5: newMD5,
revision: newRevision
}
if (dirtyItem.isPermanent) resp.isPermanent = true
return resp
}
safeIncFolderCounter () {
this._folderCounter++
if (this._folderCounter > MAX_COUNTER) this._folderCounter = 100
return this._folderCounter
}
/**
* @override
* @param {UBEntityAttribute} attribute
* @param {number} ID
* @param {BlobStoreItem} blobInfo
*/
doDeletion (attribute, ID, blobInfo) {
let fn
try {
fn = this.getPermanentFileName(blobInfo)
if (fn && fs.existsSync(fn)) {
fs.unlinkSync(fn)
console.info(`removes blob data for ${attribute.entity.code}.${attribute.name} row ID ${ID}`)
}
} catch (e) {
console.error(`BLOB store "${this.name}" - can't delete file "${fn}":`, e)
}
}
/**
* Calculate a relative path & file name for a new BLOB item.
* If new folder does not exist - create it
* @protected
* @param {UBEntityAttribute} attribute
* @param {BlobStoreItem} dirtyItem
* @param {number} ID
* @param {object} options
* @param {boolean} [options.forceFolder=true]
* @returns {{fn: string, ext: string, relPath: string, fullFn: string}}
*/
genNewPlacement (attribute, dirtyItem, ID, options) {
// generate file name for storing file
const forceFolder = !options || (options.forceFolder !== false)
let fn = this.keepOriginalFileNames ? dirtyItem.origName : ''
if (this.keepOriginalFileNames) BlobStoreCustom.validateFileName(fn)
const ext = path.extname(dirtyItem.origName)
if (!fn) {
const entropy = (Date.now() & 0x0000FFFF).toString(16)
fn = `${attribute.entity.sqlAlias || attribute.entity.code}-${attribute.code}${ID}${entropy}${ext}`
}
let l1subfolder = ''
let l2subfolder = ''
if (this.storeSize === this.SIZES.Medium) {
const c = this.safeIncFolderCounter()
l1subfolder = '' + (c % STORE_SUBFOLDER_COUNT + 100)
} else if (this.storeSize === this.SIZES.Large) {
const c = this.safeIncFolderCounter()
l1subfolder = '' + (Math.floor(c / STORE_SUBFOLDER_COUNT) % STORE_SUBFOLDER_COUNT + 100)
l2subfolder = '' + (c % STORE_SUBFOLDER_COUNT + 100)
} else if (this.storeSize === this.SIZES.Monthly || this.storeSize === this.SIZES.Daily || this.storeSize === this.SIZES.Hourly) {
const today = new Date()
const year = today.getFullYear().toString()
const month = (today.getMonth() + 1).toString().padStart(2, '0') // in JS month starts from 0
l1subfolder = `${year}${month}`
if (this.storeSize === this.SIZES.Daily) {
l2subfolder = today.getDate().toString().padStart(2, '0')
} else if (this.storeSize === this.SIZES.Hourly) {
l2subfolder = `${today.getDate().toString().padStart(2, '0')}${path.sep}${today.getHours().toString().padStart(2, '0')}`
// for testing - seconds
// l2subfolder = `${today.getDate().toString().padStart(2, '0')}${path.sep}${today.getSeconds().toString().padStart(2, '0')}`
}
}
// check target folder exists. Create if possible
// use a global cache for already verified folder
let fullFn = this.fullStorePath
let relPath = ''
let mtKey = ''
const mtCfg = this.App.serverConfig.security.multitenancy
const mtIsUsed = mtCfg && mtCfg.enabled && this.Session.tenantID
if (mtIsUsed) {
const tenantFolder = 'T' + this.Session.tenantID
fullFn = path.join(fullFn, tenantFolder)
mtKey = `#MT_${tenantFolder}`
if (forceFolder) {
const cacheKey = `BSFCACHE${this.name}${mtKey}`
const verified = this.App.globalCacheGet(cacheKey) === '1'
if (!verified) {
if (!fs.existsSync(fullFn)) fs.mkdirSync(fullFn, '0777')
this.App.globalCachePut(cacheKey, '1')
}
}
relPath = tenantFolder
}
if (l1subfolder) {
if (this.LUCount) {
const LUN = ('' + this.LUCount).padStart(2, '0')
l1subfolder = `LU${LUN}${path.sep}${l1subfolder}`
}
fullFn = path.join(fullFn, l1subfolder)
let cacheKey = `BSFCACHE#${this.name}${mtKey}#${l1subfolder}`
if (forceFolder) {
const verified = this.App.globalCacheGet(cacheKey) === '1'
if (!verified) {
if (!fs.existsSync(fullFn)) fs.mkdirSync(fullFn, '0777')
this.App.globalCachePut(cacheKey, '1')
}
}
relPath = mtIsUsed ? path.join(relPath, l1subfolder) : l1subfolder
if (l2subfolder) {
fullFn = path.join(fullFn, l2subfolder)
cacheKey = `BSFCACHE#${this.name}${mtKey}#${l1subfolder}#${l2subfolder}`
if (forceFolder) {
const verified = this.App.globalCacheGet(cacheKey) === '1'
if (!verified) {
if (!fs.existsSync(fullFn)) fs.mkdirSync(fullFn, '0777')
this.App.globalCachePut(cacheKey, '1')
}
}
relPath = path.join(relPath, l2subfolder)
}
}
fullFn = path.join(fullFn, fn)
return {
fn,
ext,
relPath,
fullFn
}
}
/**
* For file based store:
* - store.path + relativePath + fileName
* @protected
* @param {BlobStoreItem} blobItem
* @param {BlobStoreRequest} [request] Optional request to get a revision
* @returns {string} In case of item not exists - return empty string ''
*/
getPermanentFileName (blobItem, request) {
let fn = blobItem.fName
let relPath = blobItem.relPath || ''
// v:0 create a folder FormatUTF8('% % %%', [Attribute.Entity.name, Attribute.name, Request.ID])
// and place where file with name = revisionNumber+ext
// sample: {"store":"documents","fName":"doc_outdoc document 3000019161319.pdf","origName":"3000019161319.pdf","relPath":"101\\","ct":"application/pdf","size":499546,"md5":"5224456db8d3c47f5681c5e970826211","revision":5}
if (!blobItem.v) { // UB <5
const ext = path.extname(fn)
const fileFolder = path.basename(fn, ext) // file name without ext
fn = `${blobItem.revision || 0}${ext}` // actual file name is `revision number + ext`
relPath = path.join(relPath, fileFolder)
}
if (process.platform === 'nix') {
if (relPath.indexOf('\\') !== -1) { // in case file written from Windows relPath contains win separator
relPath = relPath.replace(/\\/g, '/')
}
}
if ((this.LUCount) && !relPath.startsWith('LU')) { // for non-lun`ed stores transformed to lune`s old store MUST be mounted into LU00
relPath = `LU00${path.sep}${relPath}`
}
return path.join(this.fullStorePath, relPath, fn)
}
}
module.exports = FileSystemBlobStore