/**
* Utils for load data from different formats. You can find many examples of the use inside models `_initialData` folders.
*
* Sample:
*
csvLoader = require('dataLoader'),
conn = session.connection;
csvLoader.loadSimpleCSVData(conn, __dirname + '/ubm_enum-CDN.csv',
'ubm_enum', 'eGroup;code;name;sortOrder'.split(';'), [0, 1, 2, 3]
);
* Sample with data transformation - in this case we pass transformation function to mapping
* array instead of CSV column index:
*
* var ukraineID = conn.lookup('cdn_country', 'ID', {expression: 'code', condition: 'equal', values: {code: 'UKR'}});
if (!ukraineID) {
throw new Error('Country with code UKR not found');
}
// CSV columns: code,regionType,name,fullName
// we map:
// - parentAdminUnitID to id of Ukraine (constant)
// - regionTypeID fo function what lookup region ID using region code from CSV file
csvLoader.loadSimpleCSVData(conn, __dirname + '/cdn_region_ukraine.csv', 'cdn_region',
['parentAdminUnitID', 'code', 'regionTypeID', 'name', 'caption', 'fullName'],
[
function(){return ukraineID;},
0,
function(row){
var regionType;
regionType = conn.lookup('cdn_regiontype', 'ID', {expression: 'code', condition: 'equal', values: {code: row[1]}});
if (!regionType){
throw new Error('Unknown region type ' + row[1]);
}
return regionType;
},
2, 2, 3
],
1, ','
);
* @module dataLoader
* @author pavel.mash
*/
var csv = require('csv1'),
fs = require('fs');
module.exports = {
loadSimpleCSVData: loadSimpleCSVData,
loadArrayData: loadArrayData,
localizeEntity: localizeEntity
};
/**
* Load data from CSV with delimiter (";" by default)
* @param {UBConnection} conn Connection to UnityBase server
* @param {String} fileName Full path to file
* @param {String} entityName Entity code to load data into
* @param {Array<string>} ettAttributes Array of attribute codes
* @param {Array<Number|Object|Function>} mapping Mapping of CSV file columns to attributes. Can be:
*
* - either numeric (zero based) index of column is CSV file
* - or lookup configuration
* - or function what take a array representing current row in CSW file on input and return a attribute value to bi inserted
*
* @param {Number} [startRow=0] Start from this CSV file row
* @param {String} [delimiter=';'] CSV file delimiter
* @param {Number} [transLen=1000} Maximum rows count to be inserted on the single database transaction
*/
function loadSimpleCSVData(conn, fileName, entityName, ettAttributes, mapping, startRow, delimiter, transLen){
var
fContent, csvData;
if (ettAttributes.length !== mapping.length){ throw new Error('Length of ettAttributes and mapping arrays must be identical'); }
delimiter = delimiter || ";";
transLen = transLen || 1000;
startRow = startRow || 0;
fContent = fs.readFileSync(fileName);
if (!fContent) { throw new Error('File ' + fileName + ' is empty or not exist'); }
fContent = fContent.trim();
csvData = csv.parse(fContent, delimiter);
if( !Array.isArray(csvData) ) {
throw new Error('Invalid CSV format or file ' + fileName + ' not found');
}
if( csvData.length < startRow ){
throw new Error('Length of CSVData (' + csvData.length + ') is smaller then startRow' + startRow);
}
if(startRow>0){
csvData.splice(0, startRow);
}
loadArrayData(conn, csvData, entityName, ettAttributes, mapping, transLen);
}
/**
* Load data from a array (rows) of array (columns data).
* @param {UBConnection} conn Connection to UnityBase server
* @param {Array} dataArray - array to load
* @param {String} entityName Entity code to load data into
* @param {Array<string>} ettAttributes Array of attribute codes
* @param {Array<Number|Object|Function>} mapping Mapping of CSV file columns to attributes. Can be:
* @param {Array<Number|Object|Function>} mapping Mapping of CSV file columns to attributes. Can be:
*
* - either numeric (zero based) index of column is CSV file
* - or lookup configuration
* - or function (currentRowAsArray, newRecordID) what take a array representing current row in CSV file & new RecordID on input and return a attribute value to be inserted
*
* @param {Number} [transLen=1000} Maximum rows count to be inserted on the single database transaction
*/
function loadArrayData(conn, dataArray, entityName, ettAttributes, mapping, transLen){
var
currentRecord,
attrCnt = ettAttributes.length,
curTransCnt = 0,
cmdArray = [],
cmd,
valToInsert,
cmdIdx,
idList,
i, a, curMapObj,
dataLength = dataArray.length;
for (i=0; i < dataLength; true){
cmdArray = [];
// fill add new array and get ID's
for(curTransCnt = 0; (curTransCnt<transLen) && (i+curTransCnt<dataLength); ++curTransCnt){
cmdArray.push({entity: entityName, method: 'addnew', fieldList: ['ID'], __nativeDatasetFormat: true});
}
idList = conn.query(cmdArray);
// fill insert array
cmdArray = [];
for(curTransCnt = 0; (curTransCnt<transLen) && (i<dataLength); ++curTransCnt, ++i){
currentRecord = dataArray[i];
cmdIdx = cmdArray.push({entity: entityName, method: 'insert', execParams:{ID: idList[curTransCnt].resultData[0].ID}});
cmd = cmdArray[cmdIdx-1];
for(a = 0; a < attrCnt; ++a){
curMapObj = mapping[a];
if(_.isNumber(curMapObj) ){
valToInsert = currentRecord[curMapObj];
}else if (_.isFunction(curMapObj) ){
valToInsert = curMapObj(currentRecord, cmd.execParams.ID);
}else {
throw new Error('Invalid mapping definition in element#' + a);
}
cmd.execParams[ettAttributes[a]] = _.isUndefined(valToInsert) ? null : valToInsert;
}
}
conn.query(cmdArray);
}
}
var path = require('path');
/**
* Perform localization of entities data based on config & locale. See *.js in models `_initialData/locale` folder for usage samples.
*
var
loader = require('dataLoader'),
localizationConfig = {
entity: 'ubm_enum',
keyAttribute: 'eGroup;code',
localization: [
{keyValue: 'UBS_MESSAGE_TYPE;user', execParams: {name: 'Користувачів'}},
{keyValue: 'UBS_MESSAGE_TYPE;system', execParams: {name: 'Система'}},
{keyValue: 'UBS_MESSAGE_TYPE;warning', execParams: {name: 'Попереждення'}},
{keyValue: 'UBS_MESSAGE_TYPE;information', execParams: {name: 'Інформація'}}
]
};
loader.localizeEntity(session, localizationConfig, __filename);
* @param {cmd.argv.serverSession} session
* @param {Object} config
* @param {String} config.entity Entity to localize
* @param {String} config.keyAttribute Unique key attribute (language independent) we search row for localize. If the key is a component that values should be separated by a ";"
* @param {Array.<Object>} config.localization Array of object {keyValue: valueOfKeyAttribute}, execParams: {attrToLocalize1: 'localized value', ...}} If config.keyAttribute is complex, key value must be a ; separated string
* @param {String} locale Locale to localize to. Either locale file name (contain file start with locale^ (uk^my_data.js)) or locale ("uk")
*/
function localizeEntity(session, config, locale){
"use strict";
var idValue, command = [],
conn = session.connection,
defaultLang = session.appInfo.defaultLang,
lang = path.basename(locale).split('^')[0],
keys = config.keyAttribute.split(';');
console.info('\tRun localize for', config.entity, 'to locale', lang);
_.forEach(config.localization, function(oneRow){
var execParams = {}, lookupValue = {}, whereCondition = {},
keyValues = oneRow.keyValue ? oneRow.keyValue.split(';') : [];
_.forEach(keys, function(key, idx){
var lookupValue = {};
lookupValue[key] = keyValues[idx];
whereCondition['F'+ key] = {
expression: '[' + key + ']', condition: 'equal', values: lookupValue
};
});
idValue = conn.lookup(config.entity, 'ID', whereCondition);
if (idValue){
// add language prefix
_.forEach(oneRow.execParams, function(value, key){
if (lang === defaultLang) {
execParams[key] = value; // update language
} else {
execParams[key + '_' + lang + '^'] = value; // update language
}
});
execParams.ID = idValue;
command.push({
entity: config.entity,
method: 'update',
__skipOptimisticLock: true,
execParams: execParams
});
} else {
console.warn('\toriginal row do not found for localization condition ' + JSON.stringify(lookupValue));
}
});
conn.query(command);
}