Skip to content

Commit

Permalink
moved all config and json data files loading to new data.manager api …
Browse files Browse the repository at this point in the history 
…and json.data.provider

+ fleshed out data.manager and data.provider interfaces for local data reads
  • Loading branch information
@RandomFractals
RandomFractals committed Aug 7, 2019
1 parent 68adeca commit ac95ce6
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 75 deletions.
68 changes: 46 additions & 22 deletions src/data.manager.ts
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
import * as config from './config';
import {Logger} from './logger';
import {Logger, LogLevel} from './logger';
import {JsonDataProvider} from './data.providers/json.data.provider';

/**
* Data manager api interface.
* Data Manager API interface.
*/
export interface IDataManager {

/**
* Gets IDataProvider instance for the specified file type/extension.
* @param fileType The data file type/extension to get data provider instance for.
* Gets IDataProvider instance for the specified data file mime type or extension.
* @param fileType Data file mime type or extension to get IDataProvider instance for.
*/
getDataProvider(fileType: string): IDataProvider;

/**
* Gets local or remote data.
* @param dataUrl Local data file path or remote data url.
* @param dataParseOptions Optional data parsing options.
*/
getData(dataUrl: string, dataParseOptions?: any): any;
}

/**
Expand All @@ -20,31 +27,29 @@ export interface IDataManager {
export interface IDataProvider {

/**
* Data provider name.
* Supported data provider file mime types or extensions.
*/
name: string;
supportedDataFileTypes: Array<string>;

/**
* Gets data format data.
* Gets local or remote data.
* @param dataUrl Local data file path or remote data url.
* @param parseFunction Optional data parse function override.
* @param parseOptions Optional data parsing options.
*/
getData(dataUrl: string, parseFunction: Function, parseOptions: any): any;
getData(dataUrl: string, parseOptions?: any): any;


/**
* Saves raw Data Provider data.
* @param filePath Data file path.
* Saves raw data provider data.
* @param filePath Local data file path.
* @param fileData Raw data to save.
* @param stringifyFunction Optional stringify function override.
*/
saveData(filePath: string, fileData: any, stringifyFunction: Function): void;
saveData(filePath: string, fileData: any, stringifyFunction?: Function): void;
}

/**
* IDataManager implementation.
* TODO: make this pluggable via data.preview.data.manager setting later on.
* TODO: make this pluggable via data.preview.data.manager setting later.
*/
export class DataManager implements IDataManager {

Expand All @@ -54,15 +59,15 @@ export class DataManager implements IDataManager {
private _logger: Logger = new Logger('data.manager:', config.logLevel);

/**
* Creates new Data manager instance and loads IDataProvider's
* Creates new data manager instance and loads IDataProvider's
* for the supported data formats listed in package.json.
*/
private constructor() {
this._dataProviders = this.loadDataProviders();
}

/**
* Creates Data manager singleton instance.
* Creates data manager singleton instance.
*/
public static get Instance() {
if (!this._instance) {
Expand All @@ -79,23 +84,42 @@ export class DataManager implements IDataManager {

// create data providers instances for the supported data formats
const dataProviders: any = {};
const jsonDataProvider: IDataProvider = new JsonDataProvider('.json');
dataProviders['.json'] = jsonDataProvider;
const jsonDataProvider: IDataProvider = new JsonDataProvider();
jsonDataProvider.supportedDataFileTypes.forEach(fileType => {
dataProviders[fileType] = jsonDataProvider;
});
// TODO: add other data providers loading and initialization here:
// text.data.provider, excel.data.provider, markdown, arrow, avro, etc.
// ...
this._logger.debug('loadDataProviders(): loaded data providers:', Object.keys(dataProviders));
if (this._logger.logLevel === LogLevel.Debug) {
this._logger.debug('loadDataProviders(): loaded data providers:', Object.keys(dataProviders));
}
return dataProviders;
}

/**
* Gets IDataProvider instance for the specified file type/extension.
* @param fileType The data file type/extension to get data provider instance for.
* Gets IDataProvider instance for the specified file mime type or extension.
* @param fileType The data file mime type or extension to get data provider instance for.
*/
getDataProvider(fileType: string): IDataProvider {
public getDataProvider(fileType: string): IDataProvider {
if (this._dataProviders.hasOwnProperty(fileType)) {
return this._dataProviders[fileType];
}
throw new Error(`No matching data provider found for file type: ${fileType}`);
}

/**
* Gets local or remote data.
* @param dataUrl Local data file path or remote data url.
* @param parseOptions Optional data parsing options.
*/
public getData(dataUrl: string, parseOptions?: any): any {
// TODO: add mime types later for remote http data loading
const dataFileType: string = dataUrl.substr(dataUrl.lastIndexOf('.')); // get file extension for now
const dataProvider: IDataProvider = this.getDataProvider(dataFileType);
return dataProvider.getData(dataUrl, parseOptions);
}

}

// export Data manager singleton
Expand Down
52 changes: 11 additions & 41 deletions src/data.preview.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,12 @@ import * as props from 'properties';

// local ext. imports
import * as config from './config';
import * as fileUtils from './utils/file.utils';
import * as jsonUtils from './utils/json.utils';
import {Logger, LogLevel} from './logger';
import {dataManager} from './data.manager';
import {previewManager} from './preview.manager';
import {Template} from './template.manager';
import * as fileUtils from './utils/file.utils';
import * as jsonUtils from './utils/json.utils';

/**
* Data preview web panel serializer for restoring previews on vscode reload.
Expand Down Expand Up @@ -632,28 +633,26 @@ export class DataPreview {
data = this.getExcelData(dataUrl);
break;
case '.env':
data = this.getJsonData(dataUrl, props.parse, {sections: true, comments: ['#']});
data = dataManager.getData(dataUrl, {sections: true, comments: ['#']});
this.logDataStats(data);
break;
case '.ini':
// NOTE: some INI files consider # as a comment
data = this.getJsonData(dataUrl, props.parse, {sections: true, comments: [';', '#']});
data = dataManager.getData(dataUrl, {sections: true, comments: [';', '#']});
this.logDataStats(data);
break;
case '.properties':
data = this.getJsonData(dataUrl, props.parse, {sections: true});
data = dataManager.getData(dataUrl, {sections: true});
this.logDataStats(data);
break;
case '.config':
case '.json':
data = this.getJsonData(dataUrl, JSON.parse);
break;
case '.json5':
data = this.getJsonData(dataUrl, json5.parse);
break;
case '.hjson':
data = this.getJsonData(dataUrl, hjson.parse);
break;
case '.yaml':
case '.yml':
data = this.getJsonData(dataUrl, yaml.load);
data = dataManager.getData(dataUrl);
this.logDataStats(data);
break;
case '.md':
data = this.getMarkdownData(dataUrl);
Expand Down Expand Up @@ -738,35 +737,6 @@ export class DataPreview {
return dataRows;
} // end of getExcelData()

/**
* Gets JSON data array or config object.
* @param dataFilePath Data file path.
* @param parseFunction Data parse function for the supported json/config files.
* @param options Data parsing options.
*/
private getJsonData(dataFilePath: string,
parseFunction: Function,
options: any = null): any {
let data: any = [];
try {
let content: string = fileUtils.readDataFile(dataFilePath, 'utf8');
if (dataFilePath.endsWith('.json')) {
// strip out comments for vscode settings .json config files loading :)
const comments: RegExp = new RegExp(/\/\*[\s\S]*?\*\/|\/\/.*/g);
content = content.replace(comments, '');
}
data = (options) ? parseFunction(content, options) : parseFunction(content);
// convert json configs to properties
data = jsonUtils.convertJsonData(data);
this.logDataStats(data);
}
catch (error) {
this._logger.error(`getJsonData(): Error parsing '${this._dataUrl}'. \n\t Error:`, error.message);
window.showErrorMessage(`Unable to parse data file: '${this._dataUrl}'. \n\t Error: ${error.message}`);
}
return data;
}

/**
* Gets markdown data tables array or config object.
* @param dataFilePath Data file path.
Expand Down
71 changes: 59 additions & 12 deletions src/data.providers/json.data.provider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,17 @@
// vscode imports
import {window} from 'vscode';

// data loading imports
import * as hjson from 'hjson';
import * as json5 from 'json5';
import * as yaml from 'js-yaml';
import * as props from 'properties';

import * as config from '../config';
import * as fileUtils from '../utils/file.utils';
import * as jsonUtils from '../utils/json.utils';
import {Logger, LogLevel} from '../logger';

import {IDataProvider} from '../data.manager';

/**
Expand All @@ -17,29 +24,69 @@ export class JsonDataProvider implements IDataProvider {
private logger: Logger = new Logger('json.data.provider:', config.logLevel);

/**
* Creates new JSON data provider for .json, .json5, and .hjson data files.
* @param name Data provider name.
* Creates new JSON data provider for .json, .json5 and .hjson data files.
*/
constructor() {
this.logger.debug('created for:', this.supportedDataFileTypes);
}

/**
* Gets supported json data file mime types or extensions.
*/
constructor(public name: string = '.json') {
public get supportedDataFileTypes(): Array<string> {
// TODO: add mime types later for http data loading
// TODO: consider implementing separate data provider for each config/json data file type
return ['.config', '.env', '.ini', '.json', '.json5', '.hjson', '.properties', '.yaml', '.yml'];
}

/**
* Gets data provider parse function for the specified data url.
* @param dataUrl Local data file path or remote data url.
*/
public getDataParseFunction(dataUrl: string): Function {
// TODO: add mime types later for remote http data loading
const dataFileType: string = dataUrl.substr(dataUrl.lastIndexOf('.')); // get local file extension for now
let dataParseFunction: Function = JSON.parse; // default
switch (dataFileType) {
case '.env':
case '.ini':
case '.properties':
dataParseFunction = props.parse;
break;
case '.json5':
dataParseFunction = json5.parse;
break;
case '.hjson':
dataParseFunction = hjson.parse;
break;
case '.yaml':
case '.yml':
dataParseFunction = yaml.load;
break;
}
return dataParseFunction;
}

/**
* Gets data format data.
* Gets local or remote data.
* @param dataUrl Local data file path or remote data url.
* @param parseFunction Optional data parse function override.
* @param parseOptions Optional data parsing options.
* TODO: change this to async later.
*/
public getData(dataUrl: string,
parseFunction: Function,
parseOptions: any = null): any {
public getData(dataUrl: string, dataParseOptions?: any): any {
let data: any = [];
try {
const content: string = fileUtils.readDataFile(dataUrl, 'utf8');
data = (parseOptions) ? parseFunction(content, parseOptions) : parseFunction(content);
let content: string = fileUtils.readDataFile(dataUrl, 'utf8');
if (dataUrl.endsWith('.json')) {
// strip out comments for vscode settings .json config files loading :)
const comments: RegExp = new RegExp(/\/\*[\s\S]*?\*\/|\/\/.*/g);
content = content.replace(comments, '');
}
const parseFunction: Function = this.getDataParseFunction(dataUrl);
data = (dataParseOptions) ? parseFunction(content, dataParseOptions) : parseFunction(content);
}
catch (error) {
this.logger.logMessage(LogLevel.Error,
`getData(): Error parsing '${dataUrl}' \n\t Error:`, error.message);
this.logger.logMessage(LogLevel.Error, `getData(): Error parsing '${dataUrl}' \n\t Error:`, error.message);
window.showErrorMessage(`Unable to parse data file: '${dataUrl}'. \n\t Error: ${error.message}`);
}
return jsonUtils.convertJsonData(data);
Expand Down

0 comments on commit ac95ce6

Please sign in to comment.