Skip to content

Commit

Permalink
[Fixes #925 & #1018] The upload box should accept xml and sld files a…
Browse files Browse the repository at this point in the history 
…long with the dataset itself (#1032)
  • Loading branch information
@DavidQuartz
DavidQuartz authored Jun 21, 2022
1 parent 81b455b commit 42d2f7a
Show file tree
Hide file tree
Showing 5 changed files with 311 additions and 46 deletions.
86 changes: 48 additions & 38 deletions geonode_mapstore_client/client/js/routes/UploadDataset.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import React, { useState, useEffect, useRef, useCallback } from 'react';
import PropTypes from 'prop-types';
import omit from 'lodash/omit';
import pick from 'lodash/pick';
import merge from 'lodash/merge';
import { connect } from 'react-redux';
import { createSelector } from 'reselect';
Expand All @@ -23,44 +22,64 @@ import axios from '@mapstore/framework/libs/ajax';
import UploadListContainer from '@js/routes/upload/UploadListContainer';
import UploadContainer from '@js/routes/upload/UploadContainer';
import { getConfigProp } from '@mapstore/framework/utils/ConfigUtils';
import { parseUploadResponse, processUploadResponse } from '@js/utils/ResourceUtils';
import { parseUploadResponse, processUploadResponse, parseUploadFiles } from '@js/utils/ResourceUtils';

const supportedDatasetTypes = [
{
id: 'shp',
label: 'ESRI Shapefile',
format: 'vector',
ext: ['shp'],
requires: ['shp', 'prj', 'dbf', 'shx']
requires: ['shp', 'prj', 'dbf', 'shx'],
optional: ['xml', 'sld']
},
{
id: 'tiff',
label: 'GeoTIFF',
format: 'raster',
ext: ['tiff', 'tif'],
mimeType: ['image/tiff']
mimeType: ['image/tiff'],
optional: ['xml', 'sld']
},
{
id: 'csv',
label: 'Comma Separated Value (CSV)',
format: 'vector',
ext: ['csv'],
mimeType: ['text/csv']
mimeType: ['text/csv'],
optional: ['xml', 'sld']
},
{
id: 'zip',
label: 'Zip Archive',
format: 'archive',
ext: ['zip'],
mimeType: ['application/zip']
mimeType: ['application/zip'],
optional: ['xml', 'sld']
},
{
id: 'xml',
label: 'XML Metadata File',
format: 'metadata',
ext: ['xml'],
mimeType: ['application/json'],
needsFiles: ['shp', 'prj', 'dbf', 'shx', 'csv', 'tiff', 'zip', 'sld']
},
{
id: 'sld',
label: 'Styled Layer Descriptor (SLD)',
format: 'metadata',
ext: ['sld'],
mimeType: ['application/json'],
needsFiles: ['shp', 'prj', 'dbf', 'shx', 'csv', 'tiff', 'zip', 'xml']
}
];


const supportedExtensions = supportedDatasetTypes.map(({ ext }) => ext || []).flat();
const supportedMimeTypes = supportedDatasetTypes.map(({ mimeType }) => mimeType || []).flat();
const supportedRequiresExtensions = supportedDatasetTypes.map(({ requires }) => requires || []).flat();
const supportedLabels = supportedDatasetTypes.map(({ label }) => label ).join(', ');
const supportedLabels = supportedDatasetTypes.map(({ label }) => label).join(', ');
const supportedOptionalExtensions = supportedDatasetTypes.map(({ optional }) => optional || []).flat();

function getFileNameParts(file) {
const { name } = file;
Expand Down Expand Up @@ -95,37 +114,14 @@ function UploadList({
const [loading, setLoading] = useState(false);
const [uploadContainerProgress, setUploadContainerProgress] = useState({});

function parseUploadFiles(uploadFiles) {
return Object.keys(uploadFiles)
.reduce((acc, baseName) => {
const uploadFile = uploadFiles[baseName];
const { requires = [], ext = []} = supportedDatasetTypes.find(({ id }) => id === uploadFile.type) || {};
const cleanedFiles = pick(uploadFiles[baseName].files, [...requires, ...ext]);
const filesKeys = Object.keys(cleanedFiles);
const files = requires.length > 0
? cleanedFiles
: filesKeys.length > 1
? pick(cleanedFiles, ext[0])
: cleanedFiles;
const missingExt = requires.filter((fileExt) => !filesKeys.includes(fileExt));
return {
...acc,
[baseName]: {
...uploadFile,
mainExt: filesKeys.find(key => ext.includes(key)),
files,
missingExt
}
};
}, {});
}

function updateWaitingUploads(uploadFiles) {
const newWaitingUploads = parseUploadFiles(uploadFiles);
// prepare params for parseUploadFiles
const params = { uploadFiles, supportedDatasetTypes, supportedOptionalExtensions, supportedRequiresExtensions };
const newWaitingUploads = parseUploadFiles(params);
setWaitingUploads(newWaitingUploads);
const newReadyUploads = Object.keys(newWaitingUploads)
.reduce((acc, baseName) => {
if (newWaitingUploads[baseName]?.missingExt?.length > 0) {
if (newWaitingUploads[baseName]?.missingExt?.length > 0 || newWaitingUploads[baseName]?.addMissingFiles) {
return acc;
}
return {
Expand All @@ -141,7 +137,7 @@ function UploadList({
const { type } = file;
const { ext } = getFileNameParts(file);
return {
supported: !!(supportedMimeTypes.includes(type) || supportedExtensions.includes(ext) || supportedRequiresExtensions.includes(ext)),
supported: !!(supportedMimeTypes.includes(type) || supportedExtensions.includes(ext) || supportedRequiresExtensions.includes(ext) || supportedOptionalExtensions.includes(ext)),
file
};
});
Expand All @@ -150,11 +146,14 @@ function UploadList({
.filter(({ supported }) => supported)
.reduce((acc, { file }) => {
const { ext, baseName } = getFileNameParts(file);
const type = getDatasetFileType(file);
let type = getDatasetFileType(file);
if (!type && supportedOptionalExtensions.includes(ext)) {
type = checkedFiles.length > 1 ? acc[baseName]?.type : ext;
}
return {
...acc,
[baseName]: {
type,
type: type,
files: {
...acc[baseName]?.files,
[ext]: file
Expand All @@ -173,6 +172,12 @@ function UploadList({
setUploadContainerProgress((prevFiles) => ({ ...prevFiles, [fileName]: percentCompleted }));
};

function getValidFileExt({ files }) {
const validFile = Object.keys(files).find(key => key !== 'sld' && key !== 'xml');

return validFile;
}

function handleUploadProcess() {
if (!loading) {
setLoading(true);
Expand All @@ -181,6 +186,11 @@ function UploadList({
const readyUpload = readyUploads[baseName];
cancelTokens[baseName] = axios.CancelToken;
sources[baseName] = cancelTokens[baseName].source();

const mainExt = (readyUpload.mainExt !== 'sld' && readyUpload.mainExt !== 'xml') ? readyUpload.mainExt : getValidFileExt(readyUpload);

readyUpload.mainExt = mainExt;

return uploadDataset({
file: readyUpload.files[readyUpload.mainExt],
ext: readyUpload.mainExt,
Expand Down
10 changes: 8 additions & 2 deletions geonode_mapstore_client/client/js/routes/upload/PendingUploadCard.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,13 @@ function PendingUploadCard({
progress,
size,
onAbort,
error
error,
addMissingFiles
}) {
return (
<div className="gn-upload-card">
<div className="gn-upload-card-header">
{missingExt.length > 0 ? <div className="gn-upload-card-error"><FaIcon name="exclamation" /></div> : null}
{(missingExt.length > 0 || addMissingFiles) ? <div className="gn-upload-card-error"><FaIcon name="exclamation" /></div> : null}
<div className="gn-upload-card-title">{baseName}</div>
<div>
{error ? <ErrorMessageWithTooltip tooltipId={<Message msgId="gnviewer.invalidUploadMessageErrorTooltip" />} /> : null}
Expand All @@ -53,6 +54,11 @@ function PendingUploadCard({
<Message msgId="gnviewer.missingFiles" />: {missingExt.join(', ')}
</div>
</div>}
{addMissingFiles && <div className="gn-upload-card-body">
<div className="text-danger">
<Message msgId="gnviewer.addMainFiles" />
</div>
</div>}
<div className="gn-upload-card-bottom">
<ul>
{filesExt.map(ext => {
Expand Down
3 changes: 2 additions & 1 deletion geonode_mapstore_client/client/js/routes/upload/UploadContainer.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ function UploadContainer({
{waitingUploadNames.length > 0 ? (
<ul>
{waitingUploadNames.map((baseName) => {
const { files, missingExt = [], error } = waitingUploads[baseName];
const { files, missingExt = [], error, addMissingFiles = false } = waitingUploads[baseName];
const filesExt = Object.keys(files);
const size = getSize(files, filesExt);
return (
Expand All @@ -123,6 +123,7 @@ function UploadContainer({
size={size}
onAbort={abort}
error={error}
addMissingFiles={addMissingFiles}
/>
</li>
);
Expand Down
43 changes: 39 additions & 4 deletions geonode_mapstore_client/client/js/utils/ResourceUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,7 @@ import { getConfigProp, convertFromLegacy, normalizeConfig } from '@mapstore/fra
import { parseDevHostname } from '@js/utils/APIUtils';
import { ProcessTypes, ProcessStatus } from '@js/utils/ResourceServiceUtils';
import { bboxToPolygon } from '@js/utils/CoordinatesUtils';
import uniqBy from 'lodash/uniqBy';
import orderBy from 'lodash/orderBy';
import isString from 'lodash/isString';
import isObject from 'lodash/isObject';
import { uniqBy, orderBy, isString, isObject, pick, difference } from 'lodash';
import { excludeGoogleBackground, extractTileMatrixFromSources } from '@mapstore/framework/utils/LayersUtils';

/**
Expand Down Expand Up @@ -632,3 +629,41 @@ export const cleanUrl = (targetUrl) => {
...(hash && { hash })
});
};

export const parseUploadFiles = (data) => {
const { uploadFiles = {}, supportedDatasetTypes = [], supportedOptionalExtensions = [], supportedRequiresExtensions = [] } = data;
const mainFileTypes = supportedDatasetTypes.filter(file => !file.needsFiles);
const mainFileTypeKeys = mainFileTypes.map(({ id }) => id);

return Object.keys(uploadFiles)
.reduce((acc, baseName) => {
const uploadFile = uploadFiles[baseName] || {};
const { requires = [], ext = [], optional = [], needsFiles = [] } = supportedDatasetTypes.find(({ id }) => id === uploadFile.type) || {};
const cleanedFiles = pick(uploadFile.files, [...requires, ...ext, ...optional, ...needsFiles]);
const filesKeys = Object.keys(cleanedFiles);
const files = requires.length > 0
? cleanedFiles
: filesKeys.length > 1
? pick(cleanedFiles, supportedOptionalExtensions.includes(ext[0]) ? [...needsFiles, ext[0]] : ext[0])
: cleanedFiles;
const newFileKeys = Object.keys(files);
const requiredFilesIncluded = newFileKeys.filter((id) => supportedRequiresExtensions.includes(id)) || [];
const missingExt = requires.length > 0
? requires.filter((fileExt) => !filesKeys.includes(fileExt))
: requiredFilesIncluded.length > 0 ? difference(supportedRequiresExtensions, requiredFilesIncluded) : [];

const mainExt = filesKeys.find(key => ext.includes(key));
const addMissingFiles = supportedOptionalExtensions.includes(mainExt) && missingExt?.length === 0 && !(mainFileTypeKeys.some((type) => newFileKeys.includes(type)));

return {
...acc,
[baseName]: {
...uploadFile,
mainExt,
files,
missingExt,
addMissingFiles
}
};
}, {});
};
Loading

0 comments on commit 42d2f7a

Please sign in to comment.