add new webHDFSUri in env.js.template (#3048)

microsoft · Jul 3, 2019 · 9f3ad77 · 9f3ad77
1 parent f3b95be
commit 9f3ad77
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 8 deletions.
diff --git a/src/webportal/package.json b/src/webportal/package.json
@@ -60,7 +60,6 @@
     "handlebars-loader": "~1.6.0",
     "html-loader": "~0.5.1",
     "html-webpack-plugin": "^4.0.0-beta.5",
-    "@hapi/joi": "~15.1.0",
     "joi-browser": "^13.4.0",
     "jquery": "~3.4.0",
     "js-cookie": "~2.2.0",

diff --git a/src/webportal/src/app/env.js.template b/src/webportal/src/app/env.js.template
@@ -8,6 +8,7 @@ window.ENV = {
   promScrapeTime: '${PROM_SCRAPE_TIME}',
   authnMethod: '${AUTHN_METHOD}',
   pylonAddress: '${PYLON_ADDRESS}',
+  webHDFSUri: '${WEBHDFS_URI}',
 };
 
 window.PAI_PLUGINS = [${WEBPORTAL_PLUGINS}][0] || [];
diff --git a/src/webportal/src/app/job-submission/components/data/data-component.jsx b/src/webportal/src/app/job-submission/components/data/data-component.jsx
@@ -8,7 +8,7 @@ import {MountTreeView} from './mount-tree-view';
 import {SidebarCard} from '../sidebar/sidebar-card';
 import {WebHDFSClient} from '../../utils/webhdfs';
 import {HdfsContext} from '../../models/data/hdfs-context';
-import {getHostNameFromUrl} from '../../utils/utils';
+import {getHostNameFromUrl, getPortFromUrl} from '../../utils/utils';
 import {MountDirectories} from '../../models/data/mount-directories';
 import {
   fetchUserGroup,
@@ -46,10 +46,20 @@ function reducer(state, action) {
 }
 
 export const DataComponent = React.memo((props) => {
-  // add pylonAddress to .env for local debug
-  const hdfsHost = config.hasOwnProperty('pylonAddress') ? getHostNameFromUrl(config.pylonAddress) : window.location.hostname;
-  // TODO: add a judgement whether pylon is ready
-  const hdfsClient = new WebHDFSClient(hdfsHost);
+  const envsubRegex = /^\${.*}$/; // the template string ${xx} will be reserved in envsub if not provide value
+  let hdfsHost;
+  let port;
+  let apiPath;
+  if (!config.webHDFSUri || envsubRegex.test(config.webHDFSUri)) {
+    hdfsHost = window.location.hostname;
+    port = 80;
+    apiPath = '/webhdfs/api/v1';
+  } else {
+    // add WEBHDFS_URI to .env for local debug
+    hdfsHost = getHostNameFromUrl(config.webHDFSUri);
+    port = getPortFromUrl(config.webHDFSUri);
+  }
+  const hdfsClient = new WebHDFSClient(hdfsHost, undefined, undefined, port, apiPath);
   const {onChange} = props;
   const [teamConfigs, setTeamConfigs] = useState();
   const [defaultTeamConfigs, setDefaultTeamConfigs] = useState();

diff --git a/src/webportal/src/app/job-submission/utils/utils.js b/src/webportal/src/app/job-submission/utils/utils.js
@@ -126,6 +126,11 @@ export function getHostNameFromUrl(url) {
   return parser.hostname;
 }
 
+export function getPortFromUrl(url) {
+  const parser = new URL(url);
+  return parser.port;
+}
+
 function addPreCommandsToProtocolTaskRoles(protocol, preCommands) {
   Object.keys(protocol.taskRoles).forEach((taskRoleKey) => {
     const taskRole = protocol.taskRoles[taskRoleKey];

diff --git a/src/webportal/src/app/job-submission/utils/webhdfs.js b/src/webportal/src/app/job-submission/utils/webhdfs.js
@@ -2,7 +2,7 @@ import * as webhdfs from 'webhdfs';
 import {promisify} from 'util';
 
 export class WebHDFSClient {
-  constructor(host, user, timeout, port = '80', path = `/webhdfs/api/v1`) {
+  constructor(host, user, timeout, port = '50070', path = `/webhdfs/v1`) {
     this.host = `http://${host}:${port}`;
     this.endpoint = `http://${host}:${port}${path}`;
     this.client = webhdfs.createClient({host, port, user, path}, {timeout});

diff --git a/src/webportal/yarn.lock b/src/webportal/yarn.lock
@@ -708,7 +708,7 @@
   resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-8.0.1.tgz#9712fa2ad124ac64668ab06ba847b1eaf83a03fd"
   integrity sha512-cctMYH5RLbElaUpZn3IJaUj9QNQD8iXDnl7xNY6KB1aFD2ciJrwpo3kvZowIT75uA+silJFDnSR2kGakALUymg==
 
-"@hapi/joi@~15.1.0":
+"@hapi/joi@^15.1.0":
   version "15.1.0"
   resolved "https://registry.yarnpkg.com/@hapi/joi/-/joi-15.1.0.tgz#940cb749b5c55c26ab3b34ce362e82b6162c8e7a"
   integrity sha512-n6kaRQO8S+kepUTbXL9O/UOL788Odqs38/VOfoCrATDtTvyfiO3fgjlSRaNkHabpTLgM7qru9ifqXlXbXk8SeQ==