diff --git a/pom.xml b/pom.xml index 78ab5695c..a46671e70 100644 --- a/pom.xml +++ b/pom.xml @@ -162,9 +162,9 @@ 2.8.5 3.1.0-incubating 4.4.1 - 2.10.4 - 2.10.4 - 2.9.8 + 2.6.7 + 2.6.7 + 2.6.7 0.10.7 4.11 0-10 diff --git a/utils/src/main/scala/za/co/absa/enceladus/utils/fs/FileSystemUtils.scala b/utils/src/main/scala/za/co/absa/enceladus/utils/fs/FileSystemUtils.scala index ebbf9d6bc..351a992f8 100644 --- a/utils/src/main/scala/za/co/absa/enceladus/utils/fs/FileSystemUtils.scala +++ b/utils/src/main/scala/za/co/absa/enceladus/utils/fs/FileSystemUtils.scala @@ -37,7 +37,7 @@ object FileSystemUtils { path.toSimpleS3Location match { case Some(s3Location) => // s3 over hadoop fs api - val s3BucketUri: String = s"s3://${s3Location.bucketName}" // s3:// + val s3BucketUri: String = s"${s3Location.protocol}://${s3Location.bucketName}" // s3:// val s3uri: URI = new URI(s3BucketUri) FileSystem.get(s3uri, hadoopConf) diff --git a/utils/src/test/scala/za/co/absa/enceladus/utils/fs/FileSystemUtilsSpec.scala b/utils/src/test/scala/za/co/absa/enceladus/utils/fs/FileSystemUtilsSpec.scala new file mode 100644 index 000000000..f6e72b96c --- /dev/null +++ b/utils/src/test/scala/za/co/absa/enceladus/utils/fs/FileSystemUtilsSpec.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.utils.fs + +import org.apache.hadoop.conf.Configuration +import org.scalatest.funsuite.{AnyFunSuite, AnyFunSuiteLike} +import za.co.absa.enceladus.utils.testUtils.SparkTestBase + +class FileSystemUtilsSpec extends AnyFunSuiteLike with SparkTestBase { + implicit val hadoopConf: Configuration = spark.sparkContext.hadoopConfiguration + + test("hdfs protocol default") { + val fs = FileSystemUtils.getFileSystemFromPath("hdfs://my/path") + assert(fs.getUri.toString == "file:///") + } + +// test("s3 protocol recognition and bucket set") { +// val fs = FileSystemUtils.getFileSystemFromPath("s3://my-bucket/my/path") +// assert(fs.getUri.toString == "s3://my-bucket") +// } +// +// test("s3a protocol recognition and bucket set") { +// val fs = FileSystemUtils.getFileSystemFromPath("s3a://my-bucket/my/path") +// assert(fs.getUri.toString == "s3a://my-bucket") +// } + +}