Pr 438 branch

Gemfile

@@ -1,3 +1,8 @@
 source 'https://rubygems.org'
+
+group :legacy do
+  gem 'timeliness', '0.4.3'
+end
+
 # Specify your gem's dependencies in fog-azure-rm.gemspec
 gemspec

fog-azure-rm.gemspec

@@ -22,8 +22,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'minitest', '~> 5.8.4'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'codeclimate-test-reporter' , '~> 1.0.0'
-  spec.add_dependency 'fog-core', '~> 1.43'
-  spec.add_dependency 'fog-json', '~> 1.0.2'
+  spec.add_dependency 'fog-core', '>= 1.43'
+  spec.add_dependency 'fog-json', '>= 1.0.2'
   spec.add_dependency 'azure_mgmt_compute', '~> 0.9.0'
   spec.add_dependency 'azure_mgmt_resources', '~> 0.9.0'
   spec.add_dependency 'azure_mgmt_storage', '~> 0.9.0'
@@ -32,7 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'azure_mgmt_traffic_manager', '~> 0.9.0'
   spec.add_dependency 'azure_mgmt_sql', '~> 0.9.0'
   spec.add_dependency 'azure_mgmt_key_vault', '~> 0.9.0'
-  spec.add_dependency 'azure-storage', '= 0.11.5.preview'
+  spec.add_dependency 'azure-storage-common', '~> 1.1'
+  spec.add_dependency 'azure-storage-blob', '~> 1.1'
   spec.add_dependency 'vhd', '0.0.4'
   spec.add_dependency 'mime-types', '~> 3.0'
 end

lib/fog/azurerm.rb

@@ -75,6 +75,10 @@ module AzureRM
 
   # Main AzureRM fog Provider Module
   module AzureRM
+    # We cannot necessarily set the DEBUG environment variable for fog-azure-rm because
+    # that would log the complete HTTP body since this variable is also used by the Azure storage SDK.
+    # Instead let's offer another way to enable fog-azure-rm debug logging
+    Fog::Logger[:debug] = ::STDERR if ENV["FOG_DEBUG"]
     extend Fog::Provider
     service(:resources, 'Resources')
     service(:dns, 'DNS')

lib/fog/azurerm/compute.rb

@@ -78,7 +78,7 @@ def initialize(options)
           begin
             require 'azure_mgmt_compute'
             require 'azure_mgmt_storage'
-            require 'azure/storage'
+            require 'azure/storage/common'
           rescue LoadError => e
             retry if require('rubygems')
             raise e.message

lib/fog/azurerm/constants.rb

@@ -30,6 +30,7 @@
 ENVIRONMENT_AZURE_CHINA_CLOUD = 'AzureChinaCloud'.freeze
 ENVIRONMENT_AZURE_US_GOVERNMENT = 'AzureUSGovernment'.freeze
 ENVIRONMENT_AZURE_GERMAN_CLOUD = 'AzureGermanCloud'.freeze
+ENVIRONMENT_AZURE_STACK = 'AzureStack'.freeze
 
 # MsRestAzure::AzureOperationError class Error Codes
 ERROR_CODE_RESOURCE_NOT_FOUND = 'ResourceNotFound'.freeze

lib/fog/azurerm/requests/compute/attach_data_disk_to_vm.rb

@@ -137,7 +137,7 @@ def get_unmanaged_disk_object(disk_name, disk_size, lun, storage_account_name, a
         end
 
         def check_blob_exist(storage_account_name, blob_name, access_key)
-          client = Azure::Storage::Client.new(storage_account_name: storage_account_name, storage_access_key: access_key)
+          client = Azure::Storage::Common::Client.new(storage_account_name: storage_account_name, storage_access_key: access_key)
           blob_service = Azure::Storage::Blob::BlobService.new(client: client)
           begin
             blob_prop = blob_service.get_blob_properties('vhds', blob_name)

lib/fog/azurerm/requests/storage/commit_blob_blocks.rb

@@ -4,12 +4,13 @@ class AzureRM
       # This class provides the actual implementation for service calls.
       class Real
         def commit_blob_blocks(container_name, blob_name, blocks, options = {})
-          options[:request_id] = SecureRandom.uuid
-          msg = "commit_blob_blocks: Complete uploading #{blob_name} to the container #{container_name}. options: #{options}"
+          my_options = options.clone
+          my_options[:request_id] = SecureRandom.uuid
+          msg = "commit_blob_blocks: Complete uploading #{blob_name} to the container #{container_name}. options: #{my_options}"
           Fog::Logger.debug msg
 
           begin
-            @blob_client.commit_blob_blocks(container_name, blob_name, blocks, options)
+            @blob_client.commit_blob_blocks(container_name, blob_name, blocks, my_options)
           rescue Azure::Core::Http::HTTPError => ex
             raise_azure_exception(ex, msg)
           end

lib/fog/azurerm/requests/storage/create_block_blob.rb

@@ -4,8 +4,19 @@ class AzureRM
       # This class provides the actual implementation for service calls.
       class Real
         def create_block_blob(container_name, blob_name, body, options = {})
-          options[:request_id] = SecureRandom.uuid
-          msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
+          my_options = options.clone
+          my_options[:request_id] = SecureRandom.uuid
+          if my_options[:create_block_blob_timeout]
+            Fog::Logger.debug "create_block_blob: Setting blob operation timeout to #{my_options[:create_block_blob_timeout]} seconds"
+            my_options[:timeout] = my_options[:create_block_blob_timeout]
+          else
+            # Server side default is 10 minutes per megabyte on average, lets use an avg. speed of at least 100KiB/s
+            # => 64MiB (max for create block) should be uploaded in 67108864B / 102400B/s = 655.36s
+            Fog::Logger.debug 'create_block_blob: Setting blob operation timeout to default of 656 seconds'
+            my_options[:timeout] = 656
+          end
+
+          msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{my_options}"
           Fog::Logger.debug msg
 
           begin
@@ -22,13 +33,13 @@ def create_block_blob(container_name, blob_name, body, options = {})
               data = body.read
             else
               data = Fog::Storage.parse_data(body)
-              options[:content_length] = data[:headers]['Content-Length']
-              options[:content_type] = data[:headers]['Content-Type']
+              my_options[:content_length] = data[:headers]['Content-Length']
+              my_options[:content_type] = data[:headers]['Content-Type']
               data = data[:body]
             end
 
             raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
-            blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
+            blob = @blob_client.create_block_blob(container_name, blob_name, data, my_options)
           rescue Azure::Core::Http::HTTPError => ex
             raise_azure_exception(ex, msg)
           end

lib/fog/azurerm/requests/storage/delete_blob.rb

@@ -4,12 +4,13 @@ class AzureRM
       # This class provides the actual implementation for service calls.
       class Real
         def delete_blob(container_name, blob_name, options = {})
-          options[:request_id] = SecureRandom.uuid
-          msg = "Deleting blob: #{blob_name} in container #{container_name}. options: #{options}"
+          my_options = options.clone
+          my_options[:request_id] = SecureRandom.uuid
+          msg = "Deleting blob: #{blob_name} in container #{container_name}. options: #{my_options}"
           Fog::Logger.debug msg
 
           begin
-            @blob_client.delete_blob(container_name, blob_name, options)
+            @blob_client.delete_blob(container_name, blob_name, my_options)
           rescue Azure::Core::Http::HTTPError => ex
             return true if ex.message.include?('(404)')
             raise_azure_exception(ex, msg)

lib/fog/azurerm/requests/storage/list_blobs.rb

@@ -21,7 +21,7 @@ def list_blobs(container_name, options = {})
               Fog::Logger.debug msg
               temp = @blob_client.list_blobs(container_name, options)
               # Workaround for the issue https://github.com/Azure/azure-storage-ruby/issues/37
-              raise temp unless temp.instance_of?(Azure::Service::EnumerationResults)
+              raise temp unless temp.instance_of?(Azure::Storage::Common::Service::EnumerationResults)
 
               blobs += temp unless temp.empty?
               break if temp.continuation_token.nil? || temp.continuation_token.empty?

lib/fog/azurerm/requests/storage/list_containers.rb

@@ -16,7 +16,7 @@ def list_containers
               Fog::Logger.debug msg
               temp = @blob_client.list_containers(options)
               # Workaround for the issue https://github.com/Azure/azure-storage-ruby/issues/37
-              raise temp unless temp.instance_of?(Azure::Service::EnumerationResults)
+              raise temp unless temp.instance_of?(Azure::Storage::Common::Service::EnumerationResults)
 
               containers += temp unless temp.empty?
               break if temp.continuation_token.nil? || temp.continuation_token.empty?

lib/fog/azurerm/requests/storage/put_blob_block.rb

@@ -4,12 +4,23 @@ class AzureRM
       # This class provides the actual implementation for service calls.
       class Real
         def put_blob_block(container_name, blob_name, block_id, data, options = {})
-          options[:request_id] = SecureRandom.uuid
-          msg = "put_blob_block block_id: #{block_id} / #{blob_name} to the container #{container_name}. options: #{options}"
+          my_options = options.clone
+          my_options[:request_id] = SecureRandom.uuid
+          if my_options[:put_blob_block_timeout]
+            Fog::Logger.debug "put_blob_block: Setting blob operation timeout to #{my_options[:put_blob_block_timeout]} seconds"
+            my_options[:timeout] = my_options[:put_blob_block_timeout]
+          else
+            # Server side default is 10 minutes per megabyte on average, lets use an avg. speed of at least 100KiB/s
+            default_timeout = MAXIMUM_CHUNK_SIZE / 102400
+            Fog::Logger.debug "put_blob_block: Setting blob operation timeout to default of #{default_timeout} seconds"
+            my_options[:timeout] = default_timeout
+          end
+
+          msg = "put_blob_block block_id: #{block_id} / #{blob_name} to the container #{container_name}. options: #{my_options}"
           Fog::Logger.debug msg
 
           begin
-            @blob_client.put_blob_block(container_name, blob_name, block_id, data, options)
+            @blob_client.put_blob_block(container_name, blob_name, block_id, data, my_options)
           rescue Azure::Core::Http::HTTPError => ex
             raise_azure_exception(ex, msg)
           end

lib/fog/azurerm/storage.rb

@@ -12,6 +12,7 @@ class AzureRM < Fog::Service
       # Recognizes when creating data client
       recognizes :azure_storage_account_name
       recognizes :azure_storage_access_key
+      recognizes :azure_storage_dns_suffix
       recognizes :debug
 
       request_path 'fog/azurerm/requests/storage'
@@ -90,7 +91,8 @@ class Real
         def initialize(options)
           begin
             require 'azure_mgmt_storage'
-            require 'azure/storage'
+            require 'azure/storage/common'
+            require 'azure/storage/blob'
             require 'securerandom'
             require 'vhd'
             @debug = ENV['DEBUG'] || options[:debug]
@@ -121,14 +123,24 @@ def initialize(options)
 
           @azure_storage_account_name = options[:azure_storage_account_name]
           @azure_storage_access_key = options[:azure_storage_access_key]
+          @azure_storage_dns_suffix = options[:azure_storage_dns_suffix]
+
+          client_options = {
+            storage_account_name: @azure_storage_account_name,
+            storage_access_key: @azure_storage_access_key,
+            user_agent_prefix: telemetry
+          }
+          client_options[:storage_dns_suffix] = if @environment == ENVIRONMENT_AZURE_STACK
+                                                  @azure_storage_dns_suffix.nil? ? 'local.azurestack.external' : @azure_storage_dns_suffix
+                                                else
+                                                  storage_endpoint_suffix(@environment)[1..-1]
+                                                end
 
-          azure_client = Azure::Storage::Client.create(storage_account_name: @azure_storage_account_name,
-                                                       storage_access_key: @azure_storage_access_key)
-          azure_client.storage_blob_host = get_blob_endpoint(@azure_storage_account_name, true, @environment)
-          @blob_client = azure_client.blob_client
-          @blob_client.with_filter(Azure::Storage::Core::Filter::ExponentialRetryPolicyFilter.new)
+          azure_client = Azure::Storage::Common::Client.create(client_options)
+          @blob_client = Azure::Storage::Blob::BlobService.new(client: azure_client)
+          @blob_client.with_filter(Azure::Storage::Common::Core::Filter::ExponentialRetryPolicyFilter.new)
           @blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
-          @signature_client = Azure::Storage::Core::Auth::SharedAccessSignature.new(@azure_storage_account_name,
+          @signature_client = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@azure_storage_account_name,
                                                                                     @azure_storage_access_key)
         end
       end

test/integration/blob.rb

@@ -8,11 +8,14 @@
 
 azure_credentials = YAML.load_file(File.expand_path('credentials/azure.yml', __dir__))
 
+location = azure_credentials['location']
+
 rs = Fog::Resources::AzureRM.new(
   tenant_id: azure_credentials['tenant_id'],
   client_id: azure_credentials['client_id'],
   client_secret: azure_credentials['client_secret'],
-  subscription_id: azure_credentials['subscription_id']
+  subscription_id: azure_credentials['subscription_id'],
+  environment: azure_credentials['environment']
 )
 
 storage = Fog::Storage::AzureRM.new(
@@ -40,23 +43,27 @@
 begin
   resource_group = rs.resource_groups.create(
     name: resource_group_name,
-    location: LOCATION
+    location: location
   )
 
   storage_account = storage.storage_accounts.create(
     name: storage_account_name,
-    location: LOCATION,
+    location: location,
     resource_group: resource_group_name
   )
 
   access_key = storage_account.get_access_keys[0].value
   Fog::Logger.debug access_key.inspect
-  storage_data = Fog::Storage.new(
+
+  options = {
     provider: 'AzureRM',
     azure_storage_account_name: storage_account.name,
     azure_storage_access_key: access_key,
     environment: azure_credentials['environment']
-  )
+  }
+  options[:storage_dns_suffix] = azure_credentials['azure_storage_dns_suffix'] unless azure_credentials['azure_storage_dns_suffix'].nil?
+
+  storage_data = Fog::Storage.new(options)
 
   ########################################################################################################################
   ######################                                Create Container                            ######################