From 07591f74fe743684f3a017b5a1c9838fae4e2f1b Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Tue, 27 Nov 2018 22:27:23 +0100 Subject: [PATCH] Rework the proxy settings configuration Make several changes in the code to add a new policy for the way we handle proxy settings in bloop. This new policy hopes to reduce proxy configuration problems in different clients that run in different environments in the same machine. --- frontend/src/main/scala/bloop/Cli.scala | 2 + frontend/src/main/scala/bloop/Server.scala | 4 +- .../main/scala/bloop/engine/Interpreter.scala | 1 - .../scala/bloop/util/ProxyEnvironment.scala | 152 ----------------- .../main/scala/bloop/util/ProxySetup.scala | 155 ++++++++++++++++++ website/content/faq/_index.md | 21 ++- 6 files changed, 177 insertions(+), 158 deletions(-) delete mode 100644 frontend/src/main/scala/bloop/util/ProxyEnvironment.scala create mode 100644 frontend/src/main/scala/bloop/util/ProxySetup.scala diff --git a/frontend/src/main/scala/bloop/Cli.scala b/frontend/src/main/scala/bloop/Cli.scala index f9c2b57737..d6ffda83de 100644 --- a/frontend/src/main/scala/bloop/Cli.scala +++ b/frontend/src/main/scala/bloop/Cli.scala @@ -280,6 +280,8 @@ object Cli { debugFilter ) + // Set the proxy settings right before loading the state of the build + bloop.util.ProxySetup.updateProxySettings(commonOpts.env.toMap, logger) val currentState = State.loadActiveStateFor(configDirectory, pool, cliOptions.common, logger) if (Files.exists(configDirectory.underlying)) { diff --git a/frontend/src/main/scala/bloop/Server.scala b/frontend/src/main/scala/bloop/Server.scala index 1f1f963211..cb932e2001 100644 --- a/frontend/src/main/scala/bloop/Server.scala +++ b/frontend/src/main/scala/bloop/Server.scala @@ -2,7 +2,7 @@ package bloop import java.net.InetAddress -import bloop.util.ProxyEnvironment +import bloop.util.ProxySetup import com.martiansoftware.nailgun.{Alias, NGContext, NGServer} import scala.util.Try @@ -19,7 +19,7 @@ object Server { val addr = InetAddress.getLoopbackAddress val server = new NGServer(addr, port) registerAliases(server) - ProxyEnvironment.init() + ProxySetup.init() server } diff --git a/frontend/src/main/scala/bloop/engine/Interpreter.scala b/frontend/src/main/scala/bloop/engine/Interpreter.scala index 83605d3beb..38b570a648 100644 --- a/frontend/src/main/scala/bloop/engine/Interpreter.scala +++ b/frontend/src/main/scala/bloop/engine/Interpreter.scala @@ -20,7 +20,6 @@ object Interpreter { def execute(action: Action, stateTask: Task[State]): Task[State] = { def execute(action: Action, stateTask: Task[State], inRecursion: Boolean): Task[State] = { stateTask.flatMap { state => - bloop.util.ProxyEnvironment.setJvmProxySettings(state) action match { // We keep it case because there is a 'match may not be exhaustive' false positive by scalac // Looks related to existing bug report https://github.com/scala/bug/issues/10251 diff --git a/frontend/src/main/scala/bloop/util/ProxyEnvironment.scala b/frontend/src/main/scala/bloop/util/ProxyEnvironment.scala deleted file mode 100644 index 908e46565b..0000000000 --- a/frontend/src/main/scala/bloop/util/ProxyEnvironment.scala +++ /dev/null @@ -1,152 +0,0 @@ -package bloop.util - -import java.net.URL -import java.net.MalformedURLException - -import scala.collection.JavaConverters._ -import bloop.engine.State -import bloop.logging.Logger -import monix.execution.atomic.AtomicAny - -import scala.util.Try - -object ProxyEnvironment { - - private val startupSettings = AtomicAny[Map[Proxy,ProxySettings]](startup()) - - /** - * Extract environments variables from the given state to set the proxy properties of the jvm. - * If environment variable are not found in the state, the startup properties are used. - * If the property is still no found in startup properties the corresponding property is cleared - * to let user add and remove env var. - * This function will mutate the gobal jvm properties, consequently it's do not support concurent client with differents environments. - * Information came from https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html - * */ - def setJvmProxySettings(state: State): Unit = { - - val map: Map[String, String] = state.commonOptions.env.stringPropertyNames().asScala.map(prop => prop -> state.commonOptions.env.getProperty(prop)).toMap - - val logger = state.logger - - Proxy.values.foreach{ proxy => - //get proxy settings from state environment if there is not proxy settings in state use the settings gift at server startup - val maybeSettings = getEnvProxyProperties(map, proxy.envVar, Some(logger)) orElse startupSettings.get.get(proxy) - maybeSettings match { - case Some(settings) => setGlobalProxyProperties(proxy, settings) - case None => clearGlobalProxyProperties(proxy) - } - } - } - - - - private def setGlobalProxyProperties(proxy : Proxy , settings : ProxySettings): Unit = { - System.setProperty(proxy.propertyNameHost, settings.host) - System.setProperty(proxy.propertyNamePort, settings.port.toString()) - for { - propName <- proxy.propertyNameNoProxy - noProxy <- settings.noProxy - } yield { - System.setProperty(propName, noProxy) - } - () - } - - private def clearGlobalProxyProperties(proxy : Proxy): Unit = { - System.clearProperty(proxy.propertyNameHost) - System.clearProperty(proxy.propertyNamePort) - if (proxy.propertyNameNoProxy.isEmpty) - proxy.propertyNameNoProxy.foreach(System.clearProperty) - } - - - /** Extract ProxySettings from map */ - private def getEnvProxyProperties( - env: collection.Map[String, String], - envVar: String, - logger : Option[Logger] - ): Option[ProxySettings] = { - - val maybeNoProxy = env.get("no_proxy").map(_.replace(',', '|')) - - env.get(envVar).flatMap{ proxyVar => - try { - - val url = new URL(proxyVar) - Some(ProxySettings(url.getHost(),url.getPort(),maybeNoProxy)) - } catch { - case e: MalformedURLException => - logger.foreach(_.warn(s"Unexpected non-URI format in proxy value of $proxyVar")) - None - } - } - } - - /** The goal of this function is to extract proxy settings that was set via jvm -D parameters */ - private def getJvmProxyProperties(proxy : Proxy) : Option[ProxySettings] = { - val maybeVarHost = Option(System.getProperties.getProperty(proxy.propertyNameHost)) - val maybeVarPort = Option(System.getProperties.getProperty(proxy.propertyNamePort)).flatMap(prop => Try(prop.toInt).toOption) - val maybeNoProxy = proxy.propertyNameNoProxy.flatMap(prop => Option(System.getProperties.getProperty(prop))) - maybeVarHost.map{ host => - ProxySettings(host, maybeVarPort.getOrElse(proxy.defaultPort), maybeNoProxy) - } - } - - /* Build the map containing the server startup proxy settings. jvm -D parameters are favoured over environment variables */ - private def startup(): Map[Proxy, ProxySettings] = { - val env = System.getenv().asScala - Proxy.values.flatMap{ pro => - getJvmProxyProperties(pro).orElse(getEnvProxyProperties(env, pro.envVar, None)).map(pro -> _) - }.toMap - } - - /** - * Use server startup proxy settings to parametrise the jvm properties - */ - def init() : Unit = { - startupSettings.get.foreach(x => setGlobalProxyProperties(x._1, x._2)) - } - -} - -sealed trait Proxy { - val propertyNameHost : String - val propertyNamePort : String - val propertyNameNoProxy : Option[String] - val defaultPort : Int - val envVar : String -} -case object HttpProxy extends Proxy { - val propertyNameHost = "http.proxyHost" - val propertyNamePort = "http.proxyPort" - val propertyNameNoProxy = Some("http.nonProxyHosts") - val defaultPort = 80 - val envVar = "http_proxy" -} -case object FtpProxy extends Proxy { - val propertyNameHost = "ftp.proxyHost" - val propertyNamePort = "ftp.proxyPort" - val propertyNameNoProxy = Some("ftp.nonProxyHosts") - val defaultPort = 80 - val envVar = "ftp_proxy" -} -case object HttpsProxy extends Proxy { - val propertyNameHost = "https.proxyHost" - val propertyNamePort = "https.proxyPort" - val propertyNameNoProxy = None - val defaultPort = 443 - val envVar = "https_proxy" -} -case object SocksProxy extends Proxy{ - val propertyNameHost = "socksProxyHost" - val propertyNamePort = "socksProxyHost" - val propertyNameNoProxy = None - val defaultPort = 1080 - val envVar = "socks_proxy" -} -object Proxy{ - val values = List(HttpProxy,FtpProxy, HttpsProxy,SocksProxy) -} - -//Should we fantomize this ? -case class ProxySettings(host: String, port : Int, noProxy : Option[String]) diff --git a/frontend/src/main/scala/bloop/util/ProxySetup.scala b/frontend/src/main/scala/bloop/util/ProxySetup.scala new file mode 100644 index 0000000000..a224576399 --- /dev/null +++ b/frontend/src/main/scala/bloop/util/ProxySetup.scala @@ -0,0 +1,155 @@ +package bloop.util + +import java.net.URL +import java.net.MalformedURLException + +import bloop.logging.Logger +import monix.execution.atomic.{Atomic, AtomicAny} + +import scala.util.Try +import scala.collection.JavaConverters._ + +object ProxySetup { + private case class ProxySettings(host: String, port: Int, nonProxyHosts: Option[String]) + + /** Build map of startup proxy settings set via Java System properties in the server. */ + private val configuredProxySettings: AtomicAny[Map[Proxy, ProxySettings]] = { + def getJvmProxyProperties(proxy: Proxy): Option[ProxySettings] = { + def sysprop(key: String): Option[String] = Option(System.getProperty(key)) + val maybeVarHost = sysprop(proxy.propertyNameHost) + val maybeVarPort = sysprop(proxy.propertyNamePort).flatMap(prop => Try(prop.toInt).toOption) + val maybeNoProxy = proxy.propertyNameNoProxyHosts.flatMap(p => sysprop(p)) + maybeVarHost.map { host => + ProxySettings(host, maybeVarPort.getOrElse(proxy.defaultPort), maybeNoProxy) + } + } + + lazy val env = System.getenv().asScala.toMap + Atomic { + Proxy.supportedProxySettings.flatMap { p => + getJvmProxyProperties(p) + .orElse(getEnvProxyProperties(env, p.envVar, None)) + .map(p -> _) + }.toMap + } + } + + sealed trait Proxy { + val propertyNameHost: String + val propertyNamePort: String + val propertyNameNoProxyHosts: Option[String] + val defaultPort: Int + val envVar: String + } + + object Proxy { + final case object HttpProxy extends Proxy { + val propertyNameHost = "http.proxyHost" + val propertyNamePort = "http.proxyPort" + val propertyNameNoProxyHosts = Some("http.nonProxyHosts") + val defaultPort = 80 + val envVar = "http_proxy" + } + + final case object FtpProxy extends Proxy { + val propertyNameHost = "ftp.proxyHost" + val propertyNamePort = "ftp.proxyPort" + val propertyNameNoProxyHosts = Some("ftp.nonProxyHosts") + val defaultPort = 80 + val envVar = "ftp_proxy" + } + + final case object HttpsProxy extends Proxy { + val propertyNameHost = "https.proxyHost" + val propertyNamePort = "https.proxyPort" + val propertyNameNoProxyHosts = None + val defaultPort = 443 + val envVar = "https_proxy" + } + + final case object SocksProxy extends Proxy { + val propertyNameHost = "socksProxyHost" + val propertyNamePort = "socksProxyHost" + val propertyNameNoProxyHosts = None + val defaultPort = 1080 + val envVar = "socks_proxy" + } + + val supportedProxySettings: Seq[Proxy] = List(HttpProxy, FtpProxy, HttpsProxy, SocksProxy) + } + + /** Forces the initialization of environment variables, called in `Server`. */ + def init(): Unit = () + + /** + * Update proxy settings per client environment variables. + * + * A bloop client can change its environment variables to modify the proxy settings + * globally in the bloop server. However, there is no mechanism to clean proxy + * settings. Users that want to do so must shut down the server and start it again. + * This policy is taken to avoid bad interactions with clients that run on different + * environments where proxy settings are not set (e.g. inside editors like emacs/vscode). + * + * The update process affects all the current bloop clients. Unfortunately, there is no + * way to mitigate this behavior because ivy/coursier/etc do not expose interfaces to + * configure the proxy settings directly. If users define different proxy settings in + * different clients, bloop will have an undefined behavior. Note that users are entitled + * to have a mix of clients with no proxy settings and with the same proxy settings, + * in which case bloop will consistently use the proxy settings set by one of the configured + * clients. + * + * Whenever remote compilation is added, the handling of environment variables must + * be disabled as it's unsafe (it's the only scenario where it's legit to have different + * proxy settings). + * + * Ref https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html + * + * @param environment The environment of the bloop client. + * @param logger A logger where we give feedback to the user. + */ + def updateProxySettings(environment: Map[String, String], logger: Logger): Unit = { + Proxy.supportedProxySettings.foreach { proxy => + val settingsFromEnvironment = + getEnvProxyProperties(environment, proxy.envVar, Some(logger)) + settingsFromEnvironment.foreach { proxySettings => + configuredProxySettings.transform { settingsMap => + settingsMap.get(proxy) match { + // Do nothing if the settings are the same, otherwise update globally + case Some(setSettings) if proxySettings == setSettings => settingsMap + case None => + System.setProperty(proxy.propertyNameHost, proxySettings.host) + System.setProperty(proxy.propertyNamePort, proxySettings.port.toString()) + proxy.propertyNameNoProxyHosts match { + case None => () + case Some(k) => + proxySettings.nonProxyHosts match { + case Some(v) => System.setProperty(k, v) + case None => System.clearProperty(k) + } + } + settingsMap + (proxy -> proxySettings) + } + } + } + } + } + + /** Populate proxy settings from the environment variables. */ + private def getEnvProxyProperties( + env: Map[String, String], + key: String, + logger: Option[Logger] + ): Option[ProxySettings] = { + env.get(key).flatMap { proxyVar => + try { + val url = new URL(proxyVar) + val maybeNoProxy = env.get("no_proxy").map(_.replace(',', '|')) + Some(ProxySettings(url.getHost(), url.getPort(), maybeNoProxy)) + } catch { + case e: MalformedURLException => + logger.foreach(_.warn(s"Expected valid URI format in proxy value of $proxyVar")) + None + } + } + } +} diff --git a/website/content/faq/_index.md b/website/content/faq/_index.md index f5869283f0..c4e712fc47 100644 --- a/website/content/faq/_index.md +++ b/website/content/faq/_index.md @@ -64,9 +64,24 @@ By default the sbt plugin exports only `Compile` and `Test` configurations. If you want to export other sbt configurations too, please read about [advanced sbt configuration]({{}}). -## Bloop can't download some file -You are maybe behind a proxy server. -Set `http_proxy` and `https_proxy` environments variable, Bool will used them to download needed ressources. +## Configure bloop behind a proxy + +Bloop can be configured behind a proxy in two different ways: + +1. Run the bloop server with the right proxy settings. +2. Set the proxy settings as environment variables in the terminal where + you invoke a bloop client (such as the command-line application). + +You can change the proxy settings as many times as you want, however you +need to keep in mind the following considerations: + +1. Changes in the proxy settings of a client affect all bloop clients. +2. It's not possible to clear proxy settings (if the proxy settings are + empty in one of the bloop clients, nothing is done). + +To configure bloop behind a proxy, you can set `http_proxy` and +`https_proxy`. If you need a more advanced configuration, consult the +[Oracle Proxy documentation](https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html). ## Is Bloop open source?