Support 2FA and any .netrc file (#61)

* Support 2FA on CDSE datahub

* Enable to use .netrc in non-standard places

* Use specified netrc file with ASF as well

---------

Co-authored-by: Luc Hermitte <[email protected]>

LICENSE.txt

@@ -1,6 +1,7 @@
 MIT License
 
 Copyright (c) 2018-2020 Scott Staniewicz
+Copyright (c) 2024 Luc Hermitte, CS Group, support for double authentication on CDSE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -109,13 +109,17 @@ Options:
                                   not provided the program asks for it
   --cdse-password TEXT            Copernicus Data Space Ecosystem password. If
                                   not provided the program asks for it
+  --cdse-2fa-token TEXT           Copernicus Data Space Ecosystem Two-Factor
+                                  Token. Optional, unless 2FA Authentification
+                                  has been enabled in user profile.
   --asf-user TEXT                 ASF username. If not provided the program
                                   asks for it
   --asf-password TEXT             ASF password. If not provided the program
                                   asks for it
   --ask-password                  ask for passwords interactively if needed
   --update-netrc                  save credentials provided interactively in
                                   the ~/.netrc file if necessary
+  --netrc-file TEXT               Path to .netrc file. Default: ~/.netrc
   --max-workers INTEGER           Number of parallel downloads to run. Note
                                   that CDSE has a limit of 4
   --help                          Show this message and exit.

eof/_auth.py

@@ -4,6 +4,7 @@
 import netrc
 import os
 from pathlib import Path
+from typing import Optional
 
 from ._types import Filename
 from .log import logger as _logger
@@ -23,11 +24,12 @@ def check_netrc(netrc_file: Filename = "~/.netrc"):
 
 
 def setup_netrc(
-    netrc_file: Filename = "~/.netrc",
+    netrc_file: Optional[Filename] = None,
     host: str = NASA_HOST,
     dryrun: bool = False,
 ):
     """Prompt user for NASA/Dataspace username/password, store as attribute of ~/.netrc."""
+    netrc_file = netrc_file or "~/.netrc"
     netrc_file = Path(netrc_file).expanduser()
     try:
         n = netrc.netrc(netrc_file)
@@ -68,7 +70,7 @@ def setup_netrc(
     username, password = _get_username_pass(host)
     if not dryrun:
         # Add account to netrc file
-        n.hosts[host] = (username, None, password)
+        n.hosts[host] = (username, '', password)
         print(f"Saving credentials to {netrc_file} (machine={host}).")
         with open(netrc_file, "w") as f:
             f.write(str(n))
@@ -84,9 +86,12 @@ def _file_is_0600(filename: Filename):
     return oct(Path(filename).stat().st_mode)[-4:] == "0600"
 
 
-def get_netrc_credentials(host: str) -> tuple[str, str]:
+def get_netrc_credentials(host: str, netrc_file: Optional[Filename] = None) -> tuple[str, str]:
     """Get username and password from netrc file for a given host."""
-    n = netrc.netrc()
+    netrc_file = netrc_file or "~/.netrc"
+    netrc_file = Path(netrc_file).expanduser()
+    _logger.debug(f"Using {netrc_file=!r}")
+    n = netrc.netrc(netrc_file)
     auth = n.authenticators(host)
     if auth is None:
         raise ValueError(f"No username/password found for {host} in ~/.netrc")

eof/asf_client.py

@@ -36,6 +36,7 @@ def __init__(
         cache_dir: Optional[Filename] = None,
         username: str = "",
         password: str = "",
+        netrc_file: Optional[Filename] = None,
     ):
         self._cache_dir = cache_dir
         if username and password:
@@ -46,7 +47,7 @@ def __init__(
             self._username = ""
             self._password = ""
             try:
-                self._username, self._password = get_netrc_credentials(NASA_HOST)
+                self._username, self._password = get_netrc_credentials(NASA_HOST, netrc_file)
             except FileNotFoundError:
                 logger.warning("No netrc file found.")
             except ValueError as e:

eof/cli.py

@@ -4,8 +4,10 @@
 from __future__ import annotations
 
 import logging
+from typing import Optional
 
 import click
+from ._types import Filename
 
 from eof import download, log
 from eof._auth import NASA_HOST, DATASPACE_HOST, setup_netrc
@@ -74,6 +76,11 @@
     help="Copernicus Data Space Ecosystem password. "
     "If not provided the program asks for it",
 )
+@click.option(
+    "--cdse-2fa-token",
+    help="Copernicus Data Space Ecosystem Two-Factor Token. "
+    "Optional, unless 2FA Authentification has been enabled in user profile.",
+)
 @click.option(
     "--asf-user",
     help="ASF username. If not provided the program asks for it",
@@ -92,6 +99,10 @@
     is_flag=True,
     help="save credentials provided interactively in the ~/.netrc file if necessary",
 )
+@click.option(
+    "--netrc-file",
+    help="Path to .netrc file. Default: ~/.netrc",
+)
 @click.option(
     "--max-workers",
     type=int,
@@ -111,8 +122,10 @@ def cli(
     asf_password: str = "",
     cdse_user: str = "",
     cdse_password: str = "",
+    cdse_2fa_token: str = "",
     ask_password: bool = False,
     update_netrc: bool = False,
+    netrc_file: Optional[Filename] = None,
     max_workers: int = 3,
 ):
     """Download Sentinel precise orbit files.
@@ -127,9 +140,9 @@ def cli(
     if ask_password:
         dryrun = not update_netrc
         if not force_asf and not (cdse_user and cdse_password):
-            cdse_user, cdse_password = setup_netrc(host=DATASPACE_HOST, dryrun=dryrun)
+            cdse_user, cdse_password = setup_netrc(netrc_file=netrc_file, host=DATASPACE_HOST, dryrun=dryrun)
         if not (cdse_user and cdse_password) and not (asf_user and asf_password):
-            asf_user, asf_password = setup_netrc(host=NASA_HOST, dryrun=dryrun)
+            asf_user, asf_password = setup_netrc(netrc_file=netrc_file, host=NASA_HOST, dryrun=dryrun)
 
     download.main(
         search_path=search_path,
@@ -143,5 +156,7 @@ def cli(
         asf_password=asf_password,
         cdse_user=cdse_user,
         cdse_password=cdse_password,
+        cdse_2fa_token=cdse_2fa_token,
+        netrc_file=netrc_file,
         max_workers=max_workers,
     )

eof/dataspace_client.py

@@ -31,22 +31,30 @@ class DataspaceClient:
     T0 = timedelta(seconds=T_ORBIT + 60)
     T1 = timedelta(seconds=60)
 
-    def __init__(self, username: str = "", password: str = ""):
+    def __init__(
+            self,
+            username: str = "",
+            password: str = "",
+            token_2fa: str = "",
+            netrc_file: Optional[Filename] = None,
+    ):
         if not (username and password):
             logger.debug("Get credentials form netrc")
             try:
-                username, password = get_netrc_credentials(DATASPACE_HOST)
+                username, password = get_netrc_credentials(DATASPACE_HOST, netrc_file)
             except FileNotFoundError:
                 logger.warning("No netrc file found.")
             except ValueError as e:
                 if DATASPACE_HOST not in e.args[0]:
                     raise e
                 logger.warning(
-                    f"No CDSE credentials found in netrc file. Please create one using {SIGNUP_URL}"
+                    f"No CDSE credentials found in netrc file {netrc_file!r}. Please create one using {SIGNUP_URL}"
                 )
 
         self._username = username
         self._password = password
+        self._token_2fa = token_2fa
+        self._netrc_file = netrc_file
 
     def query_orbit(
         self,
@@ -169,6 +177,7 @@ def download_all(
         self,
         query_results: list[dict],
         output_directory: Filename,
+        netrc_file : Optional[Filename] = None,
         max_workers: int = 3,
     ):
         """Download all the specified orbit products."""
@@ -177,6 +186,8 @@ def download_all(
             output_directory=output_directory,
             username=self._username,
             password=self._password,
+            token_2fa=self._token_2fa,
+            netrc_file=netrc_file,
             max_workers=max_workers,
         )
 
@@ -276,15 +287,15 @@ def query_orbit_file_service(query: str) -> list[dict]:
     return query_results
 
 
-def get_access_token(username, password) -> Optional[str]:
+def get_access_token(username, password, token_2fa, netrc_file) -> Optional[str]:
     """Get an access token for the Copernicus Data Space Ecosystem (CDSE) API.
 
     Code from https://documentation.dataspace.copernicus.eu/APIs/Token.html
     """
     if not (username and password):
         logger.debug("Get credentials form netrc")
         try:
-            username, password = get_netrc_credentials(DATASPACE_HOST)
+            username, password = get_netrc_credentials(DATASPACE_HOST, netrc_file)
         except FileNotFoundError:
             logger.warning("No netrc file found.")
             return None
@@ -295,6 +306,8 @@ def get_access_token(username, password) -> Optional[str]:
         "password": password,
         "grant_type": "password",
     }
+    if token_2fa:  # Double authentication is used
+        data["totp"] = token_2fa
 
     try:
         r = requests.post(AUTH_URL, data=data)
@@ -378,6 +391,8 @@ def download_all(
     output_directory: Filename,
     username: str = "",
     password: str = "",
+    token_2fa: str = "",
+    netrc_file: Optional[Filename] = None,
     max_workers: int = 3,
 ) -> list[Path]:
     """Download all the specified orbit products.
@@ -392,6 +407,8 @@ def download_all(
         CDSE username
     password : str
         CDSE password
+    token_2fa : str
+        2FA Token used in profiles with double authentication
     max_workers : int, default = 3
         Maximum parallel downloads from CDSE.
         Note that >4 connections will result in a HTTP 429 Error
@@ -404,7 +421,7 @@ def download_all(
     # )
     # Obtain an access token the download request from the provided credentials
 
-    access_token = get_access_token(username, password)
+    access_token = get_access_token(username, password, token_2fa, netrc_file)
     output_names = []
     download_urls = []
     for query_result in query_results:

eof/download.py

@@ -28,10 +28,12 @@
 import os
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
+from typing import Optional
 
 from dateutil.parser import parse
 from requests.exceptions import HTTPError
 
+from ._types import Filename
 from .asf_client import ASFClient
 from .dataspace_client import DataspaceClient
 from .log import logger
@@ -51,6 +53,8 @@ def download_eofs(
     asf_password: str = "",
     cdse_user: str = "",
     cdse_password: str = "",
+    cdse_2fa_token: str = "",
+    netrc_file: Optional[Filename] = None,
     max_workers: int = MAX_WORKERS,
 ) -> list[Path]:
     """Downloads and saves EOF files for specific dates
@@ -89,7 +93,7 @@ def download_eofs(
 
     # First, check that Scihub isn't having issues
     if not force_asf:
-        client = DataspaceClient(username=cdse_user, password=cdse_password)
+        client = DataspaceClient(username=cdse_user, password=cdse_password, token_2fa=cdse_2fa_token, netrc_file=netrc_file)
         if client._username and client._password:
             # try to search on scihub
             if sentinel_file:
@@ -122,7 +126,7 @@ def download_eofs(
         if not force_asf:
             logger.warning("Dataspace failed, trying ASF")
 
-        asf_client = ASFClient(username=asf_user, password=asf_password)
+        asf_client = ASFClient(username=asf_user, password=asf_password, netrc_file=netrc_file)
         urls = asf_client.get_download_urls(orbit_dts, missions, orbit_type=orbit_type)
         # Download and save all links in parallel
         pool = ThreadPool(processes=max_workers)
@@ -210,6 +214,8 @@ def main(
     asf_password: str = "",
     cdse_user: str = "",
     cdse_password: str = "",
+    cdse_2fa_token: str = "",
+    netrc_file: Optional[Filename] = None,
     max_workers: int = MAX_WORKERS,
 ):
     """Function used for entry point to download eofs"""
@@ -254,5 +260,7 @@ def main(
         asf_password=asf_password,
         cdse_user=cdse_user,
         cdse_password=cdse_password,
+        cdse_2fa_token=cdse_2fa_token,
+        netrc_file=netrc_file,
         max_workers=max_workers,
     )