fix: new AppleID auth with srp (icloud-photos-downloader#972)

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- fix: new AppleID auth with srp [#970](https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues/970)
 - feature: when ran without parameters, `icloudpd` shows help [#963](https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues/963)
 - fix: force_size should not skip subsequent sizes [#955](https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues/955)
 

pyproject.toml

@@ -38,7 +38,8 @@ dependencies = [
     "pytz==2024.1",
     "certifi==2022.12.7",
     "keyring==25.2.1",
-    "keyrings-alt==5.0.1"
+    "keyrings-alt==5.0.1",
+    "srp==1.0.21",
 ]
 
 [project.optional-dependencies]
@@ -102,7 +103,7 @@ where = ["src"]  # list of folders that contain the packages (["."] by default)
 exclude = ["starters"]
 
 [[tool.mypy.overrides]]
-module = ['piexif.*', 'vcr.*']
+module = ['piexif.*', 'vcr.*', 'srp.*']
 ignore_missing_imports = true
 
 [tool.ruff]

src/pyicloud_ipd/base.py

@@ -9,6 +9,9 @@
 from re import match
 import http.cookiejar as cookielib
 import getpass
+import srp
+import base64
+import hashlib
 
 from requests import PreparedRequest, Request, Response
 
@@ -190,30 +193,81 @@ def authenticate(self, force_refresh:bool=False, service:Optional[Any]=None) ->
         if not login_successful:
             LOGGER.debug("Authenticating as %s", self.user["accountName"])
 
-            data = dict(self.user)
-
-            data["rememberMe"] = True
-            data["trustTokens"] = []
-            if self.session_data.get("trust_token"):
-                data["trustTokens"] = [self.session_data.get("trust_token")]
-
             headers = self._get_auth_headers()
-
             scnt = self.session_data.get("scnt")
             if scnt:
                 headers["scnt"] = scnt
-            
+
             session_id = self.session_data.get("session_id")
             if session_id:
                 headers["X-Apple-ID-Session-Id"] = session_id
 
+            class SrpPassword():
+                # srp uses the encoded password at process_challenge(), thus set_encrypt_info() should be called before that
+                def __init__(self, password: str):
+                    self.pwd = password
+
+                def set_encrypt_info(self, salt: bytes, iterations: int) -> None:
+                    self.salt = salt
+                    self.iterations = iterations
+
+                def encode(self) -> bytes:
+                    key_length = 32
+                    return hashlib.pbkdf2_hmac('sha256', hashlib.sha256(self.pwd.encode()).digest(), self.salt, self.iterations, key_length)
+
+            # Step 1: client generates private key a (stored in srp.User) and public key A, sends to server
+            srp_password = SrpPassword(self.user["password"])
+            srp.rfc5054_enable()
+            srp.no_username_in_x()
+            usr = srp.User(self.user["accountName"], srp_password, hash_alg=srp.SHA256)
+            uname, A = usr.start_authentication()
+            data = {
+                'a': base64.b64encode(A).decode(),
+                'accountName': uname,
+                'protocols': ['s2k', 's2k_fo']
+            }
+
+            try:
+                response = self.session.post("%s/signin/init" % self.AUTH_ENDPOINT, data=json.dumps(data), headers=headers)
+                if response.status_code == 401:
+                    raise PyiCloudAPIResponseException(response.text, str(response.status_code))
+            except PyiCloudAPIResponseException as error:
+                msg = "Failed to initiate srp authentication."
+                raise PyiCloudFailedLoginException(msg, error) from error
+
+            # Step 2: server sends public key B, salt, and c to client
+            body = response.json()
+            salt = base64.b64decode(body['salt'])
+            b = base64.b64decode(body['b'])
+            c = body['c']
+            iterations = body['iteration']
+
+            # Step 3: client generates session key M1 and M2 with salt and b, sends to server
+            srp_password.set_encrypt_info(salt, iterations)
+            m1 = usr.process_challenge( salt, b )
+            m2 = usr.H_AMK
+
+            data = {
+                "accountName": uname,
+                "c": c,
+                "m1": base64.b64encode(m1).decode(),
+                "m2": base64.b64encode(m2).decode(),
+                "rememberMe": True,
+                "trustTokens": [],
+            }
+
+            if self.session_data.get("trust_token"):
+                data["trustTokens"] = [self.session_data.get("trust_token")]
+
             try:
-                self.session.post(
-                    "%s/signin" % self.AUTH_ENDPOINT,
+                response = self.session.post(
+                    "%s/signin/complete" % self.AUTH_ENDPOINT,
                     params={"isRememberMeEnabled": "true"},
                     data=json.dumps(data),
                     headers=headers,
                 )
+                if response.status_code == 401:
+                    raise PyiCloudAPIResponseException(response.text, str(response.status_code))
             except PyiCloudAPIResponseException as error:
                 msg = "Invalid email/password combination."
                 raise PyiCloudFailedLoginException(msg, error) from error
@@ -284,7 +338,7 @@ def _validate_token(self) -> Dict[str, Any]:
 
     def _get_auth_headers(self, overrides: Optional[Dict[str, str]]=None) -> Dict[str, str]:
         headers = {
-            "Accept": "*/*",
+            "Accept": "application/json, text/javascript",
             "Content-Type": "application/json",
             "X-Apple-OAuth-Client-Id": "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d",
             "X-Apple-OAuth-Client-Type": "firstPartyAuth",

tests/test_authentication.py

@@ -1,5 +1,6 @@
 import inspect
 import os
+import shutil
 from typing import NamedTuple
 from unittest import TestCase
 
@@ -126,28 +127,11 @@ def test_2fa_required(self) -> None:
     def test_successful_token_validation(self) -> None:
         base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3])
         cookie_dir = os.path.join(base_dir, "cookie")
+        cookie_master_path = os.path.join(self.root_path, "cookie")
 
-        for dir in [base_dir, cookie_dir]:
-            recreate_path(dir)
+        recreate_path(base_dir)
 
-        # We need to create a session file first before we test the auth token validation
-        with vcr.use_cassette(os.path.join(self.vcr_path, "2sa_flow_valid_code.yml")):
-            runner = CliRunner(env={"CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321"})
-            result = runner.invoke(
-                main,
-                [
-                    "--username",
-                    "[email protected]",
-                    "--password",
-                    "password1",
-                    "--no-progress-bar",
-                    "--cookie-directory",
-                    cookie_dir,
-                    "--auth-only",
-                ],
-                input="0\n654321\n",
-            )
-            assert result.exit_code == 0
+        shutil.copytree(cookie_master_path, cookie_dir)
 
         with vcr.use_cassette(os.path.join(self.vcr_path, "successful_auth.yml")):
             runner = CliRunner(env={"CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321"})

tests/vcr_cassettes/2fa_flow_invalid_code.yml

@@ -1,12 +1,11 @@
 interactions:
 - request:
-    body: !!python/unicode '{"accountName": "[email protected]", "password": "password1",
-      "rememberMe": true, "trustTokens": []}'
+    body: !!python/unicode '{"accountName": "[email protected]", "protocols": ["s2k", "s2k_fo"]}'
     headers:
       Accept: ['*/*']
       Accept-Encoding: ['gzip, deflate']
       Connection: ['keep-alive']
-      Content-Length: ['111']
+      Content-Length: ['98']
       Content-Type: ['application/json']
       Origin: ['https://www.icloud.com']
       Referer: ['https://www.icloud.com/']
@@ -20,16 +19,62 @@ interactions:
       X-Apple-OAuth-State: ['DE309E26-942E-11E8-92F5-14109FE0B321']
       X-Apple-Widget-Key: ['d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d']
     method: POST
-    uri: https://idmsa.apple.com/appleauth/auth/signin?isRememberMeEnabled=true
+    uri: https://idmsa.apple.com/appleauth/auth/signin/init
   response:
-    body:
-      string: !!python/unicode '{}'
+    body: {string: '{"iteration":20064,"salt":"UUN/abcdefghijklmnopqr==","protocol":"s2k","version":1,"b":"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcd==","c":"d-123-456789ab-cdef-0123-4567-89abcdef0123:MSA"}'}
     headers:
       Cache-Control:
       - 'no-cache'
       - 'no-store'
       Connection: ['keep-alive']
       Content-Type: ['text/html;charset=UTF-8']
+      Date: ['Fri, 15 Dec 2023 17:28:03 GMT']
+      Pragma: ['no-cache']
+      Referrer-Policy: ['origin']
+      Server: ['Apple']
+      Strict-Transport-Security: ['max-age=31536000; includeSubDomains; preload']
+      Transfer-Encoding: ['chunked']
+      X-Apple-I-Request-ID: ['12345678-1234-1234-1234-123456789012']
+      X-BuildVersion: ['R4_1']
+      X-Content-Type-Options: ['nosniff']
+      X-FRAME-OPTIONS: ['DENY']
+      X-XSS-Protection: ['1; mode=block']
+      content-length: ['23705']
+      scnt: ['scnt-1234567890']
+      vary: ['accept-encoding']
+    status:
+      code: 200
+      message: ''
+- request:
+    body: !!python/unicode '{"accountName": "[email protected]", "rememberMe": true, "trustTokens": []}'
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: ['keep-alive']
+      Content-Length: ['98']
+      Content-Type: ['application/json']
+      Origin: ['https://www.icloud.com']
+      Referer: ['https://www.icloud.com/']
+      User-Agent: ['Opera/9.52 (X11; Linux i686; U; en)']
+      X-Apple-OAuth-Client-Id: ['d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d']
+      X-Apple-OAuth-Client-Type: ['firstPartyAuth']
+      X-Apple-OAuth-Redirect-URI: ['https://www.icloud.com']
+      X-Apple-OAuth-Require-Grant-Code: ['true']
+      X-Apple-OAuth-Response-Mode: ['web_message']
+      X-Apple-OAuth-Response-Type: ['code']
+      X-Apple-OAuth-State: ['DE309E26-942E-11E8-92F5-14109FE0B321']
+      X-Apple-Widget-Key: ['d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d']
+    method: POST
+    uri: https://idmsa.apple.com/appleauth/auth/signin/complete?isRememberMeEnabled=true
+  response:
+    body:
+      string: !!python/unicode '{"auth_type":"hsa2"}'
+    headers:
+      Cache-Control:
+      - 'no-cache'
+      - 'no-store'
+      Connection: ['keep-alive']
+      Content-Type: ['application/json;charset=UTF-8']
       Date: ['Wed, 13 Dec 2023 05:06:31 GMT']
       Location: ['/auth']
       Pragma: ['no-cache']
@@ -39,7 +84,6 @@ interactions:
       Transfer-Encoding: ['chunked']
       X-Apple-Auth-Attributes: ['123456789abcdefg']
       X-Apple-I-Request-ID: ['12345678-1234-1234-1234-123456789012']
-      X-Apple-I-Rscd: ['409']
       X-Apple-ID-Account-Country: ['USA']
       X-Apple-ID-Session-Id: ['sess-1234567890']
       X-Apple-Session-Token: ['token-1234567890']
@@ -75,13 +119,13 @@ interactions:
     uri: https://idmsa.apple.com/appleauth/auth
   response:
     body:
-      string: <script type="application/json" class="boot_args">{"direct":{"scriptSk7Url":"https://appleid.cdn-apple.com/appleauth/static/module-assets/home-45556079a0eee434f5ce.js","scriptUrl":"https://appleid.cdn-apple.com/appleauth/static/jsj/1720813638/widget/auth/hsa2.js","module":"widget/auth/components/hsa2/hsa2","isReact":true,"authUserType":"hsa2","hasTrustedDevices":true,"twoSV":{"supportedPushModes":["voice","sms"],"phoneNumberVerification":{"trustedPhoneNumbers":[{"numberWithDialCode":"+1 (•••) •••-••81","pushMode":"sms","obfuscatedNumber":"(•••) •••-••81","lastTwoDigits":"81","id":1}],"securityCode":{"length":6,"tooManyCodesSent":false,"tooManyCodesValidated":false,"securityCodeLocked":false,"securityCodeCooldown":false},"authenticationType":"hsa2","recoveryUrl":"https://iforgot.apple.com/phone/add?prs_account_nm=dummy%40dummy.com\u0026autoSubmitAccount=true\u0026appId=142","cantUsePhoneNumberUrl":"https://iforgot.apple.com/iforgot/phone/add?context=cantuse\u0026prs_account_nm=dummy%40dummy.com\u0026autoSubmitAccount=true\u0026appId=142","recoveryWebUrl":"https://iforgot.apple.com/password/verify/appleid?prs_account_nm=dummy%40dummy.com\u0026autoSubmitAccount=true\u0026appId=142","repairPhoneNumberUrl":"https://gsa.apple.com/appleid/account/manage/repair/verify/phone","repairPhoneNumberWebUrl":"https://appleid.apple.com/widget/account/repair?#!repair","aboutTwoFactorAuthenticationUrl":"https://support.apple.com/kb/HT204921","autoVerified":false,"showAutoVerificationUI":false,"supportsCustodianRecovery":false,"hideSendSMSCodeOption":false,"supervisedChangePasswordFlow":false,"supportsRecovery":true,"trustedPhoneNumber":{"numberWithDialCode":"+1 (•••) •••-••81","pushMode":"sms","obfuscatedNumber":"(•••) •••-••81","lastTwoDigits":"81","id":1},"hsa2Account":true,"restrictedAccount":false,"managedAccount":false},"authFactors":["robocall","sms","generatedcode"],"source_returnurl":"https://idmsa.apple.com/","sourceAppId":1159},"referrerQuery":"","urlContext":"/appleauth","tag":"\u003Chsa2 class=\u0027auth-v1\u0027 suppress-iforgot=\"{suppressIforgot}\" skip-trust-browser-step=\"{skipTrustBrowserStep}\"\u003E\u003C/hsa2\u003E","authType":"hsa2","authInitialRoute":"auth/verify/phone","appleIDUrl":"https://appleid.apple.com"},"additional":{"canRoute2sv":true}}</script>
+      string: !!python/unicode '<script type="application/json" class="boot_args">{"direct":{"scriptSk7Url":"https://appleid.cdn-apple.com/appleauth/static/module-assets/home-45556079a0eee434f5ce.js","scriptUrl":"https://appleid.cdn-apple.com/appleauth/static/jsj/1720813638/widget/auth/hsa2.js","module":"widget/auth/components/hsa2/hsa2","isReact":true,"authUserType":"hsa2","hasTrustedDevices":true,"twoSV":{"supportedPushModes":["voice","sms"],"phoneNumberVerification":{"trustedPhoneNumbers":[{"numberWithDialCode":"+1 (•••) •••-••81","pushMode":"sms","obfuscatedNumber":"(\u2022\u2022\u2022) \u2022\u2022\u2022-\u2022\u202281","lastTwoDigits":"81","id":1}],"securityCode":{"length":6,"tooManyCodesSent":false,"tooManyCodesValidated":false,"securityCodeLocked":false,"securityCodeCooldown":false},"authenticationType":"hsa2","recoveryUrl":"https://iforgot.apple.com/phone/add?prs_account_nm=dummy%40dummy.com\u0026autoSubmitAccount=true\u0026appId=142","cantUsePhoneNumberUrl":"https://iforgot.apple.com/iforgot/phone/add?context=cantuse\u0026prs_account_nm=dummy%40dummy.com\u0026autoSubmitAccount=true\u0026appId=142","recoveryWebUrl":"https://iforgot.apple.com/password/verify/appleid?prs_account_nm=dummy%40dummy.com\u0026autoSubmitAccount=true\u0026appId=142","repairPhoneNumberUrl":"https://gsa.apple.com/appleid/account/manage/repair/verify/phone","repairPhoneNumberWebUrl":"https://appleid.apple.com/widget/account/repair?#!repair","aboutTwoFactorAuthenticationUrl":"https://support.apple.com/kb/HT204921","autoVerified":false,"showAutoVerificationUI":false,"supportsCustodianRecovery":false,"hideSendSMSCodeOption":false,"supervisedChangePasswordFlow":false,"supportsRecovery":true,"trustedPhoneNumber":{"numberWithDialCode":"+1 (•••) •••-••81","pushMode":"sms","obfuscatedNumber":"(•••) •••-••81","lastTwoDigits":"81","id":1},"hsa2Account":true,"restrictedAccount":false,"managedAccount":false},"authFactors":["robocall","sms","generatedcode"],"source_returnurl":"https://idmsa.apple.com/","sourceAppId":1159},"referrerQuery":"","urlContext":"/appleauth","tag":"\u003Chsa2 class=\u0027auth-v1\u0027 suppress-iforgot=\"{suppressIforgot}\" skip-trust-browser-step=\"{skipTrustBrowserStep}\"\u003E\u003C/hsa2\u003E","authType":"hsa2","authInitialRoute":"auth/verify/phone","appleIDUrl":"https://appleid.apple.com"},"additional":{"canRoute2sv":true}}</script>'
     headers:
       Access-Control-Allow-Credentials: ['true']
       Access-Control-Allow-Origin: ['https://www.icloud.com']
       Cache-Control: ['no-cache, no-store, private']
       Connection: ['keep-alive']
-      Content-Type: ['application/json; charset=UTF-8']
+      Content-Type: ['text/html; charset=UTF-8']
       Date: ['Fri, 15 Dec 2023 09:54:34 GMT']
       Server: ['AppleHttpServer/2f080fc0']
       Strict-Transport-Security: ['max-age=31536000; includeSubDomains']