Merge pull request #172 from neuralinternet/dev

Add rental API functionality

.env.example

@@ -1 +1,3 @@
 WANDB_API_KEY="your_api_key"
+DEALLOCATION_NOTIFY_URL="https://dev.neuralinternet.ai/api/gpus/webhook/deallocation"
+STATUS_NOTIFY_URL="https://dev.neuralinternet.ai/api/gpus/webhook/status-change-warning"

cert/gen_ca.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+#############################################################
+# for generating root CA, server, client private key and cert
+#############################################################
+
+ca_key_bits="4096"
+ca_cert_expire_days="365"
+pem_password="bittensor"
+local_ip=$1
+
+if [ "$local_ip" = "" ]; then
+    echo "Usage: ./gen_ca.sh <local_ip>"
+    exit 1
+fi
+
+echo "1.1. generate root CA"
+openssl req -newkey rsa:"$ca_key_bits" -keyform PEM -keyout ca.key -x509 --days "$ca_cert_expire_days" -outform PEM -passout pass:"$pem_password" -out ca.cer -subj "/C=US/ST=NY/CN=ca.neuralinternet.ai/O=NI"
+
+#for register_api server
+echo "2.1 Generate a server private key."
+openssl genrsa -out server.key "$ca_key_bits"
+
+echo "2.2 Use the server private key to generate a certificate generation request."
+openssl req -new -key server.key -out server.req -sha256 -subj "/C=US/ST=NY/CN=server.neuralinternet.ai/O=NI"
+
+echo "2.3 Use the certificate generation request and the CA cert to generate the server cert."
+openssl x509 -req -in server.req -CA ca.cer -CAkey ca.key -CAcreateserial -set_serial 100 -days "$ca_cert_expire_days" -outform PEM -passin pass:"$pem_password" -out server.cer -sha256 -extensions v3_req -extfile <(
+cat << EOF
+[ v3_req ]
+subjectAltName = @alt_names
+
+[ alt_names ]
+IP.1 = 127.0.0.1
+IP.2 = 0.0.0.0
+IP.3 = "$local_ip"
+EOF
+)
+
+
+echo "2.4 Convert the cer to PEM CRT format"
+openssl x509 -inform PEM -in server.cer -out server.crt
+
+echo "2.5 Clean up – now that the cert has been created, we no longer need the request"
+rm server.req
+
+#for frontend server
+echo "3.1 Generate a client private key."
+openssl genrsa -out client.key "$ca_key_bits"
+
+echo "3.2 Use the client private key to generate a certificate generation request."
+openssl req -new -key client.key -out client.req -subj "/C=US/ST=NY/CN=client.neuralinternet.ai/O=NI"
+
+echo "3.3 Use the certificate generation request and the CA cert to generate the client cert."
+openssl x509 -req -in client.req -CA ca.cer -CAkey ca.key -CAcreateserial -set_serial 101 -days "$ca_cert_expire_days" -outform PEM -out client.cer -passin pass:"$pem_password" -extensions v3_req -extfile <(
+cat << EOF
+[ v3_req ]
+subjectAltName = @alt_names
+
+[ alt_names ]
+IP.1 = 127.0.0.1
+IP.2 = 0.0.0.0
+IP.3 = "$local_ip"
+EOF
+)
+
+echo "3.4 Convert the client certificate and private key to pkcs#12 format for use by browsers."
+openssl pkcs12 -export -inkey client.key -in client.cer -out client.p12 -passout pass:"$pem_password"
+
+echo "3.5. Convert the cer to PEM CRT format"
+openssl x509 -inform PEM -in client.cer -out client.crt
+
+echo "3.6. Clean up – now that the cert has been created, we no longer need the request."
+rm client.req

compute/__init__.py

@@ -18,9 +18,9 @@
 import string
 
 # Define the version of the template module.
-__version__ = "1.4.4"
-__minimal_miner_version__ = "1.4.2"
-__minimal_validator_version__ = "1.4.4"
+__version__ = "1.4.5"
+__minimal_miner_version__ = "1.4.5"
+__minimal_validator_version__ = "1.4.5"
 
 version_split = __version__.split(".")
 __version_as_int__ = (100 * int(version_split[0])) + (10 * int(version_split[1])) + (1 * int(version_split[2]))

compute/protocol.py

@@ -76,6 +76,18 @@ class Allocate(bt.Synapse):
     checking: bool = True
     output: dict = {}
     public_key: str = ""
+    docker_requirement: dict = {
+        "base_image": "ubuntu",
+        "ssh_key": "",
+        "ssh_port": 4444,
+        "volume_path": "/tmp",
+        "dockerfile": ""
+    }
+    docker_change: bool = False
+    docker_action: dict = {
+        "action": "",
+        "ssh_key": "",
+    }
 
     def deserialize(self) -> dict:
         """

compute/utils/parser.py

@@ -55,7 +55,6 @@ def __init__(self, description=None):
             help="List of coldkeys to whitelist. Default: [].",
             default=[],
         )
-
         self.add_validator_argument()
         self.add_miner_argument()
 
@@ -109,9 +108,9 @@ def add_validator_argument(self):
         self.add_argument(
             "--validator.whitelist.updated.threshold",
             dest="validator_whitelist_updated_threshold",
-            help="Total quorum before starting the whitelist. Default: 70.",
+            help="Total quorum before starting the whitelist. Default: 90.",
             type=int,
-            default=60,
+            default=90,
         )
 
     def add_miner_argument(self):
@@ -147,15 +146,22 @@ def add_miner_argument(self):
             "--miner.whitelist.not.updated",
             action="store_true",
             dest="miner_whitelist_not_updated",
-            help="Whitelist validators not using the last version of the code. Default: False.",
-            default=False,
+            help="Whitelist validators not using the last version of the code. Default: True.",
+            default=True,
         )
         self.add_argument(
             "--miner.whitelist.updated.threshold",
             dest="miner_whitelist_updated_threshold",
-            help="Total quorum before starting the whitelist. Default: 50.",
+            help="Total quorum before starting the whitelist. Default: 90.",
+            type=int,
+            default=90,
+        )
+        # add ssh port argument
+        self.add_argument(
+            "--ssh.port",
             type=int,
-            default=60,
+            default=4444,
+            help="The ssh port for the allocation service.",
         )
 
     @staticmethod

compute/utils/version.py

@@ -26,20 +26,21 @@
 import git
 import requests
 import sys
+from packaging import version as packaging_version
 
 
 def get_remote_version_to_number(pattern: str = "__version__"):
     latest_version = version2number(get_remote_version(pattern=pattern))
     if not latest_version:
-        bt.logging.error(f"Github API call failed or version string is incorrect!")
+        bt.logging.error("Github API call failed or version string is incorrect!")
     return latest_version
 
 
 def version2number(version: str):
     try:
-        if version and type(version) is str:
-            version = version.split(".")
-            return (100 * int(version[0])) + (10 * int(version[1])) + (1 * int(version[2]))
+        if version and isinstance(version, str):
+            version_parts = version.split(".")
+            return (100 * int(version_parts[0])) + (10 * int(version_parts[1])) + (1 * int(version_parts[2]))
     except Exception as _:
         pass
     return None
@@ -49,35 +50,36 @@ def get_remote_version(pattern: str = "__version__"):
     url = "https://raw.githubusercontent.com/neuralinternet/compute-subnet/main/compute/__init__.py"
     try:
         response = requests.get(url, timeout=30)
-
         if response.status_code == 200:
             lines = response.text.split("\n")
             for line in lines:
                 if line.startswith(pattern):
                     version_info = line.split("=")[1].strip(" \"'").replace('"', "")
                     return version_info
         else:
-            print("Failed to get file content with status code:", response.status_code)
+            bt.logging.error(f"Failed to get file content with status code: {response.status_code}")
             return None
     except requests.exceptions.Timeout:
-        print("The request timed out after 30 seconds.")
+        bt.logging.error("The request timed out after 30 seconds.")
         return None
     except requests.exceptions.RequestException as e:
-        print("There was an error while handling the request:", e)
+        bt.logging.error(f"There was an error while handling the request: {e}")
         return None
 
 
 def get_local_version():
     try:
-        # loading version from __init__.py
         here = path.abspath(path.dirname(__file__))
         parent = here.rsplit("/", 1)[0]
-        with codecs.open(os.path.join(parent, "__init__.py"), encoding="utf-8") as init_file:
-            version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.M)
+        init_file_path = os.path.join(parent, "__init__.py")
+
+        with codecs.open(init_file_path, encoding="utf-8") as init_file:
+            content = init_file.read()
+            version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", content, re.M)
             version_string = version_match.group(1)
         return version_string
     except Exception as e:
-        bt.logging.error(f"Error getting local version. : {e}")
+        bt.logging.error(f"Error getting local version: {e}")
         return ""
 
 
@@ -86,8 +88,8 @@ def check_version_updated():
     local_version = get_local_version()
     bt.logging.info(f"Version check - remote_version: {remote_version}, local_version: {local_version}")
 
-    if version2number(local_version) < version2number(remote_version):
-        bt.logging.info(f"👩‍👦Update to the latest version is required")
+    if packaging_version.parse(local_version) < packaging_version.parse(remote_version):
+        bt.logging.info("👩‍👦Update to the latest version is required")
         return True
     else:
         return False
@@ -96,24 +98,43 @@ def check_version_updated():
 def update_repo():
     try:
         repo = git.Repo(search_parent_directories=True)
-
         origin = repo.remotes.origin
 
+        bt.logging.info(f"Current repository path: {repo.working_dir}")
+
+        # Check for detached HEAD state
+        if repo.head.is_detached:
+            bt.logging.info("Repository is in a detached HEAD state. Switching to the main branch.")
+            repo.git.checkout('main')
+
+        bt.logging.info(f"Current branch: {repo.active_branch.name}")
+
+        stashed = False
         if repo.is_dirty(untracked_files=True):
-            bt.logging.error("Update failed: Uncommited changes detected. Please commit changes or run `git stash`")
-            return False
+            bt.logging.info("Stashing uncommitted changes")
+            repo.git.stash('push', '-m', 'Auto stash before updating')
+            stashed = True
+
         try:
             bt.logging.info("Try pulling remote repository")
-            origin.pull()
-            bt.logging.info("pulling success")
+            origin.pull(rebase=True)
+            bt.logging.info("Pulling success")
+
+            if stashed:
+                bt.logging.info("Applying stashed changes")
+                repo.git.stash('apply', '--index')
+
+                # Restore the specific file from remote to ensure it is not overwritten by stash
+                repo.git.checkout('origin/main', '--', 'compute/__init__.py')
+
             return True
         except git.exc.GitCommandError as e:
-            bt.logging.info(f"update : Merge conflict detected: {e} Recommend you manually commit changes and update")
+            bt.logging.info(f"Update: Merge conflict detected: {e}. Recommend you manually commit changes and update")
+            if stashed:
+                repo.git.stash('pop')
             return handle_merge_conflict(repo)
-
     except Exception as e:
-        bt.logging.error(f"update failed: {e} Recommend you manually commit changes and update")
-
+        bt.logging.error(f"Update failed: {e}. Recommend you manually commit changes and update")
     return False
 
 
@@ -129,16 +150,16 @@ def handle_merge_conflict(repo):
             bt.logging.info(f"Resolving conflict in file: {file_path}")
             repo.git.checkout("--theirs", file_path)
         repo.index.commit("Resolved merge conflicts automatically")
-        bt.logging.info(f"Merge conflicts resolved, repository updated to remote state.")
-        bt.logging.info(f"✅ Repo update success")
+        bt.logging.info("Merge conflicts resolved, repository updated to remote state.")
+        bt.logging.info("✅ Repo update success")
         return True
     except git.GitCommandError as e:
-        bt.logging.error(f"update failed: {e} Recommend you manually commit changes and update")
+        bt.logging.error(f"Update failed: {e}. Recommend you manually commit changes and update")
         return False
 
 
 def restart_app():
-    bt.logging.info("👩‍🦱app restarted due to the update")
+    bt.logging.info("👩‍🦱App restarted due to the update")
 
     python = sys.executable
     os.execl(python, python, *sys.argv)
@@ -153,44 +174,55 @@ def try_update_packages(force=False):
 
         requirements_path = os.path.join(repo_path, "requirements.txt")
 
+        if not os.path.exists(requirements_path):
+            bt.logging.error("Requirements file does not exist.")
+            return
+
         python_executable = sys.executable
 
         if force:
-            subprocess.check_call(
-                [python_executable], "-m", "pip", "install", "--force-reinstall", "--ignore-installed", "--no-deps", "-r", requirements_path
-            )
-            subprocess.check_call([python_executable], "-m", "pip", "install", "--force-reinstall", "--ignore-installed", "--no-deps", "-e", ".")
+            subprocess.check_call([
+                python_executable, "-m", "pip", "install", "--force-reinstall", "--ignore-installed", "--no-deps", "-r", requirements_path
+            ])
+            subprocess.check_call([
+                python_executable, "-m", "pip", "install", "--force-reinstall", "--ignore-installed", "--no-deps", "-e", repo_path
+            ])
         else:
-            subprocess.check_call([python_executable], "-m", "pip", "install", "-r", requirements_path)
-            subprocess.check_call([python_executable], "-m", "pip", "install", "-e", ".")
+            subprocess.check_call([python_executable, "-m", "pip", "install", "-r", requirements_path])
+            subprocess.check_call([python_executable, "-m", "pip", "install", "-e", repo_path])
 
         bt.logging.info("📦Updating packages finished.")
-
-    except Exception as e:
+    except subprocess.CalledProcessError as e:
+        bt.logging.error(f"Updating packages failed: {e}")
         if not force:
             try_update_packages(force=True)
-        bt.logging.info(f"Updating packages failed {e}")
 
 
 def try_update():
     try:
-        if check_version_updated() is True:
-            bt.logging.info("found the latest version in the repo. try ♻️update...")
-            if update_repo() is True:
+        if check_version_updated():
+            bt.logging.info("Found the latest version in the repo. Try ♻️update...")
+            if update_repo():
                 try_update_packages()
-                restart_app()
+                # Check if the update was successful by comparing the version again
+                if not check_version_updated():
+                    bt.logging.info("Update process completed successfully.")
+                    # Restart the app only if necessary (for example, after all processes are done)
+                    restart_app()
+                else:
+                    bt.logging.info("Update process failed to update the version.")
     except Exception as e:
-        bt.logging.info(f"Try updating failed {e}")
+        bt.logging.error(f"Try updating failed: {e}")
 
 
 def check_hashcat_version(hashcat_path: str = "hashcat"):
     try:
         process = subprocess.run([hashcat_path, "--version"], capture_output=True, check=True)
         if process and process.stdout:
-            bt.logging.info(f"Version of hashcat found: {process.stdout.decode()}".strip("\n"))
+            bt.logging.info(f"Version of hashcat found: {process.stdout.decode().strip()}")
         return True
     except subprocess.CalledProcessError:
         bt.logging.error(
-            f"Hashcat is not available nor installed on the machine. Please make sure hashcat is available in your PATH or give the explicit location using the following argument: --miner.hashcat.path"
+            "Hashcat is not available nor installed on the machine. Please make sure hashcat is available in your PATH or give the explicit location using the following argument: --miner.hashcat.path"
         )
-        exit()
+        exit()