From c79d9277ac1e06f6efa93495e5cf9603f4be34ba Mon Sep 17 00:00:00 2001 From: donggeon Date: Thu, 23 Jan 2025 19:18:03 +0900 Subject: [PATCH 1/7] feat: add end-of-file-fixer hook --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8cf0c865..bb125230 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,9 +3,10 @@ repos: rev: v5.0.0 hooks: - id: trailing-whitespace + - id: end-of-file-fixer - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook rev: v9.20.0 hooks: - id: commitlint stages: [commit-msg] - additional_dependencies: ['@commitlint/config-angular'] \ No newline at end of file + additional_dependencies: ['@commitlint/config-angular'] From 0ba36f4ce13e5c6da61e55eb88d49f9c02329de8 Mon Sep 17 00:00:00 2001 From: Ivan Anishchuk Date: Tue, 4 Feb 2025 02:58:00 +0700 Subject: [PATCH 2/7] feat(CSN-505): Add pre-commit workflow config Enable pre-commit PR check. --- .github/workflows/pre-commit.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000..2b11178b --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.1 From d4662ec50280115abffc47b952cf99703bfcce6d Mon Sep 17 00:00:00 2001 From: Ivan Anishchuk Date: Tue, 4 Feb 2025 13:07:19 +0700 Subject: [PATCH 3/7] fix(autolint): run pre-commit on all files --- .circleci/config.yml | 2 +- .github/workflows/codeql.yml | 4 +- .gitignore | 2 +- CONTRIBUTING.md | 1 - .../SN27 Installer Instructions.md | 2 +- Installation Script/install_sn27.sh | 38 ++--- README.md | 10 +- cert/gen_ca.sh | 2 +- commitlint-config.js | 2 +- compute/utils/socket.py | 2 +- compute/utils/version.py | 6 +- compute/wandb/wandb.py | 38 ++--- docs/hardware_scoring.md | 2 +- docs/running_on_staging.md | 36 ++--- docs/running_on_testnet.md | 24 ++-- docs/what_are_subnets.md | 4 +- neurons/Miner/allocate.py | 6 +- neurons/Miner/container.py | 4 +- neurons/Miner/http_server.py | 11 +- neurons/Validator/database/allocate.py | 8 +- .../Validator/miner_script_m_merkletree.py | 24 ++-- neurons/Validator/pog.py | 12 +- neurons/Validator/script.py | 14 +- neurons/miner.py | 4 +- neurons/miner_checker.py | 130 +++++++++--------- neurons/register.py | 30 ++-- neurons/register_api.py | 20 +-- neurons/validator.py | 10 +- scripts/opencompute/main.py | 1 - test-scripts/benchmark.py | 2 +- 30 files changed, 224 insertions(+), 227 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5ac17228..76fffdba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -131,7 +131,7 @@ workflows: - check_compatibility: python_version: "3.11" name: check-compatibility-3.11 - + pr-requirements: jobs: - black: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 634fb1e4..54ec03f5 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -34,7 +34,7 @@ jobs: include: - language: python build-mode: none - # CodeQL supports the following values for 'language': + # CodeQL supports the following values for 'language': # 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' # Use `c-cpp` to analyze code written in C, C++ or both # Use 'java-kotlin' to analyze code written in Java, Kotlin or both @@ -51,7 +51,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.10' - + - name: Cache Python dependencies uses: actions/cache@v3 with: diff --git a/.gitignore b/.gitignore index 0bdb20f8..e191b2e8 100644 --- a/.gitignore +++ b/.gitignore @@ -257,5 +257,5 @@ cert/ # wandb wandb/ -# neural internet register api +# neural internet register api neurons/register-api/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87ed8332..5ec1b07a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,4 +93,3 @@ We are committed to providing a welcoming and inclusive environment. All partici If you have any questions or need further guidance, don't hesitate to ask for help in our [community channels](https://discord.gg/t7BMee4w). Our community and maintainers are here to support you. Thank you for contributing to the Compute Subnet! Together, we're building the future of decentralized computing. - diff --git a/Installation Script/SN27 Installer Instructions.md b/Installation Script/SN27 Installer Instructions.md index 0dc45bd1..b834a9cf 100644 --- a/Installation Script/SN27 Installer Instructions.md +++ b/Installation Script/SN27 Installer Instructions.md @@ -1,5 +1,5 @@ # Bittensor/Compute Subnet Installer Script -This repository contains an installation script for setting up a Bittensor miner with the necessary dependencies and configurations for SN27 (Subnet 27) of the Bittensor network. This installation process requires Ubuntu 22.04. You are limited to one external IP per UID. There is automatic blacklisting in place if validators detect anomalous behavior. +This repository contains an installation script for setting up a Bittensor miner with the necessary dependencies and configurations for SN27 (Subnet 27) of the Bittensor network. This installation process requires Ubuntu 22.04. You are limited to one external IP per UID. There is automatic blacklisting in place if validators detect anomalous behavior. ## Features diff --git a/Installation Script/install_sn27.sh b/Installation Script/install_sn27.sh index 14454e05..71401adf 100644 --- a/Installation Script/install_sn27.sh +++ b/Installation Script/install_sn27.sh @@ -71,7 +71,7 @@ ohai() { cd "/usr" || exit 1 linux_install_pre() { - sudo apt-get update + sudo apt-get update sudo apt-get install --no-install-recommends --no-install-suggests -y apt-utils curl git cmake build-essential ca-certificates # Add Docker's official GPG key: @@ -110,10 +110,10 @@ linux_install_python() { ohai "Updating python" sudo apt-get install --only-upgrade $python fi - exit_on_error $? + exit_on_error $? ohai "Installing python tools" - sudo apt-get install --no-install-recommends --no-install-suggests -y $python-pip $python-dev - exit_on_error $? + sudo apt-get install --no-install-recommends --no-install-suggests -y $python-pip $python-dev + exit_on_error $? } linux_update_pip() { @@ -129,7 +129,7 @@ linux_install_bittensor() { git clone https://github.com/opentensor/bittensor.git ~/.bittensor/bittensor/ 2> /dev/null || (cd ~/.bittensor/bittensor/ ; git fetch origin master ; git checkout master ; git pull --ff-only ; git reset --hard ; git clean -xdf) ohai "Installing bittensor" $python -m pip install -e ~/.bittensor/bittensor/ - exit_on_error $? + exit_on_error $? } linux_increase_ulimit(){ @@ -158,20 +158,20 @@ linux_install_compute_subnet() { ohai "Cloning Compute-Subnet into ~/Compute-Subnet" mkdir -p ~/Compute-Subnet git clone https://github.com/neuralinternet/Compute-Subnet.git ~/Compute-Subnet/ 2> /dev/null || (cd ~/Compute-Subnet/ ; git pull --ff-only ; git reset --hard ; git clean -xdf) - + ohai "Installing Compute-Subnet dependencies" cd ~/Compute-Subnet $python -m pip install -r requirements.txt $python -m pip install --no-deps -r requirements-compute.txt $python -m pip install -e . sudo apt -y install ocl-icd-libopencl1 pocl-opencl-icd - + ohai "Starting Docker service, adding user to docker, and installing 'at' package" sudo groupadd docker sudo usermod -aG docker $USER sudo systemctl start docker sudo apt install -y at - + cd ~ exit_on_error $? } @@ -240,15 +240,15 @@ if [[ "$OS" == "Linux" ]]; then abort "This linux based install requires apt. To run with other distros (centos, arch, etc), you will need to manually install the requirements" fi echo """ - - ░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓████████▓▒░ -░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ -░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ - ░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ - ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ - ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ -░▒▓███████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓████████▓▒░ ░▒▓█▓▒░ - + + ░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓████████▓▒░ +░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ +░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ + ░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ + ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ + ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ +░▒▓███████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓████████▓▒░ ░▒▓█▓▒░ + - Bittensor; Mining a new element. """ ohai "This script will install:" @@ -297,14 +297,14 @@ if [[ "$OS" == "Linux" ]]; then elif [[ "$OS" == "Darwin" ]]; then echo """ - + ██████╗░██╗████████╗████████╗███████╗███╗░░██╗░██████╗░█████╗░██████╗░ ██╔══██╗██║╚══██╔══╝╚══██╔══╝██╔════╝████╗░██║██╔════╝██╔══██╗██╔══██╗ ██████╦╝██║░░░██║░░░░░░██║░░░█████╗░░██╔██╗██║╚█████╗░██║░░██║██████╔╝ ██╔══██╗██║░░░██║░░░░░░██║░░░██╔══╝░░██║╚████║░╚═══██╗██║░░██║██╔══██╗ ██████╦╝██║░░░██║░░░░░░██║░░░███████╗██║░╚███║██████╔╝╚█████╔╝██║░░██║ ╚═════╝░╚═╝░░░╚═╝░░░░░░╚═╝░░░╚══════╝╚═╝░░╚══╝╚═════╝░░╚════╝░╚═╝░░╚═╝ - + - Mining a new element. """ ohai "This script will install:" diff --git a/README.md b/README.md index 71b4c773..b3b7c672 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ This repository serves as a compute-composable subnet, integrating various cloud ## Installation -This repository requires python3.8 or higher. To install, simply clone this repository and install the requirements. You are limited to one external IP per UID. There is automatic blacklisting in place if validators detect anomalous behavior. +This repository requires python3.8 or higher. To install, simply clone this repository and install the requirements. You are limited to one external IP per UID. There is automatic blacklisting in place if validators detect anomalous behavior. ### Bittensor @@ -143,7 +143,7 @@ If you have more complicated needs, see the [subtensor](https://github.com/opent # Running a Miner / Validator Prior to running a miner or validator, you must [create a wallet](https://github.com/opentensor/docs/blob/main/reference/btcli.md) -and [register the wallet to a netuid](https://github.com/opentensor/docs/blob/main/subnetworks/registration.md). +and [register the wallet to a netuid](https://github.com/opentensor/docs/blob/main/subnetworks/registration.md). Once you have done so, you can run the miner and validator with the following commands. ## Running Miner @@ -157,7 +157,7 @@ It is important to ensure that port 4444 is open on the host machine or that an ```bash # To run the miner cd neurons -python -m miner.py +python -m miner.py --netuid # The subnet id you want to connect to --subtensor.network # blockchain endpoint you want to connect --wallet.name # name of your wallet @@ -371,8 +371,8 @@ Enter any additional options for hashcat to use. It's recommended to use the ``` > "I don't receive any request, 'Challenge' or 'Specs' or 'Allocation', what could be the reason ?" Starting from v1.6.0, hashcat challenge benchmarking is no longer performed. -Most probably you are running into a **network issue**. -- check your ports +Most probably you are running into a **network issue**. +- check your ports - check your firewall > "I have been deregistered, why ?" diff --git a/cert/gen_ca.sh b/cert/gen_ca.sh index 103ca052..edee2f50 100644 --- a/cert/gen_ca.sh +++ b/cert/gen_ca.sh @@ -78,4 +78,4 @@ echo "3.5. Convert the cer to PEM CRT format" openssl x509 -inform PEM -in client.cer -out client.crt echo "3.6. Clean up - now that the cert has been created, we no longer need the request." -rm client.req \ No newline at end of file +rm client.req diff --git a/commitlint-config.js b/commitlint-config.js index 6eaf62bb..28fe5c5b 100644 --- a/commitlint-config.js +++ b/commitlint-config.js @@ -1 +1 @@ -module.exports = {extends: ['@commitlint/config-conventional']} \ No newline at end of file +module.exports = {extends: ['@commitlint/config-conventional']} diff --git a/compute/utils/socket.py b/compute/utils/socket.py index 92ded007..0e32e12b 100644 --- a/compute/utils/socket.py +++ b/compute/utils/socket.py @@ -15,4 +15,4 @@ def check_port(host, port): return None except socket.error: bt.logging.error(f"API: Couldn't connect to server {host}") - return None \ No newline at end of file + return None diff --git a/compute/utils/version.py b/compute/utils/version.py index 105b22d3..22e967a1 100644 --- a/compute/utils/version.py +++ b/compute/utils/version.py @@ -72,7 +72,7 @@ def get_local_version(): here = path.abspath(path.dirname(__file__)) parent = here.rsplit("/", 1)[0] init_file_path = os.path.join(parent, "__init__.py") - + with codecs.open(init_file_path, encoding="utf-8") as init_file: content = init_file.read() version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", content, re.M) @@ -123,7 +123,7 @@ def update_repo(): if stashed: bt.logging.info("Applying stashed changes") repo.git.stash('apply', '--index') - + # Restore the specific file from remote to ensure it is not overwritten by stash repo.git.checkout('origin/main', '--', 'compute/__init__.py') @@ -225,4 +225,4 @@ def check_hashcat_version(hashcat_path: str = "hashcat"): bt.logging.error( "Hashcat is not available nor installed on the machine. Please make sure hashcat is available in your PATH or give the explicit location using the following argument: --miner.hashcat.path" ) - exit() \ No newline at end of file + exit() diff --git a/compute/wandb/wandb.py b/compute/wandb/wandb.py index 4a9ccad3..10c89b8d 100644 --- a/compute/wandb/wandb.py +++ b/compute/wandb/wandb.py @@ -41,7 +41,7 @@ def __init__(self, config: bt.config, wallet: bt.wallet, role: str): if not wandb_api_key and not netrc_path.exists(): raise ValueError("Please log in to wandb using `wandb login` or set the WANDB_API_KEY environment variable.") - + self.api = wandb.Api() self.project = self.api.project(PUBLIC_WANDB_NAME, entity=PUBLIC_WANDB_ENTITY) self.project_run_id = f"{self.entity}/{self.project.name}" @@ -148,7 +148,7 @@ def update_specs(self): "specs": get_perf_info(encrypted=False), } self.run.config.update(update_dict, allow_val_change=True) - + # Sign the run self.sign_run() @@ -264,7 +264,7 @@ def update_penalized_hotkeys(self, hotkey_list): # Sign the run self.sign_run() - + def update_miner_port_open(self, is_port_open): """ This function updates the port on miner side. @@ -278,7 +278,7 @@ def update_miner_port_open(self, is_port_open): # Track is_port_open self.run.log({"is_port_open": self.run.config["is_port_open"]}) - + # Sign the run self.sign_run() @@ -316,7 +316,7 @@ def get_allocated_hotkeys(self, valid_validator_hotkeys, flag): allocated_keys = run_config.get('allocated_hotkeys') valid_validator_hotkey = hotkey in valid_validator_hotkeys - + # Allow all validator hotkeys for data retrieval only if flag == false if not flag: valid_validator_hotkey = True @@ -483,7 +483,7 @@ def get_penalized_hotkeys(self, valid_validator_hotkeys, flag): bt.logging.info(f"Run ID: {run.id}, Name: {run.name}, Error: {e}") return penalized_keys_list - + def get_penalized_hotkeys_checklist_bak(self, valid_validator_hotkeys, flag): """ This function gets all penalized hotkeys checklist from all validators. @@ -548,21 +548,21 @@ def get_miner_specs(self, queryable_uids): run_config = run.config hotkey = run_config.get('hotkey') specs = run_config.get('specs') - + # check the signature if self.verify_run(run) and specs: # Add the index and (hotkey, specs) tuple to the db_specs_dict if hotkey is valid valid_hotkeys = [axon.hotkey for axon in queryable_uids.values() if axon.hotkey] if hotkey in valid_hotkeys: db_specs_dict[index] = (hotkey, specs) - + except Exception as e: # Handle the exception by logging an error message bt.logging.error(f"An error occurred while getting specs from wandb: {e}") - + # Return the db_specs_dict for further use or inspection return db_specs_dict - + def sign_run(self): # Include the run ID in the data to be signed data_to_sign = self.run_id @@ -617,21 +617,21 @@ def sync_allocated(self, hotkey): else: return False - def get_penalized_hotkeys_checklist(self, valid_validator_hotkeys, flag): + def get_penalized_hotkeys_checklist(self, valid_validator_hotkeys, flag): """ This function gets penalized hotkeys checklist from a specific hardcoded validator. """ # Hardcoded run ID run_id = "neuralinternet/opencompute/0djlnjjs" # Fetch the specific run by its ID self.api.flush() - run = self.api.run(run_id) - if not run: - bt.logging.info(f"No run info found for ID {run_id}.") + run = self.api.run(run_id) + if not run: + bt.logging.info(f"No run info found for ID {run_id}.") return [] # Access the run's configuration - try: - run_config = run.config + try: + run_config = run.config penalized_hotkeys_checklist = run_config.get('penalized_hotkeys_checklist') - return penalized_hotkeys_checklist - except Exception as e: - bt.logging.info(f"Run ID: {run.id}, Name: {run.name}, Error: {e}") + return penalized_hotkeys_checklist + except Exception as e: + bt.logging.info(f"Run ID: {run.id}, Name: {run.name}, Error: {e}") return [] diff --git a/docs/hardware_scoring.md b/docs/hardware_scoring.md index 761e0931..f5270db3 100644 --- a/docs/hardware_scoring.md +++ b/docs/hardware_scoring.md @@ -103,4 +103,4 @@ Applying the weights: It is important to note that the role of validators, in contrast to miners, does not require the integration of GPU instances. Their function revolves around data integrity and accuracy verification, involving relatively modest network traffic and lower computational demands. As a result, their hardware requirements are less intensive, focusing more on -stability and reliability rather than high-performance computation. \ No newline at end of file +stability and reliability rather than high-performance computation. diff --git a/docs/running_on_staging.md b/docs/running_on_staging.md index aafa44bf..bb140972 100644 --- a/docs/running_on_staging.md +++ b/docs/running_on_staging.md @@ -6,7 +6,7 @@ This tutorial will guide you through setting up a local subtensor chain, creatin Begin by installing the required dependencies for running a substrate node. ```bash # Update your system packages -sudo apt update +sudo apt update # Install additional required libraries and tools sudo apt install --assume-yes make build-essential git clang curl libssl-dev llvm libudev-dev protobuf-compiler @@ -82,23 +82,23 @@ btcli wallet new_hotkey --wallet.name validator --wallet.hotkey default ``` ### 9. Mint yourself tokens. -You will need tokens to initialize the intentive mechanism on the chain as well as registering a network (below). +You will need tokens to initialize the intentive mechanism on the chain as well as registering a network (below). Run the following command to mint yourself tokens on your chain. ```bash # Mint tokens for the owner -btcli wallet faucet --wallet.name owner --subtensor.chain_endpoint ws://127.0.0.1:9946 +btcli wallet faucet --wallet.name owner --subtensor.chain_endpoint ws://127.0.0.1:9946 >> Balance: τ0.000000000 ➡ τ100.000000000 # Mint tokens to your validator. -btcli wallet faucet --wallet.name validator --subtensor.chain_endpoint ws://127.0.0.1:9946 +btcli wallet faucet --wallet.name validator --subtensor.chain_endpoint ws://127.0.0.1:9946 >> Balance: τ0.000000000 ➡ τ100.000000000 ``` ### 10. Create a Subnetwork The commands below establish a new subnetwork on the local chain. The cost will be exactly τ100.000000000 for the first network you create. ```bash -btcli subnet create --wallet.name owner --subtensor.chain_endpoint ws://127.0.0.1:9946 +btcli subnet create --wallet.name owner --subtensor.chain_endpoint ws://127.0.0.1:9946 >> Your balance is: τ200.000000000 ->> Do you want to register a subnet for τ100.000000000? [y/n]: +>> Do you want to register a subnet for τ100.000000000? [y/n]: >> Enter password to unlock key: [YOUR_PASSWORD] >> ✅ Registered subnetwork with netuid: 1 ``` @@ -136,24 +136,24 @@ btcli stake add --wallet.name validator --wallet.hotkey default --subtensor.chai Ensure both the miner and validator keys are successfully registered. ```bash btcli subnet list --subtensor.chain_endpoint ws://127.0.0.1:9946 - Subnets - finney -NETUID NEURONS MAX_N DIFFICULTY TEMPO CON_REQ EMISSION BURN(τ) - 1 2 256.00 10.00 M 1000 None 0.00% τ1.00000 - 2 128 + Subnets - finney +NETUID NEURONS MAX_N DIFFICULTY TEMPO CON_REQ EMISSION BURN(τ) + 1 2 256.00 10.00 M 1000 None 0.00% τ1.00000 + 2 128 btcli wallet overview --wallet.name validator --subtensor.chain_endpoint ws://127.0.0.1:9946 -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 +Subnet: 1 +COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 miner default 0 True 100.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ100.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 +1 1 2 τ100.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 + Wallet balance: τ0.0 btcli wallet overview --wallet.name miner --subtensor.chain_endpoint ws://127.0.0.1:9946 -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 +Subnet: 1 +COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 miner default 1 True 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 +1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 + Wallet balance: τ0.0 ``` diff --git a/docs/running_on_testnet.md b/docs/running_on_testnet.md index 167336be..7884529b 100644 --- a/docs/running_on_testnet.md +++ b/docs/running_on_testnet.md @@ -1,5 +1,5 @@ # Running on the Testing Network -This tutorial shows how to use the bittensor testnetwork to create a subnetwork and connect your mechanism to it. It is highly recommended that you run `running_on_staging` first before testnet. Mechanisms running on the testnet are open to anyone, and although they do not emit real TAO, they cost test TAO to create and you should be careful not to expose your private keys, to only use your testnet wallet, not use the same passwords as your mainnet wallet, and make sure your mechanism is resistant to abuse. +This tutorial shows how to use the bittensor testnetwork to create a subnetwork and connect your mechanism to it. It is highly recommended that you run `running_on_staging` first before testnet. Mechanisms running on the testnet are open to anyone, and although they do not emit real TAO, they cost test TAO to create and you should be careful not to expose your private keys, to only use your testnet wallet, not use the same passwords as your mainnet wallet, and make sure your mechanism is resistant to abuse. Note: This will require the `revolution` branch for Bittensor @@ -47,11 +47,11 @@ btcli wallet faucet --wallet.name owner --subtensor.network test ``` 3. Purchasing a slot -Using the test TAO from the previous step you can register your subnet to the chain. This will create a new subnet on the chain and give you the owner permissions to it. The code below shows how to purchase a slot. +Using the test TAO from the previous step you can register your subnet to the chain. This will create a new subnet on the chain and give you the owner permissions to it. The code below shows how to purchase a slot. *Note: Slots cost TAO, you wont get this TAO back, it is instead recycled back into the mechanism to be later mined.* ```bash # Run the register subnetwork command on the locally running chain. -btcli subnet create --subtensor.network test +btcli subnet create --subtensor.network test # Enter the owner wallet name which gives permissions to the coldkey to later define running hyper parameters. >> Enter wallet name (default): owner # Enter your owner wallet name >> Enter password to unlock key: # Enter your wallet password. @@ -89,19 +89,19 @@ This returns information about your registered keys. ```bash # Check that your validator key has been registered. btcli wallet overview --wallet.name validator --subtensor.network test -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 +Subnet: 1 +COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 miner default 0 True 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 +1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 + Wallet balance: τ0.0 # Check that your miner has been registered. btcli wallet overview --wallet.name miner --subtensor.network test -Subnet: 1 -COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 +Subnet: 1 +COLDKEY HOTKEY UID ACTIVE STAKE(τ) RANK TRUST CONSENSUS INCENTIVE DIVIDENDS EMISSION(ρ) VTRUST VPERMIT UPDATED AXON HOTKEY_SS58 miner default 1 True 0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 0 0.00000 14 none 5GTFrsEQfvTsh3WjiEVFeKzFTc2xcf… -1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 - Wallet balance: τ0.0 +1 1 2 τ0.00000 0.00000 0.00000 0.00000 0.00000 0.00000 ρ0 0.00000 + Wallet balance: τ0.0 ``` 12. Edit the default `NETUID=1` and `CHAIN_ENDPOINT=ws://127.0.0.1:9946` arguments in `template/__init__.py` to match your created subnetwork. @@ -128,4 +128,4 @@ btcli root register --subtensor.network test # ensure on testnet Then set your weights for the subnet: ```bash btcli root weights --subtensor.network test # ensure on testnet -``` \ No newline at end of file +``` diff --git a/docs/what_are_subnets.md b/docs/what_are_subnets.md index eee24ce2..9b6631d8 100644 --- a/docs/what_are_subnets.md +++ b/docs/what_are_subnets.md @@ -7,7 +7,7 @@ Bittensor is a library of machine intelligence that continuously grows and share # What is a subnet? -Bittensor is releasing its own language for creating incentive mechanisms. This allows developers to build incentive systems on Bittensor, tapping into our web of intelligence to develop markets of the developer’s choosings +Bittensor is releasing its own language for creating incentive mechanisms. This allows developers to build incentive systems on Bittensor, tapping into our web of intelligence to develop markets of the developer’s choosings Subnet 1, an incentive system for machine intelligence production, showcases the enormous potential of markets to procure huge amounts of resources. Releasing user-created subnets is set to create a cambrian explosion of additional resources into the Bittensor ecosystem @@ -24,4 +24,4 @@ By incentivizing developers to create their own markets, Bittensor is set to bec # Deeper dive Check out the Bittensor about page [here](https://bittensor.com/about) for more details about what the bittensor paradigm is and why subnets are revolutionary technology. -Also see our [linktree](https://linktr.ee/opentensor) for more information. \ No newline at end of file +Also see our [linktree](https://linktr.ee/opentensor) for more information. diff --git a/neurons/Miner/allocate.py b/neurons/Miner/allocate.py index f72bd200..da26c1cf 100644 --- a/neurons/Miner/allocate.py +++ b/neurons/Miner/allocate.py @@ -55,7 +55,7 @@ def register_allocation(timeline, device_requirement, public_key, docker_require # Kill container when it meets timeline start(timeline) return run_status - + except Exception as e: bt.logging.info(f"Error allocating container {e}") return {"status": False} @@ -79,7 +79,7 @@ def deregister_allocation(public_key): # Remove the key from the file after successful deallocation with open(file_path, 'w') as file: file.truncate(0) # Clear the file - + bt.logging.info("Successfully de-allocated container.") return {"status": True} else: @@ -133,4 +133,4 @@ def check_if_allocated(public_key): except Exception as e: # Handle any exceptions that occur # Log the exception or handle it as needed - return {"status": False} \ No newline at end of file + return {"status": False} diff --git a/neurons/Miner/container.py b/neurons/Miner/container.py index 4ce1d3f2..32f696b7 100644 --- a/neurons/Miner/container.py +++ b/neurons/Miner/container.py @@ -207,7 +207,7 @@ def run_container(cpu_usage, ram_usage, hard_disk_usage, gpu_usage, public_key, # Open the file in write mode ('w') and write the data with open(file_path, 'w') as file: file.write(allocation_key) - + return {"status": True, "info": encrypted_info} else: bt.logging.info(f"Container falied with status : {container.status}") @@ -517,4 +517,4 @@ def exchange_key_container(new_ssh_key: str, public_key: str, key_type: str = "u return {"status": False} except Exception as e: bt.logging.info(f"Error changing SSH key on container {e}") - return {"status": False} \ No newline at end of file + return {"status": False} diff --git a/neurons/Miner/http_server.py b/neurons/Miner/http_server.py index 81e1a795..b9bd30c9 100644 --- a/neurons/Miner/http_server.py +++ b/neurons/Miner/http_server.py @@ -16,22 +16,21 @@ def kill_process_on_port(port): else: print(f"No process found using port {port}.") except subprocess.CalledProcessError: - print(f"Port {port} is not in use.") + print(f"Port {port} is not in use.") def start_server(port) -> TCPServer: kill_process_on_port(port) - + handler = http.server.SimpleHTTPRequestHandler httpd: TCPServer = socketserver.TCPServer(("", int(port)), handler) - + server_thread = threading.Thread(target=httpd.serve_forever) server_thread.daemon = True server_thread.start() - + return httpd def stop_server(httpd: TCPServer) -> None: if httpd: httpd.shutdown() - httpd.server_close() - \ No newline at end of file + httpd.server_close() diff --git a/neurons/Validator/database/allocate.py b/neurons/Validator/database/allocate.py index 458876c3..d5ccad77 100644 --- a/neurons/Validator/database/allocate.py +++ b/neurons/Validator/database/allocate.py @@ -132,7 +132,7 @@ def update_miner_details(db: ComputeDb, hotkey_list, benchmark_responses: Tuple[ cursor.execute("ALTER TABLE new_miner_details RENAME TO miner_details;") db.conn.commit() - # Update miner_details + # Update miner_details for hotkey, response in benchmark_responses: # Print current values in the row before updating cursor.execute(""" @@ -156,18 +156,18 @@ def update_miner_details(db: ComputeDb, hotkey_list, benchmark_responses: Tuple[ INSERT INTO miner_details (hotkey, details, no_specs_count) VALUES (?, '{}', 1) ON CONFLICT(hotkey) DO UPDATE SET - no_specs_count = + no_specs_count = CASE WHEN miner_details.no_specs_count >= 5 THEN 5 ELSE miner_details.no_specs_count + 1 END, - details = + details = CASE WHEN miner_details.no_specs_count >= 5 THEN '{}' ELSE excluded.details END; """, (hotkey, '{}')) - db.conn.commit() + db.conn.commit() except Exception as e: db.conn.rollback() bt.logging.error(f"Error while updating miner_details: {e}") diff --git a/neurons/Validator/miner_script_m_merkletree.py b/neurons/Validator/miner_script_m_merkletree.py index 2342e72b..b8946660 100644 --- a/neurons/Validator/miner_script_m_merkletree.py +++ b/neurons/Validator/miner_script_m_merkletree.py @@ -96,15 +96,15 @@ def build_merkle_tree_rows(C, hash_func=hashlib.sha256, num_threads=None): num_threads = 8 n = C.shape[0] - + # Hash each row of C using the specified hash function def hash_row(i): return hash_func(C[i, :].tobytes()).digest() - + # Parallelize row hashing with ThreadPool(num_threads) as pool: leaves = pool.map(hash_row, range(n)) - + tree = leaves.copy() num_leaves = len(leaves) offset = 0 @@ -226,7 +226,7 @@ def process_gpu(gpu_id, s_A, s_B, n): # Set the current device torch.cuda.set_device(gpu_id) device = torch.device(f'cuda:{gpu_id}') - + # Initialize timing dictionary gpu_timing = {} @@ -317,7 +317,7 @@ def run_compute(): for gpu_id in range(num_gpus): s_A, s_B = seeds[gpu_id] futures.append(executor.submit(process_gpu, gpu_id, s_A, s_B, n)) - + for future in as_completed(futures): root_hash_result, gpu_timing_result = future.result() if root_hash_result: @@ -332,27 +332,27 @@ def run_compute(): def run_proof_gpu(gpu_id, indices, num_gpus): # Set the GPU device torch.cuda.set_device(gpu_id) - + # Load data for the specific GPU gpu_indices = indices[gpu_id] merkle_tree = np.load(f'/dev/shm/merkle_tree_gpu_{gpu_id}.npy', allow_pickle=True) C = np.load(f'/dev/shm/C_gpu_{gpu_id}.npy') - + # Start proof generation start_time_proof = time.time() responses = {'rows': [], 'proofs': [], 'indices': gpu_indices} total_leaves = C.shape[0] - + for idx, (i, j) in enumerate(gpu_indices): row = C[i, :] proof = get_merkle_proof_row(merkle_tree, i, total_leaves) responses['rows'].append(row) responses['proofs'].append(proof) - + end_time_proof = time.time() proof_time = end_time_proof - start_time_proof print(f"GPU {gpu_id}: Proof generation time: {proof_time:.2f} seconds") - + # Save responses to shared memory np.save(f'/dev/shm/responses_gpu_{gpu_id}.npy', responses) @@ -360,7 +360,7 @@ def run_proof(): # Get the challenge indices indices = get_challenge_indices() num_gpus = torch.cuda.device_count() - + # Use ThreadPoolExecutor for parallel GPU processing with ThreadPoolExecutor(max_workers=num_gpus) as executor: futures = [ @@ -373,7 +373,7 @@ def run_proof(): if __name__ == "__main__": parser = argparse.ArgumentParser(description='Miner script for GPU proof.') - parser.add_argument('--mode', type=str, default='benchmark', + parser.add_argument('--mode', type=str, default='benchmark', choices=['benchmark', 'compute', 'proof', 'gpu_info'], help='Mode to run: benchmark, compute, proof, or gpu_info') args = parser.parse_args() diff --git a/neurons/Validator/pog.py b/neurons/Validator/pog.py index 8b39073d..b8e71264 100644 --- a/neurons/Validator/pog.py +++ b/neurons/Validator/pog.py @@ -55,7 +55,7 @@ def identify_gpu(fp16_tflops, fp32_tflops, estimated_avram, gpu_data, reported_n combined_score = (fp16_deviation + fp32_deviation + avram_deviation) / 3 combined_scores.append((gpu, combined_score)) - + # Sort by the lowest deviation identified_gpu = sorted(combined_scores, key=lambda x: x[1])[0][0] @@ -183,7 +183,7 @@ def receive_responses(ssh_client, num_gpus): for gpu_id in range(num_gpus): remote_path = f'/dev/shm/responses_gpu_{gpu_id}.npy' local_path = f'{temp_dir}/responses_gpu_{gpu_id}.npy' - + try: sftp.get(remote_path, local_path) response = np.load(local_path, allow_pickle=True) @@ -193,7 +193,7 @@ def receive_responses(ssh_client, num_gpus): responses[gpu_id] = None except Exception as e: print(f"SFTP connection error: {e}") - + return responses def xorshift32_numpy(state): @@ -321,7 +321,7 @@ def verify_merkle_proof_row(row, proof, root_hash, index, total_leaves, hash_fun computed_hash = hash_func(row.tobytes()).digest() idx = index num_leaves = total_leaves - + # Iterate through each sibling hash in the proof for sibling_hash in proof: if idx % 2 == 0: @@ -334,7 +334,7 @@ def verify_merkle_proof_row(row, proof, root_hash, index, total_leaves, hash_fun computed_hash = hash_func(combined).digest() # Move up to the next level idx = idx // 2 - + # Compare the computed hash with the provided root hash return computed_hash == root_hash @@ -363,4 +363,4 @@ def get_remote_gpu_info(ssh_client): if error: raise RuntimeError(f"Failed to get GPU info: {error}") - return json.loads(output) \ No newline at end of file + return json.loads(output) diff --git a/neurons/Validator/script.py b/neurons/Validator/script.py index d3bafa22..7eb6f046 100644 --- a/neurons/Validator/script.py +++ b/neurons/Validator/script.py @@ -178,7 +178,7 @@ def check_docker_availability() -> Tuple[bool, str]: result = subprocess.run(["docker", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True) # If the command was successful, Docker is installed docker_version = result.stdout.strip() - + if check_docker_container('sn27-check-container') is True: return True, docker_version else: @@ -193,12 +193,12 @@ def check_docker_availability() -> Tuple[bool, str]: "Note: running a miner within containerized instances is not supported." ) return False, error_message - + def check_docker_container(container_id_or_name: str): try: # Start the container - subprocess.run(["docker", "start", container_id_or_name], + subprocess.run(["docker", "start", container_id_or_name], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -208,7 +208,7 @@ def check_docker_container(container_id_or_name: str): check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - + # Get the logs from the container logs_result = subprocess.run( ["docker", "logs", container_id_or_name], @@ -247,18 +247,18 @@ def get_perf_info(encrypted=True): except (Exception, RuntimeError) as e: print(e) return "" - + def check_ssh_login(host, port, username, password): try: # Create an SSH client instance ssh_client = paramiko.SSHClient() - + # Automatically add the server's host key (you can adjust this if needed) ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Attempt to connect to the SSH server using the provided credentials ssh_client.connect(hostname=host, port=port, username=username, password=password, timeout=10) - + # If login is successful print("SSH login successful") return True diff --git a/neurons/miner.py b/neurons/miner.py index ab1b75a6..bfc1b14d 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -655,7 +655,7 @@ async def start(self): self.current_block + 75 ) # 75 ~ every 15 minutes self.sync_status() - + # check allocation status self.__check_alloaction_errors() @@ -711,4 +711,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/neurons/miner_checker.py b/neurons/miner_checker.py index bb6c75b2..805a3cd9 100644 --- a/neurons/miner_checker.py +++ b/neurons/miner_checker.py @@ -23,28 +23,28 @@ def __init__(self, config): self.penalized_hotkeys_checklist = [] # List of dictionaries for penalized miners self.allocated_hotkeys = [] # Allocated miners that shouldn't be checked - def get_metagraph(self): - """Retrieves the metagraph from subtensor.""" + def get_metagraph(self): + """Retrieves the metagraph from subtensor.""" subtensor = bt.subtensor(config=self.config) return subtensor.metagraph(self.config.netuid) - def get_miners(self): + def get_miners(self): """Retrieves a list of miners (axons) from the metagraph.""" return self.metagraph.axons - def sync_checklist(self): - """Batch process miners using threads, and generate a new penalized hotkey list.""" - self.threads = [] - self.penalized_hotkeys_checklist.clear() # Reset the penalized list for each run - self.metagraph = self.get_metagraph() # Retrieve metagraph state + def sync_checklist(self): + """Batch process miners using threads, and generate a new penalized hotkey list.""" + self.threads = [] + self.penalized_hotkeys_checklist.clear() # Reset the penalized list for each run + self.metagraph = self.get_metagraph() # Retrieve metagraph state self.axons = self.get_miners() # Retrieve the list of axons (miners) #Step 1: Fetch allocated hotkeys from wandb with an empty validator list and flag set to False self.allocated_hotkeys = self.wandb.get_allocated_hotkeys(VALID_VALIDATOR_HOTKEYS, True) # Get allocated miners # Step 2: Create threads for miners that are NOT allocated - for i in range(0, len(self.axons), self.validator_challenge_batch_size): - for axon in self.axons[i: i + self.validator_challenge_batch_size]: - if axon.hotkey in self.allocated_hotkeys: + for i in range(0, len(self.axons), self.validator_challenge_batch_size): + for axon in self.axons[i: i + self.validator_challenge_batch_size]: + if axon.hotkey in self.allocated_hotkeys: bt.logging.info(f"Skipping allocated miner: {axon.hotkey}") continue # skip this miner since it's allocated @@ -52,15 +52,15 @@ def sync_checklist(self): self.threads.append(thread) # Start and join all threads - for thread in self.threads: - thread.start() - for thread in self.threads: + for thread in self.threads: + thread.start() + for thread in self.threads: thread.join() # Update penalized hotkeys via wandb # self.wandb.update_penalized_hotkeys_checklist(self.penalized_hotkeys_checklist) self.write_penalized_hotkeys_to_file() - + bt.logging.info(f"Length of penalized hotkeys checklist: {len(self.penalized_hotkeys_checklist)}") def write_penalized_hotkeys_to_file(self, file_path="penalized_hotkeys.json"): @@ -72,24 +72,24 @@ def write_penalized_hotkeys_to_file(self, file_path="penalized_hotkeys.json"): except Exception as e: bt.logging.error(f"Error writing penalized hotkeys to file: {e}") - def penalize_miner(self, hotkey, status_code, description): - """Adds a miner to the penalized list if it's not already penalized.""" - if not any(p['hotkey'] == hotkey for p in self.penalized_hotkeys_checklist): + def penalize_miner(self, hotkey, status_code, description): + """Adds a miner to the penalized list if it's not already penalized.""" + if not any(p['hotkey'] == hotkey for p in self.penalized_hotkeys_checklist): self.penalized_hotkeys_checklist.append({ "hotkey": hotkey, "status_code": status_code, "description": description}) - bt.logging.info(f"Penalized miner {hotkey}: {status_code} - {description}") + bt.logging.info(f"Penalized miner {hotkey}: {status_code} - {description}") else: bt.logging.info(f"Miner {hotkey} already penalized, skipping.") - def miner_checking_thread(self, axon): - """Handles allocation, SSH access, and deallocation of a miner.""" - wallet = bt.wallet(config=self.config) - dendrite = bt.dendrite(wallet=wallet) - bt.logging.info(f"Quering for miner: {axon.hotkey}") + def miner_checking_thread(self, axon): + """Handles allocation, SSH access, and deallocation of a miner.""" + wallet = bt.wallet(config=self.config) + dendrite = bt.dendrite(wallet=wallet) + bt.logging.info(f"Quering for miner: {axon.hotkey}") - is_ssh_access = True - allocation_status = False - private_key, public_key = rsa.generate_key_pair() + is_ssh_access = True + allocation_status = False + private_key, public_key = rsa.generate_key_pair() device_requirement = {"cpu": {"count": 1}, "gpu": {}, "hard_disk": {"capacity": 1073741824}, "ram": {"capacity": 1073741824}} @@ -119,28 +119,28 @@ def miner_checking_thread(self, axon): self.penalize_miner(axon.hotkey, "ALLOCATION_ERROR", f"Error during allocation: {str(e)}") # Deallocate resources if allocated, with a max retry count of 3 - retry_count = 0 - max_retries = 3 - while allocation_status and retry_count < max_retries: + retry_count = 0 + max_retries = 3 + while allocation_status and retry_count < max_retries: try: # Deallocation query - deregister_response = dendrite.query(axon, Allocate(timeline=0, checking=False, public_key=public_key), timeout=60) - if deregister_response and deregister_response["status"] is True: + deregister_response = dendrite.query(axon, Allocate(timeline=0, checking=False, public_key=public_key), timeout=60) + if deregister_response and deregister_response["status"] is True: allocation_status = False - bt.logging.info(f"Deallocated miner {axon.hotkey}") - break - else: + bt.logging.info(f"Deallocated miner {axon.hotkey}") + break + else: retry_count += 1 - bt.logging.error(f"Failed to deallocate miner {axon.hotkey} (attempt {retry_count}/{max_retries})") - if retry_count >= max_retries: + bt.logging.error(f"Failed to deallocate miner {axon.hotkey} (attempt {retry_count}/{max_retries})") + if retry_count >= max_retries: bt.logging.error(f"Max retries reached for deallocating miner {axon.hotkey}.") - self.penalize_miner(axon.hotkey, "DEALLOCATION_FAILED", "Failed to deallocate after max retries") + self.penalize_miner(axon.hotkey, "DEALLOCATION_FAILED", "Failed to deallocate after max retries") time.sleep(5) - except Exception as e: - retry_count += 1 - bt.logging.error(f"Error while trying to deallocate miner {axon.hotkey} (attempt {retry_count}/{max_retries}): {e}") + except Exception as e: + retry_count += 1 + bt.logging.error(f"Error while trying to deallocate miner {axon.hotkey} (attempt {retry_count}/{max_retries}): {e}") if retry_count >= max_retries: - bt.logging.error(f"Max retries reached for deallocating miner {axon.hotkey}.") + bt.logging.error(f"Max retries reached for deallocating miner {axon.hotkey}.") self.penalize_miner(axon.hotkey, "DEALLOCATION_FAILED", "Failed to deallocate after max retries") time.sleep(5) @@ -148,46 +148,46 @@ def miner_checking_thread(self, axon): # Penalize if SSH access fails self.penalize_miner(axon.hotkey, "SSH_ACCESS_DISABLED", "Failed SSH access") - def check_ssh_login(self, host, port, username, password): - """Check SSH login using Paramiko.""" + def check_ssh_login(self, host, port, username, password): + """Check SSH login using Paramiko.""" try: ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh_client.connect(hostname=host, port=port, username=username, password=password, timeout=10) - bt.logging.info(f"SSH login successful for {host}") + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(hostname=host, port=port, username=username, password=password, timeout=10) + bt.logging.info(f"SSH login successful for {host}") return True - except paramiko.AuthenticationException: + except paramiko.AuthenticationException: bt.logging.error(f"Authentication failed for {host}") - return False - except paramiko.SSHException as ssh_exception: - bt.logging.error(f"Unable to establish SSH connection: {ssh_exception}") + return False + except paramiko.SSHException as ssh_exception: + bt.logging.error(f"Unable to establish SSH connection: {ssh_exception}") return False except Exception as e: - bt.logging.error(f"Exception in connecting to the server: {e}") - return False + bt.logging.error(f"Exception in connecting to the server: {e}") + return False finally: ssh_client.close() def get_config(): - """Set up configuration using argparse.""" - parser = argparse.ArgumentParser() - parser.add_argument("--netuid", type=int, default=1, help="The chain subnet uid.") - bt.subtensor.add_args(parser) - bt.logging.add_args(parser) - bt.wallet.add_args(parser) + """Set up configuration using argparse.""" + parser = argparse.ArgumentParser() + parser.add_argument("--netuid", type=int, default=1, help="The chain subnet uid.") + bt.subtensor.add_args(parser) + bt.logging.add_args(parser) + bt.wallet.add_args(parser) config = bt.config(parser) # Ensure the logging directory exists config.full_path = os.path.expanduser( "{}/{}/{}/netuid{}/{}".format( config.logging.logging_dir, config.wallet.name, config.wallet.hotkey, config.netuid, "validator",)) return config -def main(): - """Main function to run the miner checker loop.""" - config = get_config() +def main(): + """Main function to run the miner checker loop.""" + config = get_config() miner_checker = MinerChecker(config) - while True: - miner_checker.sync_checklist() - bt.logging.info("Sleeping before next loop...") + while True: + miner_checker.sync_checklist() + bt.logging.info("Sleeping before next loop...") time.sleep(900) # Sleep for 15 minutes before re-checking miners if __name__ == "__main__": diff --git a/neurons/register.py b/neurons/register.py index 093dcfb3..068c8d40 100644 --- a/neurons/register.py +++ b/neurons/register.py @@ -86,7 +86,7 @@ def get_config_cli(): bt.subtensor.add_args(parser) bt.logging.add_args(parser) bt.wallet.add_args(parser) - + # Parse the initial config to check for provided arguments config = bt.config(parser) @@ -229,7 +229,7 @@ def allocate_container_hotkey(config, hotkey, timeline, public_key): def allocate(wandb): config = get_config_cli() - + device_requirement = {"cpu": {"count": 1}, "gpu": {}, "hard_disk": {"capacity": 1073741824}, "ram": {"capacity": 1073741824}} if config.gpu_type != "" and config.gpu_size != 0: device_requirement["gpu"] = {"count": 1, "capacity": config.gpu_size, "type": config.gpu_type} @@ -292,7 +292,7 @@ def allocate_hotkey(wandb): hotkey = input("Enter the hotkey of the resource to allocate: ") config = get_config() - + timeline = 30 private_key, public_key = rsa.generate_key_pair() result = allocate_container_hotkey(config, hotkey, timeline, public_key) @@ -528,7 +528,7 @@ def list_allocations_hotkeys(wandb): cursor.close() db.close() -def list_resources(wandb): +def list_resources(wandb): db = ComputeDb() specs_details = get_miner_details(db) @@ -566,19 +566,19 @@ def list_resources(wandb): gpu_capacity = "{:.2f}".format((gpu_miner['capacity']/1024)) gpu_name = str(gpu_miner['details'][0]['name']).lower() gpu_count = gpu_miner['count'] - + # Extract CPU details cpu_miner = details['cpu'] cpu_count = cpu_miner['count'] - + # Extract RAM details ram_miner = details['ram'] ram = "{:.2f}".format(ram_miner['available']/1024.0**3) - + # Extract Hard Disk details hard_disk_miner = details['hard_disk'] hard_disk = "{:.2f}".format(hard_disk_miner['free']/1024.0**3) - + # Update the GPU instances count gpu_key = (gpu_name, gpu_count) gpu_instances[gpu_key] = gpu_instances.get(gpu_key, 0) + 1 @@ -598,11 +598,11 @@ def list_resources(wandb): cpu_count = "N/A" ram = "N/A" hard_disk = "N/A" - + # Allocation status status = "N/A" allocated_hotkeys = wandb.get_allocated_hotkeys([], False) - + if hotkey in allocated_hotkeys: status = "Res." else: @@ -626,7 +626,7 @@ def list_resources(wandb): summary_data = [gpu_name, gpu_count, instances_count] summary_line = '|'.join(str(d).ljust(w) for d, w in zip(summary_data, [30, 10, 15])) print(summary_line) - + # Print the summary table for total GPU counts print("\nSUMMARY (Total GPU Counts):") summary_headers = ['GPU Name', 'Total GPU Count'] @@ -643,7 +643,7 @@ def list_resources(wandb): def update_allocation_wandb(wandb): hotkey_list = [] - + # Instantiate the connection to the db db = ComputeDb() cursor = db.get_cursor() @@ -652,7 +652,7 @@ def update_allocation_wandb(wandb): # Retrieve all records from the allocation table cursor.execute("SELECT id, hotkey, details FROM allocation") rows = cursor.fetchall() - + for row in rows: id, hotkey, details = row hotkey_list.append(hotkey) @@ -674,7 +674,7 @@ def penalize_hotkey(wandb): # Split the input by commas and strip any extra whitespace to create a list of hotkeys hotkey_list = [hotkey.strip() for hotkey in hotkeys_input.split(',')] - + # Instantiate the connection to the db db = ComputeDb() cursor = db.get_cursor() @@ -779,7 +779,7 @@ def print_welcome_message(): print(f"Version: {get_local_version()}\n") def main(): - + # Check wandb API-Key config = get_config() wallet = bt.wallet(config=config) diff --git a/neurons/register_api.py b/neurons/register_api.py index f2029995..d85f6ef1 100644 --- a/neurons/register_api.py +++ b/neurons/register_api.py @@ -126,7 +126,7 @@ async def dispatch(self, request: Request, call_next): if client_ip not in self.whitelisted_ips: bt.logging.info(f"Access attempt from IP: {client_ip}") raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail="Access forbidden: IP not whitelisted") - + # Process the request and get the response response = await call_next(request) return response @@ -330,13 +330,13 @@ def __init__( # Optional: Initialize per-hotkey locks if necessary self.hotkey_locks = {} self.hotkey_locks_lock = threading.Lock() - + # Initialize executor for the thread execution cpu_cores = os.cpu_count() or 1 configured_max_workers = 32 safe_max_workers = min((cpu_cores + 4)*4, configured_max_workers) self.executor = ThreadPoolExecutor(max_workers=safe_max_workers) - + def _setup_routes(self): # Define a custom validation error handler @@ -587,7 +587,7 @@ async def allocate_hotkey(hotkey: str, ssh_key: Optional[str] = None, "success": False, "message": "Fail to allocate resource, blacklisted", "err_detail": "blacklisted", - }, + }, ) if hotkey: # client_host = request.client.host @@ -611,7 +611,7 @@ async def allocate_hotkey(hotkey: str, ssh_key: Optional[str] = None, run_start = time.time() result = await self._allocate_container_hotkey(requirements, hotkey,requirements.timeline, public_key, docker_requirement.dict()) - + if result["status"] is False: bt.logging.error(f"API: Allocation {hotkey} Failed : {result['msg']}") return JSONResponse( @@ -1725,8 +1725,8 @@ async def count_all_gpus() -> JSONResponse: """ Count all GPUs on the compute subnet """ - bt.logging.info(f"API: Count Gpus(wandb) on compute subnet") - GPU_COUNTS = 0 + bt.logging.info(f"API: Count Gpus(wandb) on compute subnet") + GPU_COUNTS = 0 specs_details , running_hotkey = await get_wandb_running_miners() try: if specs_details: @@ -1777,7 +1777,7 @@ async def count_all_model(model: str , cpu_count: Optional[int] = None, ram_size """ Count all GPUs on the compute subnet """ - bt.logging.info(f"API: Count Gpus by model(wandb) on compute subnet") + bt.logging.info(f"API: Count Gpus by model(wandb) on compute subnet") counter = 0 specs_details , running_hotkey = await get_wandb_running_miners() try: @@ -1854,7 +1854,7 @@ async def list_resources_wandb(query: ResourceQuery = None, query: The query parameter to filter the resources.
""" - bt.logging.info(f"API: List resources(wandb) on compute subnet") + bt.logging.info(f"API: List resources(wandb) on compute subnet") self.wandb.api.flush() specs_details,running_hotkey = await get_wandb_running_miners() @@ -2847,7 +2847,7 @@ async def _allocate_container_hotkey(self, requirements, hotkey, timeline, publi bt.logging.warning( f"API: Allocation check failed for hotkey: {hotkey} result: {check_allocation}, axon: {axon.ip}:{axon.port}") await asyncio.sleep(3) - continue # Move to the next axon if allocation check failed + continue # Move to the next axon if allocation check failed else: bt.logging.info(f"API: Allocation check passed for hotkey: {hotkey}") break diff --git a/neurons/validator.py b/neurons/validator.py index 2260c88a..0575438f 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -245,9 +245,9 @@ def init_prometheus(self, force_update: bool = False): """ Register the prometheus information on metagraph. :return: bool - """ + """ # extrinsic prometheus is removed at 8.2.1 - + bt.logging.info("Extrinsic prometheus information on metagraph.") success = True # TODO : remove all the related code from the code base @@ -1086,13 +1086,13 @@ async def start(self): block_next_pog = 1 block_next_sync_status = 1 block_next_set_weights = self.current_block + weights_rate_limit - block_next_hardware_info = 1 + block_next_hardware_info = 1 block_next_miner_checking = 1 time_next_pog = None time_next_sync_status = None time_next_set_weights = None - time_next_hardware_info = None + time_next_hardware_info = None bt.logging.info("Starting validator loop.") while True: @@ -1204,4 +1204,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/opencompute/main.py b/scripts/opencompute/main.py index c9334a24..0f921556 100644 --- a/scripts/opencompute/main.py +++ b/scripts/opencompute/main.py @@ -113,4 +113,3 @@ def display_hardware_specs(specs_details, allocated_keys, penalized_keys): except: st.write("Unable to connect to the server. Please try again later.") print("Error: ConnectionError occurred while attempting to connect to the server.") - diff --git a/test-scripts/benchmark.py b/test-scripts/benchmark.py index 66d06ec0..fe76dead 100644 --- a/test-scripts/benchmark.py +++ b/test-scripts/benchmark.py @@ -680,4 +680,4 @@ def main(): print("") if __name__ == "__main__": - main() \ No newline at end of file + main() From fed2aaff921b7b3af25c984017f15222d3c2e47d Mon Sep 17 00:00:00 2001 From: Ivan Anishchuk Date: Tue, 4 Feb 2025 13:08:35 +0700 Subject: [PATCH 4/7] docs(CSN-505): explain how to use pre-commit Add a new document QA.md that will contain other helpful bits like this. Add a new file .git-blame-ignore-revs to improve git blame handling of mass-linting commits with a first revision to be ignored. --- .git-blame-ignore-revs | 1 + QA.md | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 .git-blame-ignore-revs create mode 100644 QA.md diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..c3d070a0 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1 @@ +d4662ec50280115abffc47b952cf99703bfcce6d diff --git a/QA.md b/QA.md new file mode 100644 index 00000000..f07fe934 --- /dev/null +++ b/QA.md @@ -0,0 +1,12 @@ +# Style guide and linting tools + +We are starting to introduce pre-commit and adding checks to it and enabling GitHub Actions. + +Locally, start by installing pre-commit package and running `pre-commit install --install-hooks`. It will ensure checksare being run before each commit is made. + +Autolinting commits (made after running `pre-commit run -a` and fixing all files with new checks) are to be recorded in `.git-blame-ignore-revs` and that file can be used with git blame and git config snippet like this (or command-line `--ignore-revs-file`) to skip such commits when examining history. + +``` +[blame] + ignoreRevsFile = .git-blame-ignore-revs +``` From 22f8b8381f4687b454821c96944f7f4380e3114e Mon Sep 17 00:00:00 2001 From: Ivan Anishchuk Date: Tue, 4 Feb 2025 13:19:28 +0700 Subject: [PATCH 5/7] fix(CSN-505): attempt to fix Actions workflow --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 2b11178b..0597de7d 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -3,7 +3,7 @@ name: pre-commit on: pull_request: push: - branches: [main] + branches: ["dev", "main"] jobs: pre-commit: From a6e5c05f21d2304f7ece2273b45f63ff6cde51a6 Mon Sep 17 00:00:00 2001 From: Ivan Anishchuk Date: Tue, 4 Feb 2025 13:22:29 +0700 Subject: [PATCH 6/7] fix(autolint): rerun pre-commit after dev merge --- compute/axon.py | 8 ++++---- requirements.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compute/axon.py b/compute/axon.py index 842134ee..3d725c12 100644 --- a/compute/axon.py +++ b/compute/axon.py @@ -52,7 +52,7 @@ from bittensor.core.types import AxonServeCallParams from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor - + def custom_serve_extrinsic( subtensor: "Subtensor", wallet: "Wallet", @@ -188,7 +188,7 @@ def __init__( external_port (:type:`Optional[int]`): The external port of the server to broadcast to the network. max_workers (:type:`Optional[int]`): Used to create the threadpool if not passed, specifies the number of active threads servicing requests. """ - + # Build and check config. if config is None: config = axon.config() @@ -220,13 +220,13 @@ def __init__( ) self.full_address = str(self.config.axon.ip) + ":" + str(self.config.axon.port) # type: ignore self.started = False - + # Build middleware self.thread_pool = PriorityThreadPoolExecutor( max_workers=self.config.axon.max_workers # type: ignore ) self.nonces: dict[str, int] = {} - + # Request default functions. self.forward_class_types: dict[str, list[Signature]] = {} self.blacklist_fns: dict[str, Callable | None] = {} diff --git a/requirements.txt b/requirements.txt index 15d8b68b..8b29d6a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ requests==2.31.0 paramiko==3.4.1 blake3 ipwhois==1.3.0 -torch==2.5.1 \ No newline at end of file +torch==2.5.1 From 1cec92dc000ae8eaad1f389d6b71dc02a6dc3acd Mon Sep 17 00:00:00 2001 From: Ivan Anishchuk Date: Tue, 4 Feb 2025 13:23:05 +0700 Subject: [PATCH 7/7] fix: update .git-blame-ignore-revs --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c3d070a0..a3ee93ac 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1 +1,2 @@ d4662ec50280115abffc47b952cf99703bfcce6d +a6e5c05f21d2304f7ece2273b45f63ff6cde51a6