Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[hotfix] fix flags for run command, fix hotkeys flag for overview, and [feature] CUDA reg #877

Merged
merged 7 commits into from
Aug 16, 2022
Merged

[hotfix] fix flags for run command, fix hotkeys flag for overview, and [feature] CUDA reg #877

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4288c3a
[feature] cpu register faster (#854)
Aug 11, 2022
4629b82
[hotfix] fix flags for multiproc register limit (#876)
Aug 12, 2022
3ade241
Merge remote-tracking branch 'origin/master' into nobunaga
Eugene-hu Aug 12, 2022
224719c
Fix/diff unpack bit shift (#878)
Aug 15, 2022
c546d3f
[Feature] [cubit] CUDA registration solver (#868)
Aug 15, 2022
fb6be7c
Fix/move overview args to cli (#867)
Aug 15, 2022
b9ca298
Add/address CUDA reg changes (#879)
Aug 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 54 additions & 22 deletions bittensor/_cli/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,18 @@
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import argparse
import os
import sys
import argparse
from typing import List

import bittensor
from rich.prompt import Prompt
from rich.prompt import Confirm
import torch
from rich.prompt import Confirm, Prompt
from substrateinterface.utils.ss58 import ss58_decode, ss58_encode

from . import cli_impl

console = bittensor.__console__

class cli:
Expand Down Expand Up @@ -88,6 +91,27 @@ def config() -> 'bittensor.config':
help='''Set the output width of the overview. Defaults to automatic width from terminal.''',
default=None,
)
overview_parser.add_argument(
'--sort_by',
'--wallet.sort_by',
dest='sort_by',
required=False,
action='store',
default="",
type=str,
help='''Sort the hotkeys by the specified column title (e.g. name, uid, axon).'''
)
overview_parser.add_argument(
'--sort_order',
'--wallet.sort_order',
dest="sort_order",
required=False,
action='store',
default="ascending",
type=str,
help='''Sort the hotkeys in the specified ordering. (ascending/asc or descending/desc/reverse)'''
)

bittensor.wallet.add_args( overview_parser )
bittensor.subtensor.add_args( overview_parser )

Expand Down Expand Up @@ -250,6 +274,7 @@ def config() -> 'bittensor.config':
'register',
help='''Register a wallet to a network.'''
)

unstake_parser = cmd_parsers.add_parser(
'unstake',
help='''Unstake from hotkey accounts.'''
Expand Down Expand Up @@ -497,6 +522,7 @@ def config() -> 'bittensor.config':
help='''Set true to avoid prompting the user.''',
default=False,
)

bittensor.wallet.add_args( unstake_parser )
bittensor.subtensor.add_args( unstake_parser )

Expand Down Expand Up @@ -535,6 +561,7 @@ def config() -> 'bittensor.config':
help='''Set true to avoid prompting the user.''',
default=False,
)

bittensor.wallet.add_args( stake_parser )
bittensor.subtensor.add_args( stake_parser )

Expand Down Expand Up @@ -571,23 +598,6 @@ def config() -> 'bittensor.config':
help='''Set true to avoid prompting the user.''',
default=False,
)
register_parser.add_argument(
'--num_processes',
'--num',
'-n',
dest='num_processes',
help="Number of processors to use for registration",
type=int,
default=None,
)
register_parser.add_argument(
'--update_interval',
'-u',
dest='update_interval',
help="The number of nonces to process before checking for next block during registration",
type=int,
default=None,
)

bittensor.wallet.add_args( register_parser )
bittensor.subtensor.add_args( register_parser )
Expand Down Expand Up @@ -709,7 +719,7 @@ def check_unstake_config( config: 'bittensor.Config' ):
if config.wallet.get('all_hotkeys'):
hotkeys = "all hotkeys"
elif config.wallet.get('hotkeys'):
hotkeys = str(config.hotkeys).replace('[', '').replace(']', '')
hotkeys = str(config.wallet.hotkeys).replace('[', '').replace(']', '')
else:
hotkeys = str(config.wallet.hotkey)
if not Confirm.ask("Unstake all Tao from: [bold]'{}'[/bold]?".format(hotkeys)):
Expand Down Expand Up @@ -819,6 +829,28 @@ def check_register_config( config: 'bittensor.Config' ):
hotkey = Prompt.ask("Enter hotkey name", default = bittensor.defaults.wallet.hotkey)
config.wallet.hotkey = str(hotkey)

if not config.no_prompt and config.subtensor.register.cuda.use_cuda == bittensor.defaults.subtensor.register.cuda.use_cuda:
# Ask about cuda registration only if a CUDA device is available.
if torch.cuda.is_available():
cuda = Confirm.ask("Detected CUDA device, use CUDA for registration?\n")
config.subtensor.register.cuda.use_cuda = cuda
# Only ask about which CUDA device if the user has more than one CUDA device.
if cuda and config.subtensor.register.cuda.get('dev_id') is None and torch.cuda.device_count() > 0:
devices: List[str] = [str(x) for x in range(torch.cuda.device_count())]
device_names: List[str] = [torch.cuda.get_device_name(x) for x in range(torch.cuda.device_count())]
console.print("Available CUDA devices:")
choices_str: str = ""
for i, device in enumerate(devices):
choices_str += (" {}: {}\n".format(device, device_names[i]))
console.print(choices_str)
dev_id = Prompt.ask("Which GPU would you like to use?", choices=devices, default=str(bittensor.defaults.subtensor.register.cuda.dev_id))
camfairchild marked this conversation as resolved.
Show resolved Hide resolved
try:
dev_id = int(dev_id)
except ValueError:
console.error(":cross_mark:[red]Invalid GPU device[/red] [bold white]{}[/bold white]\nAvailable CUDA devices:{}".format(dev_id, choices_str))
sys.exit(1)
config.subtensor.register.cuda.dev_id = dev_id

def check_new_coldkey_config( config: 'bittensor.Config' ):
if config.wallet.get('name') == bittensor.defaults.wallet.name and not config.no_prompt:
wallet_name = Prompt.ask("Enter wallet name", default = bittensor.defaults.wallet.name)
Expand Down Expand Up @@ -902,4 +934,4 @@ def check_help_config( config: 'bittensor.Config'):
def check_update_config( config: 'bittensor.Config'):
if not config.no_prompt:
answer = Prompt.ask('This will update the local bittensor package', choices = ['Y','N'], default = 'Y')
config.answer = answer
config.answer = answer
65 changes: 32 additions & 33 deletions bittensor/_cli/cli_impl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import os
import sys
from typing import List, Union
from typing import List, Union, Optional

from cachetools import Cache

Expand Down Expand Up @@ -239,7 +239,15 @@ def register( self ):
"""
wallet = bittensor.wallet( config = self.config )
subtensor = bittensor.subtensor( config = self.config )
subtensor.register( wallet = wallet, prompt = not self.config.no_prompt, num_processes = self.config.num_processes, update_interval = self.config.update_interval )
subtensor.register(
wallet = wallet,
prompt = not self.config.no_prompt,
TPB = self.config.subtensor.register.cuda.get('TPB', None),
update_interval = self.config.subtensor.register.get('update_interval', None),
num_processes = self.config.subtensor.register.get('num_processes', None),
cuda = self.config.subtensor.register.cuda.get('use_cuda', None),
dev_id = self.config.subtensor.register.cuda.get('dev_id', None)
)

def transfer( self ):
r""" Transfer token of amount to destination.
Expand All @@ -251,7 +259,6 @@ def transfer( self ):
def unstake( self ):
r""" Unstake token of amount from hotkey(s).
"""
# TODO: Implement this without re-unlocking the coldkey.
config = self.config.copy()
config.hotkey = None
wallet = bittensor.wallet( config = self.config )
Expand All @@ -263,7 +270,7 @@ def unstake( self ):
all_hotkeys: List[bittensor.wallet] = self._get_hotkey_wallets_for_wallet( wallet = wallet )
# Exclude hotkeys that are specified.
wallets_to_unstake_from = [
wallet for wallet in all_hotkeys if wallet.hotkey_str not in self.config.wallet.get('hotkeys')
wallet for wallet in all_hotkeys if wallet.hotkey_str not in self.config.wallet.get('hotkeys', [])
]

elif self.config.wallet.get('hotkeys'):
Expand All @@ -276,9 +283,7 @@ def unstake( self ):
subtensor.unstake( wallet, amount = None if self.config.get('unstake_all') else self.config.get('amount'), wait_for_inclusion = True, prompt = not self.config.no_prompt )
return None

wallet_0: 'bittensor.wallet' = wallets_to_unstake_from[0]
# Decrypt coldkey for all wallet(s) to use
wallet_0.coldkey


final_wallets: List['bittensor.wallet'] = []
final_amounts: List[Union[float, Balance]] = []
Expand All @@ -287,9 +292,6 @@ def unstake( self ):
if not wallet.is_registered():
# Skip unregistered hotkeys.
continue
# Assign decrypted coldkey from wallet_0
# so we don't have to decrypt again
wallet._coldkey = wallet_0._coldkey

unstake_amount_tao: float = self.config.get('amount')
if self.config.get('max_stake'):
Expand All @@ -307,19 +309,17 @@ def unstake( self ):
if not self.config.no_prompt:
if not Confirm.ask("Do you want to unstake from the following keys:\n" + \
"".join([
f" [bold white]- {wallet.hotkey_str}: {amount}𝜏[/bold white]\n" for wallet, amount in zip(final_wallets, final_amounts)
f" [bold white]- {wallet.hotkey_str}: {amount.tao}𝜏[/bold white]\n" for wallet, amount in zip(final_wallets, final_amounts)
])
):
return None

for wallet, amount in zip(final_wallets, final_amounts):
subtensor.unstake( wallet, amount = None if self.config.get('unstake_all') else amount, wait_for_inclusion = True, prompt = False )

subtensor.unstake_multiple( wallets = final_wallets, amounts = None if self.config.get('unstake_all') else final_amounts, wait_for_inclusion = True, prompt = False )


def stake( self ):
r""" Stake token of amount to hotkey(s).
"""
# TODO: Implement this without re-unlocking the coldkey.
config = self.config.copy()
config.hotkey = None
wallet = bittensor.wallet( config = config )
Expand All @@ -331,7 +331,7 @@ def stake( self ):
all_hotkeys: List[bittensor.wallet] = self._get_hotkey_wallets_for_wallet( wallet = wallet )
# Exclude hotkeys that are specified.
wallets_to_stake_to = [
wallet for wallet in all_hotkeys if wallet.hotkey_str not in self.config.wallet.get('hotkeys')
wallet for wallet in all_hotkeys if wallet.hotkey_str not in self.config.wallet.get('hotkeys', [])
]

elif self.config.wallet.get('hotkeys'):
Expand Down Expand Up @@ -386,8 +386,7 @@ def stake( self ):
):
return None

for wallet, amount in zip(final_wallets, final_amounts):
subtensor.add_stake( wallet, amount = None if self.config.get('stake_all') else amount, wait_for_inclusion = True, prompt = False )
subtensor.add_stake_multiple( wallets = final_wallets, amounts = None if self.config.get('stake_all') else final_amounts, wait_for_inclusion = True, prompt = False )


def set_weights( self ):
Expand Down Expand Up @@ -596,24 +595,14 @@ def overview(self):

all_hotkeys = []
total_balance = bittensor.Balance(0)

# We are printing for every wallet.
# We are printing for every coldkey.
if self.config.all:
cold_wallets = CLI._get_coldkey_wallets_for_path(self.config.wallet.path)
for cold_wallet in tqdm(cold_wallets, desc="Pulling balances"):
if cold_wallet.coldkeypub_file.exists_on_device() and not cold_wallet.coldkeypub_file.is_encrypted():
total_balance = total_balance + subtensor.get_balance( cold_wallet.coldkeypub.ss58_address )
all_hotkeys = CLI._get_all_wallets_for_path( self.config.wallet.path )

# We are printing for a select number of hotkeys.
elif self.config.wallet.hotkeys:
# Only show hotkeys for wallets in the list
all_hotkeys = [hotkey for hotkey in all_hotkeys if hotkey.hotkey_str in self.config.wallet.hotkeys]
coldkey_wallet = bittensor.wallet( config = self.config )
if coldkey_wallet.coldkeypub_file.exists_on_device() and not coldkey_wallet.coldkeypub_file.is_encrypted():
total_balance = subtensor.get_balance( coldkey_wallet.coldkeypub.ss58_address )

# We are printing for all keys under the wallet.
else:
# We are only printing keys for a single coldkey
coldkey_wallet = bittensor.wallet( config = self.config )
Expand All @@ -624,6 +613,16 @@ def overview(self):
return
all_hotkeys = CLI._get_hotkey_wallets_for_wallet( coldkey_wallet )

# We are printing for a select number of hotkeys from all_hotkeys.

if self.config.wallet.get('hotkeys', []):
if not self.config.get('all_hotkeys', False):
# We are only showing hotkeys that are specified.
all_hotkeys = [hotkey for hotkey in all_hotkeys if hotkey.hotkey_str in self.config.wallet.hotkeys]
else:
# We are excluding the specified hotkeys from all_hotkeys.
all_hotkeys = [hotkey for hotkey in all_hotkeys if hotkey.hotkey_str not in self.config.wallet.hotkeys]

# Check we have keys to display.
if len(all_hotkeys) == 0:
console.print("[red]No wallets found.[/red]")
Expand Down Expand Up @@ -732,10 +731,10 @@ def overview(self):

console.clear()

sort_by: str = self.config.wallet.sort_by
sort_order: str = self.config.wallet.sort_order
sort_by: Optional[str] = self.config.get('sort_by', None)
sort_order: Optional[str] = self.config.get('sort_order', None)

if sort_by != "":
if sort_by is not None and sort_by != "":
column_to_sort_by: int = 0
highest_matching_ratio: int = 0
sort_descending: bool = False # Default sort_order to ascending
Expand Down
3 changes: 1 addition & 2 deletions bittensor/_dataset/dataset_impl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,7 @@ def __init__(

self.build_hash_table()

if not os.path.isdir(os.path.expanduser(data_dir)):
os.makedirs(os.path.expanduser(data_dir))
os.makedirs(os.path.expanduser(data_dir), exist_ok=True)

self.data_queue = ThreadQueue(
producer_target = self.reserve_multiple_data,
Expand Down
17 changes: 17 additions & 0 deletions bittensor/_subtensor/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,14 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ):
help='''The subtensor endpoint flag. If set, overrides the --network flag.
''')
parser.add_argument('--' + prefix_str + 'subtensor._mock', action='store_true', help='To turn on subtensor mocking for testing purposes.', default=bittensor.defaults.subtensor._mock)

parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest='subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes)
parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval)
# registration args. Used for register and re-register and anything that calls register.
parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.use_cuda', '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', default=bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set true to use CUDA.''', action='store_true', required=False )
parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=int, default=argparse.SUPPRESS, help='''Set the CUDA device id. Goes by the order of speed. (i.e. 0 is the fastest).''', required=False )
parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False )

except argparse.ArgumentError:
# re-parsing arguments.
pass
Expand All @@ -197,6 +205,15 @@ def add_defaults(cls, defaults ):
defaults.subtensor.chain_endpoint = os.getenv('BT_SUBTENSOR_CHAIN_ENDPOINT') if os.getenv('BT_SUBTENSOR_CHAIN_ENDPOINT') != None else None
defaults.subtensor._mock = os.getenv('BT_SUBTENSOR_MOCK') if os.getenv('BT_SUBTENSOR_MOCK') != None else False

defaults.subtensor.register = bittensor.Config()
defaults.subtensor.register.num_processes = os.getenv('BT_SUBTENSOR_REGISTER_NUM_PROCESSES') if os.getenv('BT_SUBTENSOR_REGISTER_NUM_PROCESSES') != None else None # uses processor count by default within the function
defaults.subtensor.register.update_interval = os.getenv('BT_SUBTENSOR_REGISTER_UPDATE_INTERVAL') if os.getenv('BT_SUBTENSOR_REGISTER_UPDATE_INTERVAL') != None else 50_000

defaults.subtensor.register.cuda = bittensor.Config()
defaults.subtensor.register.cuda.dev_id = 0
defaults.subtensor.register.cuda.use_cuda = False
defaults.subtensor.register.cuda.TPB = 256

@staticmethod
def check_config( config: 'bittensor.Config' ):
assert config.subtensor
Expand Down
Loading