Skip to content

Commit

Permalink
Add some bug fixes (#15)
Browse files Browse the repository at this point in the history
* Update namespace

* Fix shuffle assignment
  • Loading branch information
ljvmiranda921 authored Aug 3, 2024
1 parent a6638f4 commit 008b533
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions scripts/run_generative.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ def format_judgements(batch, optional_chat_template=None):
answer_b,
multi_turn=mult_turn,
model_modifier=model_modifier,
include_langs=args.include_langs,
include_langs=args.include_languages,
)

if optional_chat_template is not None:
Expand Down Expand Up @@ -302,7 +302,12 @@ def format_judgements(batch, optional_chat_template=None):
winners = [process_judgement(a, is_prometheus=is_prometheus) for a in answers]

def process_shuffled(win, shuffle):
winner_text, loser_text = "B", "A" if shuffle else "A", "B"
if shuffle:
winner_text = "B"
loser_text = "A"
else:
winner_text = "A"
loser_text = "B"
if win == winner_text:
return 1
elif win == loser_text:
Expand Down

0 comments on commit 008b533

Please sign in to comment.