这是indexloc提供的服务，不要输入任何密码
Skip to content

Fix bug in evaluation code #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix bug in evaluation code #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions eval/eval_hrbench.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import io
from openai import OpenAI
import requests
import copy
import pandas as pd


parser = argparse.ArgumentParser()
Expand Down
9 changes: 6 additions & 3 deletions eval/eval_vstar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import io
from openai import OpenAI
import requests

from random import shuffle

parser = argparse.ArgumentParser()
parser.add_argument('--model_name', type=str, default='qwen', help='Model name for result save')
Expand Down Expand Up @@ -127,7 +127,9 @@ def process(img_arg):
anno = json.load(f)
question = anno['question']
options = anno['options']

correct_answer = anno['options'][0]
shuffle(options)

option_str = "\n"
for i in range(len(options)):
option_str += abc_map[i + 1] + '. ' + options[i] + '\n'
Expand Down Expand Up @@ -270,7 +272,8 @@ def process(img_arg):
save_info = {}
save_info['image'] = img
save_info['question'] = question
save_info['answer'] = anno['options'][0]
save_info['answer'] = correct_answer
save_info['answer_choice'] = chr(ord('A') + options.index(correct_answer))
save_info['pred_ans'] = output_text
save_info['pred_output'] = print_messages
save_info['status'] = status
Expand Down
7 changes: 4 additions & 3 deletions eval/judge_result.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,23 +134,24 @@ def process(line):
line = line.strip()
data = json.loads(line)
question = data['question']
choice = data['answer_choice']
answer = data['answer']
pred_ans = data['pred_ans']
pred_output = data['pred_output']
answer = 'A. ' + answer
answer = f"{choice}. {answer}"

if '\\boxed' in pred_ans:
pred_ans = pred_ans.split('\\boxed{')[1].split('}')[0]

# rule base check
acc_reward = 0.0
if len(pred_ans)==1:
if pred_ans == 'A':
if pred_ans == choice: #'A':
acc_reward = 1.0
else:
acc_reward = 0.0
elif len(pred_ans) == 2 and '.' in pred_ans:
if 'A' in pred_ans:
if choice in pred_ans: #'A' in pred_ans:
acc_reward = 1.0
else:
acc_reward = 0.0
Expand Down
7 changes: 5 additions & 2 deletions eval/judge_result_hrbench.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
else:
eval_model_name = args.eval_model_name

hrbench_path = args.vstar_bench_path
hrbench_path = args.hrbench_path
result_root_path = args.save_path
result_root_path = os.path.join(result_root_path, args.model_name)

Expand Down Expand Up @@ -129,6 +129,7 @@ def process(line):
data = json.loads(line)
question = data['question']
answer = data['answer']
answer_str = data['answer_str']
pred_ans = data['pred_ans']
pred_output = data['pred_output']
category = data['category']
Expand All @@ -148,7 +149,9 @@ def process(line):
acc_reward = 1.0
else:
acc_reward = 0.0
elif answer in pred_ans:
# elif answer in pred_ans:
# acc_reward = 1.0
elif answer_str in pred_ans:
acc_reward = 1.0
else:
full_prompt = get_prompt(pred_ans, answer, question)
Expand Down