Skip to content

Commit

Permalink
Merge pull request #14 from Me1oyy/2402
Browse files Browse the repository at this point in the history 
FLARE_PIXIU2.0
  • Loading branch information
@jiminHuang
jiminHuang authored Feb 26, 2024
2 parents 415fa3e + f0e0d42 commit 6bb1156
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 122 deletions.
176 changes: 61 additions & 115 deletions app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,104 +9,58 @@

ENG_COLS = [
("Model", "str"),
("FPB-acc", "number"),
("FPB-F1", "number"),
("FPB-missing", "number"),
("FPB-Acc", "number"),
("FiQA-SA-F1", "number"),
("FiQA-SA-missing", "number"),
("Headline-AvgF1", "number"),
("NER-EntityF1", "number"),
("ConvFinQA-EmAcc", "number"),
("FinQA-EmAcc", "number"),
("BigData22-Acc", "number"),
("BigData22-MCC", "number"),
("BigData22-missing", "number"),
("ACL18-Acc", "number"),
("ACL18-MCC", "number"),
("ACL18-missing", "number"),
("CIKM18-Acc", "number"),
("CIKM18-MCC", "number"),
("CIKM18-missing", "number"),
("FOMC-acc", "number"),
("FOMC-F1", "number"),
("FOMC-missing", "number"),
("FinerOrd-EntityF1", "number"),
("FinerOrd-F1", "number"),
("German-Acc", "number"),
("German-MCC", "number"),
("German-missing", "number"),
("Australian-Acc", "number"),
("Australian-MCC", "number"),
("Australian-missing", "number"),
("TSA-RMSE", "number"),
("TSA-missing", "number"),
("MLESG-F1", "number"),
("MLESG-missing", "number"),
("FSRL-entity-F1", "number"),
("FSRL-F1", "number"),
("CFA-acc", "number"),
("CFA-F1", "number"),
("CFA-missing", "number"),
("Finarg-ECCAUC-F1", "number"),
("Finarg-ECCAUC-missing", "number"),
("Finarg-ECCARC-F1", "number"),
("Finarg_ECCARC-missing", "number"),
("CD-Entity-F1", "number"),
("CD-F1", "number"),
("MultiFin-EN-acc", "number"),
("MultiFin-EN-F1", "number"),
("MultiFin-EN-missing", "number"),
("MA-acc", "number"),
("MA-F1", "number"),
("MA-missing", "number"),
("Causal20-sc-acc", "number"),
("Causal20-sc-F1", "number"),
("Causal20-sc-missing", "number"),
("TATQA-EmAcc", "number"),
("FNXL-entity-F1", "number"),
("FNXL-F1", "number"),
("FinRED-precision", "number"),
("FinRED-recall", "number"),
("FinRED-F1", "number"),
("ECTSUM-Rouge1", "number"),
("ECTSUM-Rouge2", "number"),
("ECTSUM-RougeL", "number"),
("ECTSUM-BertScore", "number"),
("ECTSUM-BARTScore", "number"),
("EDTSUM-Rouge1", "number"),
("EDTSUM-Rouge2", "number"),
("EDTSUM-RougeL", "number"),
("EDTSUM-BertScore", "number"),
("EDTSUM-BARTScore", "number"),
("German-f1", "number"),
("Australian-f1", "number"),
("Lendingclub-Acc", "number"),
("Lendingclub-f1", "number"),
("Lendingclub-missing", "number"),
("ccf-Acc", "number"),
("ccf-MCC", "number"),
("ccf-f1", "number"),
("ccf-missing", "number"),
("ccfraud-Acc", "number"),
("ccfraud-MCC", "number"),
("ccfraud-f1", "number"),
("ccfraud-missing", "number"),
("polish-Acc", "number"),
("polish-MCC", "number"),
("polish-f1", "number"),
("polish-missing", "number"),
("taiwan-Acc", "number"),
("taiwan-MCC", "number"),
("taiwan-f1", "number"),
("taiwan-missing", "number"),
("portoseguro-Acc", "number"),
("portoseguro-MCC", "number"),
("portoseguro-f1", "number"),
("portoseguro-missing", "number"),
("travelinsurace-Acc", "number"),
("travelinsurace-MCC", "number"),
("travelinsurace-f1", "number"),
("travelinsurace-missing", "number"),
("TSA-RMSE","number"),
("Headlines-AvgF1","number"),
("FOMC-F1","number"),
("FOMC-Acc","number"),
("FinArg-ACC-MicroF1","number"),
("FinArg-ARC-MicroF1","number"),
("MultiFin-MicroF1","number"),
("MA-MicroF1","number"),
("MLESG-MicroF1","number"),
("NER-EntityF1","number"),
("FINER-ORD-EntityF1","number"),
("FinRED-F1","number"),
("SC-F1","number"),
("CD-F1","number"),
("FinQA-EmAcc","number"),
("TATQA-EmAcc","number"),
("ConvFinQA-EmAcc","number"),
("FNXL-EntityF1","number"),
("FSRL-EntityF1","number"),
("EDTSUM-Rouge1","number"),
("EDTSUM-BertScore","number"),
("EDTSUM-BartScore","number"),
("ECTSUM-Rouge1","number"),
("ECTSUM-BertScore","number"),
("ECTSUM-BartScore","number"),
("BigData22-Acc","number"),
("BigData22-MCC","number"),
("ACL18-Acc","number"),
("ACL18-MCC","number"),
("CIKM18-Acc","number"),
("CIKM18-MCC","number"),
("German-F1","number"),
("German-MCC","number"),
("Australian-F1","number"),
("Australian-MCC","number"),
("LendingClub-F1","number"),
("LendingClub-MCC","number"),
("ccf-F1","number"),
("ccf-MCC","number"),
("ccfraud-F1","number"),
("ccfraud-MCC","number"),
("polish-F1","number"),
("polish-MCC","number"),
("taiwan-F1","number"),
("taiwan-MCC","number"),
("portoseguro-F1","number"),
("portoseguro-MCC","number"),
("travelinsurance-F1","number"),
("travelinsurance-MCC","number"),
]

SPA_COLS = [
Expand Down Expand Up @@ -182,26 +136,18 @@
# Extract column names
eng_cols = [col_name for col_name, _ in ENG_COLS]
eng_cates = {
"Sentiment Analysis": ["Model", "FPB-acc", "FPB-F1", "FPB-missing",
"FiQA-SA-F1", "FiQA-SA-missing", "Headline-AvgF1", "TSA-RMSE",
"TSA-missing", "FOMC-acc", "FOMC-F1", "FOMC-missing"],
"NER": ["Model", "NER-EntityF1", "FinerOrd-EntityF1", "FinerOrd-F1"],
"Number Understanding": ["Model", "FinQA-EmAcc", "ConvFinQA-EmAcc"],
"Text Summarization": ["Model", "ECTSUM-Rouge1", "ECTSUM-Rouge2",
"ECTSUM-RougeL", "ECTSUM-BertScore", "ECTSUM-BARTScore",
"EDTSUM-Rouge1", "EDTSUM-Rouge2", "EDTSUM-RougeL", "EDTSUM-BertScore", "EDTSUM-BARTScore",],
"Sentiment Analysis": ["Model", "FPB-F1", "FPB-Acc", "FiQA-SA-F1",
"TSA-RMSE","Headlines-AvgF1","Headlines-F1","FOMC-F1","FOMC-F1","FinArg-ACC-MicroF1"
,"FinArg-ARC-MicroF1","MultiFin-MicroF1","MA-MicroF1","MLESG-MicroF1"],
"NER": ["Model", "NER-EntityF1", "FINER-ORD-EntityF1", "FinRED-F1","SC-F1","CD-F1"],
"Question Answering":["Model","FinQA-EmAcc","TATQA-EmAcc","ConvFinQA-EmAcc","FNXL-EntityF1,FSRL-EntityF1"],
"Text Summarization": ["Model", "EDTSUM-Rouge1", "EDTSUM-BertScore","EDTSUM-BartScore","ECTSUM-Rouge1","ECTSUM-BertScore","ECTSUM-BartScore"],
"Stock Movement Prediction": ["Model", "BigData22-Acc",
"BigData22-MCC", "BigData22-missing", "ACL18-Acc", "ACL18-MCC",
"ACL18-missing", "CIKM18-Acc", "CIKM18-MCC", "CIKM18-missing", ],
"Credit Scoring": ["Model", "German-Acc", "German-MCC", "German-f1", "German-missing",
"Australian-Acc", "Australian-MCC", "Australian-f1", "Australian-missing",
"Lendingclub-Acc", "Lendingclub-f1", "Lendingclub-missing",
"ccf-Acc", "ccf-MCC", "ccf-f1", "ccf-missing",
"ccfraud-Acc", "ccfraud-MCC", "ccfraud-f1", "ccfraud-missing",
"polish-Acc", "polish-MCC", "polish-f1", "polish-missing",
"taiwan-Acc", "taiwan-MCC", "taiwan-f1", "taiwan-missing",
"portoseguro-Acc", "portoseguro-MCC", "portoseguro-f1", "portoseguro-missing",
"travelinsurace-Acc", "travelinsurace-MCC", "travelinsurace-f1", "travelinsurace-missing",],
"BigData22-MCC", "ACL18-Acc", "ACL18-MCC", "CIKM18-Acc",
"CIKM18-MCC", "CIKM18-Acc", "CIKM18-MCC"],
"Credit Scoring": ["Model", "German-Acc", "German-MCC", "Australian-Acc", "Australian-MCC",
"LendingClub-F1","LendingClub-MCC","ccf-F1","ccf-MCC","ccfraud-F1","ccfraud-MCC","polish-F1","polish-MCC"
"taiwan-F1","taiwan-MCC","portoseguro-F1","portoseguro-MCC","travelinsurance-F1","travelinsurance-MCC"]
}

spa_cols = [col_name for col_name, _ in SPA_COLS]
Expand Down
22 changes: 15 additions & 7 deletions english_result.csv
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
ChatGPT,0.78,0.78,0,0.6,0.04,0.77,0.77,0.58,0.58,0.53,-0.025,0,0.5,0.005,0,0.55,0.01,0,0.6,0.64,0,0.28,0.02,0.2,-0.1,0,0.41,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,0,0,0,-5.18,0.17,0.08,0.13,0.66,-3.64,0.41,0.257,0.386,0.401,0,0.998,-0.001,0.998,0,0.173,0.066,0.214,0,0.93,0.019,0.917,0,0.968,0,0.952,0,0.97,0,0.955,0,0.981,-0.008,0.975,0
Vicuna-7B,0.55,0.52,0,0.75,0,0.6,0.01,0,0,0.44,-0.05,0,0.5,0.02,0,0.44,-0.03,0,0.19,0.28,0,0,0,0.52,0,0,0.26,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,0,0,0,-5.18,0.22,0.1,0.17,0.61,-4.13,0.505,0.513,0.808,0.723,0,0.999,0,0.998,0,0.608,-0.095,0.651,0,0.949,0,0.924,0,0.167,0.023,0.266,0.644,0,0,0,0.947,0.015,0,0.13,0
GPT-4,0.76,0.78,0,0.8,0,0.86,0.83,0.63,0.63,0.54,0.03,0,0.52,0.02,0,0.57,0.01,0,0.69,0.71,0,0.77,0.78,0.55,-0.02,0,0.74,0.47,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,0,0,0,-5.18,0.2,0.09,0.15,0.67,-3.62,0.513,0.746,0.762,0.74,0,0.81,0.331,0.878,0.11,0.58,0.113,0.587,0.21,0.65,-0.026,0.623,0,0.73,0.15,0.75,0.01,0.79,-0.03,0.778,0,0.835,0.153,0.897,0
FinMA-7B,0.86,0.86,0,0.84,0,0.98,0.75,0.25,0.06,0.44,-0.05,0,0.5,0.02,0,0.44,-0.03,0,0.28,0.19,0,0,0,0.17,0,0,0.41,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,0,0,0,-5.18,0.09,0.04,0.08,0.56,-6.12,0.17,0.41,0.19,0.06,0,0.04,0.0068,0.07,0,0.06,0,0.0065,0,0.95,0,0.92,0,0.97,0,0.95,0,0.97,0,0.95,0,0.02,0.0065,0.006,0
FinMA-7B-trade,0.03,0.12,0,0.16,0,0.27,0,0,0,0.45,0,0,0.49,0.03,0,0.43,-0.003,0,0.1,0.25,0,0,0,0.52,-0.07,0,0.26,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,0,0,0,-5.18,0.05,0.02,0.05,0.51,-6.91,0.52,0.26,0.81,0.72,0,1,0,1,0,0.06,0.016,0.02,0,0.95,-0.008,0.92,0,0.97,0,0.95,0,0.93,0.0008,0.94,0,0.98,0.035,0.98,0
FinMA-30B,0.88,0.87,0,0.87,0,0.97,0.62,0.4,0.11,0.47,0.04,0,0.49,0,0,0.43,-0.05,0,0.43,0.53,0,0,0,0.53,-0.07,0,0.46,-0.01,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,0,0,0,-5.18,0.17,0.08,0.14,0.54,-5.24,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
FinMA-7B-full,0.88,0.88,0,0.79,0,0.97,0.69,0.2,0.04,0.51,0.02,0,0.51,0.03,0,0.5,0.08,0,0.46,0.49,0,0,0,0.17,0,0,0.41,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,0,0,0,-5.18,0.13,0.06,0.1,0.38,-5.71,0.17,0.41,0.57,0.61,0,0.0031,0.0015,0.0035,0,0.06,-0.061,0.0065,0,0.95,-0.011,0.92,0,0.97,0,0.95,0,0.05,0.0079,0.04,0,0.0015,0,0.0004,0
ChatGPT,0.78,0.78,0.6,0.53,0.77,0.64,0.6,0.5,0.39,0.59,0.85,0.25,0.77,0.28,0,0.8,0,0.58,0,0.6,0,0,0.17,0.66,-3.64,0,0,-5.18,0.53,-0.025,0.5,0.005,0.55,0.01,0.2,-0.1,0.41,0,0.2,-0.1,0.2,-0.1,0.2,-0.1,0.2,-0.1,0.2,-0.1,0.2,-0.1,0.2,-0.1
GPT4,0.78,0.76,0.8,0.5,0.86,0.71,0.69,0.6,0.4,0.65,0.79,0.35,0.83,0.77,0.02,0.81,0.01,0.63,0.13,0.76,0,0.01,0.2,0.67,-3.62,0,0,-5.18,0.54,0.03,0.52,0.02,0.57,0.02,0.55,-0.02,0.74,0.47,0.55,-0.02,0.55,-0.02,0.55,-0.02,0.55,-0.02,0.55,-0.02,0.55,-0.02,0.55,-0.02
Gemini,0.77,0.77,0.81,0.37,0.78,0.53,0.6,0.31,0.6,0.62,0.84,0.34,0.61,0.14,0,0.74,0.03,0,0.18,0.43,0,0.03,0.39,0.72,-3.87,0,0,-4.93,0.55,0.04,0.52,0.04,0.54,0.02,0.52,0,0.26,0,0.65,0.19,0.96,-0.01,0.9,0,0.86,0.14,0.95,0,0.95,0,0,0
LLaMA2-7B-chat,0.35,0.29,0,0.36,0.6,0.53,0,0.51,0.28,0.56,0.81,0.18,0,0,0,0.2,0,0,0,0,0,0,0.11,0.59,-4.26,0,0,-5.18,0.55,0.03,0.51,0.01,0.54,-0.01,0.52,0,0.26,0,0.73,0.04,0,0,0.88,-0.05,0.82,0.08,0.75,-0.07,0.01,0,0,0
LLaMA2-70B,0.73,0.72,0.83,0.57,0.63,0.49,0.47,0.58,0.36,0.63,0.86,0.31,0.04,0.03,0,0.61,0.01,0.06,0.01,0.25,0,0.01,0.25,0.68,-3.81,0,0,-4.86,0.47,0,0.51,0.01,0.49,-0.07,0.17,0,0.41,0,0.17,0,0.17,0,0.17,0,0.17,0,0.17,0,0.17,0,0.17,0
ChatGLM3-6B,0.37,0.38,0.6,0.34,0.6,0.47,0.55,0.5,0.3,0.22,0.5,0.14,0.01,0.07,0,0.4,0,0,0,0,0,0,0.14,0.46,-4.94,0,0,-5.18,0.47,0.03,0.49,0,0.42,0.04,0.52,0,0.26,0,0.65,0.19,0.96,-0.01,0.9,0,0.01,0.01,0.95,0,0.97,0,0,0
FinMA-7B,0.88,0.88,0.79,0.8,0.97,0.49,0.46,0.27,0.08,0.14,0.45,0,0.69,0,0,0.19,0,0.04,0,0.2,0,0,0.13,0.38,-5.71,0,0,-5.18,0.51,0.02,0.51,0.03,0.5,0.08,0.17,0,0.41,0,0.61,0,0,0,0.01,-0.06,0.92,-0.01,0.95,0,0.04,0.01,0,0
FinGPT-7b-lora,0,0,0,0,0.6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.5,-7.25,0,0,-5.18,0.45,0,0.49,0,0.49,0,0.52,0,0.41,0,0,0,0,0,0,0,0.07,0.02,0.82,0.01,0,0,0.98,0
InternLM-7B,0.27,0.2,0,0.33,0.6,0,0,0.47,0.37,0.33,0.74,0.24,0,0,0,0.58,0,0,0,0,0,0,0.12,0.51,-4.51,0,0,-5.18,0.52,0.02,0.52,0.02,0.57,0.01,0.44,-0.01,0.27,-0.02,0.71,0.18,1,0,0.77,-0.05,0.9,-0.01,0.48,0.04,0.95,0,0.79,0
Falcon-7B,0.07,0.05,0.77,0.5,0.45,0.3,0.3,0.23,0.32,0.09,0.39,0.06,0,0,0,0.67,0,0,0,0,0,0,0.15,0.57,-6.1,0,0,-5.18,0.55,0,0.51,0,0.47,-0.06,0.23,-0.07,0.26,0,0.02,-0.01,0.1,0,0.62,-0.02,0.76,0.05,0,0,0.95,0,0.77,-0.03
Mixtral-7B,0.29,0.37,0.16,0.5,0.6,0.37,0.35,0.39,0.57,0.37,0.34,0.17,0.24,0.05,0,0.83,0,0,0.01,0.31,0,0,0.12,0.61,-4.47,0,0,-5.18,0.46,0.02,0.49,0,0.42,-0.05,0.53,0,0.26,0,0.61,0.08,0,0,0.48,0.16,0.92,0,0.95,0,0.72,0.01,0,0
CFGPT-sft-7B-Full,0.35,0.26,0.42,1.05,0.61,0.16,0.21,0.05,0.05,0.05,0.25,0.01,0,0,0,0.15,0,0,0,0.01,0,0,0.01,0.51,-7.08,0,0,-5.18,0.45,0.03,0.48,-0.03,0.41,-0.07,0.53,0,0.29,-0.1,0.05,0.01,0,0,0.03,0.01,0.4,-0.02,0.7,0,0,0,0.03,0.01
Baichuan-7B,0.36,0.32,0.17,1.07,0.6,0.16,0.25,0.34,0.17,0.06,0.02,0,0,0,0,0.57,0,0,0,0,0,0,0.22,0.54,-4.57,0,0,-5.18,0.53,-0.01,0.5,0,0.53,-0.05,0.52,0,0.26,0,0.72,-0.01,0.97,0,0,0,0.91,0.02,0.7,-0.02,0.01,0.01,0.03,-0.09
CodeLLaMA-7B,0.34,0.39,0.66,0.43,0.6,0.14,0.27,0.28,0.25,0.21,0.54,0.1,0.07,0,0,0.85,0,0,0,0,0,0,0.1,0.67,-3.62,0,0,-5.18,0.52,-0.01,0.51,0,0.51,0.02,0.66,0,0.43,0,0.81,0,0,0,0.06,0,0.47,0.04,0.36,-0.03,0.88,-0.01,0.02,0
DISC-FinLLM,0.29,0.26,0.32,0.32,0.6,0.19,0.28,0.29,0.29,0.29,0.29,0.29,0.12,0,0,0,0,0,0,0,0,0,0.22,0.61,-4.13,0,0,-5.18,0.44,-0.05,0.5,0.02,0.44,-0.03,0.52,0,0.26,0,0.72,0,0.66,-0.04,0.46,0.02,0.92,0,0.95,0,0.63,-0.02,0,0

0 comments on commit 6bb1156

Please sign in to comment.