Add 32B SFT

data/evaluation_results.csv

@@ -2,6 +2,9 @@ model,category,accuracy,precision,recall,num_samples
 Qwen2.5-VL-32B-BASE,masterCategory,0.909,0.9196051103650724,0.909,1000
 Qwen2.5-VL-32B-BASE,gender,0.546,0.9259626959624715,0.546,1000
 Qwen2.5-VL-32B-BASE,subCategory,0.432,0.7070035848765855,0.432,1000
+Qwen2.5-VL-32B-SFT,masterCategory,0.7898550724637681,0.8517880287498527,0.7898550724637681,690
+Qwen2.5-VL-32B-SFT,gender,0.9,0.8784316770186336,0.9,690
+Qwen2.5-VL-32B-SFT,subCategory,0.9173913043478261,0.9378322745486548,0.9173913043478261,690
 Qwen2-VL-72B-BASE,masterCategory,0.968968968968969,0.9711267688093789,0.968968968968969,999
 Qwen2-VL-72B-BASE,gender,0.7607607607607607,0.9354341592843324,0.7607607607607607,999
 Qwen2-VL-72B-BASE,subCategory,0.34134134134134136,0.6784829173652965,0.34134134134134136,999

generate_eval_results.py

@@ -14,6 +14,7 @@ test_df = pd.read_csv(DATA_PATH / "test.csv")
 # Define model prediction files and their display names
 model_files = {
     "Qwen2.5-VL-32B-BASE": "df_pred_FireworksAI_qwen2p5-vl-32b-instruct-ralh0ben.csv",
+    "Qwen2.5-VL-32B-SFT": "df_pred_FireworksAI_qwen-32b-SFT-fashion-catalog-c6fhxibo.csv",
     "Qwen2-VL-72B-BASE": "df_pred_FireworksAI_qwen2-vl-72b-BASE-instruct-yaxztv7t.csv",
     "Qwen2-VL-72B-SFT": "df_pred_FireworksAI_qwen-72b-SFT-fashion-catalog-oueqouqs.csv",
     "GPT-5-Mini": "df_pred_OpenAI_gpt-5-mini-2025-08-07.csv",

notebooks/02-model-evals.ipynb

@@ -245,10 +245,59 @@
     "3. Run test set through deployment for base model and save results"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "19",
+   "metadata": {},
+   "source": [
+    "#### Run evals on Qwen 32B SFT\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "19",
+   "id": "20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! firectl -a pyroworks create deployment accounts/pyroworks/models/qwen-32b-fashion-catalog --min-replica-count 1 --max-replica-count 1 --accelerator-type NVIDIA_H100_80GB"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run with concurrent requests using await directly in Jupyter\n",
+    "df_predictions_qwen_32b_fine_tuned = await run_inference_on_dataframe_async(\n",
+    "    df_test,\n",
+    "    model=\"accounts/pyroworks/deployedModels/qwen-32b-fashion-catalog-c6fhxibo\",\n",
+    "    provider=\"FireworksAI\",\n",
+    "    api_key=FIREWORKS_API_KEY,\n",
+    "    max_concurrent_requests=20,  # Adjust based on rate limits\n",
+    ")\n",
+    "\n",
+    "results_qwen_fine_tuned_32b = evaluate_all_categories(\n",
+    "    df_ground_truth=df_test,\n",
+    "    df_predictions=df_predictions_qwen_32b_fine_tuned,\n",
+    "    categories=[\"masterCategory\", \"gender\", \"subCategory\"]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "22",
+   "metadata": {},
+   "source": [
+    "#### Run evals on Qwen 72B SFT"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -258,7 +307,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "20",
+   "id": "24",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -268,7 +317,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "21",
+   "id": "25",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -290,7 +339,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "22",
+   "id": "26",
    "metadata": {},
    "source": [
     "#### Run test set through closed source model"
@@ -299,7 +348,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "23",
+   "id": "27",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -322,7 +371,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "24",
+   "id": "28",
    "metadata": {},
    "source": [
     "### Compare eval metrics across models"
@@ -331,7 +380,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "25",
+   "id": "29",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -352,7 +401,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "26",
+   "id": "30",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -367,7 +416,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "27",
+   "id": "31",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -404,7 +453,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "28",
+   "id": "32",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -414,7 +463,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "29",
+   "id": "33",
    "metadata": {},
    "outputs": [],
    "source": [