added transformer without flash attention

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/.gitattributes

@@ -0,0 +1,4 @@
+plots_checkpoint*/** filter=lfs diff=lfs merge=lfs -text
+preds_checkpoint*/** filter=lfs diff=lfs merge=lfs -text
+runs/** filter=lfs diff=lfs merge=lfs -text
+checkpoints/** filter=lfs diff=lfs merge=lfs -text

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/attn_conv_id_0.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a63abfc80b7f06bc1a30f569ebe6b0f2582fc52537f81ab9383f6f98d8007e3b
+size 53954056

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/attn_conv_id_1.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bfd7fe6f462d0a20f33b9339230293a9bff861bab298705b093818ea6b66589
+size 53954056

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/attn_conv_id_2.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c06335a26bf1a5d471dd8d3cbe2918e0df8e419a292cb2f9e1392508364842fe
+size 53954056

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/attn_conv_reg_0.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:461a35ceb8a37e628760b030057172b93d72b1db3eaf0c8632ba6068c8743fee
+size 53954056

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/attn_conv_reg_1.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca4e194fd988ca9a2e5d45bcd00ac871f84ebd244609a7cfb9d27bbf47abb611
+size 53954056

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/attn_conv_reg_2.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19707abfbe8d50df0bf7ef99e6f2e376221ebc3ad6fba6a184543b4ba9753888
+size 53954056

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/batch0_epoch1.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28ec201b526fe31241c35e2a958366be385e8d1ae763bc817b6be29e3d5c2531
+size 2544690

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/batch0_epoch2.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:707a0fd205e0e90b651c07851c16ba8bf0596c9c545dabb7b504175019ad050d
+size 2546577

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/checkpoints/checkpoint-02-2.220678.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:830e94648658e79374dce0bb86d8e8075bb5d2e60cf85c8ce556af9d5b80d587
+size 631766830

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/history/epoch_1.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.13023509053755675, "Regression_eta": 0.00017174951638457376, "Regression_sin_phi": 0.00011457915233879325, "Regression_cos_phi": 0.00011486207141154782, "Regression_energy": 0.12988839610747568, "Classification_binary": 2.387953783610299, "Classification": 0.018947793490660586, "ispu": 1.029574128254712e-05, "MET": 4.840134144022062, "Sliced_Wasserstein_Loss": 44.11957064403395, "Total": 2.667434859615339}, "valid": {"Regression_pt": 0.11223718119366508, "Regression_eta": 0.00015769626439491338, "Regression_sin_phi": 0.00010136838289356569, "Regression_cos_phi": 0.00010135367144438447, "Regression_energy": 0.11233068963888815, "Classification_binary": 2.159941232815342, "Classification": 0.015957186792679136, "ispu": 1.366309797016072e-13, "MET": 3.7326260263988447, "Sliced_Wasserstein_Loss": 40.70979324917086, "Total": 2.4008256743473844}, "epoch_train_time": 55047.92060089111, "epoch_valid_time": 2225.9467058181763, "epoch_total_time": 57273.867307662964}

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/history/epoch_2.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.10502994345906735, "Regression_eta": 0.00014515724342650365, "Regression_sin_phi": 9.354014535309955e-05, "Regression_cos_phi": 9.345064557029725e-05, "Regression_energy": 0.10467700374661222, "Classification_binary": 2.0461803574400306, "Classification": 0.015114094111808477, "ispu": 1.7147474985948907e-13, "MET": 3.708865273518294, "Sliced_Wasserstein_Loss": 39.59190381095024, "Total": 2.271334813256306}, "valid": {"Regression_pt": 0.1044524137611834, "Regression_eta": 0.0001451477744789242, "Regression_sin_phi": 9.296932513285056e-05, "Regression_cos_phi": 9.291878202656087e-05, "Regression_energy": 0.10473331130225608, "Classification_binary": 1.9969109824676368, "Classification": 0.014254206678529877, "ispu": 5.824024262750208e-14, "MET": 3.6014254239328127, "Sliced_Wasserstein_Loss": 38.68739970043865, "Total": 2.220677632194822}, "epoch_train_time": 55223.50347614288, "epoch_valid_time": 2253.887442588806, "epoch_total_time": 57477.39092040062}

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/hyperparameters.json

@@ -0,0 +1 @@
+{"num_mlpf_params": 52630547, "train": true, "test": null, "make_plots": null, "comet": true, "save_attention": true, "dataset": "clic", "sort_data": false, "data_dir": "/scratch/persistent/joosep/tensorflow_datasets", "gpus": 1, "gpu_batch_multiplier": 256, "load": null, "finetune": null, "num_epochs": 10, "patience": 20, "lr": 0.0001, "lr_schedule": "cosinedecay", "conv_type": "attention", "ntrain": null, "ntest": 2000, "nvalid": null, "num_workers": 8, "prefetch_factor": 100, "checkpoint_freq": 1, "comet_name": "particleflow-pt", "comet_offline": false, "comet_step_freq": 100, "dtype": "bfloat16", "val_freq": null, "model": {"trainable": "all", "learned_representation_mode": "last", "input_encoding": "split", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "gnn_lsh": {"conv_type": "gnn_lsh", "embedding_dim": 1024, "width": 1024, "num_convs": 3, "activation": "elu", "bin_size": 32, "max_num_bins": 200, "distance_dim": 128, "layernorm": true, "num_node_messages": 2, "ffn_dist_hidden_dim": 128, "ffn_dist_num_layers": 2}, "attention": {"conv_type": "attention", "num_convs": 3, "dropout_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "activation": "relu", "head_dim": 32, "num_heads": 32, "attention_type": "math", "use_pre_layernorm": true}, "mamba": {"conv_type": "mamba", "embedding_dim": 128, "width": 128, "num_convs": 2, "dropout": 0.0, "activation": "elu", "num_heads": 2, "d_state": 16, "d_conv": 4, "expand": 2}}, "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "raytune": {"local_dir": null, "sched": null, "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"clic": {"physical": {"batch_size": 1, "samples": {"clic_edm_qq_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ttbar_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ww_fullhad_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}}}}}, "valid_dataset": {"clic": {"physical": {"batch_size": 1, "samples": {"clic_edm_qq_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ttbar_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ww_fullhad_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}}}}}, "test_dataset": {"clic_edm_qq_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ttbar_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ww_fullhad_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}}, "enabled_test_datasets": ["clic_edm_qq_pf"]}

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/model_kwargs.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91f1453782879f7bbfadc7bb37aa54724149a9c98687dcf8466556fed6cf3dba
+size 552

clic/clusters/v2.3.0/pyg-clic_20250211_145811_219129/overridden_config.yaml

@@ -0,0 +1,225 @@
+checkpoint_freq: 1
+comet: true
+comet_name: particleflow-pt
+comet_offline: false
+comet_step_freq: 100
+conv_type: attention
+data_dir: /scratch/persistent/joosep/tensorflow_datasets
+dataset: clic
+dtype: bfloat16
+enabled_test_datasets:
+- clic_edm_qq_pf
+finetune: null
+gpu_batch_multiplier: 256
+gpus: 1
+load: null
+lr: 0.0001
+lr_schedule: cosinedecay
+lr_schedule_config:
+  onecycle:
+    pct_start: 0.3
+make_plots: null
+model:
+  attention:
+    activation: relu
+    attention_type: math
+    conv_type: attention
+    dropout_conv_id_ff: 0.0
+    dropout_conv_id_mha: 0.0
+    dropout_conv_reg_ff: 0.0
+    dropout_conv_reg_mha: 0.0
+    dropout_ff: 0.0
+    head_dim: 32
+    num_convs: 3
+    num_heads: 32
+    use_pre_layernorm: true
+  cos_phi_mode: linear
+  energy_mode: direct-elemtype-split
+  eta_mode: linear
+  gnn_lsh:
+    activation: elu
+    bin_size: 32
+    conv_type: gnn_lsh
+    distance_dim: 128
+    embedding_dim: 1024
+    ffn_dist_hidden_dim: 128
+    ffn_dist_num_layers: 2
+    layernorm: true
+    max_num_bins: 200
+    num_convs: 3
+    num_node_messages: 2
+    width: 1024
+  input_encoding: split
+  learned_representation_mode: last
+  mamba:
+    activation: elu
+    conv_type: mamba
+    d_conv: 4
+    d_state: 16
+    dropout: 0.0
+    embedding_dim: 128
+    expand: 2
+    num_convs: 2
+    num_heads: 2
+    width: 128
+  pt_mode: direct-elemtype-split
+  sin_phi_mode: linear
+  trainable: all
+ntest: 2000
+ntrain: null
+num_epochs: 10
+num_workers: 8
+nvalid: null
+patience: 20
+prefetch_factor: 100
+raytune:
+  asha:
+    brackets: 1
+    grace_period: 10
+    max_t: 200
+    reduction_factor: 4
+  default_metric: val_loss
+  default_mode: min
+  hyperband:
+    max_t: 200
+    reduction_factor: 4
+  hyperopt:
+    n_random_steps: 10
+  local_dir: null
+  nevergrad:
+    n_random_steps: 10
+  sched: null
+  search_alg: null
+save_attention: true
+sort_data: false
+test: null
+test_dataset:
+  clic_edm_qq_pf:
+    splits:
+    - 1
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    version: 2.5.0
+  clic_edm_ttbar_pf:
+    splits:
+    - 1
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    version: 2.5.0
+  clic_edm_ww_fullhad_pf:
+    splits:
+    - 1
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    version: 2.5.0
+train: true
+train_dataset:
+  clic:
+    physical:
+      batch_size: 1
+      samples:
+        clic_edm_qq_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+        clic_edm_ttbar_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+        clic_edm_ww_fullhad_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+val_freq: null
+valid_dataset:
+  clic:
+    physical:
+      batch_size: 1
+      samples:
+        clic_edm_qq_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+        clic_edm_ttbar_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+        clic_edm_ww_fullhad_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0