richarderkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf overview

{
  "metadata": {},
  "card_data": {
    "frontmatter": {},
    "hero_image_url": "https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png",
    "summary": "This model is a fine-tuned version of macadeliccc/magistrate-3.2-3b-base on the None dataset. It achieves the following results on the evaluation set:   See axolotl config axolotl version: 0.4.1 ``yaml base_model: macadeliccc/magistrate-3.2-3b-base model_type: LlamaForCausalLM tokenizer_type: AutoTokenizer load_in_8bit: false load_in_4bit: false strict: false datasets: type: sharegpt conversation: chatml data_files: train/hermes-2.5.jsonl # - path: json #   type: sharegpt #   conversation: chatml #   data_files: train/financial_instructions_cleaned_2.json type: sharegpt conversation: chatml data_files: train/glaive-function-calling-5k.json type: sharegpt conversation: chatml data_files: train/func-calling-singleturn.json type: sharegpt conversation: chatml data_files: train/func-calling.json type: sharegpt conversation: chatml data_files: train/json-mode-agentic.json type: sharegpt conversation: chatml data_files: train/json-mode-singleturn.json type: sharegpt conversation: chatml data_files: train/reasoning_sharegpt.json type: sharegpt conversation: chatml data_files: train/systemchat_2_0_small.json type: sharegpt conversation: chatml data_files: train/argument_dataset/303_creative_llc_v__elenis_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/abitron_austria_gmbh_v__hetronic_international__inc__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/acheson_hotels__llc_v__laufer_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/alexander_v__sc_conference_of_naacp_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/amgen_inc__v__sanofi_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/andy_warhol_found___inc__v__goldsmith_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/arizona_v__navajo_nation_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/becerra__sec__of_h_hs_v__san_carlos_apache_tribe_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/biden_v__nebraska_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/bissonnette_v__lepage_bakeries_park_st___llc_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/bittner_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/brown_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/cantero_v__bank_of_america__n_a__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/cfpb_v__com__fin__services_assn__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/chiaverini_v__city_of_napoleon_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/ciminelli_v__united_state_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/city_of_grants_pass_v__johnson_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/coinbase__inc__v__bielski_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/coinbase__inc__v__suski_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/connelly_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/corner_post__inc__v__bd__of_governors__frs_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/counterman_v__colorado_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/cruz_v__arizona_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/culley_v__marshall_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/dept__of_agric__rural_dev__v__kirtz_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/dept__of_education_v__brown_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/dept__of_state_v__munoz_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/devillier_v__texas_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/diaz_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/dubin_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/dupree_v__younger_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/erlinger_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/fbi_v__fikre_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/fda_v__alliance_hippocratic_medicine_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/financial_oversight_board_v__cpi_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/fischer_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/garland__att_y_gen__v__cargill_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/glacier_northwest__inc__v__int_l_brotherhood_of_teamsters_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/gonzalez_v__google_llc_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/gonzalez_v__trevino_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/great_lakes_insurance_se_v__raiders_retreat_realty_co___llc_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/groff_v__dejoy_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/harrington_v__purdue_pharma_l_p__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/harrow_v__dept__of_defense_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/health_and_hospital_corp__v__talevski_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/helix_energy_solutions_v__hewitt_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/in_re_grand_jury_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/jack_daniel_s_properties__inc__v__vip_products_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/jones_v__hendrix_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/karcho_polselli_v__irs_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/lac_du_flambeau_band_v__coughlin_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/lindke_v__freed_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/loper_bright_enterprises__inc__v__raimondo__sec__of_comm__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/lora_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/macquarie_infrastructure_corp__v__moab_partners__l_p__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/mallory_v__norfolk_southern_railway_co__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/mcintosh_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/merrill_v__milligan_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/moore_v__harper_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/moore_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/moyle_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/muldrow_v__st__louis_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/murray_v__ubs_securities__llc_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/murthy__surgeon_gen__v__missouri_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/netchoice__llc_v__paxton_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/new_york_v__new_jersey_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/nra_v__vullo_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/o_connor_ratcliff_v__garnier_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/oh_adjutant_gen__s_dept__v__flra_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/ohio_v__epa_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/perez_v__sturgis_public_schools_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/pugin_v__garland_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/pulsifer_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/relentless__inc__v__dept__of_commerce_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/rudisill_v__mcdonough__sec__of_va_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/sackett_v__epa_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/samia_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/santos_zacaria_v__garland__att_y_gen__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/sec_v__cochran_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/sec_v__jarkesy_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/sheetz_v__county_of_el_dorado_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/slack_technologies__llc_v__pirani_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/smith_v__arizona_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/smith_v__spizzirri_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/smith_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/snyder_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/starbucks_corp__v__mckinney_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/students_for_fair_admissions_v__university_of_nc_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/texas_v__new_mexico_and_colorado_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/thornell_v__jones_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/truck_insurance_exchange_v__kaiser_gypsum_co__inc__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/trump_v__anderson_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/turkiye_halk_bankasi_a_s__v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/twitter__inc__v__taamneh_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/tyler_v__hennepin_county_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/u_s___ex_rel__polansky_v__executive_health_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/u_s___ex_rel__schutte_v__supervalu_inc__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/united_states_trustee_v__john_q__hammons_fall_2006__llc_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/united_states_v__hansen_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/united_states_v__rahimi_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/united_states_v__texas_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/vidal__under_sec__of_comm__v__elster_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/warner_chappell_music__inc__v__nealy_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/wilkins_v__united_states_sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/wilkinson_v__garland__att_y_gen__sharegpt.json type: sharegpt conversation: chatml data_files: train/argument_dataset/yegiazaryan_v__smagin_sharegpt.json chat_template: chatml unfrozen_parameters: # input_layernorm layers # mlp.down_proj layers # mlp.gate_proj layers # mlp.up_proj layers # post_attention_layernorm layers # self_attn.k_proj layers # self_attn.o_proj layers # self_attn.q_proj layers # model.norm layers # self_attn.v_proj layers val_set_size: 0.05 output_dir: ./outputs/magistrate-3.2-3b sequence_len: 8192 sample_packing: true eval_sample_packing: false pad_to_sequence_len: true adapter: wandb_project: wandb_entity: wandb_watch: wandb_name: wandb_log_model: gradient_accumulation_steps: 8 micro_batch_size: 1 num_epochs: 3 optimizer: paged_adamw_32bit lr_scheduler: cosine learning_rate: 2e-4 train_on_inputs: false group_by_length: false bf16: auto fp16: tf32: false gradient_checkpointing: true early_stopping_patience: resume_from_checkpoint: local_rank: logging_steps: 1 xformers_attention: flash_attention: true s2_attention: warmup_steps: 1000 evals_per_epoch: 2 eval_table_size: eval_max_new_tokens: 128 saves_per_epoch: 1 debug: deepspeed: deepspeed_configs/zero3.json weight_decay: 0.0 fsdp: fsdp_config: special_tokens: eos_token: \"\" pad_token: \"\" tokens: ``",
    "quick_links": [],
    "benchmark_table_html": "",
    "readme_markdown": "Quantization made by Richard Erkhov.\n\n[Github](https://github.com/RichardErkhov)\n\n[Discord](https://discord.gg/pvy7H8DZMG)\n\n[Request more models](https://github.com/RichardErkhov/quant_request)\n\n\nmagistrate-3.2-3b-it - GGUF\n- Model creator: https://huggingface.co/macadeliccc/\n- Original model: https://huggingface.co/macadeliccc/magistrate-3.2-3b-it/\n\n\n| Name | Quant method | Size |\n| ---- | ---- | ---- |\n| [magistrate-3.2-3b-it.Q2_K.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q2_K.gguf) | Q2_K | 1.27GB |\n| [magistrate-3.2-3b-it.IQ3_XS.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.IQ3_XS.gguf) | IQ3_XS | 1.38GB |\n| [magistrate-3.2-3b-it.IQ3_S.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.IQ3_S.gguf) | IQ3_S | 1.44GB |\n| [magistrate-3.2-3b-it.Q3_K_S.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q3_K_S.gguf) | Q3_K_S | 1.44GB |\n| [magistrate-3.2-3b-it.IQ3_M.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.IQ3_M.gguf) | IQ3_M | 1.49GB |\n| [magistrate-3.2-3b-it.Q3_K.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q3_K.gguf) | Q3_K | 1.57GB |\n| [magistrate-3.2-3b-it.Q3_K_M.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q3_K_M.gguf) | Q3_K_M | 1.57GB |\n| [magistrate-3.2-3b-it.Q3_K_L.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q3_K_L.gguf) | Q3_K_L | 1.69GB |\n| [magistrate-3.2-3b-it.IQ4_XS.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.IQ4_XS.gguf) | IQ4_XS | 1.71GB |\n| [magistrate-3.2-3b-it.Q4_0.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q4_0.gguf) | Q4_0 | 1.79GB |\n| [magistrate-3.2-3b-it.IQ4_NL.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.IQ4_NL.gguf) | IQ4_NL | 1.79GB |\n| [magistrate-3.2-3b-it.Q4_K_S.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q4_K_S.gguf) | Q4_K_S | 1.8GB |\n| [magistrate-3.2-3b-it.Q4_K.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q4_K.gguf) | Q4_K | 1.88GB |\n| [magistrate-3.2-3b-it.Q4_K_M.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q4_K_M.gguf) | Q4_K_M | 1.88GB |\n| [magistrate-3.2-3b-it.Q4_1.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q4_1.gguf) | Q4_1 | 1.95GB |\n| [magistrate-3.2-3b-it.Q5_0.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q5_0.gguf) | Q5_0 | 2.11GB |\n| [magistrate-3.2-3b-it.Q5_K_S.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q5_K_S.gguf) | Q5_K_S | 2.11GB |\n| [magistrate-3.2-3b-it.Q5_K.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q5_K.gguf) | Q5_K | 2.16GB |\n| [magistrate-3.2-3b-it.Q5_K_M.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q5_K_M.gguf) | Q5_K_M | 2.16GB |\n| [magistrate-3.2-3b-it.Q5_1.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q5_1.gguf) | Q5_1 | 2.28GB |\n| [magistrate-3.2-3b-it.Q6_K.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q6_K.gguf) | Q6_K | 2.46GB |\n| [magistrate-3.2-3b-it.Q8_0.gguf](https://huggingface.co/RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf/blob/main/magistrate-3.2-3b-it.Q8_0.gguf) | Q8_0 | 3.19GB |\n\n\n\n\nOriginal model description:\n---\nlibrary_name: transformers\nlicense: llama3.2\nbase_model: macadeliccc/magistrate-3.2-3b-base\ndatasets:\n- teknium/OpenHermes-2.5\n- NousResearch/hermes-function-calling-v1\n- arcee-ai/The-Tome\n- cognitivecomputations/SystemChat-2.0\ntags:\n- spectrum\n- llama-3\n- axolotl\n- legal\n- HFforLegal\nlanguage:\n- en\npipeline_tag: text-generation\n---\n# magistrate-3.2-3b-it\n\nThis model is a fine-tuned version of [macadeliccc/magistrate-3.2-3b-base](https://huggingface.co/macadeliccc/magistrate-3.2-3b-base) on the None dataset.\nIt achieves the following results on the evaluation set:\n- Loss: 0.8067\n\n<!-- This model card has been generated automatically according to the information the Trainer had access to. You\nshould probably proofread and complete it, then remove this comment. -->\n\n[<img src=\"https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png\" alt=\"Built with Axolotl\" width=\"200\" height=\"32\"/>](https://github.com/axolotl-ai-cloud/axolotl)\n<details><summary>See axolotl config</summary>\n\naxolotl version: `0.4.1`\n```yaml\nbase_model: macadeliccc/magistrate-3.2-3b-base\nmodel_type: LlamaForCausalLM\ntokenizer_type: AutoTokenizer\n\nload_in_8bit: false\nload_in_4bit: false\nstrict: false\n\ndatasets:\n  - path: json \n    type: sharegpt\n    conversation: chatml\n    data_files: train/hermes-2.5.jsonl\n  # - path: json \n  #   type: sharegpt\n  #   conversation: chatml\n  #   data_files: train/financial_instructions_cleaned_2.json\n  - path: json \n    type: sharegpt\n    conversation: chatml\n    data_files: train/glaive-function-calling-5k.json\n  - path: json \n    type: sharegpt\n    conversation: chatml\n    data_files: train/func-calling-singleturn.json\n  - path: json \n    type: sharegpt\n    conversation: chatml\n    data_files: train/func-calling.json\n  - path: json \n    type: sharegpt\n    conversation: chatml\n    data_files: train/json-mode-agentic.json\n  - path: json \n    type: sharegpt\n    conversation: chatml\n    data_files: train/json-mode-singleturn.json\n  - path: json \n    type: sharegpt\n    conversation: chatml\n    data_files: train/reasoning_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/systemchat_2_0_small.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/303_creative_llc_v__elenis_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/abitron_austria_gmbh_v__hetronic_international__inc__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/acheson_hotels__llc_v__laufer_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/alexander_v__sc_conference_of_naacp_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/amgen_inc__v__sanofi_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/andy_warhol_found___inc__v__goldsmith_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/arizona_v__navajo_nation_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/becerra__sec__of_h_hs_v__san_carlos_apache_tribe_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/biden_v__nebraska_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/bissonnette_v__lepage_bakeries_park_st___llc_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/bittner_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/brown_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/cantero_v__bank_of_america__n_a__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/cfpb_v__com__fin__services_assn__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/chiaverini_v__city_of_napoleon_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/ciminelli_v__united_state_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/city_of_grants_pass_v__johnson_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/coinbase__inc__v__bielski_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/coinbase__inc__v__suski_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/connelly_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/corner_post__inc__v__bd__of_governors__frs_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/counterman_v__colorado_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/cruz_v__arizona_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/culley_v__marshall_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/dept__of_agric__rural_dev__v__kirtz_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/dept__of_education_v__brown_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/dept__of_state_v__munoz_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/devillier_v__texas_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/diaz_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/dubin_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/dupree_v__younger_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/erlinger_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/fbi_v__fikre_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/fda_v__alliance_hippocratic_medicine_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/financial_oversight_board_v__cpi_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/fischer_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/garland__att_y_gen__v__cargill_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/glacier_northwest__inc__v__int_l_brotherhood_of_teamsters_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/gonzalez_v__google_llc_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/gonzalez_v__trevino_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/great_lakes_insurance_se_v__raiders_retreat_realty_co___llc_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/groff_v__dejoy_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/harrington_v__purdue_pharma_l_p__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/harrow_v__dept__of_defense_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/health_and_hospital_corp__v__talevski_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/helix_energy_solutions_v__hewitt_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/in_re_grand_jury_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/jack_daniel_s_properties__inc__v__vip_products_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/jones_v__hendrix_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/karcho_polselli_v__irs_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/lac_du_flambeau_band_v__coughlin_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/lindke_v__freed_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/loper_bright_enterprises__inc__v__raimondo__sec__of_comm__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/lora_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/macquarie_infrastructure_corp__v__moab_partners__l_p__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/mallory_v__norfolk_southern_railway_co__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/mcintosh_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/merrill_v__milligan_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/moore_v__harper_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/moore_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/moyle_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/muldrow_v__st__louis_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/murray_v__ubs_securities__llc_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/murthy__surgeon_gen__v__missouri_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/netchoice__llc_v__paxton_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/new_york_v__new_jersey_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/nra_v__vullo_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/o_connor_ratcliff_v__garnier_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/oh_adjutant_gen__s_dept__v__flra_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/ohio_v__epa_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/perez_v__sturgis_public_schools_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/pugin_v__garland_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/pulsifer_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/relentless__inc__v__dept__of_commerce_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/rudisill_v__mcdonough__sec__of_va_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/sackett_v__epa_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/samia_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/santos_zacaria_v__garland__att_y_gen__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/sec_v__cochran_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/sec_v__jarkesy_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/sheetz_v__county_of_el_dorado_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/slack_technologies__llc_v__pirani_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/smith_v__arizona_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/smith_v__spizzirri_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/smith_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/snyder_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/starbucks_corp__v__mckinney_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/students_for_fair_admissions_v__university_of_nc_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/texas_v__new_mexico_and_colorado_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/thornell_v__jones_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/truck_insurance_exchange_v__kaiser_gypsum_co__inc__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/trump_v__anderson_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/turkiye_halk_bankasi_a_s__v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/twitter__inc__v__taamneh_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/tyler_v__hennepin_county_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/u_s___ex_rel__polansky_v__executive_health_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/u_s___ex_rel__schutte_v__supervalu_inc__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/united_states_trustee_v__john_q__hammons_fall_2006__llc_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/united_states_v__hansen_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/united_states_v__rahimi_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/united_states_v__texas_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/vidal__under_sec__of_comm__v__elster_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/warner_chappell_music__inc__v__nealy_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/wilkins_v__united_states_sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/wilkinson_v__garland__att_y_gen__sharegpt.json\n  - path: json\n    type: sharegpt\n    conversation: chatml\n    data_files: train/argument_dataset/yegiazaryan_v__smagin_sharegpt.json\n\nchat_template: chatml\n\nunfrozen_parameters:\n- ^lm_head.weight$\n- ^model.embed_tokens.weight$\n# input_layernorm layers\n- model.layers.0.input_layernorm\n- model.layers.1.input_layernorm\n- model.layers.2.input_layernorm\n- model.layers.3.input_layernorm\n- model.layers.4.input_layernorm\n- model.layers.5.input_layernorm\n- model.layers.6.input_layernorm\n- model.layers.7.input_layernorm\n- model.layers.8.input_layernorm\n- model.layers.9.input_layernorm\n- model.layers.10.input_layernorm\n- model.layers.11.input_layernorm\n- model.layers.12.input_layernorm\n- model.layers.13.input_layernorm\n# mlp.down_proj layers\n- model.layers.0.mlp.down_proj\n- model.layers.1.mlp.down_proj\n- model.layers.17.mlp.down_proj\n- model.layers.19.mlp.down_proj\n- model.layers.18.mlp.down_proj\n- model.layers.5.mlp.down_proj\n- model.layers.20.mlp.down_proj\n- model.layers.2.mlp.down_proj\n- model.layers.4.mlp.down_proj\n- model.layers.6.mlp.down_proj\n- model.layers.3.mlp.down_proj\n- model.layers.16.mlp.down_proj\n- model.layers.15.mlp.down_proj\n- model.layers.13.mlp.down_proj\n# mlp.gate_proj layers\n- model.layers.0.mlp.gate_proj\n- model.layers.1.mlp.gate_proj\n- model.layers.2.mlp.gate_proj\n- model.layers.3.mlp.gate_proj\n- model.layers.22.mlp.gate_proj\n- model.layers.21.mlp.gate_proj\n- model.layers.20.mlp.gate_proj\n- model.layers.23.mlp.gate_proj\n- model.layers.19.mlp.gate_proj\n- model.layers.4.mlp.gate_proj\n- model.layers.18.mlp.gate_proj\n- model.layers.17.mlp.gate_proj\n- model.layers.5.mlp.gate_proj\n- model.layers.24.mlp.gate_proj\n# mlp.up_proj layers\n- model.layers.4.mlp.up_proj\n- model.layers.3.mlp.up_proj\n- model.layers.5.mlp.up_proj\n- model.layers.6.mlp.up_proj\n- model.layers.7.mlp.up_proj\n- model.layers.2.mlp.up_proj\n- model.layers.8.mlp.up_proj\n- model.layers.14.mlp.up_proj\n- model.layers.13.mlp.up_proj\n- model.layers.11.mlp.up_proj\n- model.layers.9.mlp.up_proj\n- model.layers.1.mlp.up_proj\n- model.layers.15.mlp.up_proj\n- model.layers.12.mlp.up_proj\n# post_attention_layernorm layers\n- model.layers.0.post_attention_layernorm\n- model.layers.1.post_attention_layernorm\n- model.layers.2.post_attention_layernorm\n- model.layers.3.post_attention_layernorm\n- model.layers.4.post_attention_layernorm\n- model.layers.5.post_attention_layernorm\n- model.layers.6.post_attention_layernorm\n- model.layers.7.post_attention_layernorm\n- model.layers.8.post_attention_layernorm\n- model.layers.9.post_attention_layernorm\n- model.layers.10.post_attention_layernorm\n- model.layers.11.post_attention_layernorm\n- model.layers.12.post_attention_layernorm\n- model.layers.13.post_attention_layernorm\n# self_attn.k_proj layers\n- model.layers.25.self_attn.k_proj\n- model.layers.22.self_attn.k_proj\n- model.layers.19.self_attn.k_proj\n- model.layers.20.self_attn.k_proj\n- model.layers.17.self_attn.k_proj\n- model.layers.24.self_attn.k_proj\n- model.layers.23.self_attn.k_proj\n- model.layers.18.self_attn.k_proj\n- model.layers.21.self_attn.k_proj\n- model.layers.27.self_attn.k_proj\n- model.layers.15.self_attn.k_proj\n- model.layers.10.self_attn.k_proj\n- model.layers.6.self_attn.k_proj\n- model.layers.5.self_attn.k_proj\n# self_attn.o_proj layers\n- model.layers.13.self_attn.o_proj\n- model.layers.7.self_attn.o_proj\n- model.layers.12.self_attn.o_proj\n- model.layers.10.self_attn.o_proj\n- model.layers.5.self_attn.o_proj\n- model.layers.21.self_attn.o_proj\n- model.layers.6.self_attn.o_proj\n- model.layers.19.self_attn.o_proj\n- model.layers.8.self_attn.o_proj\n- model.layers.20.self_attn.o_proj\n- model.layers.22.self_attn.o_proj\n- model.layers.9.self_attn.o_proj\n- model.layers.17.self_attn.o_proj\n- model.layers.11.self_attn.o_proj\n# self_attn.q_proj layers\n- model.layers.12.self_attn.q_proj\n- model.layers.13.self_attn.q_proj\n- model.layers.9.self_attn.q_proj\n- model.layers.8.self_attn.q_proj\n- model.layers.10.self_attn.q_proj\n- model.layers.14.self_attn.q_proj\n- model.layers.11.self_attn.q_proj\n- model.layers.15.self_attn.q_proj\n- model.layers.26.self_attn.q_proj\n- model.layers.6.self_attn.q_proj\n- model.layers.7.self_attn.q_proj\n- model.layers.16.self_attn.q_proj\n- model.layers.5.self_attn.q_proj\n- model.layers.25.self_attn.q_proj\n# model.norm layers\n# self_attn.v_proj layers\n- model.layers.23.self_attn.v_proj\n- model.layers.14.self_attn.v_proj\n- model.layers.15.self_attn.v_proj\n- model.layers.19.self_attn.v_proj\n- model.layers.3.self_attn.v_proj\n- model.layers.18.self_attn.v_proj\n- model.layers.25.self_attn.v_proj\n- model.layers.4.self_attn.v_proj\n- model.layers.17.self_attn.v_proj\n- model.layers.22.self_attn.v_proj\n- model.layers.20.self_attn.v_proj\n- model.layers.13.self_attn.v_proj\n- model.layers.6.self_attn.v_proj\n- model.layers.27.self_attn.v_proj\n\nval_set_size: 0.05\noutput_dir: ./outputs/magistrate-3.2-3b\n\nsequence_len: 8192\nsample_packing: true\neval_sample_packing: false\npad_to_sequence_len: true\n\nadapter: \n\nwandb_project:\nwandb_entity:\nwandb_watch:\nwandb_name:\nwandb_log_model:\n\ngradient_accumulation_steps: 8\nmicro_batch_size: 1\nnum_epochs: 3\noptimizer: paged_adamw_32bit\nlr_scheduler: cosine\nlearning_rate: 2e-4\n\ntrain_on_inputs: false\ngroup_by_length: false\nbf16: auto\nfp16:\ntf32: false\n\ngradient_checkpointing: true\nearly_stopping_patience:\nresume_from_checkpoint:\nlocal_rank:\nlogging_steps: 1\nxformers_attention:\nflash_attention: true\ns2_attention:\n\nwarmup_steps: 1000\nevals_per_epoch: 2\neval_table_size:\neval_max_new_tokens: 128\nsaves_per_epoch: 1\ndebug:\ndeepspeed: deepspeed_configs/zero3.json\nweight_decay: 0.0\nfsdp:\nfsdp_config:\nspecial_tokens:\n  eos_token: \"<|im_end|>\"\n  pad_token: \"<|end_of_text|>\"\ntokens:\n  - \"<|im_start|>\"\n  - \"<|im_end|>\"\n```\n\n</details><br>\n\n\n## Model description\n\nMagistrate-3.2-3b-it is a legal assistant specializing in US Supreme Court case law and US Federal regulations. \n\nThe base model is pretrained with ~250M tokens containing no synthetic legal data. The instruct model does contain synthetic data.\n\n## Intended uses & limitations\n\nThis model is for research purposes and for continued development of the legal specialty. You are liable for all model outputs. \n\n## Training and evaluation data\n\nThis model was trained on a variety of standard open source datasets like OpenHermes-2.5, hermes-function-calling, and some select entries from the Tome.\nAdditionally, I have included a comprehensive, non-synthetic argument dataset. This is a work in progress but has shown promising results so far.\n\n## Training procedure\n\nSpectrum top 35% finetune for both pretrain and SFT. Thanks to the cognitive computations team for the work done with spectrum. \n\n+ Pretraining methodology based on Cohere's paper: [To Code, or Not To Code? Exploring Impact of Code in Pre-training](https://arxiv.org/abs/2408.10914)\n+ Instruct finetune largely based on OpenHermes-2.5 and hermes-function-calling\n\n### Training hyperparameters\n\nThe following hyperparameters were used during training:\n- learning_rate: 0.0002\n- train_batch_size: 1\n- eval_batch_size: 1\n- seed: 42\n- distributed_type: multi-GPU\n- num_devices: 2\n- gradient_accumulation_steps: 8\n- total_train_batch_size: 16\n- total_eval_batch_size: 2\n- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08\n- lr_scheduler_type: cosine\n- lr_scheduler_warmup_steps: 1000\n- num_epochs: 3\n\n### Training results\n\n| Training Loss | Epoch  | Step | Validation Loss |\n|:-------------:|:------:|:----:|:---------------:|\n| 1.3754        | 0.0005 | 1    | 1.7429          |\n| 1.0           | 0.5002 | 1017 | 0.8864          |\n| 0.9482        | 1.0005 | 2034 | 0.8395          |\n| 0.6817        | 1.4987 | 3051 | 0.8063          |\n| 0.697         | 1.9991 | 4068 | 0.7580          |\n| 0.3769        | 2.4966 | 5085 | 0.8140          |\n| 0.4278        | 2.9965 | 6102 | 0.8067          |\n\n\n### Framework versions\n\n- Transformers 4.45.0\n- Pytorch 2.3.1+cu121\n- Datasets 2.21.0\n- Tokenizers 0.20.0\n\n\nAdditional thanks to @nicoboss for giving me access to his private supercomputer, enabling me to provide many more quants, at much higher speed, than I would otherwise be able to.",
    "related_quantizations": []
  },
  "tags": [
    "gguf",
    "arxiv:2408.10914",
    "endpoints_compatible",
    "region:us",
    "conversational"
  ],
  "likes": 0,
  "downloads": 2159,
  "gated": false,
  "private": false,
  "last_modified": "2024-10-21T05:22:03.000Z",
  "created_at": "2024-10-21T04:54:40.000Z",
  "pipeline_tag": "",
  "library_name": ""
}

{
  "_id": "6715de9010fe127d8969e770",
  "id": "RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf",
  "modelId": "RichardErkhov/macadeliccc_-_magistrate-3.2-3b-it-gguf",
  "sha": "24303ddedadb7d95155de87b0be798b6e4a9bc3f",
  "createdAt": "2024-10-21T04:54:40.000Z",
  "lastModified": "2024-10-21T05:22:03.000Z",
  "author": "RichardErkhov",
  "downloads": 2159,
  "likes": 0,
  "gated": false,
  "private": false,
  "pipeline_tag": "",
  "library_name": "",
  "siblings_count": 24
}