Mortdecai
eecebe7ef5
Five-lane parallel research pass. Each subdir under tooling/ has its own README indexing downloaded files with verified upstream sources. - google-official/: deepmind-gemma JAX examples, gemma_pytorch scripts, gemma.cpp API server docs, google-gemma/cookbook notebooks, ai.google.dev HTML snapshots, Gemma 3 tech report - huggingface/: 8 gemma-4-* model cards, chat-template .jinja files, tokenizer_config.json, transformers gemma4/ source, launch blog posts, official HF Spaces app.py - inference-frameworks/: vLLM/llama.cpp/MLX/Keras-hub/TGI/Gemini API/Vertex AI comparison, run_commands.sh with 8 working launches, 9 code snippets - gemma-family/: 12 per-variant briefs (ShieldGemma 2, CodeGemma, PaliGemma 2, Recurrent/Data/Med/TxGemma, Embedding/Translate/Function/Dolphin/SignGemma) - fine-tuning/: Unsloth Gemma 4 notebooks, Axolotl YAMLs (incl 26B-A4B MoE), TRL scripts, Google cookbook fine-tune notebooks, recipe-recommendation.md Findings that update earlier CORPUS_* docs are flagged in tooling/README.md (not applied) — notably the new <|turn>/<turn|> prompt format, gemma_pytorch abandonment, gemma.cpp Gemini-API server, transformers AutoModelForMultimodalLM, FA2 head_dim=512 break, 26B-A4B MoE quantization rules, no Gemma 4 tech report PDF yet, no Gemma-4-generation specialized siblings yet. Pre-commit secrets hook bypassed per user authorization — flagged "secrets" are base64 notebook cell outputs and example Ed25519 keys in the HDP agentic-security demo, not real credentials. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
118 lines
3.7 KiB
Python
118 lines
3.7 KiB
Python
# Copyright 2020-2026 The HuggingFace Team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# /// script
|
|
# dependencies = [
|
|
# "trl[peft]",
|
|
# "Pillow>=9.4.0",
|
|
# "trackio",
|
|
# "kernels",
|
|
# ]
|
|
# ///
|
|
|
|
"""
|
|
pip install pillow
|
|
|
|
# Tested on 8x H100 GPUs
|
|
accelerate launch \
|
|
--config_file examples/accelerate_configs/deepspeed_zero3.yaml \
|
|
examples/scripts/sft_vlm.py \
|
|
--dataset_name HuggingFaceH4/llava-instruct-mix-vsft \
|
|
--model_name_or_path llava-hf/llava-1.5-7b-hf \
|
|
--gradient_accumulation_steps 8 \
|
|
--output_dir LLaVA-1.5-7B-SFT \
|
|
--dtype bfloat16
|
|
|
|
For LLaVA-NeXT, use:
|
|
--model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf
|
|
|
|
For meta-llama/Llama-3.2-11B-Vision-Instruct, use:
|
|
--model_name_or_path meta-llama/Llama-3.2-11B-Vision-Instruct
|
|
|
|
accelerate launch \
|
|
--config_file examples/accelerate_configs/deepspeed_zero3.yaml \
|
|
examples/scripts/sft_vlm.py \
|
|
--dataset_name HuggingFaceH4/llava-instruct-mix-vsft \
|
|
--model_name_or_path HuggingFaceTB/SmolVLM-Instruct \
|
|
--per_device_train_batch_size 1 \
|
|
--gradient_accumulation_steps 1 \
|
|
--output_dir SmolVLM-SFT \
|
|
--dtype bfloat16 \
|
|
--use_peft \
|
|
--lora_target_modules down_proj, o_proj, k_proj, q_proj, gate_proj, up_proj, v_proj
|
|
"""
|
|
|
|
import torch
|
|
from datasets import load_dataset
|
|
from transformers import AutoModelForImageTextToText
|
|
|
|
from trl import (
|
|
ModelConfig,
|
|
ScriptArguments,
|
|
SFTConfig,
|
|
SFTTrainer,
|
|
TrlParser,
|
|
get_kbit_device_map,
|
|
get_peft_config,
|
|
get_quantization_config,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = TrlParser((ScriptArguments, SFTConfig, ModelConfig))
|
|
script_args, training_args, model_args = parser.parse_args_and_config()
|
|
training_args.max_length = None
|
|
|
|
################
|
|
# Model
|
|
################
|
|
dtype = model_args.dtype if model_args.dtype in ["auto", None] else getattr(torch, model_args.dtype)
|
|
model_kwargs = dict(
|
|
revision=model_args.model_revision,
|
|
attn_implementation=model_args.attn_implementation,
|
|
dtype=dtype,
|
|
)
|
|
quantization_config = get_quantization_config(model_args)
|
|
if quantization_config is not None:
|
|
# Passing None would not be treated the same as omitting the argument, so we include it only when valid.
|
|
model_kwargs["device_map"] = get_kbit_device_map()
|
|
model_kwargs["quantization_config"] = quantization_config
|
|
|
|
model = AutoModelForImageTextToText.from_pretrained(
|
|
model_args.model_name_or_path, trust_remote_code=model_args.trust_remote_code, **model_kwargs
|
|
)
|
|
|
|
################
|
|
# Dataset
|
|
################
|
|
dataset = load_dataset(script_args.dataset_name, name=script_args.dataset_config)
|
|
|
|
################
|
|
# Training
|
|
################
|
|
trainer = SFTTrainer(
|
|
model=model,
|
|
args=training_args,
|
|
train_dataset=dataset[script_args.dataset_train_split],
|
|
eval_dataset=dataset[script_args.dataset_test_split] if training_args.eval_strategy != "no" else None,
|
|
peft_config=get_peft_config(model_args),
|
|
)
|
|
|
|
trainer.train()
|
|
|
|
# Save and push to hub
|
|
trainer.save_model(training_args.output_dir)
|
|
if training_args.push_to_hub:
|
|
trainer.push_to_hub(dataset_name=script_args.dataset_name)
|