No official torrent yet β requesting it tells the flock what to host next. You cannot seed a catalog-only page.
Cataloged from RedHatAI/GLM-5.2-NVFP4
All models by RedHatAI βThis is a quantized version of zai-org/GLM-5.2 with MoE layers quantized to NVFP4.
This model is intended for deployment with vLLM and requires the following branch: https://github.com/vllm-project/vllm/pull/41276. You can serve the model using
This model was created using LLM Compressor. The example script can be found in examples/quantizing_moe/glm5_example.py [Example] GLM5.2 Example. Quantizing the model with data parallelism and 6xA100 takes about 3 hours.
vllm serve RedHatAI/GLM-5.2-NVFP4 \ --tensor-parallel-size 8 \ --kv_cache_dtype=fp8import torchfrom compressed_tensors.offload import init_distfrom compressed_tensors.quantization.quant_scheme import ( NVFP4, QuantizationScheme,)from datasets import load_datasetfrom transformers import AutoModelForCausalLM, AutoTokenizer
from llmcompressor import oneshotfrom llmcompressor.datasets.utils import get_rank_partitionfrom llmcompressor.modifiers.quantization import QuantizationModifierfrom llmcompressor.utils import load_context
# Load the modelinit_dist()model_id = "zai-org/GLM-5.2"with load_context(): model = AutoModelForCausalLM.from_pretrained( model_id, device_map="auto_offload", max_memory={}, offload_folder="/mnt/nvme-data/engine/kylesayrs/offload_folder", )tokenizer = AutoTokenizer.from_pretrained(model_id)
# Select calibration dataset.DATASET_ID = "HuggingFaceH4/ultrachat_200k"DATASET_SPLIT = "train_sft"
# Select number of samples. 512 samples is a good place to start.# Increasing the number of samples can improve accuracy.NUM_CALIBRATION_SAMPLES = 512MAX_SEQUENCE_LENGTH = 2048
# Load dataset and preprocess.ds = load_dataset( DATASET_ID, split=get_rank_partition(DATASET_SPLIT, NUM_CALIBRATION_SAMPLES))ds = ds.shuffle(seed=42)
def preprocess(example): return { "text": tokenizer.apply_chat_template( example["messages"], tokenize=False, ) }
ds = ds.map(preprocess)
# Tokenize inputs.def tokenize(sample): return tokenizer( sample["text"], padding=False, max_length=MAX_SEQUENCE_LENGTH, truncation=True, add_special_tokens=False, )
ds = ds.map(tokenize, remove_columns=ds.column_names)
# Configure the quantization algorithm to run.recipe = QuantizationModifier( config_groups={ "mlp": QuantizationScheme( targets=[r"re:.*mlp\..*"], **NVFP4, ), }, ignore=[ r"re:^model\.layers\.[0-2]\..*" r"re:.*mlp\.gate.*", # not technically necessary r"re:.*shared_experts.*", r"lm_head", ],)
# Apply algorithms.oneshot( model=model, dataset=ds, batch_size=4, recipe=recipe, shuffle_calibration_samples=False,)
# Save to disk compressed.# Note: base checkpoint generation_config needs fixing for newer transformers versionsmodel.generation_config.top_p = NoneSAVE_DIR = ( "/mnt/nvme-data/engine/kylesayrs/" + model_id.rstrip("/").split("/")[-1] + "-NVFP4")model.save_pretrained(SAVE_DIR, save_compressed=True)tokenizer.save_pretrained(SAVE_DIR)
torch.distributed.destroy_process_group()