RedHatAI/GLM-5.2-NVFP4-FP8

text-generationtransformersUpstream checksumstransformerssafetensorsglm_moe_dsatext-generationcompressed-tensorsllm-compressor

License: mit
Size: not mirrored yet
Downloads in the last 30 days: 0 / 30 days

No official torrent yet — requesting it tells the flock what to host next. You cannot seed a catalog-only page.

Cataloged from RedHatAI/GLM-5.2-NVFP4-FP8

All models by RedHatAI →

RedHatAI/GLM-5.2-NVFP4-FP8

This is a quantized version of zai-org/GLM-5.2 with MoE layers quantized to NVFP4 and attention layers quantized to FP8 block

Usage

This model is intended for deployment with vLLM and requires the following fix: https://github.com/vllm-project/vllm/pull/47780. You can serve the model using

Creation Process

This model was created using LLM Compressor. The example script can be found in examples/quantizing_moe/glm5_example.py [Example] GLM5.2 Example. Quantizing the model with data parallelism and 6xA100 takes about 3 hours.

LLM Compressor Creation Script

python

import torchfrom compressed_tensors.offload import init_distfrom compressed_tensors.quantization.quant_scheme import (    FP8_BLOCK,    NVFP4,    QuantizationScheme,)from datasets import load_datasetfrom transformers import AutoModelForCausalLM, AutoTokenizer
from llmcompressor import oneshotfrom llmcompressor.datasets.utils import get_rank_partitionfrom llmcompressor.modifiers.quantization import QuantizationModifierfrom llmcompressor.utils import load_context
# Load the modelinit_dist()model_id = "zai-org/GLM-5.2"with load_context():    model = AutoModelForCausalLM.from_pretrained(        model_id,        device_map="auto_offload",        max_memory={},        offload_folder="/mnt/nvme-data/engine/kylesayrs/offload_folder",    )tokenizer = AutoTokenizer.from_pretrained(model_id)
# Select calibration dataset.DATASET_ID = "HuggingFaceH4/ultrachat_200k"DATASET_SPLIT = "train_sft"
# Select number of samples. 512 samples is a good place to start.# Increasing the number of samples can improve accuracy.NUM_CALIBRATION_SAMPLES = 512MAX_SEQUENCE_LENGTH = 2048
# Load dataset and preprocess.ds = load_dataset(    DATASET_ID, split=get_rank_partition(DATASET_SPLIT, NUM_CALIBRATION_SAMPLES))ds = ds.shuffle(seed=42)

def preprocess(example):    return {        "text": tokenizer.apply_chat_template(            example["messages"],            tokenize=False,        )    }

ds = ds.map(preprocess)

# Tokenize inputs.def tokenize(sample):    return tokenizer(        sample["text"],        padding=False,        max_length=MAX_SEQUENCE_LENGTH,        truncation=True,        add_special_tokens=False,    )

ds = ds.map(tokenize, remove_columns=ds.column_names)
# Configure the quantization algorithm to run.recipe = QuantizationModifier(    config_groups={        "attention_shared_experts": QuantizationScheme(            targets=[r"re:.*self_attn\..*"],            **FP8_BLOCK,        ),        "mlp": QuantizationScheme(            targets=[r"re:.*mlp\..*"],            **NVFP4,        ),    },    ignore=[        r"re:^model\.layers\.[0-2]\..*"        r"re:.*mlp\.gate.*",  # not technically necessary        r"re:.*indexer\.weights_proj$",  # sensitive to quantization        r"lm_head",    ],)
# Apply algorithms.oneshot(    model=model,    dataset=ds,    batch_size=4,    recipe=recipe,    shuffle_calibration_samples=False,)
# Save to disk compressed.# Note: base checkpoint generation_config needs fixing for newer transformers versionsmodel.generation_config.top_p = NoneSAVE_DIR = (    "/mnt/nvme-data/engine/kylesayrs/"    + model_id.rstrip("/").split("/")[-1]    + "-NVFP4-FP8")model.save_pretrained(SAVE_DIR, save_compressed=True)tokenizer.save_pretrained(SAVE_DIR)
torch.distributed.destroy_process_group()

Evaluation

Benchmark	`zai-org/GLM-5.2`	`RedHatAI/GLM-5.2-NVFP4-FP8`
GPQA-Diamond	91.2	89.1

Usage

This model is intended for deployment with vLLM and requires the following fix: https://github.com/vllm-project/vllm/pull/47780. You can serve the model using

Creation Process

LLM Compressor Creation Script

python

import torchfrom compressed_tensors.offload import init_distfrom compressed_tensors.quantization.quant_scheme import (    FP8_BLOCK,    NVFP4,    QuantizationScheme,)from datasets import load_datasetfrom transformers import AutoModelForCausalLM, AutoTokenizer
from llmcompressor import oneshotfrom llmcompressor.datasets.utils import get_rank_partitionfrom llmcompressor.modifiers.quantization import QuantizationModifierfrom llmcompressor.utils import load_context
# Load the modelinit_dist()model_id = "zai-org/GLM-5.2"with load_context():    model = AutoModelForCausalLM.from_pretrained(        model_id,        device_map="auto_offload",        max_memory={},        offload_folder="/mnt/nvme-data/engine/kylesayrs/offload_folder",    )tokenizer = AutoTokenizer.from_pretrained(model_id)
# Select calibration dataset.DATASET_ID = "HuggingFaceH4/ultrachat_200k"DATASET_SPLIT = "train_sft"
# Select number of samples. 512 samples is a good place to start.# Increasing the number of samples can improve accuracy.NUM_CALIBRATION_SAMPLES = 512MAX_SEQUENCE_LENGTH = 2048
# Load dataset and preprocess.ds = load_dataset(    DATASET_ID, split=get_rank_partition(DATASET_SPLIT, NUM_CALIBRATION_SAMPLES))ds = ds.shuffle(seed=42)

def preprocess(example):    return {        "text": tokenizer.apply_chat_template(            example["messages"],            tokenize=False,        )    }

ds = ds.map(preprocess)

# Tokenize inputs.def tokenize(sample):    return tokenizer(        sample["text"],        padding=False,        max_length=MAX_SEQUENCE_LENGTH,        truncation=True,        add_special_tokens=False,    )

ds = ds.map(tokenize, remove_columns=ds.column_names)
# Configure the quantization algorithm to run.recipe = QuantizationModifier(    config_groups={        "attention_shared_experts": QuantizationScheme(            targets=[r"re:.*self_attn\..*"],            **FP8_BLOCK,        ),        "mlp": QuantizationScheme(            targets=[r"re:.*mlp\..*"],            **NVFP4,        ),    },    ignore=[        r"re:^model\.layers\.[0-2]\..*"        r"re:.*mlp\.gate.*",  # not technically necessary        r"re:.*indexer\.weights_proj$",  # sensitive to quantization        r"lm_head",    ],)
# Apply algorithms.oneshot(    model=model,    dataset=ds,    batch_size=4,    recipe=recipe,    shuffle_calibration_samples=False,)
# Save to disk compressed.# Note: base checkpoint generation_config needs fixing for newer transformers versionsmodel.generation_config.top_p = NoneSAVE_DIR = (    "/mnt/nvme-data/engine/kylesayrs/"    + model_id.rstrip("/").split("/")[-1]    + "-NVFP4-FP8")model.save_pretrained(SAVE_DIR, save_compressed=True)tokenizer.save_pretrained(SAVE_DIR)
torch.distributed.destroy_process_group()

import torchfrom compressed_tensors.offload import init_distfrom compressed_tensors.quantization.quant_scheme import (    FP8_BLOCK,    NVFP4,    QuantizationScheme,)from datasets import load_datasetfrom transformers import AutoModelForCausalLM, AutoTokenizer
from llmcompressor import oneshotfrom llmcompressor.datasets.utils import get_rank_partitionfrom llmcompressor.modifiers.quantization import QuantizationModifierfrom llmcompressor.utils import load_context
# Load the modelinit_dist()model_id = "zai-org/GLM-5.2"with load_context():    model = AutoModelForCausalLM.from_pretrained(        model_id,        device_map="auto_offload",        max_memory={},        offload_folder="/mnt/nvme-data/engine/kylesayrs/offload_folder",    )tokenizer = AutoTokenizer.from_pretrained(model_id)
# Select calibration dataset.DATASET_ID = "HuggingFaceH4/ultrachat_200k"DATASET_SPLIT = "train_sft"
# Select number of samples. 512 samples is a good place to start.# Increasing the number of samples can improve accuracy.NUM_CALIBRATION_SAMPLES = 512MAX_SEQUENCE_LENGTH = 2048
# Load dataset and preprocess.ds = load_dataset(    DATASET_ID, split=get_rank_partition(DATASET_SPLIT, NUM_CALIBRATION_SAMPLES))ds = ds.shuffle(seed=42)

def preprocess(example):    return {        "text": tokenizer.apply_chat_template(            example["messages"],            tokenize=False,        )    }

ds = ds.map(preprocess)

# Tokenize inputs.def tokenize(sample):    return tokenizer(        sample["text"],        padding=False,        max_length=MAX_SEQUENCE_LENGTH,        truncation=True,        add_special_tokens=False,    )

ds = ds.map(tokenize, remove_columns=ds.column_names)
# Configure the quantization algorithm to run.recipe = QuantizationModifier(    config_groups={        "attention_shared_experts": QuantizationScheme(            targets=[r"re:.*self_attn\..*"],            **FP8_BLOCK,        ),        "mlp": QuantizationScheme(            targets=[r"re:.*mlp\..*"],            **NVFP4,        ),    },    ignore=[        r"re:^model\.layers\.[0-2]\..*"        r"re:.*mlp\.gate.*",  # not technically necessary        r"re:.*indexer\.weights_proj$",  # sensitive to quantization        r"lm_head",    ],)
# Apply algorithms.oneshot(    model=model,    dataset=ds,    batch_size=4,    recipe=recipe,    shuffle_calibration_samples=False,)
# Save to disk compressed.# Note: base checkpoint generation_config needs fixing for newer transformers versionsmodel.generation_config.top_p = NoneSAVE_DIR = (    "/mnt/nvme-data/engine/kylesayrs/"    + model_id.rstrip("/").split("/")[-1]    + "-NVFP4-FP8")model.save_pretrained(SAVE_DIR, save_compressed=True)tokenizer.save_pretrained(SAVE_DIR)
torch.distributed.destroy_process_group()

Benchmark	`zai-org/GLM-5.2`	`RedHatAI/GLM-5.2-NVFP4-FP8`
GPQA-Diamond	91.2	89.1

import torchfrom compressed_tensors.offload import init_distfrom compressed_tensors.quantization.quant_scheme import ( FP8_BLOCK, NVFP4, QuantizationScheme,)from datasets import load_datasetfrom transformers import AutoModelForCausalLM, AutoTokenizer from llmcompressor import oneshotfrom llmcompressor.datasets.utils import get_rank_partitionfrom llmcompressor.modifiers.quantization import QuantizationModifierfrom llmcompressor.utils import load_context # Load the modelinit_dist()model_id = "zai-org/GLM-5.2"with load_context(): model = AutoModelForCausalLM.from_pretrained( model_id, device_map="auto_offload", max_memory={}, offload_folder="/mnt/nvme-data/engine/kylesayrs/offload_folder", )tokenizer = AutoTokenizer.from_pretrained(model_id) # Select calibration dataset.DATASET_ID = "HuggingFaceH4/ultrachat_200k"DATASET_SPLIT = "train_sft" # Select number of samples. 512 samples is a good place to start.# Increasing the number of samples can improve accuracy.NUM_CALIBRATION_SAMPLES = 512MAX_SEQUENCE_LENGTH = 2048 # Load dataset and preprocess.ds = load_dataset( DATASET_ID, split=get_rank_partition(DATASET_SPLIT, NUM_CALIBRATION_SAMPLES))ds = ds.shuffle(seed=42) def preprocess(example): return { "text": tokenizer.apply_chat_template( example["messages"], tokenize=False, ) } ds = ds.map(preprocess) # Tokenize inputs.def tokenize(sample): return tokenizer( sample["text"], padding=False, max_length=MAX_SEQUENCE_LENGTH, truncation=True, add_special_tokens=False, ) ds = ds.map(tokenize, remove_columns=ds.column_names) # Configure the quantization algorithm to run.recipe = QuantizationModifier( config_groups={ "attention_shared_experts": QuantizationScheme( targets=[r"re:.*self_attn\..*"], **FP8_BLOCK, ), "mlp": QuantizationScheme( targets=[r"re:.*mlp\..*"], **NVFP4, ), }, ignore=[ r"re:^model\.layers\.[0-2]\..*" r"re:.*mlp\.gate.*", # not technically necessary r"re:.*indexer\.weights_proj$", # sensitive to quantization r"lm_head", ],) # Apply algorithms.oneshot( model=model, dataset=ds, batch_size=4, recipe=recipe, shuffle_calibration_samples=False,) # Save to disk compressed.# Note: base checkpoint generation_config needs fixing for newer transformers versionsmodel.generation_config.top_p = NoneSAVE_DIR = ( "/mnt/nvme-data/engine/kylesayrs/" + model_id.rstrip("/").split("/")[-1] + "-NVFP4-FP8")model.save_pretrained(SAVE_DIR, save_compressed=True)tokenizer.save_pretrained(SAVE_DIR) torch.distributed.destroy_process_group()