Skip to content

Commit e45c1f3

Browse files
committed
fix: Make position embedding optional for split GGUF
Critical fixes for split GGUF models: 1. Position embedding made OPTIONAL: - Split GGUF models lack v.pos_embed.weight - Dimension mismatch (768 vs 1152) makes it unusable - Now skipped gracefully instead of failing ensureVisionReady 2. Defensive nil check in VisionPositionEmbedding.Forward: - Returns hiddenStates unchanged if PositionEmbedding is nil - Prevents panic in split GGUF path 3. RoPE provides positional info in attention layers - Position embedding not critical for Qwen3-VL - M-RoPE in attention handles spatial/temporal positions This should allow split GGUF to proceed past ensureVisionReady.
1 parent 25ef4f6 commit e45c1f3

File tree

3 files changed

+11
-3
lines changed

3 files changed

+11
-3
lines changed
662 KB
Binary file not shown.

model/models/qwen3vl/model.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,12 @@ func (m *Model) ensureVisionReady() error {
122122
pos = m.GetTensor("v.position_embd.weight")
123123
}
124124
if pos == nil {
125-
slog.Debug("ensureVisionReady missing position embedding")
126-
return model.ErrNoVisionModel
125+
// Position embedding is OPTIONAL for split GGUF models
126+
// We skip it during forward pass when dimensions don't match
127+
slog.Info("SPLIT GGUF: Position embedding not found (optional for split models)")
128+
} else {
129+
vm.PositionEmbedding.PositionEmbedding = &nn.Embedding{Weight: pos}
127130
}
128-
vm.PositionEmbedding.PositionEmbedding = &nn.Embedding{Weight: pos}
129131
}
130132

131133
if vm.PatchMerger == nil {

model/models/qwen3vl/model_vision.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,12 @@ func makeSlice2D[T int32 | float32](n0, n1 int) iter.Seq[[]T] {
163163
}
164164

165165
func (m *VisionPositionEmbedding) Forward(ctx ml.Context, hiddenStates ml.Tensor, grid *Grid, opts VisionOptions) ml.Tensor {
166+
// Guard: if position embedding is nil (split GGUF), return hiddenStates unchanged
167+
if m.PositionEmbedding == nil {
168+
logutil.Trace("split GGUF: skipping position embedding (nil weight)")
169+
return hiddenStates
170+
}
171+
166172
indexSlice := slices.Collect(makeSlice2D[int32](4, grid.Height*grid.Width))
167173
weightSlice := slices.Collect(makeSlice2D[float32](4, grid.Height*grid.Width))
168174

0 commit comments

Comments
 (0)