Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit b1d1f2a

Browse files
authored
chore(importers): small logic enhancements (#7262)
* chore(import): import mmproj files to specific folder Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Slightly enhance logics Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 3728552 commit b1d1f2a

File tree

2 files changed

+82
-24
lines changed

2 files changed

+82
-24
lines changed

‎core/gallery/importers/importers_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,48 @@ var _ = Describe("DiscoverModelConfig", func() {
2727
Expect(modelConfig.Files[0].URI).To(Equal("https://huggingface.co/mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF/resolve/main/localai-functioncall-qwen2.5-7b-v0.5-q4_k_m.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
2828
Expect(modelConfig.Files[0].SHA256).To(Equal("4e7b7fe1d54b881f1ef90799219dc6cc285d29db24f559c8998d1addb35713d4"), fmt.Sprintf("Model config: %+v", modelConfig))
2929
})
30+
31+
It("should discover and import using LlamaCPPImporter", func() {
32+
uri := "https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF"
33+
preferences := json.RawMessage(`{}`)
34+
35+
modelConfig, err := importers.DiscoverModelConfig(uri, preferences)
36+
37+
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err))
38+
Expect(modelConfig.Name).To(Equal("Qwen3-VL-2B-Instruct-GGUF"), fmt.Sprintf("Model config: %+v", modelConfig))
39+
Expect(modelConfig.Description).To(Equal("Imported from https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF"), fmt.Sprintf("Model config: %+v", modelConfig))
40+
Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: llama-cpp"), fmt.Sprintf("Model config: %+v", modelConfig))
41+
Expect(modelConfig.ConfigFile).To(ContainSubstring("mmproj: mmproj/mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
42+
Expect(modelConfig.ConfigFile).To(ContainSubstring("model: Qwen3VL-2B-Instruct-Q4_K_M.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
43+
Expect(len(modelConfig.Files)).To(Equal(2), fmt.Sprintf("Model config: %+v", modelConfig))
44+
Expect(modelConfig.Files[0].Filename).To(Equal("Qwen3VL-2B-Instruct-Q4_K_M.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
45+
Expect(modelConfig.Files[0].URI).To(Equal("https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF/resolve/main/Qwen3VL-2B-Instruct-Q4_K_M.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
46+
Expect(modelConfig.Files[0].SHA256).ToNot(BeEmpty(), fmt.Sprintf("Model config: %+v", modelConfig))
47+
Expect(modelConfig.Files[1].Filename).To(Equal("mmproj/mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
48+
Expect(modelConfig.Files[1].URI).To(Equal("https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF/resolve/main/mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
49+
Expect(modelConfig.Files[1].SHA256).ToNot(BeEmpty(), fmt.Sprintf("Model config: %+v", modelConfig))
50+
})
51+
52+
It("should discover and import using LlamaCPPImporter", func() {
53+
uri := "https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF"
54+
preferences := json.RawMessage(`{ "quantizations": "Q8_0", "mmproj_quantizations": "f16" }`)
55+
56+
modelConfig, err := importers.DiscoverModelConfig(uri, preferences)
57+
58+
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err))
59+
Expect(modelConfig.Name).To(Equal("Qwen3-VL-2B-Instruct-GGUF"), fmt.Sprintf("Model config: %+v", modelConfig))
60+
Expect(modelConfig.Description).To(Equal("Imported from https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF"), fmt.Sprintf("Model config: %+v", modelConfig))
61+
Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: llama-cpp"), fmt.Sprintf("Model config: %+v", modelConfig))
62+
Expect(modelConfig.ConfigFile).To(ContainSubstring("mmproj: mmproj/mmproj-Qwen3VL-2B-Instruct-F16.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
63+
Expect(modelConfig.ConfigFile).To(ContainSubstring("model: Qwen3VL-2B-Instruct-Q8_0.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
64+
Expect(len(modelConfig.Files)).To(Equal(2), fmt.Sprintf("Model config: %+v", modelConfig))
65+
Expect(modelConfig.Files[0].Filename).To(Equal("Qwen3VL-2B-Instruct-Q8_0.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
66+
Expect(modelConfig.Files[0].URI).To(Equal("https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF/resolve/main/Qwen3VL-2B-Instruct-Q8_0.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
67+
Expect(modelConfig.Files[0].SHA256).ToNot(BeEmpty(), fmt.Sprintf("Model config: %+v", modelConfig))
68+
Expect(modelConfig.Files[1].Filename).To(Equal("mmproj/mmproj-Qwen3VL-2B-Instruct-F16.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
69+
Expect(modelConfig.Files[1].URI).To(Equal("https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF/resolve/main/mmproj-Qwen3VL-2B-Instruct-F16.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
70+
Expect(modelConfig.Files[1].SHA256).ToNot(BeEmpty(), fmt.Sprintf("Model config: %+v", modelConfig))
71+
})
3072
})
3173

3274
Context("with .gguf URI", func() {

‎core/gallery/importers/llama-cpp.go‎

Lines changed: 40 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
6969
}
7070

7171
preferedQuantizations, _ := preferencesMap["quantizations"].(string)
72-
quants := []string{"q4_k_m", "q4_0", "q8_0", "f16"}
72+
quants := []string{"q4_k_m"}
7373
if preferedQuantizations != "" {
7474
quants = strings.Split(preferedQuantizations, ",")
7575
}
@@ -100,7 +100,7 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
100100
Description: description,
101101
}
102102

103-
if strings.Contains(details.URI, ".gguf") {
103+
if strings.HasSuffix(details.URI, ".gguf") {
104104
cfg.Files = append(cfg.Files, gallery.File{
105105
URI: details.URI,
106106
Filename: filepath.Base(details.URI),
@@ -111,46 +111,62 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
111111
},
112112
}
113113
} else if details.HuggingFace != nil {
114-
lastMMProjFile := gallery.File{}
114+
// We want to:
115+
// Get first the chosen quants that match filenames
116+
// OR the first mmproj/gguf file found
117+
var lastMMProjFile *gallery.File
118+
var lastGGUFFile *gallery.File
115119
foundPreferedQuant := false
120+
foundPreferedMMprojQuant := false
116121

117122
for _, file := range details.HuggingFace.Files {
118-
// get the files of the prefered quants
119-
if slices.ContainsFunc(quants, func(quant string) bool {
120-
return strings.Contains(strings.ToLower(file.Path), strings.ToLower(quant))
121-
}) {
122-
cfg.Files = append(cfg.Files, gallery.File{
123-
URI: file.URL,
124-
Filename: filepath.Base(file.Path),
125-
SHA256: file.SHA256,
126-
})
127-
}
128123
// Get the mmproj prefered quants
129124
if strings.Contains(strings.ToLower(file.Path), "mmproj") {
130-
lastMMProjFile = gallery.File{
125+
lastMMProjFile = &gallery.File{
131126
URI: file.URL,
132-
Filename: filepath.Base(file.Path),
127+
Filename: filepath.Join("mmproj", filepath.Base(file.Path)),
133128
SHA256: file.SHA256,
134129
}
135130
if slices.ContainsFunc(mmprojQuantsList, func(quant string) bool {
136131
return strings.Contains(strings.ToLower(file.Path), strings.ToLower(quant))
132+
}) {
133+
cfg.Files = append(cfg.Files, *lastMMProjFile)
134+
foundPreferedMMprojQuant = true
135+
}
136+
} else if strings.HasSuffix(strings.ToLower(file.Path), "gguf") {
137+
lastGGUFFile = &gallery.File{
138+
URI: file.URL,
139+
Filename: filepath.Base(file.Path),
140+
SHA256: file.SHA256,
141+
}
142+
// get the files of the prefered quants
143+
if slices.ContainsFunc(quants, func(quant string) bool {
144+
return strings.Contains(strings.ToLower(file.Path), strings.ToLower(quant))
137145
}) {
138146
foundPreferedQuant = true
139-
cfg.Files = append(cfg.Files, lastMMProjFile)
147+
cfg.Files = append(cfg.Files, *lastGGUFFile)
140148
}
141149
}
142150
}
143151

144-
if !foundPreferedQuant && lastMMProjFile.URI != "" {
145-
cfg.Files = append(cfg.Files, lastMMProjFile)
146-
modelConfig.PredictionOptions = schema.PredictionOptions{
147-
BasicModelRequest: schema.BasicModelRequest{
148-
Model: lastMMProjFile.Filename,
149-
},
152+
// Make sure to add at least one file if not already present (which is the latest one)
153+
if lastMMProjFile != nil && !foundPreferedMMprojQuant {
154+
if !slices.ContainsFunc(cfg.Files, func(f gallery.File) bool {
155+
return f.Filename == lastMMProjFile.Filename
156+
}) {
157+
cfg.Files = append(cfg.Files, *lastMMProjFile)
158+
}
159+
}
160+
161+
if lastGGUFFile != nil && !foundPreferedQuant {
162+
if !slices.ContainsFunc(cfg.Files, func(f gallery.File) bool {
163+
return f.Filename == lastGGUFFile.Filename
164+
}) {
165+
cfg.Files = append(cfg.Files, *lastGGUFFile)
150166
}
151167
}
152168

153-
// Find first mmproj file
169+
// Find first mmproj file and configure it in the config file
154170
for _, file := range cfg.Files {
155171
if !strings.Contains(strings.ToLower(file.Filename), "mmproj") {
156172
continue
@@ -159,7 +175,7 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
159175
break
160176
}
161177

162-
// Find first non-mmproj file
178+
// Find first non-mmproj file and configure it in the config file
163179
for _, file := range cfg.Files {
164180
if strings.Contains(strings.ToLower(file.Filename), "mmproj") {
165181
continue

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /