Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit be8cf83

Browse files
authored
feat(importers): add transformers and vLLM (#7278)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 3276d1c commit be8cf83

File tree

8 files changed

+675
-13
lines changed

8 files changed

+675
-13
lines changed

‎core/gallery/importers/importers.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@ import (
1010
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
1111
)
1212

13-
var DefaultImporters = []Importer{
13+
var defaultImporters = []Importer{
1414
&LlamaCPPImporter{},
1515
&MLXImporter{},
16+
&VLLMImporter{},
17+
&TransformersImporter{},
1618
}
1719

1820
type Details struct {
@@ -52,7 +54,7 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
5254
Preferences: preferences,
5355
}
5456

55-
for _, importer := range DefaultImporters {
57+
for _, importer := range defaultImporters {
5658
if importer.Match(details) {
5759
modelConfig, err = importer.Import(details)
5860
if err != nil {

‎core/gallery/importers/llama-cpp.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,10 +80,13 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
8080
mmprojQuantsList = strings.Split(mmprojQuants, ",")
8181
}
8282

83+
embeddings, _ := preferencesMap["embeddings"].(string)
84+
8385
modelConfig := config.ModelConfig{
8486
Name: name,
8587
Description: description,
8688
KnownUsecaseStrings: []string{"chat"},
89+
Options: []string{"use_jinja:true"},
8790
Backend: "llama-cpp",
8891
TemplateConfig: config.TemplateConfig{
8992
UseTokenizerTemplate: true,
@@ -95,6 +98,11 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
9598
},
9699
}
97100

101+
if embeddings != "" && strings.ToLower(embeddings) == "true" || strings.ToLower(embeddings) == "yes" {
102+
trueV := true
103+
modelConfig.Embeddings = &trueV
104+
}
105+
98106
cfg := gallery.ModelConfig{
99107
Name: name,
100108
Description: description,

‎core/gallery/importers/llama-cpp_test.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,21 @@ import (
55
"fmt"
66

77
"github.com/mudler/LocalAI/core/gallery/importers"
8+
. "github.com/mudler/LocalAI/core/gallery/importers"
89
. "github.com/onsi/ginkgo/v2"
910
. "github.com/onsi/gomega"
1011
)
1112

1213
var _ = Describe("LlamaCPPImporter", func() {
13-
var importer *importers.LlamaCPPImporter
14+
var importer *LlamaCPPImporter
1415

1516
BeforeEach(func() {
16-
importer = &importers.LlamaCPPImporter{}
17+
importer = &LlamaCPPImporter{}
1718
})
1819

1920
Context("Match", func() {
2021
It("should match when URI ends with .gguf", func() {
21-
details := importers.Details{
22+
details := Details{
2223
URI: "https://example.com/model.gguf",
2324
}
2425

@@ -28,7 +29,7 @@ var _ = Describe("LlamaCPPImporter", func() {
2829

2930
It("should match when backend preference is llama-cpp", func() {
3031
preferences := json.RawMessage(`{"backend": "llama-cpp"}`)
31-
details := importers.Details{
32+
details := Details{
3233
URI: "https://example.com/model",
3334
Preferences: preferences,
3435
}
@@ -38,7 +39,7 @@ var _ = Describe("LlamaCPPImporter", func() {
3839
})
3940

4041
It("should not match when URI does not end with .gguf and no backend preference", func() {
41-
details := importers.Details{
42+
details := Details{
4243
URI: "https://example.com/model.bin",
4344
}
4445

@@ -48,7 +49,7 @@ var _ = Describe("LlamaCPPImporter", func() {
4849

4950
It("should not match when backend preference is different", func() {
5051
preferences := json.RawMessage(`{"backend": "mlx"}`)
51-
details := importers.Details{
52+
details := Details{
5253
URI: "https://example.com/model",
5354
Preferences: preferences,
5455
}
@@ -59,7 +60,7 @@ var _ = Describe("LlamaCPPImporter", func() {
5960

6061
It("should return false when JSON preferences are invalid", func() {
6162
preferences := json.RawMessage(`invalid json`)
62-
details := importers.Details{
63+
details := Details{
6364
URI: "https://example.com/model.gguf",
6465
Preferences: preferences,
6566
}
@@ -72,7 +73,7 @@ var _ = Describe("LlamaCPPImporter", func() {
7273

7374
Context("Import", func() {
7475
It("should import model config with default name and description", func() {
75-
details := importers.Details{
76+
details := Details{
7677
URI: "https://example.com/my-model.gguf",
7778
}
7879

@@ -89,7 +90,7 @@ var _ = Describe("LlamaCPPImporter", func() {
8990

9091
It("should import model config with custom name and description from preferences", func() {
9192
preferences := json.RawMessage(`{"name": "custom-model", "description": "Custom description"}`)
92-
details := importers.Details{
93+
details := Details{
9394
URI: "https://example.com/my-model.gguf",
9495
Preferences: preferences,
9596
}
@@ -106,7 +107,7 @@ var _ = Describe("LlamaCPPImporter", func() {
106107

107108
It("should handle invalid JSON preferences", func() {
108109
preferences := json.RawMessage(`invalid json`)
109-
details := importers.Details{
110+
details := Details{
110111
URI: "https://example.com/my-model.gguf",
111112
Preferences: preferences,
112113
}
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package importers
2+
3+
import (
4+
"encoding/json"
5+
"path/filepath"
6+
"strings"
7+
8+
"github.com/mudler/LocalAI/core/config"
9+
"github.com/mudler/LocalAI/core/gallery"
10+
"github.com/mudler/LocalAI/core/schema"
11+
"go.yaml.in/yaml/v2"
12+
)
13+
14+
var _ Importer = &TransformersImporter{}
15+
16+
type TransformersImporter struct{}
17+
18+
func (i *TransformersImporter) Match(details Details) bool {
19+
preferences, err := details.Preferences.MarshalJSON()
20+
if err != nil {
21+
return false
22+
}
23+
preferencesMap := make(map[string]any)
24+
err = json.Unmarshal(preferences, &preferencesMap)
25+
if err != nil {
26+
return false
27+
}
28+
29+
b, ok := preferencesMap["backend"].(string)
30+
if ok && b == "transformers" {
31+
return true
32+
}
33+
34+
if details.HuggingFace != nil {
35+
for _, file := range details.HuggingFace.Files {
36+
if strings.Contains(file.Path, "tokenizer.json") ||
37+
strings.Contains(file.Path, "tokenizer_config.json") {
38+
return true
39+
}
40+
}
41+
}
42+
43+
return false
44+
}
45+
46+
func (i *TransformersImporter) Import(details Details) (gallery.ModelConfig, error) {
47+
preferences, err := details.Preferences.MarshalJSON()
48+
if err != nil {
49+
return gallery.ModelConfig{}, err
50+
}
51+
preferencesMap := make(map[string]any)
52+
err = json.Unmarshal(preferences, &preferencesMap)
53+
if err != nil {
54+
return gallery.ModelConfig{}, err
55+
}
56+
57+
name, ok := preferencesMap["name"].(string)
58+
if !ok {
59+
name = filepath.Base(details.URI)
60+
}
61+
62+
description, ok := preferencesMap["description"].(string)
63+
if !ok {
64+
description = "Imported from " + details.URI
65+
}
66+
67+
backend := "transformers"
68+
b, ok := preferencesMap["backend"].(string)
69+
if ok {
70+
backend = b
71+
}
72+
73+
modelType, ok := preferencesMap["type"].(string)
74+
if !ok {
75+
modelType = "AutoModelForCausalLM"
76+
}
77+
78+
quantization, ok := preferencesMap["quantization"].(string)
79+
if !ok {
80+
quantization = ""
81+
}
82+
83+
modelConfig := config.ModelConfig{
84+
Name: name,
85+
Description: description,
86+
KnownUsecaseStrings: []string{"chat"},
87+
Backend: backend,
88+
PredictionOptions: schema.PredictionOptions{
89+
BasicModelRequest: schema.BasicModelRequest{
90+
Model: details.URI,
91+
},
92+
},
93+
TemplateConfig: config.TemplateConfig{
94+
UseTokenizerTemplate: true,
95+
},
96+
}
97+
modelConfig.ModelType = modelType
98+
modelConfig.Quantization = quantization
99+
100+
data, err := yaml.Marshal(modelConfig)
101+
if err != nil {
102+
return gallery.ModelConfig{}, err
103+
}
104+
105+
return gallery.ModelConfig{
106+
Name: name,
107+
Description: description,
108+
ConfigFile: string(data),
109+
}, nil
110+
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /