niuniu/llm-intelligence
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(import): refine official release metadata backfill

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-13 23:02:50 +08:00
parent d893d2542e
commit bed5e3aec7
5 changed files with 182 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
scripts/import_bytedance_data.go
View File

@@ -50,6 +50,21 @@ type bytedanceModelMetadata struct {
}
var bytedanceModelMetadataRules = []bytedanceModelMetadata{
{
Prefix: "bytedance-doubao-1.5-vision-pro",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
},
{
Prefix: "bytedance-doubao-1.5-pro",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
},
{
Prefix: "bytedance-doubao-1.5-lite",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
},
{
Prefix: "bytedance-doubao-1.5-thinking",
ReleaseDate: "2025-04-17",
@@ -60,6 +75,31 @@ var bytedanceModelMetadataRules = []bytedanceModelMetadata{
ReleaseDate: "2025-06-11",
ModelSourceURL: "https://developer.volcengine.com/articles/7517188354606104612",
},
{
Prefix: "bytedance-doubao-seed-1.8",
ModelSourceURL: "https://developer.volcengine.com/articles/7601918680544641034",
},
{
Prefix: "bytedance-doubao-seed-2.0-code",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
},
{
Prefix: "bytedance-doubao-seed-2.0-pro",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
},
{
Prefix: "bytedance-doubao-seed-2.0-mini",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
},
{
Prefix: "bytedance-doubao-seed-2.0-lite",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
},
{
Prefix: "bytedance-doubao-seed-code",
ReleaseDate: "2024-06-26",
ModelSourceURL: "https://developer.volcengine.com/articles/7383101327527641125",
},
{
Prefix: "bytedance-seedance-1.0-lite",
ReleaseDate: "2025-05-13",
@@ -86,6 +126,10 @@ func enrichBytedanceModelMetadata(model ModelPricing) ModelPricing {
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
}
func main() {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
@@ -183,14 +227,22 @@ func main() {
log.Printf("Model error for %s: %v", p.ModelID, err)
continue
}
_, _ = db.Exec(
if _, err := db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
SET source_url = CASE
WHEN $4 THEN $2
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
)
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}
// Insert pricing
sourceType := p.OperatorType

29
scripts/import_bytedance_data_test.go
View File

@@ -10,6 +10,16 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) {
wantReleaseDate string
wantSourceURL string
}{
{
modelID: "bytedance-doubao-1.5-pro-32k",
wantReleaseDate: "2025-01-22",
wantSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
},
{
modelID: "bytedance-doubao-1.5-vision-pro",
wantReleaseDate: "2025-01-22",
wantSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
},
{
modelID: "bytedance-doubao-seed-1.6-thinking",
wantReleaseDate: "2025-06-11",
@@ -25,6 +35,11 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) {
wantReleaseDate: "2025-05-13",
wantSourceURL: "https://developer.volcengine.com/articles/7504284064976502823",
},
{
modelID: "bytedance-doubao-seed-code-256k",
wantReleaseDate: "2024-06-26",
wantSourceURL: "https://developer.volcengine.com/articles/7383101327527641125",
},
}
for _, tc := range cases {
@@ -56,6 +71,20 @@ func TestEnrichBytedanceModelMetadataFallsBackToPricingSource(t *testing.T) {
}
}
func TestEnrichBytedanceModelMetadataSupportsSourceOnlyRules(t *testing.T) {
enriched := enrichBytedanceModelMetadata(ModelPricing{
ModelID: "bytedance-doubao-seed-2.0-pro-256k",
SourceURL: "https://www.volcengine.com/docs/82379/1099320",
})
if enriched.ReleaseDate != "" {
t.Fatalf("unexpected release date: %q", enriched.ReleaseDate)
}
if enriched.ModelSourceURL != "https://developer.volcengine.com/articles/7610285824933445675" {
t.Fatalf("model source url = %q, want 2.0 source", enriched.ModelSourceURL)
}
}
func TestBytedanceReleaseDateValueReturnsNilForUnknownDate(t *testing.T) {
if value := releaseDateValue(""); value != nil {
t.Fatalf("blank release date should stay nil, got %v", value)

55
scripts/import_phase2_data.go
View File

@@ -73,6 +73,24 @@ type baiduModelMetadata struct {
}
var baiduModelMetadataRules = []baiduModelMetadata{
{
Prefix: "baidu-ernie-5.0",
ReleaseDate: "2026-01-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba",
},
{
Prefix: "baidu-ernie-x1.1",
ReleaseDate: "2025-09-09",
ModelSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a",
},
{
Prefix: "baidu-ernie-5.1",
ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html",
},
{
Prefix: "baidu-ernie-4.5-turbo-vl",
ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html",
},
{
Prefix: "baidu-ernie-4.5-turbo",
ReleaseDate: "2025-04-25",
@@ -93,6 +111,21 @@ var baiduModelMetadataRules = []baiduModelMetadata{
ReleaseDate: "2025-03-16",
ModelSourceURL: "https://cloud.baidu.com/article/3835921",
},
{
Prefix: "baidu-ernie-character",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
},
{
Prefix: "baidu-ernie-lite-pro",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
},
{
Prefix: "baidu-ernie-speed-pro",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
},
}
func enrichBaiduModelMetadata(model ModelPricing) ModelPricing {
@@ -114,6 +147,10 @@ func enrichBaiduModelMetadata(model ModelPricing) ModelPricing {
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
}
func parseZhipuPrice(s string) float64 {
// Extract price from strings like "6元", "免费", "限时免费"
if strings.Contains(s, "免费") {
@@ -260,14 +297,22 @@ func main() {
log.Printf("Model error: %v", err)
continue
}
_, _ = db.Exec(
if _, err := db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
SET source_url = CASE
WHEN $4 THEN $2
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
)
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}
// Insert pricing
sourceType := p.OperatorType

29
scripts/import_phase2_data_test.go
View File

@@ -10,6 +10,16 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) {
wantReleaseDate string
wantSourceURL string
}{
{
modelID: "baidu-ernie-5.0",
wantReleaseDate: "2026-01-22",
wantSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba",
},
{
modelID: "baidu-ernie-x1.1",
wantReleaseDate: "2025-09-09",
wantSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a",
},
{
modelID: "baidu-ernie-4.5-turbo-128k",
wantReleaseDate: "2025-04-25",
@@ -30,6 +40,11 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) {
wantReleaseDate: "2025-03-16",
wantSourceURL: "https://cloud.baidu.com/article/3835921",
},
{
modelID: "baidu-ernie-character",
wantReleaseDate: "2024-03-22",
wantSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
},
}
for _, tc := range cases {
@@ -61,6 +76,20 @@ func TestEnrichBaiduModelMetadataFallsBackToPricingSource(t *testing.T) {
}
}
func TestEnrichBaiduModelMetadataSupportsSourceOnlyRules(t *testing.T) {
enriched := enrichBaiduModelMetadata(ModelPricing{
ModelID: "baidu-ernie-5.1",
SourceURL: "https://cloud.baidu.com/doc/qianfan/s/wmh4sv6ya",
})
if enriched.ReleaseDate != "" {
t.Fatalf("unexpected release date: %q", enriched.ReleaseDate)
}
if enriched.ModelSourceURL != "https://cloud.baidu.com/product/wenxinworkshop.html" {
t.Fatalf("model source url = %q, want product source", enriched.ModelSourceURL)
}
}
func TestBaiduReleaseDateValueReturnsNilForUnknownDate(t *testing.T) {
if value := releaseDateValue(""); value != nil {
t.Fatalf("blank release date should stay nil, got %v", value)

22
scripts/import_zhipu_data.go
View File

@@ -105,6 +105,10 @@ func enrichZhipuModelMetadata(model ModelPricing) ModelPricing {
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
}
func main() {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
@@ -233,14 +237,22 @@ func main() {
log.Printf("Model error for %s: %v", p.ModelID, err)
continue
}
_, _ = db.Exec(
if _, err := db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
SET source_url = CASE
WHEN $4 THEN $2
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
)
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}
// Insert pricing
sourceType := p.OperatorType