diff --git a/scripts/import_bytedance_data.go b/scripts/import_bytedance_data.go index 2d8b117..6909b50 100644 --- a/scripts/import_bytedance_data.go +++ b/scripts/import_bytedance_data.go @@ -50,6 +50,21 @@ type bytedanceModelMetadata struct { } var bytedanceModelMetadataRules = []bytedanceModelMetadata{ + { + Prefix: "bytedance-doubao-1.5-vision-pro", + ReleaseDate: "2025-01-22", + ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083", + }, + { + Prefix: "bytedance-doubao-1.5-pro", + ReleaseDate: "2025-01-22", + ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083", + }, + { + Prefix: "bytedance-doubao-1.5-lite", + ReleaseDate: "2025-01-22", + ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083", + }, { Prefix: "bytedance-doubao-1.5-thinking", ReleaseDate: "2025-04-17", @@ -60,6 +75,31 @@ var bytedanceModelMetadataRules = []bytedanceModelMetadata{ ReleaseDate: "2025-06-11", ModelSourceURL: "https://developer.volcengine.com/articles/7517188354606104612", }, + { + Prefix: "bytedance-doubao-seed-1.8", + ModelSourceURL: "https://developer.volcengine.com/articles/7601918680544641034", + }, + { + Prefix: "bytedance-doubao-seed-2.0-code", + ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675", + }, + { + Prefix: "bytedance-doubao-seed-2.0-pro", + ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675", + }, + { + Prefix: "bytedance-doubao-seed-2.0-mini", + ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675", + }, + { + Prefix: "bytedance-doubao-seed-2.0-lite", + ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675", + }, + { + Prefix: "bytedance-doubao-seed-code", + ReleaseDate: "2024-06-26", + ModelSourceURL: "https://developer.volcengine.com/articles/7383101327527641125", + }, { Prefix: "bytedance-seedance-1.0-lite", ReleaseDate: "2025-05-13", @@ -86,6 +126,10 @@ func enrichBytedanceModelMetadata(model ModelPricing) ModelPricing { return model } +func hasExplicitModelMetadata(model ModelPricing) bool { + return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL +} + func main() { dsn := os.Getenv("DATABASE_URL") if dsn == "" { @@ -183,14 +227,22 @@ func main() { log.Printf("Model error for %s: %v", p.ModelID, err) continue } - _, _ = db.Exec( + if _, err := db.Exec( `UPDATE models - SET source_url = COALESCE(NULLIF(source_url, ''), $2), - release_date = COALESCE(release_date, $3), + SET source_url = CASE + WHEN $4 THEN $2 + ELSE COALESCE(NULLIF(source_url, ''), $2) + END, + release_date = CASE + WHEN $4 AND $3::date IS NOT NULL THEN $3::date + ELSE COALESCE(release_date, $3::date) + END, updated_at = CURRENT_TIMESTAMP WHERE id = $1`, - modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), - ) + modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), + ); err != nil { + log.Printf("Model metadata update error for %s: %v", p.ModelID, err) + } // Insert pricing sourceType := p.OperatorType diff --git a/scripts/import_bytedance_data_test.go b/scripts/import_bytedance_data_test.go index 7018dd4..45393b6 100644 --- a/scripts/import_bytedance_data_test.go +++ b/scripts/import_bytedance_data_test.go @@ -10,6 +10,16 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) { wantReleaseDate string wantSourceURL string }{ + { + modelID: "bytedance-doubao-1.5-pro-32k", + wantReleaseDate: "2025-01-22", + wantSourceURL: "https://developer.volcengine.com/articles/7462939272262189083", + }, + { + modelID: "bytedance-doubao-1.5-vision-pro", + wantReleaseDate: "2025-01-22", + wantSourceURL: "https://developer.volcengine.com/articles/7462939272262189083", + }, { modelID: "bytedance-doubao-seed-1.6-thinking", wantReleaseDate: "2025-06-11", @@ -25,6 +35,11 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) { wantReleaseDate: "2025-05-13", wantSourceURL: "https://developer.volcengine.com/articles/7504284064976502823", }, + { + modelID: "bytedance-doubao-seed-code-256k", + wantReleaseDate: "2024-06-26", + wantSourceURL: "https://developer.volcengine.com/articles/7383101327527641125", + }, } for _, tc := range cases { @@ -56,6 +71,20 @@ func TestEnrichBytedanceModelMetadataFallsBackToPricingSource(t *testing.T) { } } +func TestEnrichBytedanceModelMetadataSupportsSourceOnlyRules(t *testing.T) { + enriched := enrichBytedanceModelMetadata(ModelPricing{ + ModelID: "bytedance-doubao-seed-2.0-pro-256k", + SourceURL: "https://www.volcengine.com/docs/82379/1099320", + }) + + if enriched.ReleaseDate != "" { + t.Fatalf("unexpected release date: %q", enriched.ReleaseDate) + } + if enriched.ModelSourceURL != "https://developer.volcengine.com/articles/7610285824933445675" { + t.Fatalf("model source url = %q, want 2.0 source", enriched.ModelSourceURL) + } +} + func TestBytedanceReleaseDateValueReturnsNilForUnknownDate(t *testing.T) { if value := releaseDateValue(""); value != nil { t.Fatalf("blank release date should stay nil, got %v", value) diff --git a/scripts/import_phase2_data.go b/scripts/import_phase2_data.go index 6807793..15ffe81 100644 --- a/scripts/import_phase2_data.go +++ b/scripts/import_phase2_data.go @@ -73,6 +73,24 @@ type baiduModelMetadata struct { } var baiduModelMetadataRules = []baiduModelMetadata{ + { + Prefix: "baidu-ernie-5.0", + ReleaseDate: "2026-01-22", + ModelSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba", + }, + { + Prefix: "baidu-ernie-x1.1", + ReleaseDate: "2025-09-09", + ModelSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a", + }, + { + Prefix: "baidu-ernie-5.1", + ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html", + }, + { + Prefix: "baidu-ernie-4.5-turbo-vl", + ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html", + }, { Prefix: "baidu-ernie-4.5-turbo", ReleaseDate: "2025-04-25", @@ -93,6 +111,21 @@ var baiduModelMetadataRules = []baiduModelMetadata{ ReleaseDate: "2025-03-16", ModelSourceURL: "https://cloud.baidu.com/article/3835921", }, + { + Prefix: "baidu-ernie-character", + ReleaseDate: "2024-03-22", + ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77", + }, + { + Prefix: "baidu-ernie-lite-pro", + ReleaseDate: "2024-03-22", + ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77", + }, + { + Prefix: "baidu-ernie-speed-pro", + ReleaseDate: "2024-03-22", + ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77", + }, } func enrichBaiduModelMetadata(model ModelPricing) ModelPricing { @@ -114,6 +147,10 @@ func enrichBaiduModelMetadata(model ModelPricing) ModelPricing { return model } +func hasExplicitModelMetadata(model ModelPricing) bool { + return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL +} + func parseZhipuPrice(s string) float64 { // Extract price from strings like "6元", "免费", "限时免费" if strings.Contains(s, "免费") { @@ -260,14 +297,22 @@ func main() { log.Printf("Model error: %v", err) continue } - _, _ = db.Exec( + if _, err := db.Exec( `UPDATE models - SET source_url = COALESCE(NULLIF(source_url, ''), $2), - release_date = COALESCE(release_date, $3), + SET source_url = CASE + WHEN $4 THEN $2 + ELSE COALESCE(NULLIF(source_url, ''), $2) + END, + release_date = CASE + WHEN $4 AND $3::date IS NOT NULL THEN $3::date + ELSE COALESCE(release_date, $3::date) + END, updated_at = CURRENT_TIMESTAMP WHERE id = $1`, - modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), - ) + modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), + ); err != nil { + log.Printf("Model metadata update error for %s: %v", p.ModelID, err) + } // Insert pricing sourceType := p.OperatorType diff --git a/scripts/import_phase2_data_test.go b/scripts/import_phase2_data_test.go index 2c828ab..16d3d42 100644 --- a/scripts/import_phase2_data_test.go +++ b/scripts/import_phase2_data_test.go @@ -10,6 +10,16 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) { wantReleaseDate string wantSourceURL string }{ + { + modelID: "baidu-ernie-5.0", + wantReleaseDate: "2026-01-22", + wantSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba", + }, + { + modelID: "baidu-ernie-x1.1", + wantReleaseDate: "2025-09-09", + wantSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a", + }, { modelID: "baidu-ernie-4.5-turbo-128k", wantReleaseDate: "2025-04-25", @@ -30,6 +40,11 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) { wantReleaseDate: "2025-03-16", wantSourceURL: "https://cloud.baidu.com/article/3835921", }, + { + modelID: "baidu-ernie-character", + wantReleaseDate: "2024-03-22", + wantSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77", + }, } for _, tc := range cases { @@ -61,6 +76,20 @@ func TestEnrichBaiduModelMetadataFallsBackToPricingSource(t *testing.T) { } } +func TestEnrichBaiduModelMetadataSupportsSourceOnlyRules(t *testing.T) { + enriched := enrichBaiduModelMetadata(ModelPricing{ + ModelID: "baidu-ernie-5.1", + SourceURL: "https://cloud.baidu.com/doc/qianfan/s/wmh4sv6ya", + }) + + if enriched.ReleaseDate != "" { + t.Fatalf("unexpected release date: %q", enriched.ReleaseDate) + } + if enriched.ModelSourceURL != "https://cloud.baidu.com/product/wenxinworkshop.html" { + t.Fatalf("model source url = %q, want product source", enriched.ModelSourceURL) + } +} + func TestBaiduReleaseDateValueReturnsNilForUnknownDate(t *testing.T) { if value := releaseDateValue(""); value != nil { t.Fatalf("blank release date should stay nil, got %v", value) diff --git a/scripts/import_zhipu_data.go b/scripts/import_zhipu_data.go index 23dd875..be90caf 100644 --- a/scripts/import_zhipu_data.go +++ b/scripts/import_zhipu_data.go @@ -105,6 +105,10 @@ func enrichZhipuModelMetadata(model ModelPricing) ModelPricing { return model } +func hasExplicitModelMetadata(model ModelPricing) bool { + return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL +} + func main() { dsn := os.Getenv("DATABASE_URL") if dsn == "" { @@ -233,14 +237,22 @@ func main() { log.Printf("Model error for %s: %v", p.ModelID, err) continue } - _, _ = db.Exec( + if _, err := db.Exec( `UPDATE models - SET source_url = COALESCE(NULLIF(source_url, ''), $2), - release_date = COALESCE(release_date, $3), + SET source_url = CASE + WHEN $4 THEN $2 + ELSE COALESCE(NULLIF(source_url, ''), $2) + END, + release_date = CASE + WHEN $4 AND $3::date IS NOT NULL THEN $3::date + ELSE COALESCE(release_date, $3::date) + END, updated_at = CURRENT_TIMESTAMP WHERE id = $1`, - modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), - ) + modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), + ); err != nil { + log.Printf("Model metadata update error for %s: %v", p.ModelID, err) + } // Insert pricing sourceType := p.OperatorType