From bb5a1ff9e578cf5848d7150edac5a0aac8164213 Mon Sep 17 00:00:00 2001 From: phamnazage-jpg Date: Wed, 13 May 2026 21:56:16 +0800 Subject: [PATCH] feat(import): add explicit zhipu release metadata --- scripts/import_zhipu_data.go | 80 +++++++++++++++++++++++++++++-- scripts/import_zhipu_data_test.go | 72 ++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 scripts/import_zhipu_data_test.go diff --git a/scripts/import_zhipu_data.go b/scripts/import_zhipu_data.go index 5bc0f1d..042e55b 100644 --- a/scripts/import_zhipu_data.go +++ b/scripts/import_zhipu_data.go @@ -6,6 +6,7 @@ import ( "database/sql" "log" "os" + "strings" "time" _ "github.com/lib/pq" @@ -25,6 +26,7 @@ type ModelPricing struct { ContextLength int IsFree bool SourceURL string + ModelSourceURL string ReleaseDate string Modality string SceneTags []string @@ -41,6 +43,68 @@ func releaseDateValue(raw string) time.Time { return parsed } +type zhipuModelMetadata struct { + Prefix string + ReleaseDate string + ModelSourceURL string +} + +var zhipuModelMetadataRules = []zhipuModelMetadata{ + { + Prefix: "glm-5-turbo", + ReleaseDate: "2026-03-15", + ModelSourceURL: "https://www.zhipuai.cn/en/research/155", + }, + { + Prefix: "glm-5.1", + ReleaseDate: "2026-04-07", + ModelSourceURL: "https://www.zhipuai.cn/zh/research", + }, + { + Prefix: "glm-5", + ReleaseDate: "2026-02-11", + ModelSourceURL: "https://www.zhipuai.cn/zh/research/154", + }, + { + Prefix: "glm-4.7-flash", + ReleaseDate: "2026-01-19", + ModelSourceURL: "https://www.zhipuai.cn/zh/news/148", + }, + { + Prefix: "glm-4.7", + ReleaseDate: "2025-12-21", + ModelSourceURL: "https://www.zhipuai.cn/zh/research", + }, + { + Prefix: "glm-4.6v", + ReleaseDate: "2025-12-07", + ModelSourceURL: "https://www.zhipuai.cn/zh/research/144", + }, + { + Prefix: "glm-tts", + ReleaseDate: "2025-12-10", + ModelSourceURL: "https://www.zhipuai.cn/zh/research", + }, +} + +func enrichZhipuModelMetadata(model ModelPricing) ModelPricing { + for _, metadata := range zhipuModelMetadataRules { + if strings.HasPrefix(model.ModelID, metadata.Prefix) { + if metadata.ReleaseDate != "" { + model.ReleaseDate = metadata.ReleaseDate + } + if metadata.ModelSourceURL != "" { + model.ModelSourceURL = metadata.ModelSourceURL + } + return model + } + } + if model.ModelSourceURL == "" { + model.ModelSourceURL = model.SourceURL + } + return model +} + func main() { dsn := os.Getenv("DATABASE_URL") if dsn == "" { @@ -125,7 +189,8 @@ func main() { log.Printf("Importing %d Zhipu AI models...", len(prices)) // Save to database - for _, p := range prices { + for _, rawPricing := range prices { + p := enrichZhipuModelMetadata(rawPricing) // Find or create provider var providerID int64 err := db.QueryRow("SELECT id FROM model_provider WHERE name = $1", p.ProviderName).Scan(&providerID) @@ -161,7 +226,7 @@ func main() { err = db.QueryRow( `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date) VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`, - p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate), + p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), ).Scan(&modelID) } if err != nil { @@ -174,7 +239,7 @@ func main() { release_date = COALESCE(release_date, $3), updated_at = CURRENT_TIMESTAMP WHERE id = $1`, - modelID, p.SourceURL, releaseDateValue(p.ReleaseDate), + modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), ) // Insert pricing @@ -211,3 +276,12 @@ func main() { log.Printf("Successfully imported %d Zhipu AI models", len(prices)) } + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} diff --git a/scripts/import_zhipu_data_test.go b/scripts/import_zhipu_data_test.go new file mode 100644 index 0000000..cccb6b0 --- /dev/null +++ b/scripts/import_zhipu_data_test.go @@ -0,0 +1,72 @@ +//go:build llm_script + +package main + +import "testing" + +func TestEnrichZhipuModelMetadataUsesSpecificFamilyRules(t *testing.T) { + cases := []struct { + modelID string + wantReleaseDate string + wantSourceURL string + }{ + { + modelID: "glm-5.1-32k", + wantReleaseDate: "2026-04-07", + wantSourceURL: "https://www.zhipuai.cn/zh/research", + }, + { + modelID: "glm-5-turbo-32k", + wantReleaseDate: "2026-03-15", + wantSourceURL: "https://www.zhipuai.cn/en/research/155", + }, + { + modelID: "glm-5-32k", + wantReleaseDate: "2026-02-11", + wantSourceURL: "https://www.zhipuai.cn/zh/research/154", + }, + { + modelID: "glm-4.7-flash", + wantReleaseDate: "2026-01-19", + wantSourceURL: "https://www.zhipuai.cn/zh/news/148", + }, + { + modelID: "glm-4.6v-flashx", + wantReleaseDate: "2025-12-07", + wantSourceURL: "https://www.zhipuai.cn/zh/research/144", + }, + { + modelID: "glm-tts-clone", + wantReleaseDate: "2025-12-10", + wantSourceURL: "https://www.zhipuai.cn/zh/research", + }, + } + + for _, tc := range cases { + enriched := enrichZhipuModelMetadata(ModelPricing{ + ModelID: tc.modelID, + SourceURL: "https://open.bigmodel.cn/pricing", + }) + + if enriched.ReleaseDate != tc.wantReleaseDate { + t.Fatalf("%s release date = %q, want %q", tc.modelID, enriched.ReleaseDate, tc.wantReleaseDate) + } + if enriched.ModelSourceURL != tc.wantSourceURL { + t.Fatalf("%s source url = %q, want %q", tc.modelID, enriched.ModelSourceURL, tc.wantSourceURL) + } + } +} + +func TestEnrichZhipuModelMetadataFallsBackToPricingSource(t *testing.T) { + enriched := enrichZhipuModelMetadata(ModelPricing{ + ModelID: "glm-unknown-preview", + SourceURL: "https://open.bigmodel.cn/pricing", + }) + + if enriched.ReleaseDate != "" { + t.Fatalf("unexpected release date: %q", enriched.ReleaseDate) + } + if enriched.ModelSourceURL != "https://open.bigmodel.cn/pricing" { + t.Fatalf("model source url = %q, want pricing source fallback", enriched.ModelSourceURL) + } +}