niuniu/llm-intelligence
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(import): track release date evidence tiers

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-13 23:27:47 +08:00
parent 569b94cb73
commit f2f68b85c1
10 changed files with 599 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
db/migrations/006_model_release_date_metadata.sql Normal file
View File

@@ -0,0 +1,34 @@
-- Phase 2.1: 模型发布日期证据元数据
-- 区分一级官方发布日期与二级权威佐证日期，避免混淆 source_url 与发布日期证据层级
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='models' AND column_name='date_confidence') THEN
ALTER TABLE models ADD COLUMN date_confidence TEXT NOT NULL DEFAULT 'unknown';
END IF;
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='models' AND column_name='date_source_kind') THEN
ALTER TABLE models ADD COLUMN date_source_kind TEXT NOT NULL DEFAULT 'unknown';
END IF;
END $$;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname='chk_models_date_confidence') THEN
ALTER TABLE models
ADD CONSTRAINT chk_models_date_confidence
CHECK (date_confidence IN ('official_primary', 'secondary_authoritative', 'inferred', 'unknown'));
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname='chk_models_date_source_kind') THEN
ALTER TABLE models
ADD CONSTRAINT chk_models_date_source_kind
CHECK (date_source_kind IN ('official_announcement', 'official_product_page', 'secondary_authoritative_report', 'catalog_backfill', 'unknown'));
END IF;
END $$;
CREATE INDEX IF NOT EXISTS idx_models_date_confidence ON models(date_confidence);
CREATE INDEX IF NOT EXISTS idx_models_date_source_kind ON models(date_source_kind);
COMMENT ON COLUMN models.date_confidence IS '发布日期证据置信度：official_primary / secondary_authoritative / inferred / unknown';
COMMENT ON COLUMN models.date_source_kind IS '发布日期证据来源类型：official_announcement / official_product_page / secondary_authoritative_report / catalog_backfill / unknown';

58
scripts/generate_daily_report.go
View File

@@ -812,6 +812,8 @@ func loadOfficialReleaseEvents(db *sql.DB, date string) ([]ModelEvent, error) {
COALESCE(lp.operator_name, 'Unknown') AS operator_name,
COALESCE(lp.operator_type, 'reseller') AS operator_type,
COALESCE(m.source_url, '') AS source_url,
COALESCE(m.date_confidence, 'unknown') AS date_confidence,
COALESCE(m.date_source_kind, 'unknown') AS date_source_kind,
COALESCE(mp.country, 'unknown') AS provider_country,
COALESCE(m.release_date, m.created_at::date) AS release_date,
COALESCE(lp.currency, 'USD') AS currency
@@ -838,6 +840,8 @@ func loadOfficialReleaseEvents(db *sql.DB, date string) ([]ModelEvent, error) {
operatorName string
operatorType string
sourceURL string
dateConfidence string
dateSourceKind string
providerCountry string
releaseDate time.Time
currency string
@@ -848,6 +852,8 @@ func loadOfficialReleaseEvents(db *sql.DB, date string) ([]ModelEvent, error) {
&operatorName,
&operatorType,
&sourceURL,
&dateConfidence,
&dateSourceKind,
&providerCountry,
&releaseDate,
&currency,
@@ -869,11 +875,11 @@ func loadOfficialReleaseEvents(db *sql.DB, date string) ([]ModelEvent, error) {
ModelName: modelName,
ProviderName: providerName,
OperatorName: operatorName,
TrustLabel: buildTrustLabel(model),
SourceKindLabel: "官方发布",
TrustLabel: buildReleaseTrustLabel(model, dateConfidence),
SourceKindLabel: buildReleaseSourceKindLabel(dateSourceKind, dateConfidence),
PrimarySource: sourceURL,
UpdatedAt: releaseDate.Format("2006-01-02 15:04"),
EvidenceDetail: "models.release_date = 今日，且 source_url 指向官方发布页",
EvidenceDetail: buildReleaseEvidenceDetail(dateSourceKind, dateConfidence),
Baseline: "官方首次发布",
Summary: fmt.Sprintf("%s 官方发布新模型，值得优先复查默认选型。", providerName),
Currency: currency,
@@ -1478,6 +1484,10 @@ func headlineItemFromModelEvent(event ModelEvent) HeadlineItem {
case "official_release":
item.Label = "官方发布"
item.Title = fmt.Sprintf("%s 官方发布", event.ModelName)
if event.SourceKindLabel == "权威佐证发布" {
item.Label = "权威佐证"
item.Title = fmt.Sprintf("%s 进入权威佐证发布时间线", event.ModelName)
}
item.Tone = "info"
case "new_model":
item.Label = "新模型"
@@ -1534,6 +1544,48 @@ func buildPriceEvidenceDetail(changePct, oldPrice, newPrice float64, currency st
)
}
func buildReleaseSourceKindLabel(dateSourceKind, dateConfidence string) string {
switch {
case dateSourceKind == "secondary_authoritative_report" || dateConfidence == "secondary_authoritative":
return "权威佐证发布"
case dateSourceKind == "official_announcement" && dateConfidence == "official_primary":
return "官方发布"
case dateSourceKind == "official_product_page":
return "官方产品页"
case dateSourceKind == "catalog_backfill":
return "目录回填"
default:
return "官方发布"
}
}
func buildReleaseEvidenceDetail(dateSourceKind, dateConfidence string) string {
switch {
case dateSourceKind == "secondary_authoritative_report" || dateConfidence == "secondary_authoritative":
return "models.release_date = 今日，发布日期采用次级权威报道佐证，模型来源页保留官方文档"
case dateSourceKind == "official_announcement" && dateConfidence == "official_primary":
return "models.release_date = 今日，且 source_url 指向官方发布页"
case dateSourceKind == "official_product_page":
return "models.release_date = 今日，来源页为官方产品页，发布日期置信度待确认"
case dateSourceKind == "catalog_backfill":
return "models.release_date = 今日，发布日期来自目录级元数据回填"
default:
return "models.release_date = 今日，且已记录发布日期证据元数据"
}
}
func buildReleaseTrustLabel(model ModelInfo, dateConfidence string) string {
base := buildTrustLabel(model)
switch dateConfidence {
case "official_primary":
return base + " / 一级证据"
case "secondary_authoritative":
return base + " / 二级佐证"
default:
return base
}
}
func buildFreeEvidenceDetail(model ModelInfo) string {
switch classifyFreeSource(model) {
case "官方免费":

21
scripts/generate_daily_report_test.go
View File

@@ -575,3 +575,24 @@ func TestHeadlineItemFromOfficialReleaseEvent(t *testing.T) {
t.Fatalf("expected primary source to be preserved, got %+v", item)
}
}
func TestHeadlineItemFromSecondaryReleaseEvent(t *testing.T) {
item := headlineItemFromModelEvent(ModelEvent{
EventType: "official_release",
ModelName: "Doubao Seed 1.8",
TrustLabel: "官方来源 / 二级佐证",
Baseline: "官方首次发布",
Summary: "模型进入正式发布日期观察池。",
SourceKindLabel: "权威佐证发布",
PrimarySource: "https://developer.volcengine.com/articles/7601918680544641034",
UpdatedAt: "2025-12-18 00:00",
EvidenceDetail: "models.release_date = 今日，发布日期采用次级权威报道佐证，模型来源页保留官方文档",
})
if item.Label != "权威佐证" {
t.Fatalf("expected label to be 权威佐证, got %+v", item)
}
if !strings.Contains(item.Title, "权威佐证发布时间线") {
t.Fatalf("expected title to mention 权威佐证发布时间线, got %+v", item)
}
}

64
scripts/import_bytedance_data.go
View File

@@ -29,6 +29,8 @@ type ModelPricing struct {
SourceURL string
ModelSourceURL string
ReleaseDate string
DateConfidence string
DateSourceKind string
Modality string
}
@@ -47,6 +49,8 @@ type bytedanceModelMetadata struct {
Prefix string
ReleaseDate string
ModelSourceURL string
DateConfidence string
DateSourceKind string
}
var bytedanceModelMetadataRules = []bytedanceModelMetadata{
@@ -54,60 +58,85 @@ var bytedanceModelMetadataRules = []bytedanceModelMetadata{
Prefix: "bytedance-doubao-1.5-vision-pro",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-pro",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-lite",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-thinking",
ReleaseDate: "2025-04-17",
ModelSourceURL: "https://developer.volcengine.com/articles/7496718897794039827",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-1.6",
ReleaseDate: "2025-06-11",
ModelSourceURL: "https://developer.volcengine.com/articles/7517188354606104612",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-1.8",
ReleaseDate: "2025-12-18",
ModelSourceURL: "https://developer.volcengine.com/articles/7601918680544641034",
DateConfidence: "secondary_authoritative",
DateSourceKind: "secondary_authoritative_report",
},
{
Prefix: "bytedance-doubao-seed-2.0-code",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-pro",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-mini",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-lite",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-code",
ReleaseDate: "2024-06-26",
ModelSourceURL: "https://developer.volcengine.com/articles/7383101327527641125",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-seedance-1.0-lite",
ReleaseDate: "2025-05-13",
ModelSourceURL: "https://developer.volcengine.com/articles/7504284064976502823",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
}
@@ -121,17 +150,32 @@ func enrichBytedanceModelMetadata(model ModelPricing) ModelPricing {
if metadata.ModelSourceURL != "" {
model.ModelSourceURL = metadata.ModelSourceURL
}
if metadata.DateConfidence != "" {
model.DateConfidence = metadata.DateConfidence
}
if metadata.DateSourceKind != "" {
model.DateSourceKind = metadata.DateSourceKind
}
return model
}
}
if model.ModelSourceURL == "" {
model.ModelSourceURL = model.SourceURL
}
if model.DateConfidence == "" {
model.DateConfidence = "unknown"
}
if model.DateSourceKind == "" {
model.DateSourceKind = "unknown"
}
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
return strings.TrimSpace(model.ReleaseDate) != "" ||
firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}
func main() {
@@ -222,9 +266,9 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
).Scan(&modelID)
}
if err != nil {
@@ -238,12 +282,20 @@ func main() {
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
WHEN $4 THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
date_confidence = CASE
WHEN $4 THEN $5
ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
END,
date_source_kind = CASE
WHEN $4 THEN $6
ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}

33
scripts/import_bytedance_data_test.go
View File

@@ -9,36 +9,50 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) {
modelID string
wantReleaseDate string
wantSourceURL string
wantConfidence string
wantSourceKind string
}{
{
modelID: "bytedance-doubao-1.5-pro-32k",
wantReleaseDate: "2025-01-22",
wantSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-doubao-1.5-vision-pro",
wantReleaseDate: "2025-01-22",
wantSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-doubao-seed-1.6-thinking",
wantReleaseDate: "2025-06-11",
wantSourceURL: "https://developer.volcengine.com/articles/7517188354606104612",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-doubao-1.5-thinking-pro",
wantReleaseDate: "2025-04-17",
wantSourceURL: "https://developer.volcengine.com/articles/7496718897794039827",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-seedance-1.0-lite",
wantReleaseDate: "2025-05-13",
wantSourceURL: "https://developer.volcengine.com/articles/7504284064976502823",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-doubao-seed-code-256k",
wantReleaseDate: "2024-06-26",
wantSourceURL: "https://developer.volcengine.com/articles/7383101327527641125",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
}
@@ -54,6 +68,12 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) {
if enriched.ModelSourceURL != tc.wantSourceURL {
t.Fatalf("%s source url = %q, want %q", tc.modelID, enriched.ModelSourceURL, tc.wantSourceURL)
}
if enriched.DateConfidence != tc.wantConfidence {
t.Fatalf("%s date confidence = %q, want %q", tc.modelID, enriched.DateConfidence, tc.wantConfidence)
}
if enriched.DateSourceKind != tc.wantSourceKind {
t.Fatalf("%s date source kind = %q, want %q", tc.modelID, enriched.DateSourceKind, tc.wantSourceKind)
}
}
}
@@ -69,6 +89,9 @@ func TestEnrichBytedanceModelMetadataFallsBackToPricingSource(t *testing.T) {
if enriched.ModelSourceURL != "https://www.volcengine.com/docs/82379/1099320" {
t.Fatalf("model source url = %q, want pricing source fallback", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "unknown" {
t.Fatalf("unexpected fallback date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestEnrichBytedanceModelMetadataSupportsSourceOnlyRules(t *testing.T) {
@@ -77,12 +100,15 @@ func TestEnrichBytedanceModelMetadataSupportsSourceOnlyRules(t *testing.T) {
SourceURL: "https://www.volcengine.com/docs/82379/1099320",
})
if enriched.ReleaseDate != "" {
t.Fatalf("unexpected release date: %q", enriched.ReleaseDate)
if enriched.ReleaseDate != "2025-12-18" {
t.Fatalf("release date = %q, want %q", enriched.ReleaseDate, "2025-12-18")
}
if enriched.ModelSourceURL != "https://developer.volcengine.com/articles/7601918680544641034" {
t.Fatalf("model source url = %q, want 1.8 source", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "secondary_authoritative" || enriched.DateSourceKind != "secondary_authoritative_report" {
t.Fatalf("unexpected 1.8 date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestEnrichBytedanceModelMetadataUsesTwoPointZeroReleaseDate(t *testing.T) {
@@ -97,6 +123,9 @@ func TestEnrichBytedanceModelMetadataUsesTwoPointZeroReleaseDate(t *testing.T) {
if enriched.ModelSourceURL != "https://developer.volcengine.com/articles/7610285824933445675" {
t.Fatalf("model source url = %q, want 2.0 source", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "official_primary" || enriched.DateSourceKind != "official_announcement" {
t.Fatalf("unexpected 2.0 date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestBytedanceReleaseDateValueReturnsNilForUnknownDate(t *testing.T) {

61
scripts/import_phase2_data.go
View File

@@ -51,6 +51,8 @@ type ModelPricing struct {
SourceURL string
ModelSourceURL string
ReleaseDate string
DateConfidence string
DateSourceKind string
Modality string
SceneTags []string
}
@@ -70,6 +72,8 @@ type baiduModelMetadata struct {
Prefix string
ReleaseDate string
ModelSourceURL string
DateConfidence string
DateSourceKind string
}
var baiduModelMetadataRules = []baiduModelMetadata{
@@ -77,54 +81,76 @@ var baiduModelMetadataRules = []baiduModelMetadata{
Prefix: "baidu-ernie-5.0",
ReleaseDate: "2026-01-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1.1",
ReleaseDate: "2025-09-09",
ModelSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-5.1",
ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html",
DateConfidence: "unknown",
DateSourceKind: "official_product_page",
},
{
Prefix: "baidu-ernie-4.5-turbo-vl",
ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html",
DateConfidence: "unknown",
DateSourceKind: "official_product_page",
},
{
Prefix: "baidu-ernie-4.5-turbo",
ReleaseDate: "2025-04-25",
ModelSourceURL: "https://cloud.baidu.com/article/3887765",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1-turbo",
ReleaseDate: "2025-04-25",
ModelSourceURL: "https://cloud.baidu.com/article/3887765",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-4.5",
ReleaseDate: "2025-03-16",
ModelSourceURL: "https://cloud.baidu.com/article/3835921",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1",
ReleaseDate: "2025-03-16",
ModelSourceURL: "https://cloud.baidu.com/article/3835921",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-character",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-lite-pro",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-speed-pro",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
}
@@ -138,17 +164,32 @@ func enrichBaiduModelMetadata(model ModelPricing) ModelPricing {
if metadata.ModelSourceURL != "" {
model.ModelSourceURL = metadata.ModelSourceURL
}
if metadata.DateConfidence != "" {
model.DateConfidence = metadata.DateConfidence
}
if metadata.DateSourceKind != "" {
model.DateSourceKind = metadata.DateSourceKind
}
return model
}
}
if model.ModelSourceURL == "" {
model.ModelSourceURL = model.SourceURL
}
if model.DateConfidence == "" {
model.DateConfidence = "unknown"
}
if model.DateSourceKind == "" {
model.DateSourceKind = "unknown"
}
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
return strings.TrimSpace(model.ReleaseDate) != "" ||
firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}
func parseZhipuPrice(s string) float64 {
@@ -288,9 +329,9 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
).Scan(&modelID)
}
if err != nil {
@@ -304,12 +345,20 @@ func main() {
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
WHEN $4 THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
date_confidence = CASE
WHEN $4 THEN $5
ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
END,
date_source_kind = CASE
WHEN $4 THEN $6
ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}

28
scripts/import_phase2_data_test.go
View File

@@ -9,41 +9,57 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) {
modelID string
wantReleaseDate string
wantSourceURL string
wantConfidence string
wantSourceKind string
}{
{
modelID: "baidu-ernie-5.0",
wantReleaseDate: "2026-01-22",
wantSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-x1.1",
wantReleaseDate: "2025-09-09",
wantSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-4.5-turbo-128k",
wantReleaseDate: "2025-04-25",
wantSourceURL: "https://cloud.baidu.com/article/3887765",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-x1-turbo-32k",
wantReleaseDate: "2025-04-25",
wantSourceURL: "https://cloud.baidu.com/article/3887765",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-4.5-8k",
wantReleaseDate: "2025-03-16",
wantSourceURL: "https://cloud.baidu.com/article/3835921",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-x1-8k",
wantReleaseDate: "2025-03-16",
wantSourceURL: "https://cloud.baidu.com/article/3835921",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-character",
wantReleaseDate: "2024-03-22",
wantSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
}
@@ -59,6 +75,12 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) {
if enriched.ModelSourceURL != tc.wantSourceURL {
t.Fatalf("%s source url = %q, want %q", tc.modelID, enriched.ModelSourceURL, tc.wantSourceURL)
}
if enriched.DateConfidence != tc.wantConfidence {
t.Fatalf("%s date confidence = %q, want %q", tc.modelID, enriched.DateConfidence, tc.wantConfidence)
}
if enriched.DateSourceKind != tc.wantSourceKind {
t.Fatalf("%s date source kind = %q, want %q", tc.modelID, enriched.DateSourceKind, tc.wantSourceKind)
}
}
}
@@ -74,6 +96,9 @@ func TestEnrichBaiduModelMetadataFallsBackToPricingSource(t *testing.T) {
if enriched.ModelSourceURL != "https://cloud.baidu.com/doc/qianfan/s/wmh4sv6ya" {
t.Fatalf("model source url = %q, want pricing source fallback", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "unknown" {
t.Fatalf("unexpected fallback date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestEnrichBaiduModelMetadataSupportsSourceOnlyRules(t *testing.T) {
@@ -88,6 +113,9 @@ func TestEnrichBaiduModelMetadataSupportsSourceOnlyRules(t *testing.T) {
if enriched.ModelSourceURL != "https://cloud.baidu.com/product/wenxinworkshop.html" {
t.Fatalf("model source url = %q, want product source", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "official_product_page" {
t.Fatalf("unexpected source-only metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestBaiduReleaseDateValueReturnsNilForUnknownDate(t *testing.T) {

53
scripts/import_zhipu_data.go
View File

@@ -28,6 +28,8 @@ type ModelPricing struct {
SourceURL string
ModelSourceURL string
ReleaseDate string
DateConfidence string
DateSourceKind string
Modality string
SceneTags []string
}
@@ -47,6 +49,8 @@ type zhipuModelMetadata struct {
Prefix string
ReleaseDate string
ModelSourceURL string
DateConfidence string
DateSourceKind string
}
var zhipuModelMetadataRules = []zhipuModelMetadata{
@@ -54,36 +58,50 @@ var zhipuModelMetadataRules = []zhipuModelMetadata{
Prefix: "glm-5-turbo",
ReleaseDate: "2026-03-15",
ModelSourceURL: "https://www.zhipuai.cn/en/research/155",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-5.1",
ReleaseDate: "2026-04-07",
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-5",
ReleaseDate: "2026-02-11",
ModelSourceURL: "https://www.zhipuai.cn/zh/research/154",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-4.7-flash",
ReleaseDate: "2026-01-19",
ModelSourceURL: "https://www.zhipuai.cn/zh/news/148",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-4.7",
ReleaseDate: "2025-12-21",
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-4.6v",
ReleaseDate: "2025-12-07",
ModelSourceURL: "https://www.zhipuai.cn/zh/research/144",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-tts",
ReleaseDate: "2025-12-10",
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
}
@@ -96,17 +114,32 @@ func enrichZhipuModelMetadata(model ModelPricing) ModelPricing {
if metadata.ModelSourceURL != "" {
model.ModelSourceURL = metadata.ModelSourceURL
}
if metadata.DateConfidence != "" {
model.DateConfidence = metadata.DateConfidence
}
if metadata.DateSourceKind != "" {
model.DateSourceKind = metadata.DateSourceKind
}
return model
}
}
if model.ModelSourceURL == "" {
model.ModelSourceURL = model.SourceURL
}
if model.DateConfidence == "" {
model.DateConfidence = "unknown"
}
if model.DateSourceKind == "" {
model.DateSourceKind = "unknown"
}
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
return strings.TrimSpace(model.ReleaseDate) != "" ||
firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}
func main() {
@@ -228,9 +261,9 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
).Scan(&modelID)
}
if err != nil {
@@ -244,12 +277,20 @@ func main() {
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
WHEN $4 THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
date_confidence = CASE
WHEN $4 THEN $5
ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
END,
date_source_kind = CASE
WHEN $4 THEN $6
ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}

23
scripts/import_zhipu_data_test.go
View File

@@ -9,36 +9,50 @@ func TestEnrichZhipuModelMetadataUsesSpecificFamilyRules(t *testing.T) {
modelID string
wantReleaseDate string
wantSourceURL string
wantConfidence string
wantSourceKind string
}{
{
modelID: "glm-5.1-32k",
wantReleaseDate: "2026-04-07",
wantSourceURL: "https://www.zhipuai.cn/zh/research",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-5-turbo-32k",
wantReleaseDate: "2026-03-15",
wantSourceURL: "https://www.zhipuai.cn/en/research/155",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-5-32k",
wantReleaseDate: "2026-02-11",
wantSourceURL: "https://www.zhipuai.cn/zh/research/154",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-4.7-flash",
wantReleaseDate: "2026-01-19",
wantSourceURL: "https://www.zhipuai.cn/zh/news/148",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-4.6v-flashx",
wantReleaseDate: "2025-12-07",
wantSourceURL: "https://www.zhipuai.cn/zh/research/144",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-tts-clone",
wantReleaseDate: "2025-12-10",
wantSourceURL: "https://www.zhipuai.cn/zh/research",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
}
@@ -54,6 +68,12 @@ func TestEnrichZhipuModelMetadataUsesSpecificFamilyRules(t *testing.T) {
if enriched.ModelSourceURL != tc.wantSourceURL {
t.Fatalf("%s source url = %q, want %q", tc.modelID, enriched.ModelSourceURL, tc.wantSourceURL)
}
if enriched.DateConfidence != tc.wantConfidence {
t.Fatalf("%s date confidence = %q, want %q", tc.modelID, enriched.DateConfidence, tc.wantConfidence)
}
if enriched.DateSourceKind != tc.wantSourceKind {
t.Fatalf("%s date source kind = %q, want %q", tc.modelID, enriched.DateSourceKind, tc.wantSourceKind)
}
}
}
@@ -69,6 +89,9 @@ func TestEnrichZhipuModelMetadataFallsBackToPricingSource(t *testing.T) {
if enriched.ModelSourceURL != "https://open.bigmodel.cn/pricing" {
t.Fatalf("model source url = %q, want pricing source fallback", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "unknown" {
t.Fatalf("unexpected fallback date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestZhipuReleaseDateValueReturnsNilForUnknownDate(t *testing.T) {

6
scripts/official_import_scripts_test.go
View File

@@ -42,5 +42,11 @@ func TestOfficialImportScriptsWriteModelSourceURLAndReleaseDate(t *testing.T) {
if !strings.Contains(content, "release_date") {
t.Fatalf("%s missing release_date in models write path", relativePath)
}
if !strings.Contains(content, "date_confidence") {
t.Fatalf("%s missing date_confidence in models write path", relativePath)
}
if !strings.Contains(content, "date_source_kind") {
t.Fatalf("%s missing date_source_kind in models write path", relativePath)
}
}
}