niuniu/llm-intelligence
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(import): track release date evidence tiers

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-13 23:27:47 +08:00
parent 569b94cb73
commit f2f68b85c1
10 changed files with 599 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
scripts/generate_daily_report.go
View File

@@ -239,14 +239,14 @@ type ModelEvent struct {
}
type Recommendation struct {
Name string
Provider string
Operator string
Usage string
Name string
Provider string
Operator string
Usage string
PriceSummary string
Evidence string
TrustLabel string
Tags []string
Evidence string
TrustLabel string
Tags []string
}
type SceneSection struct {
@@ -812,6 +812,8 @@ func loadOfficialReleaseEvents(db *sql.DB, date string) ([]ModelEvent, error) {
COALESCE(lp.operator_name, 'Unknown') AS operator_name,
COALESCE(lp.operator_type, 'reseller') AS operator_type,
COALESCE(m.source_url, '') AS source_url,
COALESCE(m.date_confidence, 'unknown') AS date_confidence,
COALESCE(m.date_source_kind, 'unknown') AS date_source_kind,
COALESCE(mp.country, 'unknown') AS provider_country,
COALESCE(m.release_date, m.created_at::date) AS release_date,
COALESCE(lp.currency, 'USD') AS currency
@@ -838,6 +840,8 @@ func loadOfficialReleaseEvents(db *sql.DB, date string) ([]ModelEvent, error) {
operatorName string
operatorType string
sourceURL string
dateConfidence string
dateSourceKind string
providerCountry string
releaseDate time.Time
currency string
@@ -848,6 +852,8 @@ func loadOfficialReleaseEvents(db *sql.DB, date string) ([]ModelEvent, error) {
&operatorName,
&operatorType,
&sourceURL,
&dateConfidence,
&dateSourceKind,
&providerCountry,
&releaseDate,
&currency,
@@ -869,11 +875,11 @@ func loadOfficialReleaseEvents(db *sql.DB, date string) ([]ModelEvent, error) {
ModelName: modelName,
ProviderName: providerName,
OperatorName: operatorName,
TrustLabel: buildTrustLabel(model),
SourceKindLabel: "官方发布",
TrustLabel: buildReleaseTrustLabel(model, dateConfidence),
SourceKindLabel: buildReleaseSourceKindLabel(dateSourceKind, dateConfidence),
PrimarySource: sourceURL,
UpdatedAt: releaseDate.Format("2006-01-02 15:04"),
EvidenceDetail: "models.release_date = 今日，且 source_url 指向官方发布页",
EvidenceDetail: buildReleaseEvidenceDetail(dateSourceKind, dateConfidence),
Baseline: "官方首次发布",
Summary: fmt.Sprintf("%s 官方发布新模型，值得优先复查默认选型。", providerName),
Currency: currency,
@@ -991,7 +997,7 @@ func loadNewModelEvents(db *sql.DB, date string) ([]ModelEvent, error) {
Summary: summary,
Currency: currency,
NewInputPrice: inputPrice,
NewOutputPrice: outputPrice,
NewOutputPrice: outputPrice,
Priority: 85 + minInt(contextLength/(1024*128), 10),
})
}
@@ -1247,7 +1253,7 @@ func buildFreeSourceBreakdown(models []ModelInfo) []FreeSourceStat {
counts := map[string]int{
"官方免费": 0,
"聚合免费": 0,
"待确认": 0,
"待确认": 0,
}
for _, model := range models {
counts[classifyFreeSource(model)]++
@@ -1478,6 +1484,10 @@ func headlineItemFromModelEvent(event ModelEvent) HeadlineItem {
case "official_release":
item.Label = "官方发布"
item.Title = fmt.Sprintf("%s 官方发布", event.ModelName)
if event.SourceKindLabel == "权威佐证发布" {
item.Label = "权威佐证"
item.Title = fmt.Sprintf("%s 进入权威佐证发布时间线", event.ModelName)
}
item.Tone = "info"
case "new_model":
item.Label = "新模型"
@@ -1534,6 +1544,48 @@ func buildPriceEvidenceDetail(changePct, oldPrice, newPrice float64, currency st
)
}
func buildReleaseSourceKindLabel(dateSourceKind, dateConfidence string) string {
switch {
case dateSourceKind == "secondary_authoritative_report" || dateConfidence == "secondary_authoritative":
return "权威佐证发布"
case dateSourceKind == "official_announcement" && dateConfidence == "official_primary":
return "官方发布"
case dateSourceKind == "official_product_page":
return "官方产品页"
case dateSourceKind == "catalog_backfill":
return "目录回填"
default:
return "官方发布"
}
}
func buildReleaseEvidenceDetail(dateSourceKind, dateConfidence string) string {
switch {
case dateSourceKind == "secondary_authoritative_report" || dateConfidence == "secondary_authoritative":
return "models.release_date = 今日，发布日期采用次级权威报道佐证，模型来源页保留官方文档"
case dateSourceKind == "official_announcement" && dateConfidence == "official_primary":
return "models.release_date = 今日，且 source_url 指向官方发布页"
case dateSourceKind == "official_product_page":
return "models.release_date = 今日，来源页为官方产品页，发布日期置信度待确认"
case dateSourceKind == "catalog_backfill":
return "models.release_date = 今日，发布日期来自目录级元数据回填"
default:
return "models.release_date = 今日，且已记录发布日期证据元数据"
}
}
func buildReleaseTrustLabel(model ModelInfo, dateConfidence string) string {
base := buildTrustLabel(model)
switch dateConfidence {
case "official_primary":
return base + " / 一级证据"
case "secondary_authoritative":
return base + " / 二级佐证"
default:
return base
}
}
func buildFreeEvidenceDetail(model ModelInfo) string {
switch classifyFreeSource(model) {
case "官方免费":

485
scripts/generate_daily_report_test.go
View File

@@ -16,146 +16,146 @@ func sampleReportForV1() *ReportV3 {
TotalModels: 504,
AllModels: []ModelInfo{
{
Name: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
ProviderCountry:"CN",
ContextLength: 262144,
InputPrice: 0.30,
OutputPrice: 1.20,
Currency: "USD",
OperatorName: "OpenRouter",
OperatorType: "reseller",
Region: "global",
SceneTags: []SceneTag{SceneCode, SceneReasoning},
Name: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
ProviderCountry: "CN",
ContextLength: 262144,
InputPrice: 0.30,
OutputPrice: 1.20,
Currency: "USD",
OperatorName: "OpenRouter",
OperatorType: "reseller",
Region: "global",
SceneTags: []SceneTag{SceneCode, SceneReasoning},
},
{
Name: "glm-5",
ProviderName: "Zhipu",
ProviderCountry:"CN",
ContextLength: 131072,
InputPrice: 0,
OutputPrice: 0,
Currency: "CNY",
IsFree: true,
OperatorName: "Zhipu",
OperatorType: "official",
Region: "cn",
SceneTags: []SceneTag{SceneWriting, SceneChat},
Name: "glm-5",
ProviderName: "Zhipu",
ProviderCountry: "CN",
ContextLength: 131072,
InputPrice: 0,
OutputPrice: 0,
Currency: "CNY",
IsFree: true,
OperatorName: "Zhipu",
OperatorType: "official",
Region: "cn",
SceneTags: []SceneTag{SceneWriting, SceneChat},
},
{
Name: "claude-3.7-sonnet",
ProviderName: "Anthropic",
ProviderCountry:"US",
ContextLength: 200000,
InputPrice: 3.0,
OutputPrice: 15.0,
Currency: "USD",
OperatorName: "Anthropic",
OperatorType: "official",
Region: "global",
SceneTags: []SceneTag{SceneWriting, SceneChat},
Name: "claude-3.7-sonnet",
ProviderName: "Anthropic",
ProviderCountry: "US",
ContextLength: 200000,
InputPrice: 3.0,
OutputPrice: 15.0,
Currency: "USD",
OperatorName: "Anthropic",
OperatorType: "official",
Region: "global",
SceneTags: []SceneTag{SceneWriting, SceneChat},
},
{
Name: "qwen-vl-max",
ProviderName: "Alibaba",
ProviderCountry:"CN",
ContextLength: 65536,
InputPrice: 0.8,
OutputPrice: 2.4,
Currency: "CNY",
OperatorName: "DashScope",
OperatorType: "cloud",
Region: "cn",
SceneTags: []SceneTag{SceneVision},
Name: "qwen-vl-max",
ProviderName: "Alibaba",
ProviderCountry: "CN",
ContextLength: 65536,
InputPrice: 0.8,
OutputPrice: 2.4,
Currency: "CNY",
OperatorName: "DashScope",
OperatorType: "cloud",
Region: "cn",
SceneTags: []SceneTag{SceneVision},
},
},
FreeModels: []ModelInfo{
{
Name: "glm-5",
ProviderName: "Zhipu",
ProviderCountry:"CN",
ContextLength: 131072,
Currency: "CNY",
IsFree: true,
OperatorName: "Zhipu",
OperatorType: "official",
Region: "cn",
SceneTags: []SceneTag{SceneWriting, SceneChat},
Name: "glm-5",
ProviderName: "Zhipu",
ProviderCountry: "CN",
ContextLength: 131072,
Currency: "CNY",
IsFree: true,
OperatorName: "Zhipu",
OperatorType: "official",
Region: "cn",
SceneTags: []SceneTag{SceneWriting, SceneChat},
},
{
Name: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
ProviderCountry:"CN",
ContextLength: 262144,
Currency: "USD",
IsFree: true,
OperatorName: "OpenRouter",
OperatorType: "reseller",
Region: "global",
SceneTags: []SceneTag{SceneCode, SceneReasoning},
Name: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
ProviderCountry: "CN",
ContextLength: 262144,
Currency: "USD",
IsFree: true,
OperatorName: "OpenRouter",
OperatorType: "reseller",
Region: "global",
SceneTags: []SceneTag{SceneCode, SceneReasoning},
},
{
Name: "mystery-free-model",
ProviderName: "Unknown",
ProviderCountry:"unknown",
ContextLength: 65536,
Currency: "USD",
IsFree: true,
OperatorName: "Unknown Gateway",
OperatorType: "self_hosted_gateway",
Region: "global",
SceneTags: []SceneTag{SceneChat},
Name: "mystery-free-model",
ProviderName: "Unknown",
ProviderCountry: "unknown",
ContextLength: 65536,
Currency: "USD",
IsFree: true,
OperatorName: "Unknown Gateway",
OperatorType: "self_hosted_gateway",
Region: "global",
SceneTags: []SceneTag{SceneChat},
},
},
FreeTop20: []ModelInfo{
{
Name: "glm-5",
ProviderName: "Zhipu",
ProviderCountry:"CN",
ContextLength: 131072,
Currency: "CNY",
IsFree: true,
OperatorName: "Zhipu",
OperatorType: "official",
Region: "cn",
Name: "glm-5",
ProviderName: "Zhipu",
ProviderCountry: "CN",
ContextLength: 131072,
Currency: "CNY",
IsFree: true,
OperatorName: "Zhipu",
OperatorType: "official",
Region: "cn",
},
},
IntlTop5: []ModelInfo{
{
Name: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
ProviderCountry:"CN",
ContextLength: 262144,
InputPrice: 0.30,
OutputPrice: 1.20,
Currency: "USD",
OperatorName: "OpenRouter",
OperatorType: "reseller",
Region: "global",
SceneTags: []SceneTag{SceneCode, SceneReasoning},
Name: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
ProviderCountry: "CN",
ContextLength: 262144,
InputPrice: 0.30,
OutputPrice: 1.20,
Currency: "USD",
OperatorName: "OpenRouter",
OperatorType: "reseller",
Region: "global",
SceneTags: []SceneTag{SceneCode, SceneReasoning},
},
},
DomesticTop10: []ModelInfo{
{
Name: "qwen-vl-max",
ProviderName: "Alibaba",
ProviderCountry:"CN",
ContextLength: 65536,
InputPrice: 0.8,
OutputPrice: 2.4,
Currency: "CNY",
OperatorName: "DashScope",
OperatorType: "cloud",
Region: "cn",
SceneTags: []SceneTag{SceneVision},
Name: "qwen-vl-max",
ProviderName: "Alibaba",
ProviderCountry: "CN",
ContextLength: 65536,
InputPrice: 0.8,
OutputPrice: 2.4,
Currency: "CNY",
OperatorName: "DashScope",
OperatorType: "cloud",
Region: "cn",
SceneTags: []SceneTag{SceneVision},
},
},
DailySignals: DailySignals{
NewModels: 2,
PriceChanges: 1,
OfficialFree: 1,
AggregatorFree:1,
UnknownFree: 1,
NewModels: 2,
PriceChanges: 1,
OfficialFree: 1,
AggregatorFree: 1,
UnknownFree: 1,
},
}
}
@@ -184,33 +184,33 @@ func TestDecorateReportV1BuildsHotDaySummary(t *testing.T) {
report := sampleReportForV1()
report.ModelEvents = []ModelEvent{
{
EventType: "new_model",
ModelName: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
OperatorName: "OpenRouter",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型进入情报池，值得重新评估低成本编码默认选择。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:30",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 95,
EventType: "new_model",
ModelName: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
OperatorName: "OpenRouter",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型进入情报池，值得重新评估低成本编码默认选择。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:30",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 95,
},
{
EventType: "price_cut",
ModelName: "qwen-vl-max",
ProviderName: "Alibaba",
OperatorName: "DashScope",
TrustLabel: "官方来源",
Baseline: "较昨日 -18%",
Summary: "价格下降已足以影响视觉模型默认选择。",
SourceKindLabel: "价格快照",
PrimarySource: "pricing_history",
UpdatedAt: "2026-05-13 10:00",
EvidenceDetail: "pricing_history 记录到输入价格较昨日下降 18%",
PriceChangePct: -18,
Priority: 90,
EventType: "price_cut",
ModelName: "qwen-vl-max",
ProviderName: "Alibaba",
OperatorName: "DashScope",
TrustLabel: "官方来源",
Baseline: "较昨日 -18%",
Summary: "价格下降已足以影响视觉模型默认选择。",
SourceKindLabel: "价格快照",
PrimarySource: "pricing_history",
UpdatedAt: "2026-05-13 10:00",
EvidenceDetail: "pricing_history 记录到输入价格较昨日下降 18%",
PriceChangePct: -18,
Priority: 90,
},
}
@@ -287,18 +287,18 @@ func TestGenerateMarkdownV3IncludesTencentSubscriptionSection(t *testing.T) {
}
report.ModelEvents = []ModelEvent{
{
EventType: "new_model",
ModelName: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
OperatorName: "OpenRouter",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型进入情报池，值得重新评估低成本编码默认选择。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:30",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 95,
EventType: "new_model",
ModelName: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
OperatorName: "OpenRouter",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型进入情报池，值得重新评估低成本编码默认选择。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:30",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 95,
},
}
decorateReportV1(report)
@@ -340,18 +340,18 @@ func TestGenerateHTMLV3IncludesTencentSubscriptionSection(t *testing.T) {
report := sampleReportForV1()
report.ModelEvents = []ModelEvent{
{
EventType: "new_model",
ModelName: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
OperatorName: "OpenRouter",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型进入情报池，值得重新评估低成本编码默认选择。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:30",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 95,
EventType: "new_model",
ModelName: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
OperatorName: "OpenRouter",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型进入情报池，值得重新评估低成本编码默认选择。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:30",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 95,
},
}
report.TencentSubscriptionPlans = []SubscriptionPlanInfo{
@@ -406,47 +406,47 @@ func TestBuildHeadlineItemsUsesModelEvents(t *testing.T) {
report := sampleReportForV1()
report.ModelEvents = []ModelEvent{
{
EventType: "official_release",
ModelName: "GLM-5",
ProviderName: "Zhipu",
OperatorName: "Zhipu",
TrustLabel: "官方来源",
Baseline: "官方首次发布",
Summary: "官方发布新模型，值得优先复查中文通用与推理场景默认选择。",
SourceKindLabel: "官方发布",
PrimarySource: "https://open.bigmodel.cn/dev/howuse/model",
UpdatedAt: "2026-05-13 08:30",
EvidenceDetail: "models.release_date = 今日，且 source_url 指向官方文档",
Priority: 120,
EventType: "official_release",
ModelName: "GLM-5",
ProviderName: "Zhipu",
OperatorName: "Zhipu",
TrustLabel: "官方来源",
Baseline: "官方首次发布",
Summary: "官方发布新模型，值得优先复查中文通用与推理场景默认选择。",
SourceKindLabel: "官方发布",
PrimarySource: "https://open.bigmodel.cn/dev/howuse/model",
UpdatedAt: "2026-05-13 08:30",
EvidenceDetail: "models.release_date = 今日，且 source_url 指向官方文档",
Priority: 120,
},
{
EventType: "price_cut",
ModelName: "glm-5",
ProviderName: "Zhipu",
OperatorName: "Zhipu",
TrustLabel: "官方来源",
Baseline: "较昨日 -25%",
Summary: "价格下降已足以影响中文通用场景默认选型。",
SourceKindLabel: "价格快照",
PrimarySource: "pricing_history",
UpdatedAt: "2026-05-13 10:00",
EvidenceDetail: "pricing_history 记录到输入价格较昨日下降 25%",
PriceChangePct: -25,
Priority: 100,
EventType: "price_cut",
ModelName: "glm-5",
ProviderName: "Zhipu",
OperatorName: "Zhipu",
TrustLabel: "官方来源",
Baseline: "较昨日 -25%",
Summary: "价格下降已足以影响中文通用场景默认选型。",
SourceKindLabel: "价格快照",
PrimarySource: "pricing_history",
UpdatedAt: "2026-05-13 10:00",
EvidenceDetail: "pricing_history 记录到输入价格较昨日下降 25%",
PriceChangePct: -25,
Priority: 100,
},
{
EventType: "new_model",
ModelName: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
OperatorName: "OpenRouter",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型进入情报池，值得重新评估低成本编码默认选择。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:30",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 90,
EventType: "new_model",
ModelName: "DeepSeek-V4-Flash",
ProviderName: "DeepSeek",
OperatorName: "OpenRouter",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型进入情报池，值得重新评估低成本编码默认选择。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:30",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 90,
},
}
@@ -470,45 +470,45 @@ func TestBuildHeadlineItemsDeduplicatesSameModel(t *testing.T) {
report := sampleReportForV1()
report.ModelEvents = []ModelEvent{
{
EventType: "price_cut",
ModelName: "OpenAI: GPT-4o",
ProviderName: "OpenAI",
TrustLabel: "官方来源",
Baseline: "较昨日 -20%",
Summary: "价格下降影响默认成本。",
SourceKindLabel: "价格快照",
PrimarySource: "pricing_history",
UpdatedAt: "2026-05-13 10:00",
EvidenceDetail: "pricing_history 记录到输入价格较昨日下降 20%",
PriceChangePct: -20,
Priority: 95,
EventType: "price_cut",
ModelName: "OpenAI: GPT-4o",
ProviderName: "OpenAI",
TrustLabel: "官方来源",
Baseline: "较昨日 -20%",
Summary: "价格下降影响默认成本。",
SourceKindLabel: "价格快照",
PrimarySource: "pricing_history",
UpdatedAt: "2026-05-13 10:00",
EvidenceDetail: "pricing_history 记录到输入价格较昨日下降 20%",
PriceChangePct: -20,
Priority: 95,
},
{
EventType: "price_increase",
ModelName: "OpenAI: GPT-4o",
ProviderName: "OpenAI",
TrustLabel: "官方来源",
Baseline: "较昨日 +5%",
Summary: "同日另有上调记录。",
SourceKindLabel: "价格快照",
PrimarySource: "pricing_history",
UpdatedAt: "2026-05-13 11:00",
EvidenceDetail: "pricing_history 记录到输入价格较昨日上涨 5%",
PriceChangePct: 5,
Priority: 80,
EventType: "price_increase",
ModelName: "OpenAI: GPT-4o",
ProviderName: "OpenAI",
TrustLabel: "官方来源",
Baseline: "较昨日 +5%",
Summary: "同日另有上调记录。",
SourceKindLabel: "价格快照",
PrimarySource: "pricing_history",
UpdatedAt: "2026-05-13 11:00",
EvidenceDetail: "pricing_history 记录到输入价格较昨日上涨 5%",
PriceChangePct: 5,
Priority: 80,
},
{
EventType: "new_model",
ModelName: "Claude Opus 4.7",
ProviderName: "Anthropic",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型上线。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:00",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 70,
EventType: "new_model",
ModelName: "Claude Opus 4.7",
ProviderName: "Anthropic",
TrustLabel: "聚合来源",
Baseline: "首次出现",
Summary: "新模型上线。",
SourceKindLabel: "模型快照",
PrimarySource: "OpenRouter / region_pricing",
UpdatedAt: "2026-05-13 09:00",
EvidenceDetail: "models.created_at = 今日，且已存在最新价格快照",
Priority: 70,
},
}
@@ -575,3 +575,24 @@ func TestHeadlineItemFromOfficialReleaseEvent(t *testing.T) {
t.Fatalf("expected primary source to be preserved, got %+v", item)
}
}
func TestHeadlineItemFromSecondaryReleaseEvent(t *testing.T) {
item := headlineItemFromModelEvent(ModelEvent{
EventType: "official_release",
ModelName: "Doubao Seed 1.8",
TrustLabel: "官方来源 / 二级佐证",
Baseline: "官方首次发布",
Summary: "模型进入正式发布日期观察池。",
SourceKindLabel: "权威佐证发布",
PrimarySource: "https://developer.volcengine.com/articles/7601918680544641034",
UpdatedAt: "2025-12-18 00:00",
EvidenceDetail: "models.release_date = 今日，发布日期采用次级权威报道佐证，模型来源页保留官方文档",
})
if item.Label != "权威佐证" {
t.Fatalf("expected label to be 权威佐证, got %+v", item)
}
if !strings.Contains(item.Title, "权威佐证发布时间线") {
t.Fatalf("expected title to mention 权威佐证发布时间线, got %+v", item)
}
}

64
scripts/import_bytedance_data.go
View File

@@ -29,6 +29,8 @@ type ModelPricing struct {
SourceURL string
ModelSourceURL string
ReleaseDate string
DateConfidence string
DateSourceKind string
Modality string
}
@@ -47,6 +49,8 @@ type bytedanceModelMetadata struct {
Prefix string
ReleaseDate string
ModelSourceURL string
DateConfidence string
DateSourceKind string
}
var bytedanceModelMetadataRules = []bytedanceModelMetadata{
@@ -54,60 +58,85 @@ var bytedanceModelMetadataRules = []bytedanceModelMetadata{
Prefix: "bytedance-doubao-1.5-vision-pro",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-pro",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-lite",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-thinking",
ReleaseDate: "2025-04-17",
ModelSourceURL: "https://developer.volcengine.com/articles/7496718897794039827",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-1.6",
ReleaseDate: "2025-06-11",
ModelSourceURL: "https://developer.volcengine.com/articles/7517188354606104612",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-1.8",
ReleaseDate: "2025-12-18",
ModelSourceURL: "https://developer.volcengine.com/articles/7601918680544641034",
DateConfidence: "secondary_authoritative",
DateSourceKind: "secondary_authoritative_report",
},
{
Prefix: "bytedance-doubao-seed-2.0-code",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-pro",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-mini",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-lite",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-code",
ReleaseDate: "2024-06-26",
ModelSourceURL: "https://developer.volcengine.com/articles/7383101327527641125",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-seedance-1.0-lite",
ReleaseDate: "2025-05-13",
ModelSourceURL: "https://developer.volcengine.com/articles/7504284064976502823",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
}
@@ -121,17 +150,32 @@ func enrichBytedanceModelMetadata(model ModelPricing) ModelPricing {
if metadata.ModelSourceURL != "" {
model.ModelSourceURL = metadata.ModelSourceURL
}
if metadata.DateConfidence != "" {
model.DateConfidence = metadata.DateConfidence
}
if metadata.DateSourceKind != "" {
model.DateSourceKind = metadata.DateSourceKind
}
return model
}
}
if model.ModelSourceURL == "" {
model.ModelSourceURL = model.SourceURL
}
if model.DateConfidence == "" {
model.DateConfidence = "unknown"
}
if model.DateSourceKind == "" {
model.DateSourceKind = "unknown"
}
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
return strings.TrimSpace(model.ReleaseDate) != "" ||
firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}
func main() {
@@ -222,9 +266,9 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
).Scan(&modelID)
}
if err != nil {
@@ -238,12 +282,20 @@ func main() {
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
WHEN $4 THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
date_confidence = CASE
WHEN $4 THEN $5
ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
END,
date_source_kind = CASE
WHEN $4 THEN $6
ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}

33
scripts/import_bytedance_data_test.go
View File

@@ -9,36 +9,50 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) {
modelID string
wantReleaseDate string
wantSourceURL string
wantConfidence string
wantSourceKind string
}{
{
modelID: "bytedance-doubao-1.5-pro-32k",
wantReleaseDate: "2025-01-22",
wantSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-doubao-1.5-vision-pro",
wantReleaseDate: "2025-01-22",
wantSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-doubao-seed-1.6-thinking",
wantReleaseDate: "2025-06-11",
wantSourceURL: "https://developer.volcengine.com/articles/7517188354606104612",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-doubao-1.5-thinking-pro",
wantReleaseDate: "2025-04-17",
wantSourceURL: "https://developer.volcengine.com/articles/7496718897794039827",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-seedance-1.0-lite",
wantReleaseDate: "2025-05-13",
wantSourceURL: "https://developer.volcengine.com/articles/7504284064976502823",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "bytedance-doubao-seed-code-256k",
wantReleaseDate: "2024-06-26",
wantSourceURL: "https://developer.volcengine.com/articles/7383101327527641125",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
}
@@ -54,6 +68,12 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) {
if enriched.ModelSourceURL != tc.wantSourceURL {
t.Fatalf("%s source url = %q, want %q", tc.modelID, enriched.ModelSourceURL, tc.wantSourceURL)
}
if enriched.DateConfidence != tc.wantConfidence {
t.Fatalf("%s date confidence = %q, want %q", tc.modelID, enriched.DateConfidence, tc.wantConfidence)
}
if enriched.DateSourceKind != tc.wantSourceKind {
t.Fatalf("%s date source kind = %q, want %q", tc.modelID, enriched.DateSourceKind, tc.wantSourceKind)
}
}
}
@@ -69,6 +89,9 @@ func TestEnrichBytedanceModelMetadataFallsBackToPricingSource(t *testing.T) {
if enriched.ModelSourceURL != "https://www.volcengine.com/docs/82379/1099320" {
t.Fatalf("model source url = %q, want pricing source fallback", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "unknown" {
t.Fatalf("unexpected fallback date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestEnrichBytedanceModelMetadataSupportsSourceOnlyRules(t *testing.T) {
@@ -77,12 +100,15 @@ func TestEnrichBytedanceModelMetadataSupportsSourceOnlyRules(t *testing.T) {
SourceURL: "https://www.volcengine.com/docs/82379/1099320",
})
if enriched.ReleaseDate != "" {
t.Fatalf("unexpected release date: %q", enriched.ReleaseDate)
if enriched.ReleaseDate != "2025-12-18" {
t.Fatalf("release date = %q, want %q", enriched.ReleaseDate, "2025-12-18")
}
if enriched.ModelSourceURL != "https://developer.volcengine.com/articles/7601918680544641034" {
t.Fatalf("model source url = %q, want 1.8 source", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "secondary_authoritative" || enriched.DateSourceKind != "secondary_authoritative_report" {
t.Fatalf("unexpected 1.8 date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestEnrichBytedanceModelMetadataUsesTwoPointZeroReleaseDate(t *testing.T) {
@@ -97,6 +123,9 @@ func TestEnrichBytedanceModelMetadataUsesTwoPointZeroReleaseDate(t *testing.T) {
if enriched.ModelSourceURL != "https://developer.volcengine.com/articles/7610285824933445675" {
t.Fatalf("model source url = %q, want 2.0 source", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "official_primary" || enriched.DateSourceKind != "official_announcement" {
t.Fatalf("unexpected 2.0 date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestBytedanceReleaseDateValueReturnsNilForUnknownDate(t *testing.T) {

61
scripts/import_phase2_data.go
View File

@@ -51,6 +51,8 @@ type ModelPricing struct {
SourceURL string
ModelSourceURL string
ReleaseDate string
DateConfidence string
DateSourceKind string
Modality string
SceneTags []string
}
@@ -70,6 +72,8 @@ type baiduModelMetadata struct {
Prefix string
ReleaseDate string
ModelSourceURL string
DateConfidence string
DateSourceKind string
}
var baiduModelMetadataRules = []baiduModelMetadata{
@@ -77,54 +81,76 @@ var baiduModelMetadataRules = []baiduModelMetadata{
Prefix: "baidu-ernie-5.0",
ReleaseDate: "2026-01-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1.1",
ReleaseDate: "2025-09-09",
ModelSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-5.1",
ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html",
DateConfidence: "unknown",
DateSourceKind: "official_product_page",
},
{
Prefix: "baidu-ernie-4.5-turbo-vl",
ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html",
DateConfidence: "unknown",
DateSourceKind: "official_product_page",
},
{
Prefix: "baidu-ernie-4.5-turbo",
ReleaseDate: "2025-04-25",
ModelSourceURL: "https://cloud.baidu.com/article/3887765",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1-turbo",
ReleaseDate: "2025-04-25",
ModelSourceURL: "https://cloud.baidu.com/article/3887765",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-4.5",
ReleaseDate: "2025-03-16",
ModelSourceURL: "https://cloud.baidu.com/article/3835921",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1",
ReleaseDate: "2025-03-16",
ModelSourceURL: "https://cloud.baidu.com/article/3835921",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-character",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-lite-pro",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-speed-pro",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
}
@@ -138,17 +164,32 @@ func enrichBaiduModelMetadata(model ModelPricing) ModelPricing {
if metadata.ModelSourceURL != "" {
model.ModelSourceURL = metadata.ModelSourceURL
}
if metadata.DateConfidence != "" {
model.DateConfidence = metadata.DateConfidence
}
if metadata.DateSourceKind != "" {
model.DateSourceKind = metadata.DateSourceKind
}
return model
}
}
if model.ModelSourceURL == "" {
model.ModelSourceURL = model.SourceURL
}
if model.DateConfidence == "" {
model.DateConfidence = "unknown"
}
if model.DateSourceKind == "" {
model.DateSourceKind = "unknown"
}
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
return strings.TrimSpace(model.ReleaseDate) != "" ||
firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}
func parseZhipuPrice(s string) float64 {
@@ -288,9 +329,9 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
).Scan(&modelID)
}
if err != nil {
@@ -304,12 +345,20 @@ func main() {
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
WHEN $4 THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
date_confidence = CASE
WHEN $4 THEN $5
ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
END,
date_source_kind = CASE
WHEN $4 THEN $6
ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}

28
scripts/import_phase2_data_test.go
View File

@@ -9,41 +9,57 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) {
modelID string
wantReleaseDate string
wantSourceURL string
wantConfidence string
wantSourceKind string
}{
{
modelID: "baidu-ernie-5.0",
wantReleaseDate: "2026-01-22",
wantSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-x1.1",
wantReleaseDate: "2025-09-09",
wantSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-4.5-turbo-128k",
wantReleaseDate: "2025-04-25",
wantSourceURL: "https://cloud.baidu.com/article/3887765",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-x1-turbo-32k",
wantReleaseDate: "2025-04-25",
wantSourceURL: "https://cloud.baidu.com/article/3887765",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-4.5-8k",
wantReleaseDate: "2025-03-16",
wantSourceURL: "https://cloud.baidu.com/article/3835921",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-x1-8k",
wantReleaseDate: "2025-03-16",
wantSourceURL: "https://cloud.baidu.com/article/3835921",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "baidu-ernie-character",
wantReleaseDate: "2024-03-22",
wantSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
}
@@ -59,6 +75,12 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) {
if enriched.ModelSourceURL != tc.wantSourceURL {
t.Fatalf("%s source url = %q, want %q", tc.modelID, enriched.ModelSourceURL, tc.wantSourceURL)
}
if enriched.DateConfidence != tc.wantConfidence {
t.Fatalf("%s date confidence = %q, want %q", tc.modelID, enriched.DateConfidence, tc.wantConfidence)
}
if enriched.DateSourceKind != tc.wantSourceKind {
t.Fatalf("%s date source kind = %q, want %q", tc.modelID, enriched.DateSourceKind, tc.wantSourceKind)
}
}
}
@@ -74,6 +96,9 @@ func TestEnrichBaiduModelMetadataFallsBackToPricingSource(t *testing.T) {
if enriched.ModelSourceURL != "https://cloud.baidu.com/doc/qianfan/s/wmh4sv6ya" {
t.Fatalf("model source url = %q, want pricing source fallback", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "unknown" {
t.Fatalf("unexpected fallback date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestEnrichBaiduModelMetadataSupportsSourceOnlyRules(t *testing.T) {
@@ -88,6 +113,9 @@ func TestEnrichBaiduModelMetadataSupportsSourceOnlyRules(t *testing.T) {
if enriched.ModelSourceURL != "https://cloud.baidu.com/product/wenxinworkshop.html" {
t.Fatalf("model source url = %q, want product source", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "official_product_page" {
t.Fatalf("unexpected source-only metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestBaiduReleaseDateValueReturnsNilForUnknownDate(t *testing.T) {

53
scripts/import_zhipu_data.go
View File

@@ -28,6 +28,8 @@ type ModelPricing struct {
SourceURL string
ModelSourceURL string
ReleaseDate string
DateConfidence string
DateSourceKind string
Modality string
SceneTags []string
}
@@ -47,6 +49,8 @@ type zhipuModelMetadata struct {
Prefix string
ReleaseDate string
ModelSourceURL string
DateConfidence string
DateSourceKind string
}
var zhipuModelMetadataRules = []zhipuModelMetadata{
@@ -54,36 +58,50 @@ var zhipuModelMetadataRules = []zhipuModelMetadata{
Prefix: "glm-5-turbo",
ReleaseDate: "2026-03-15",
ModelSourceURL: "https://www.zhipuai.cn/en/research/155",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-5.1",
ReleaseDate: "2026-04-07",
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-5",
ReleaseDate: "2026-02-11",
ModelSourceURL: "https://www.zhipuai.cn/zh/research/154",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-4.7-flash",
ReleaseDate: "2026-01-19",
ModelSourceURL: "https://www.zhipuai.cn/zh/news/148",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-4.7",
ReleaseDate: "2025-12-21",
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-4.6v",
ReleaseDate: "2025-12-07",
ModelSourceURL: "https://www.zhipuai.cn/zh/research/144",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "glm-tts",
ReleaseDate: "2025-12-10",
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
}
@@ -96,17 +114,32 @@ func enrichZhipuModelMetadata(model ModelPricing) ModelPricing {
if metadata.ModelSourceURL != "" {
model.ModelSourceURL = metadata.ModelSourceURL
}
if metadata.DateConfidence != "" {
model.DateConfidence = metadata.DateConfidence
}
if metadata.DateSourceKind != "" {
model.DateSourceKind = metadata.DateSourceKind
}
return model
}
}
if model.ModelSourceURL == "" {
model.ModelSourceURL = model.SourceURL
}
if model.DateConfidence == "" {
model.DateConfidence = "unknown"
}
if model.DateSourceKind == "" {
model.DateSourceKind = "unknown"
}
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
return strings.TrimSpace(model.ReleaseDate) != "" ||
firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}
func main() {
@@ -228,9 +261,9 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
).Scan(&modelID)
}
if err != nil {
@@ -244,12 +277,20 @@ func main() {
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
WHEN $4 THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
date_confidence = CASE
WHEN $4 THEN $5
ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
END,
date_source_kind = CASE
WHEN $4 THEN $6
ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}

23
scripts/import_zhipu_data_test.go
View File

@@ -9,36 +9,50 @@ func TestEnrichZhipuModelMetadataUsesSpecificFamilyRules(t *testing.T) {
modelID string
wantReleaseDate string
wantSourceURL string
wantConfidence string
wantSourceKind string
}{
{
modelID: "glm-5.1-32k",
wantReleaseDate: "2026-04-07",
wantSourceURL: "https://www.zhipuai.cn/zh/research",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-5-turbo-32k",
wantReleaseDate: "2026-03-15",
wantSourceURL: "https://www.zhipuai.cn/en/research/155",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-5-32k",
wantReleaseDate: "2026-02-11",
wantSourceURL: "https://www.zhipuai.cn/zh/research/154",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-4.7-flash",
wantReleaseDate: "2026-01-19",
wantSourceURL: "https://www.zhipuai.cn/zh/news/148",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-4.6v-flashx",
wantReleaseDate: "2025-12-07",
wantSourceURL: "https://www.zhipuai.cn/zh/research/144",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
{
modelID: "glm-tts-clone",
wantReleaseDate: "2025-12-10",
wantSourceURL: "https://www.zhipuai.cn/zh/research",
wantConfidence: "official_primary",
wantSourceKind: "official_announcement",
},
}
@@ -54,6 +68,12 @@ func TestEnrichZhipuModelMetadataUsesSpecificFamilyRules(t *testing.T) {
if enriched.ModelSourceURL != tc.wantSourceURL {
t.Fatalf("%s source url = %q, want %q", tc.modelID, enriched.ModelSourceURL, tc.wantSourceURL)
}
if enriched.DateConfidence != tc.wantConfidence {
t.Fatalf("%s date confidence = %q, want %q", tc.modelID, enriched.DateConfidence, tc.wantConfidence)
}
if enriched.DateSourceKind != tc.wantSourceKind {
t.Fatalf("%s date source kind = %q, want %q", tc.modelID, enriched.DateSourceKind, tc.wantSourceKind)
}
}
}
@@ -69,6 +89,9 @@ func TestEnrichZhipuModelMetadataFallsBackToPricingSource(t *testing.T) {
if enriched.ModelSourceURL != "https://open.bigmodel.cn/pricing" {
t.Fatalf("model source url = %q, want pricing source fallback", enriched.ModelSourceURL)
}
if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "unknown" {
t.Fatalf("unexpected fallback date metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind)
}
}
func TestZhipuReleaseDateValueReturnsNilForUnknownDate(t *testing.T) {

6
scripts/official_import_scripts_test.go
View File

@@ -42,5 +42,11 @@ func TestOfficialImportScriptsWriteModelSourceURLAndReleaseDate(t *testing.T) {
if !strings.Contains(content, "release_date") {
t.Fatalf("%s missing release_date in models write path", relativePath)
}
if !strings.Contains(content, "date_confidence") {
t.Fatalf("%s missing date_confidence in models write path", relativePath)
}
if !strings.Contains(content, "date_source_kind") {
t.Fatalf("%s missing date_source_kind in models write path", relativePath)
}
}
}