niuniu/llm-intelligence
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(import): track release date evidence tiers

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-13 23:27:47 +08:00
parent 569b94cb73
commit f2f68b85c1
10 changed files with 599 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
scripts/import_bytedance_data.go
View File

@@ -29,6 +29,8 @@ type ModelPricing struct {
SourceURL string
ModelSourceURL string
ReleaseDate string
DateConfidence string
DateSourceKind string
Modality string
}
@@ -47,6 +49,8 @@ type bytedanceModelMetadata struct {
Prefix string
ReleaseDate string
ModelSourceURL string
DateConfidence string
DateSourceKind string
}
var bytedanceModelMetadataRules = []bytedanceModelMetadata{
@@ -54,60 +58,85 @@ var bytedanceModelMetadataRules = []bytedanceModelMetadata{
Prefix: "bytedance-doubao-1.5-vision-pro",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-pro",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-lite",
ReleaseDate: "2025-01-22",
ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-1.5-thinking",
ReleaseDate: "2025-04-17",
ModelSourceURL: "https://developer.volcengine.com/articles/7496718897794039827",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-1.6",
ReleaseDate: "2025-06-11",
ModelSourceURL: "https://developer.volcengine.com/articles/7517188354606104612",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-1.8",
ReleaseDate: "2025-12-18",
ModelSourceURL: "https://developer.volcengine.com/articles/7601918680544641034",
DateConfidence: "secondary_authoritative",
DateSourceKind: "secondary_authoritative_report",
},
{
Prefix: "bytedance-doubao-seed-2.0-code",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-pro",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-mini",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-2.0-lite",
ReleaseDate: "2026-02-14",
ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-doubao-seed-code",
ReleaseDate: "2024-06-26",
ModelSourceURL: "https://developer.volcengine.com/articles/7383101327527641125",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "bytedance-seedance-1.0-lite",
ReleaseDate: "2025-05-13",
ModelSourceURL: "https://developer.volcengine.com/articles/7504284064976502823",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
}
@@ -121,17 +150,32 @@ func enrichBytedanceModelMetadata(model ModelPricing) ModelPricing {
if metadata.ModelSourceURL != "" {
model.ModelSourceURL = metadata.ModelSourceURL
}
if metadata.DateConfidence != "" {
model.DateConfidence = metadata.DateConfidence
}
if metadata.DateSourceKind != "" {
model.DateSourceKind = metadata.DateSourceKind
}
return model
}
}
if model.ModelSourceURL == "" {
model.ModelSourceURL = model.SourceURL
}
if model.DateConfidence == "" {
model.DateConfidence = "unknown"
}
if model.DateSourceKind == "" {
model.DateSourceKind = "unknown"
}
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
return strings.TrimSpace(model.ReleaseDate) != "" ||
firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}
func main() {
@@ -222,9 +266,9 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
).Scan(&modelID)
}
if err != nil {
@@ -238,12 +282,20 @@ func main() {
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
WHEN $4 THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
date_confidence = CASE
WHEN $4 THEN $5
ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
END,
date_source_kind = CASE
WHEN $4 THEN $6
ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}