diff --git a/cmd/server/main.go b/cmd/server/main.go index 396ebbc..6155e72 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -22,6 +22,9 @@ type modelResponse struct { ProviderCN string `json:"providerCN"` Modality string `json:"modality"` ContextLength int `json:"contextLength"` + PricingMode string `json:"pricingMode,omitempty"` + PriceUnit string `json:"priceUnit,omitempty"` + FlatPrice float64 `json:"flatPrice,omitempty"` InputPrice float64 `json:"inputPrice"` OutputPrice float64 `json:"outputPrice"` Currency string `json:"currency"` @@ -171,6 +174,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) { WITH latest_prices AS ( SELECT rp.model_id, + rp.pricing_mode, + rp.price_unit, + rp.flat_price, rp.input_price_per_mtok, rp.output_price_per_mtok, rp.currency, @@ -188,6 +194,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) { COALESCE(mp.name, split_part(m.external_id, '/', 1)), COALESCE(m.modality, 'text'), COALESCE(m.context_length, 0), + COALESCE(lp.pricing_mode, 'input_output'), + COALESCE(lp.price_unit, 'million_tokens'), + COALESCE(lp.flat_price, 0), lp.input_price_per_mtok, lp.output_price_per_mtok, COALESCE(lp.currency, 'USD'), @@ -207,6 +216,7 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) { var models []modelResponse for rows.Next() { var model modelResponse + var flatPrice sql.NullFloat64 var inputPrice sql.NullFloat64 var outputPrice sql.NullFloat64 if err := rows.Scan( @@ -216,6 +226,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) { &model.Provider, &model.Modality, &model.ContextLength, + &model.PricingMode, + &model.PriceUnit, + &flatPrice, &inputPrice, &outputPrice, &model.Currency, @@ -232,6 +245,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) { if outputPrice.Valid { model.OutputPrice = outputPrice.Float64 } + if flatPrice.Valid { + model.FlatPrice = flatPrice.Float64 + } model.Stale = model.DataConfidence == "stale" models = append(models, model) } diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go index 49d30fb..754d24f 100644 --- a/cmd/server/main_test.go +++ b/cmd/server/main_test.go @@ -10,6 +10,55 @@ import ( "testing" ) +func TestModelsHandlerReturnsFlatPricingFields(t *testing.T) { + mux := newMux( + &sql.DB{}, + func(context.Context, *sql.DB) ([]modelResponse, error) { + return []modelResponse{{ + ID: "mobile-cloud-huabei-huhehaote-cosyvoice", + Name: "CosyVoice", + Provider: "Alibaba", + ProviderCN: "阿里云", + Modality: "audio", + PricingMode: "flat", + PriceUnit: "10k_characters", + FlatPrice: 2, + Currency: "CNY", + IsFree: false, + DataConfidence: "official", + }}, nil + }, + func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) { + return nil, nil + }, + func(context.Context, *sql.DB) (*latestReportResponse, error) { + return nil, sql.ErrNoRows + }, + ) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", rec.Code) + } + + var payload struct { + Data []modelResponse `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if len(payload.Data) != 1 { + t.Fatalf("expected 1 model, got %d", len(payload.Data)) + } + got := payload.Data[0] + if got.PricingMode != "flat" || got.PriceUnit != "10k_characters" || got.FlatPrice != 2 { + t.Fatalf("unexpected flat pricing payload: %+v", got) + } +} + func TestSubscriptionPlansHandlerReturnsEnvelope(t *testing.T) { mux := newMux( &sql.DB{}, diff --git a/db/migrations/016_region_pricing_non_token_units.sql b/db/migrations/016_region_pricing_non_token_units.sql new file mode 100644 index 0000000..6371545 --- /dev/null +++ b/db/migrations/016_region_pricing_non_token_units.sql @@ -0,0 +1,31 @@ +-- Phase 2: region_pricing 扩展非 token 统一计费字段（字符/秒等） + +ALTER TABLE region_pricing + ADD COLUMN IF NOT EXISTS pricing_mode TEXT NOT NULL DEFAULT 'input_output', + ADD COLUMN IF NOT EXISTS price_unit TEXT NOT NULL DEFAULT 'million_tokens', + ADD COLUMN IF NOT EXISTS flat_price REAL; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'chk_region_pricing_pricing_mode' + ) THEN + ALTER TABLE region_pricing + ADD CONSTRAINT chk_region_pricing_pricing_mode + CHECK (pricing_mode IN ('input_output', 'flat')); + END IF; +END +$$; + +UPDATE region_pricing +SET pricing_mode = 'input_output' +WHERE coalesce(pricing_mode, '') = ''; + +UPDATE region_pricing +SET price_unit = 'million_tokens' +WHERE coalesce(price_unit, '') = ''; + +CREATE INDEX IF NOT EXISTS idx_region_pricing_pricing_mode ON region_pricing(pricing_mode); +CREATE INDEX IF NOT EXISTS idx_region_pricing_price_unit ON region_pricing(price_unit); diff --git a/scripts/import_mobile_cloud_pricing.go b/scripts/import_mobile_cloud_pricing.go new file mode 100644 index 0000000..d869f14 --- /dev/null +++ b/scripts/import_mobile_cloud_pricing.go @@ -0,0 +1,517 @@ +//go:build llm_script + +package main + +import ( + "database/sql" + "encoding/json" + "flag" + "fmt" + "html" + "io" + "net/http" + "os" + "regexp" + "strings" + "time" +) + +const ( + defaultMobileCloudOutlineTreeURL = "https://ecloud.10086.cn/op-help-center/request-api/service-api/outline/tree?outlineId=972" + defaultMobileCloudArticleInfoURL = "https://ecloud.10086.cn/op-help-center/request-api/service-api/article/info/%d" + defaultMobileCloudArticleContentURL = "https://ecloud.10086.cn/op-help-center/request-api/service-api/article/content/%s" + defaultMobileCloudDocURLPattern = "https://ecloud.10086.cn/op-help-center/doc/article/%d" + mobileCloudPricingArticleTitle = "预置模型服务-token按量计费" +) + +type mobileCloudPricingImportConfig struct { + OutlineTreeURL string + Fixture string + DryRun bool + Timeout time.Duration +} + +type mobileCloudOutlineEnvelope struct { + Code int `json:"code"` + Data mobileCloudOutlineNode `json:"data"` +} + +type mobileCloudOutlineNode struct { + ArticleID int `json:"articleId"` + ArticleTitle string `json:"articleTitle"` + ArticleContentPublished string `json:"articleContentPublished"` + Children []mobileCloudOutlineNode `json:"children"` +} + +type mobileCloudArticleInfoEnvelope struct { + Code int `json:"code"` + Data mobileCloudArticleInfo `json:"data"` +} + +type mobileCloudArticleInfo struct { + ID int `json:"id"` + Title string `json:"title"` + ContentPublished string `json:"contentPublished"` +} + +type mobileCloudArticlePayload struct { + ArticleID int + Title string + ContentPublished string + DocURL string + ContentHTML string +} + +func main() { + loadSubscriptionImportEnv() + + var outlineTreeURL string + var fixture string + var dryRun bool + var timeoutSeconds int + + flag.StringVar(&outlineTreeURL, "outline-tree-url", defaultMobileCloudOutlineTreeURL, "移动云 MoMA 文档大纲树接口") + flag.StringVar(&fixture, "fixture", "", "移动云 MoMA 价格样例文件") + flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要，不写入数据库") + flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时（秒）") + flag.Parse() + + cfg := mobileCloudPricingImportConfig{OutlineTreeURL: outlineTreeURL, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second} + + var db *sql.DB + var err error + if !cfg.DryRun { + db, err = subscriptionImportDB() + if err != nil { + fmt.Fprintf(os.Stderr, "open db: %v\n", err) + os.Exit(1) + } + defer db.Close() + } + + if err := runMobileCloudPricingImport(cfg, db, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "import_mobile_cloud_pricing: %v\n", err) + os.Exit(1) + } +} + +func runMobileCloudPricingImport(cfg mobileCloudPricingImportConfig, db *sql.DB, out io.Writer) error { + client := &http.Client{Timeout: cfg.Timeout} + payload, err := fetchMobileCloudArticlePayload(cfg, client) + if err != nil { + return err + } + records, err := parseMobileCloudPricingHTML(payload.ContentHTML, payload.DocURL) + if err != nil { + return err + } + records = dedupeOfficialPricingRecords(records) + if cfg.DryRun { + _, err = fmt.Fprintf(out, "source=mobile-cloud-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName) + return err + } + if db == nil { + return fmt.Errorf("db is required when dry-run=false") + } + if err := upsertOfficialPricingRecords(db, records, "mobile-cloud-pricing-import"); err != nil { + return err + } + var tableRows int + if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil { + return fmt.Errorf("count region_pricing: %w", err) + } + _, err = fmt.Fprintf(out, "source=mobile-cloud-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows) + return err +} + +func fetchMobileCloudArticlePayload(cfg mobileCloudPricingImportConfig, client *http.Client) (mobileCloudArticlePayload, error) { + if cfg.Fixture != "" { + data, err := os.ReadFile(cfg.Fixture) + if err != nil { + return mobileCloudArticlePayload{}, fmt.Errorf("read fixture %s: %w", cfg.Fixture, err) + } + return mobileCloudArticlePayload{ + ArticleID: 91592, + Title: mobileCloudPricingArticleTitle, + DocURL: fmt.Sprintf(defaultMobileCloudDocURLPattern, 91592), + ContentHTML: string(data), + }, nil + } + if client == nil { + client = &http.Client{Timeout: 20 * time.Second} + } + outlineRaw, err := fetchRawPricingPage(cfg.OutlineTreeURL, "", client) + if err != nil { + return mobileCloudArticlePayload{}, err + } + articleID, contentPublished, err := resolveMobileCloudPricingArticle(outlineRaw) + if err != nil { + return mobileCloudArticlePayload{}, err + } + infoURL := fmt.Sprintf(defaultMobileCloudArticleInfoURL, articleID) + infoRaw, err := fetchRawPricingPage(infoURL, "", client) + if err != nil { + return mobileCloudArticlePayload{}, err + } + articleInfo, err := parseMobileCloudArticleInfo(infoRaw) + if err != nil { + return mobileCloudArticlePayload{}, err + } + if strings.TrimSpace(contentPublished) == "" { + contentPublished = articleInfo.ContentPublished + } + contentURL := fmt.Sprintf(defaultMobileCloudArticleContentURL, contentPublished) + contentHTML, err := fetchRawPricingPage(contentURL, "", client) + if err != nil { + return mobileCloudArticlePayload{}, err + } + return mobileCloudArticlePayload{ + ArticleID: articleInfo.ID, + Title: articleInfo.Title, + ContentPublished: contentPublished, + DocURL: fmt.Sprintf(defaultMobileCloudDocURLPattern, articleInfo.ID), + ContentHTML: contentHTML, + }, nil +} + +func resolveMobileCloudPricingArticle(raw string) (int, string, error) { + var envelope mobileCloudOutlineEnvelope + if err := json.Unmarshal([]byte(raw), &envelope); err != nil { + return 0, "", fmt.Errorf("parse mobile cloud outline tree: %w", err) + } + articleID, contentPublished, ok := findMobileCloudPricingArticle(envelope.Data) + if !ok { + return 0, "", fmt.Errorf("mobile cloud pricing article %q not found in outline tree", mobileCloudPricingArticleTitle) + } + return articleID, contentPublished, nil +} + +func findMobileCloudPricingArticle(node mobileCloudOutlineNode) (int, string, bool) { + if strings.TrimSpace(node.ArticleTitle) == mobileCloudPricingArticleTitle && node.ArticleID > 0 { + return node.ArticleID, strings.TrimSpace(node.ArticleContentPublished), true + } + for _, child := range node.Children { + if articleID, contentPublished, ok := findMobileCloudPricingArticle(child); ok { + return articleID, contentPublished, true + } + } + return 0, "", false +} + +func parseMobileCloudArticleInfo(raw string) (mobileCloudArticleInfo, error) { + var envelope mobileCloudArticleInfoEnvelope + if err := json.Unmarshal([]byte(raw), &envelope); err != nil { + return mobileCloudArticleInfo{}, fmt.Errorf("parse mobile cloud article info: %w", err) + } + if envelope.Data.ID == 0 { + return mobileCloudArticleInfo{}, fmt.Errorf("unexpected mobile cloud article info content") + } + return envelope.Data, nil +} + +func parseMobileCloudPricingHTML(raw string, docURL string) ([]officialPricingRecord, error) { + sections := mobileCloudRegionSections(raw) + if len(sections) == 0 { + return nil, fmt.Errorf("no mobile cloud pricing regions found") + } + records := make([]officialPricingRecord, 0) + for _, section := range sections { + for _, table := range mobileCloudTableBlocks(section.Body) { + rows := mobileCloudTableRows(table) + if len(rows) < 2 { + continue + } + switch { + case isMobileCloudTokenPricingHeader(rows[0]): + records = append(records, buildMobileCloudRecordsFromTable(section.Region, rows[1:], docURL)...) + case isMobileCloudVoicePricingHeader(rows[0]): + records = append(records, buildMobileCloudVoiceRecordsFromTable(section.Region, rows[1:], docURL)...) + } + } + } + if len(records) == 0 { + return nil, fmt.Errorf("no mobile cloud token pricing rows found") + } + return records, nil +} + +type mobileCloudRegionSection struct { + Region string + Body string +} + +func mobileCloudRegionSections(raw string) []mobileCloudRegionSection { + headingPattern := regexp.MustCompile(`(?is)]*>(.*?)`) + matches := headingPattern.FindAllStringSubmatchIndex(raw, -1) + sections := make([]mobileCloudRegionSection, 0, len(matches)) + for i, match := range matches { + heading := cleanMobileCloudHTMLText(raw[match[2]:match[3]]) + if !strings.Contains(heading, "支持订购模型") { + continue + } + start := match[1] + end := len(raw) + if i+1 < len(matches) { + end = matches[i+1][0] + } + region := strings.TrimSpace(strings.TrimSuffix(heading, "资源池支持订购模型")) + if region == heading { + region = strings.TrimSpace(strings.TrimSuffix(heading, "支持订购模型")) + } + sections = append(sections, mobileCloudRegionSection{Region: region, Body: raw[start:end]}) + } + return sections +} + +func mobileCloudTableBlocks(raw string) []string { + return regexp.MustCompile(`(?is)`).FindAllString(raw, -1) +} + +func mobileCloudTableRows(raw string) [][]string { + rowMatches := regexp.MustCompile(`(?is)]*>(.*?)`).FindAllStringSubmatch(raw, -1) + rows := make([][]string, 0, len(rowMatches)) + for _, rowMatch := range rowMatches { + cellMatches := regexp.MustCompile(`(?is)]*>(.*?)`).FindAllStringSubmatch(rowMatch[1], -1) + cells := make([]string, 0, len(cellMatches)) + for _, cellMatch := range cellMatches { + cells = append(cells, cleanMobileCloudHTMLText(cellMatch[1])) + } + if len(cells) > 0 { + rows = append(rows, cells) + } + } + return rows +} + +func cleanMobileCloudHTMLText(raw string) string { + raw = strings.ReplaceAll(raw, "
", " ") + raw = strings.ReplaceAll(raw, "
", " ") + raw = strings.ReplaceAll(raw, "
", " ") + raw = regexp.MustCompile(`(?is)<[^>]+>`).ReplaceAllString(raw, " ") + raw = html.UnescapeString(raw) + raw = regexp.MustCompile(`\s+`).ReplaceAllString(raw, " ") + return strings.TrimSpace(raw) +} + +func isMobileCloudTokenPricingHeader(cells []string) bool { + if len(cells) < 4 { + return false + } + return cells[0] == "规格名称" && cells[1] == "输入/输出tokens" && cells[2] == "单价（元/百万tokens）" && cells[3] == "包含模型" +} + +func isMobileCloudVoicePricingHeader(cells []string) bool { + if len(cells) < 5 { + return false + } + return cells[0] == "规格名称" && cells[1] == "模型类别" && cells[2] == "资费场景" && cells[3] == "单价" && cells[4] == "包含模型" +} + +func buildMobileCloudRecordsFromTable(region string, rows [][]string, docURL string) []officialPricingRecord { + records := make([]officialPricingRecord, 0) + currentModels := make([]string, 0) + currentInputPrice := 0.0 + for _, row := range rows { + switch { + case len(row) >= 4: + billingKind := strings.TrimSpace(row[1]) + price := mustParseSubscriptionPrice(row[2]) + currentModels = mobileCloudModelNames(row[3]) + switch billingKind { + case "输入tokens": + currentInputPrice = price + case "tokens资费": + records = append(records, buildMobileCloudFlatTokenRecords(region, currentModels, price, docURL)...) + currentInputPrice = 0 + default: + currentInputPrice = 0 + } + case len(row) >= 2 && strings.TrimSpace(row[0]) == "输出tokens": + if currentInputPrice <= 0 || len(currentModels) == 0 { + continue + } + outputPrice := mustParseSubscriptionPrice(row[1]) + records = append(records, buildMobileCloudInputOutputRecords(region, currentModels, currentInputPrice, outputPrice, docURL)...) + currentInputPrice = 0 + } + } + return records +} + +func buildMobileCloudInputOutputRecords(region string, modelNames []string, inputPrice float64, outputPrice float64, docURL string) []officialPricingRecord { + records := make([]officialPricingRecord, 0, len(modelNames)) + for _, modelName := range modelNames { + providerName := mobileCloudProviderName(modelName) + providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName) + records = append(records, officialPricingRecord{ + ModelID: normalizeExternalID("mobile-cloud", mobileCloudRegionCode(region), modelName), + ModelName: modelName, + ProviderName: providerName, + ProviderNameCn: providerNameCn, + ProviderCountry: providerCountry, + ProviderWebsite: providerWebsite, + OperatorName: "Mobile Cloud", + OperatorNameCn: "移动云", + OperatorCountry: "CN", + OperatorWebsite: "https://ecloud.10086.cn/portal/product/MaaS", + OperatorType: "official", + Region: region, + Currency: "CNY", + InputPrice: inputPrice, + OutputPrice: outputPrice, + SourceURL: docURL, + ModelSourceURL: docURL, + DateConfidence: "unknown", + DateSourceKind: "official_pricing", + Modality: detectModality(modelName), + }) + } + return records +} + +func buildMobileCloudFlatTokenRecords(region string, modelNames []string, price float64, docURL string) []officialPricingRecord { + records := make([]officialPricingRecord, 0, len(modelNames)) + for _, modelName := range modelNames { + providerName := mobileCloudProviderName(modelName) + providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName) + records = append(records, officialPricingRecord{ + ModelID: normalizeExternalID("mobile-cloud", mobileCloudRegionCode(region), modelName), + ModelName: modelName, + ProviderName: providerName, + ProviderNameCn: providerNameCn, + ProviderCountry: providerCountry, + ProviderWebsite: providerWebsite, + OperatorName: "Mobile Cloud", + OperatorNameCn: "移动云", + OperatorCountry: "CN", + OperatorWebsite: "https://ecloud.10086.cn/portal/product/MaaS", + OperatorType: "official", + Region: region, + Currency: "CNY", + InputPrice: price, + OutputPrice: price, + SourceURL: docURL, + ModelSourceURL: docURL, + DateConfidence: "unknown", + DateSourceKind: "official_pricing", + Modality: detectModality(modelName), + }) + } + return records +} + +func buildMobileCloudVoiceRecordsFromTable(region string, rows [][]string, docURL string) []officialPricingRecord { + records := make([]officialPricingRecord, 0, len(rows)) + for _, row := range rows { + if len(row) < 5 { + continue + } + modelNames := mobileCloudModelNames(row[4]) + if len(modelNames) == 0 { + modelNames = []string{strings.TrimSpace(row[0])} + } + flatPrice := mobileCloudInlinePrice(row[3]) + priceUnit := mobileCloudVoicePriceUnit(row[2], row[3]) + for _, modelName := range modelNames { + providerName := mobileCloudProviderName(modelName) + providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName) + records = append(records, officialPricingRecord{ + ModelID: normalizeExternalID("mobile-cloud", mobileCloudRegionCode(region), modelName), + ModelName: modelName, + ProviderName: providerName, + ProviderNameCn: providerNameCn, + ProviderCountry: providerCountry, + ProviderWebsite: providerWebsite, + OperatorName: "Mobile Cloud", + OperatorNameCn: "移动云", + OperatorCountry: "CN", + OperatorWebsite: "https://ecloud.10086.cn/portal/product/MaaS", + OperatorType: "official", + Region: region, + Currency: "CNY", + PricingMode: "flat", + PriceUnit: priceUnit, + FlatPrice: flatPrice, + SourceURL: docURL, + ModelSourceURL: docURL, + DateConfidence: "unknown", + DateSourceKind: "official_pricing", + Modality: "audio", + }) + } + } + return records +} + +func mobileCloudVoicePriceUnit(scene string, price string) string { + text := strings.ToLower(strings.TrimSpace(scene + " " + price)) + switch { + case strings.Contains(text, "万字符"), strings.Contains(text, "字符"): + return "10k_characters" + case strings.Contains(text, "元/秒"), strings.Contains(text, "秒"): + return "second" + default: + return "flat" + } +} + +func mobileCloudInlinePrice(raw string) float64 { + matches := regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)`).FindStringSubmatch(raw) + if len(matches) != 2 { + return 0 + } + return mustParseSubscriptionPrice(matches[1]) +} + +func mobileCloudModelNames(raw string) []string { + parts := strings.Fields(strings.TrimSpace(raw)) + models := make([]string, 0, len(parts)) + for _, part := range parts { + cleaned := strings.TrimSpace(strings.TrimSuffix(part, "、")) + if cleaned != "" { + models = append(models, cleaned) + } + } + return models +} + +func mobileCloudProviderName(modelName string) string { + lower := strings.ToLower(strings.TrimSpace(modelName)) + switch { + case strings.HasPrefix(lower, "minimax"): + return "MiniMax" + case strings.HasPrefix(lower, "deepseek"): + return "DeepSeek" + case strings.HasPrefix(lower, "qwen"), strings.HasPrefix(lower, "qwq"): + return "Qwen" + case strings.HasPrefix(lower, "bge"): + return "BAAI" + case strings.HasPrefix(lower, "cosyvoice"), strings.HasPrefix(lower, "sensevoice"): + return "Alibaba" + default: + return "China Mobile" + } +} + +func mobileCloudRegionCode(region string) string { + switch strings.TrimSpace(region) { + case "华北-呼和浩特": + return "huabei-huhehaote" + case "东北-哈尔滨": + return "dongbei-haerbin" + case "华中-郑州": + return "huazhong-zhengzhou" + case "黑龙江-哈尔滨": + return "heilongjiang-haerbin" + case "华东-上海5": + return "huadong-shanghai5" + case "江西-南昌": + return "jiangxi-nanchang" + case "湖北-武汉": + return "hubei-wuhan" + case "华南-广州8": + return "huanan-guangzhou8" + default: + return normalizeExternalID(region) + } +} diff --git a/scripts/import_mobile_cloud_pricing_test.go b/scripts/import_mobile_cloud_pricing_test.go new file mode 100644 index 0000000..77759d3 --- /dev/null +++ b/scripts/import_mobile_cloud_pricing_test.go @@ -0,0 +1,89 @@ +//go:build llm_script + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestResolveMobileCloudPricingArticle(t *testing.T) { + raw := `{"code":200,"data":{"children":[{"articleTitle":"其他文档","articleId":1,"articleContentPublished":"x"},{"children":[{"articleTitle":"预置模型服务-token按量计费","articleId":91592,"articleContentPublished":"64ec46cbfd7c535db501aff43df5a788"}]}]}}` + articleID, contentPublished, err := resolveMobileCloudPricingArticle(raw) + if err != nil { + t.Fatalf("resolveMobileCloudPricingArticle 返回错误: %v", err) + } + if articleID != 91592 || contentPublished != "64ec46cbfd7c535db501aff43df5a788" { + t.Fatalf("解析价格文章失败: articleID=%d contentPublished=%q", articleID, contentPublished) + } +} + +func TestParseMobileCloudPricingHTMLBuildsRecords(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "mobile_cloud_pricing_sample.html")) + if err != nil { + t.Fatalf("读取 fixture 失败: %v", err) + } + records, err := parseMobileCloudPricingHTML(string(raw), "https://ecloud.10086.cn/op-help-center/doc/article/91592") + if err != nil { + t.Fatalf("parseMobileCloudPricingHTML 返回错误: %v", err) + } + if len(records) != 8 { + t.Fatalf("期望 8 条移动云价格记录，实际 %d", len(records)) + } + recordMap := make(map[string]officialPricingRecord, len(records)) + for _, record := range records { + recordMap[record.ModelID] = record + } + if recordMap["mobile-cloud-huabei-huhehaote-minimax-m2-5"].Region != "华北-呼和浩特" { + t.Fatalf("华北 MiniMax region 错误: %+v", recordMap["mobile-cloud-huabei-huhehaote-minimax-m2-5"]) + } + if recordMap["mobile-cloud-huabei-huhehaote-deepseek-v3-0324"].InputPrice != 2 || recordMap["mobile-cloud-huabei-huhehaote-deepseek-v3-0324"].OutputPrice != 8 { + t.Fatalf("DeepSeek-V3-0324 价格错误: %+v", recordMap["mobile-cloud-huabei-huhehaote-deepseek-v3-0324"]) + } + if recordMap["mobile-cloud-huabei-huhehaote-qwq-32b"].ProviderName != "Qwen" || recordMap["mobile-cloud-huabei-huhehaote-qwq-32b"].OutputPrice != 6 { + t.Fatalf("QwQ-32B provider/价格错误: %+v", recordMap["mobile-cloud-huabei-huhehaote-qwq-32b"]) + } + if recordMap["mobile-cloud-huabei-huhehaote-bge-m3"].ProviderName != "BAAI" || recordMap["mobile-cloud-huabei-huhehaote-bge-m3"].InputPrice != 0.5 || recordMap["mobile-cloud-huabei-huhehaote-bge-m3"].OutputPrice != 0.5 { + t.Fatalf("bge-m3 平价 token 记录错误: %+v", recordMap["mobile-cloud-huabei-huhehaote-bge-m3"]) + } + cosyVoice, ok := recordMap["mobile-cloud-huabei-huhehaote-cosyvoice"] + if !ok { + t.Fatalf("缺少 CosyVoice 语音计费记录") + } + if cosyVoice.ProviderName != "Alibaba" || cosyVoice.Modality != "audio" || cosyVoice.PricingMode != "flat" || cosyVoice.FlatPrice != 2 || cosyVoice.PriceUnit != "10k_characters" { + t.Fatalf("CosyVoice 语音计费记录错误: %+v", cosyVoice) + } + senseVoice, ok := recordMap["mobile-cloud-huabei-huhehaote-sensevoice"] + if !ok { + t.Fatalf("缺少 SenseVoice 语音计费记录") + } + if senseVoice.ProviderName != "Alibaba" || senseVoice.Modality != "audio" || senseVoice.PricingMode != "flat" || senseVoice.FlatPrice != 0.0007 || senseVoice.PriceUnit != "second" { + t.Fatalf("SenseVoice 语音计费记录错误: %+v", senseVoice) + } +} + +func TestRunMobileCloudPricingImportDryRunPrintsSummary(t *testing.T) { + var out bytes.Buffer + err := runMobileCloudPricingImport(mobileCloudPricingImportConfig{ + OutlineTreeURL: defaultMobileCloudOutlineTreeURL, + Fixture: filepath.Join("testdata", "mobile_cloud_pricing_sample.html"), + DryRun: true, + }, nil, &out) + if err != nil { + t.Fatalf("runMobileCloudPricingImport 返回错误: %v", err) + } + output := out.String() + for _, want := range []string{ + "source=mobile-cloud-pricing-import", + "models=8", + "operator=Mobile Cloud", + "dry_run=true", + } { + if !strings.Contains(output, want) { + t.Fatalf("输出缺少 %q，实际: %q", want, output) + } + } +} diff --git a/scripts/official_pricing_import_common.go b/scripts/official_pricing_import_common.go index d9da41f..ba615b0 100644 --- a/scripts/official_pricing_import_common.go +++ b/scripts/official_pricing_import_common.go @@ -34,6 +34,9 @@ type officialPricingRecord struct { OperatorType string Region string Currency string + PricingMode string + PriceUnit string + FlatPrice float64 InputPrice float64 OutputPrice float64 ContextLength int @@ -81,16 +84,21 @@ func upsertOfficialPricingRecords(db *sql.DB, records []officialPricingRecord, b _, err = db.Exec( `INSERT INTO region_pricing ( model_id, operator_id, region, currency, + pricing_mode, price_unit, flat_price, input_price_per_mtok, output_price_per_mtok, is_free, effective_date, source_url, source_type, free_quota, free_limitations, rate_limit ) VALUES ( $1, $2, $3, $4, - $5, $6, $7, CURRENT_DATE, $8, $9, - $10, $11, $12 + $5, $6, $7, + $8, $9, $10, CURRENT_DATE, $11, $12, + $13, $14, $15 ) ON CONFLICT (model_id, operator_id, region, currency, effective_date) DO UPDATE SET + pricing_mode = EXCLUDED.pricing_mode, + price_unit = EXCLUDED.price_unit, + flat_price = EXCLUDED.flat_price, input_price_per_mtok = EXCLUDED.input_price_per_mtok, output_price_per_mtok = EXCLUDED.output_price_per_mtok, is_free = EXCLUDED.is_free, @@ -101,6 +109,7 @@ func upsertOfficialPricingRecords(db *sql.DB, records []officialPricingRecord, b rate_limit = EXCLUDED.rate_limit, updated_at = CURRENT_TIMESTAMP`, modelID, operatorID, record.Region, record.Currency, + fallbackPricingMode(record.PricingMode), fallbackPriceUnit(record.PriceUnit), nullIfZeroFloat(record.FlatPrice), record.InputPrice, record.OutputPrice, record.IsFree, record.SourceURL, sourceType, nullIfBlank(freeQuota), freeLimitations, rateLimit, ) @@ -260,6 +269,29 @@ func fallbackModality(raw string) string { return value } +func fallbackPricingMode(raw string) string { + value := strings.TrimSpace(raw) + if value == "" { + return "input_output" + } + return value +} + +func fallbackPriceUnit(raw string) string { + value := strings.TrimSpace(raw) + if value == "" { + return "million_tokens" + } + return value +} + +func nullIfZeroFloat(value float64) any { + if value == 0 { + return nil + } + return value +} + func fetchRawPricingPage(url string, fixture string, client *http.Client) (string, error) { return fetchRawPricingPageWithOptions(url, fixture, client, officialPricingFetchOptions{ AcceptLanguage: "zh-CN,zh;q=0.9,en;q=0.8", @@ -399,6 +431,8 @@ func detectModality(modelName string) string { switch { case strings.Contains(lower, "coder"), strings.Contains(lower, "code"): return "code" + case strings.Contains(lower, "voice"), strings.Contains(lower, "audio"), strings.Contains(lower, "speech"): + return "audio" case strings.Contains(lower, "vision"), strings.Contains(lower, "vl"), strings.Contains(lower, "omni"), strings.Contains(lower, "multi"), strings.Contains(lower, "live"): return "multimodal" default: @@ -414,8 +448,14 @@ func providerMetadata(providerName string) (string, string, string) { return "亚马逊", "US", "https://aws.amazon.com" case "Anthropic": return "Anthropic", "US", "https://www.anthropic.com" + case "BAAI": + return "智源", "CN", "https://www.baai.ac.cn" case "Baidu": return "百度", "CN", "https://cloud.baidu.com" + case "ByteDance": + return "字节跳动", "CN", "https://www.volcengine.com" + case "China Mobile": + return "中国移动", "CN", "https://ecloud.10086.cn" case "Cloudflare": return "Cloudflare", "US", "https://www.cloudflare.com" case "Cohere": diff --git a/scripts/testdata/mobile_cloud_pricing_sample.html b/scripts/testdata/mobile_cloud_pricing_sample.html new file mode 100644 index 0000000..1c1577a --- /dev/null +++ b/scripts/testdata/mobile_cloud_pricing_sample.html @@ -0,0 +1,32 @@ +
+
+

华北-呼和浩特资源池支持订购模型

+
+ + + + + + + + + +
规格名称输入/输出tokens单价（元/百万tokens）包含模型
MiniMax-M2.5输入tokens2.1

MiniMax-M2.5

输出tokens8.4
DeepSeek-V3输入tokens2

DeepSeek-V3

DeepSeek-V3-0324

输出tokens8
QwQ-32B（深度思考）输入tokens2

QwQ-32B

输出tokens6
bge-m3tokens资费0.5

bge-m3

+ + + + +
规格名称模型类别资费场景单价包含模型
CosyVoice语音生成按照处理字符数收费2元/万字符CosyVoice
SenseVoice语音识别按照处理的语音时长秒数收费0.0007 元/秒SenseVoice
+
+
+
+

华南-广州8支持订购模型

+
+ + + + +
规格名称输入/输出tokens单价（元/百万tokens）包含模型
MiniMax-M2.5输入tokens2.1

MiniMax-M2.5

输出tokens8.4
+
+
+