llm-intelligence/scripts/verification_executor.go

// verification_executor.go
// Reads TASKS.md, runs each task's verification.command,
// matches expected_evidence, outputs pass/fail report.
//
// Usage: go run scripts/verification_executor.go [--dry-run] [--task T-Q2-1.1]
package main

import (
	"bufio"
	"bytes"
	"context"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"time"
)

type Verification struct {
	Mode            string
	Command         string
	ExpectedEvidence string
	TimeoutSeconds  int
}

type TaskResult struct {
	TaskID    string
	TaskName  string
	Verified  bool
	Command   string
	ExitCode  int
	Stdout    string
	Stderr    string
	Error     string
	Reason    string
}

func main() {
	dryRun := flag.Bool("dry-run", false, "print commands without executing")
	taskFilter := flag.String("task", "", "filter by task ID (e.g. T-Q2-1.1)")
	tasksPathFlag := flag.String("tasks", "", "path to TASKS.md")
	flag.Parse()

	tasksPath := resolveTasksPath(*tasksPathFlag)

	f, err := os.Open(tasksPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "open TASKS.md: %v\n", err)
		os.Exit(1)
	}
	defer f.Close()

	tasks := parseTasks(f)
	if *taskFilter != "" {
		var filtered []taskEntry
		for _, t := range tasks {
			if t.ID == *taskFilter {
				filtered = append(filtered, t)
			}
		}
		tasks = filtered
	}

	fmt.Printf("=== Verification Report (%s) ===\n", time.Now().Format("2006-01-02 15:04"))
	fmt.Printf("Tasks checked: %d | Dry-run: %v | TASKS: %s\n\n", len(tasks), *dryRun, tasksPath)

	var passed, failed int
	var results []TaskResult

	for _, t := range tasks {
		r := verifyTask(t, *dryRun)
		results = append(results, r)
		if r.Verified {
			passed++
		} else {
			failed++
		}
	}

	for _, r := range results {
		icon := "✅"
		if !r.Verified {
			icon = "❌"
		}
		fmt.Printf("%s [%s] %s\n", icon, r.TaskID, r.TaskName)
		if r.Error != "" {
			fmt.Printf("   ERROR: %s\n", r.Error)
		} else {
			if r.Command != "" {
				fmt.Printf("   cmd: %s\n", r.Command)
			}
			if r.ExitCode != 0 && r.Stdout != "" {
				fmt.Printf("   output: %s\n", strings.TrimSpace(r.Stdout))
			} else if r.Reason != "" {
				fmt.Printf("   reason: %s\n", r.Reason)
			}
		}
	}

	fmt.Printf("\n=== Summary: %d passed, %d failed ===\n", passed, failed)
	if failed > 0 {
		os.Exit(1)
	}
}

func resolveTasksPath(flagValue string) string {
	candidates := []string{}
	if flagValue != "" {
		candidates = append(candidates, flagValue)
	}
	if envValue := os.Getenv("TASKS_PATH"); envValue != "" {
		candidates = append(candidates, envValue)
	}

	if wd, err := os.Getwd(); err == nil {
		candidates = append(candidates,
			filepath.Join(wd, "TASKS.md"),
			filepath.Join(wd, "..", "TASKS.md"),
		)
	}

	if _, sourcePath, _, ok := runtime.Caller(0); ok {
		scriptDir := filepath.Dir(sourcePath)
		candidates = append(candidates, filepath.Join(scriptDir, "..", "TASKS.md"))
	}

	candidates = append(candidates, "/home/long/.openclaw/workspace/TASKS.md")

	seen := map[string]struct{}{}
	for _, candidate := range candidates {
		if candidate == "" {
			continue
		}
		cleaned := filepath.Clean(candidate)
		if _, ok := seen[cleaned]; ok {
			continue
		}
		seen[cleaned] = struct{}{}
		if _, err := os.Stat(cleaned); err == nil {
			return cleaned
		}
	}

	if flagValue != "" {
		return filepath.Clean(flagValue)
	}
	if envValue := os.Getenv("TASKS_PATH"); envValue != "" {
		return filepath.Clean(envValue)
	}
	return "/home/long/.openclaw/workspace/TASKS.md"
}

type taskEntry struct {
	ID            string
	Name          string
	Verification  Verification
	HasVerification bool
}

func parseTasks(f *os.File) []taskEntry {
	var tasks []taskEntry
	var currentTask *taskEntry
	inVerification := false
	scanner := bufio.NewScanner(f)

	for scanner.Scan() {
		line := scanner.Text()

		// Match task header: ### T-1.1 🔶 Phase 1 范围冻结
		taskRe := regexp.MustCompile(`^### (T-[A-Za-z0-9.-]+)\s+[^\s]+\s+(.+)`)
		if m := taskRe.FindStringSubmatch(line); m != nil {
			if currentTask != nil {
				tasks = append(tasks, *currentTask)
			}
			currentTask = &taskEntry{ID: m[1], Name: m[2]}
			inVerification = false
			continue
		}

		if currentTask == nil {
			continue
		}

		// Check for verification block
		if strings.Contains(line, "**verification**") || strings.Contains(line, "**verification**:") {
			inVerification = true
			currentTask.HasVerification = true
			continue
		}

		if !inVerification {
			continue
		}

		// Parse verification fields (indented under **verification**)
		//  - mode: `artifact_present`
		modeRe := regexp.MustCompile(`^\s+- mode:\s+` + "`" + `([^` + "`" + `]+)` + "`")
		if m := modeRe.FindStringSubmatch(line); m != nil {
			currentTask.Verification.Mode = m[1]
			continue
		}

		cmdRe := regexp.MustCompile(`^\s+- command:\s+` + "`" + `([^` + "`" + `]+)` + "`")
		if m := cmdRe.FindStringSubmatch(line); m != nil {
			currentTask.Verification.Command = m[1]
			continue
		}

		expRe := regexp.MustCompile(`^\s+- expected_evidence:\s+` + "`" + `([^` + "`" + `]+)` + "`")
		if m := expRe.FindStringSubmatch(line); m != nil {
			currentTask.Verification.ExpectedEvidence = m[1]
			continue
		}

		timeoutRe := regexp.MustCompile(`^\s+- timeout_seconds:\s+(\d+)`)
		if m := timeoutRe.FindStringSubmatch(line); m != nil {
			fmt.Sscanf(m[1], "%d", &currentTask.Verification.TimeoutSeconds)
			continue
		}

		// Blank line or new top-level field ends verification block
		if strings.TrimSpace(line) == "" || (strings.HasPrefix(strings.TrimSpace(line), "**") && !strings.Contains(line, "verification")) {
			inVerification = false
		}
	}

	if currentTask != nil {
		tasks = append(tasks, *currentTask)
	}

	return tasks
}

func verifyTask(t taskEntry, dryRun bool) TaskResult {
	r := TaskResult{TaskID: t.ID, TaskName: t.Name}

	if !t.HasVerification {
		r.Reason = "no verification block"
		r.Verified = true // No verification = trivially pass
		return r
	}

	if t.Verification.Command == "" {
		r.Reason = "verification.command is empty"
		r.Verified = false
		return r
	}

	r.Command = t.Verification.Command

	if t.Verification.TimeoutSeconds == 0 {
		t.Verification.TimeoutSeconds = 30
	}

	if dryRun {
		r.Stdout = "(dry-run, command not executed)"
		r.Verified = true
		return r
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(t.Verification.TimeoutSeconds)*time.Second)
	defer cancel()

	cmd := exec.CommandContext(ctx, "sh", "-c", t.Verification.Command)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err := cmd.Run()
	r.ExitCode = 0
	if err != nil {
		r.ExitCode = -1
		if ctx.Err() == context.DeadlineExceeded {
			r.Error = fmt.Sprintf("timeout after %ds", t.Verification.TimeoutSeconds)
		} else {
			r.Error = err.Error()
		}
	}

	r.Stdout = stdout.String()
	r.Stderr = stderr.String()

	if r.ExitCode != 0 && t.Verification.Mode == "test_pass" {
		r.Verified = false
		return r
	}

	// Match expected_evidence
	if t.Verification.ExpectedEvidence != "" {
		evidence := t.Verification.ExpectedEvidence
		matched := false

		if strings.HasPrefix(evidence, "[") && strings.HasSuffix(evidence, "]") {
			// Regex range like [4-9]
			re := regexp.MustCompile(`\[(\d+)-(\d+)\]`)
			if m := re.FindStringSubmatch(evidence); m != nil {
				var lo, hi int
				fmt.Sscanf(m[1], "%d", &lo)
				fmt.Sscanf(m[2], "%d", &hi)
				reOut := regexp.MustCompile(fmt.Sprintf(`^\s*(\d+)\s*$`))
				if numMatch := reOut.FindStringSubmatch(strings.TrimSpace(r.Stdout)); numMatch != nil {
					var n int
					fmt.Sscanf(numMatch[1], "%d", &n)
					matched = n >= lo && n <= hi
				}
			}
		} else if strings.Contains(r.Stdout, evidence) {
			matched = true
		}

		r.Verified = matched
		if !matched {
			r.Reason = fmt.Sprintf("expected_evidence '%s' not found in output", evidence)
		}
	} else if r.ExitCode == 0 {
		r.Verified = true
	} else {
		r.Verified = false
		r.Reason = fmt.Sprintf("exit code %d", r.ExitCode)
	}

	return r
}