feat: 添加本地缓存功能,减少API调用
- 实现SQLite缓存模块,支持高效查询和存储 - 添加缓存键生成策略(基于原文+语言对的SHA256哈希) - 集成缓存到Translator类,先查缓存再调用API - 添加缓存管理命令:cache clear, cache stats, cache cleanup - 实现组合缓存清理策略(数量限制+时间过期) - 添加完整的单元测试 - 更新配置文件模板,添加缓存配置 - 更新文档和版本记录 版本: v0.5.1
This commit is contained in:
75
internal/cache/cache.go
vendored
Normal file
75
internal/cache/cache.go
vendored
Normal file
@@ -0,0 +1,75 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Cache 缓存接口
|
||||
type Cache interface {
|
||||
// Get 获取缓存
|
||||
Get(ctx context.Context, key string) (*CacheEntry, error)
|
||||
|
||||
// Set 设置缓存
|
||||
Set(ctx context.Context, entry *CacheEntry) error
|
||||
|
||||
// Delete 删除缓存
|
||||
Delete(ctx context.Context, key string) error
|
||||
|
||||
// Clear 清空缓存
|
||||
Clear(ctx context.Context) error
|
||||
|
||||
// Stats 获取缓存统计信息
|
||||
Stats(ctx context.Context) (*CacheStats, error)
|
||||
|
||||
// Cleanup 清理过期缓存
|
||||
Cleanup(ctx context.Context) error
|
||||
|
||||
// Close 关闭缓存
|
||||
Close() error
|
||||
}
|
||||
|
||||
// CacheEntry 缓存条目
|
||||
type CacheEntry struct {
|
||||
ID int64 `json:"id"`
|
||||
CacheKey string `json:"cache_key"`
|
||||
OriginalText string `json:"original_text"`
|
||||
TranslatedText string `json:"translated_text"`
|
||||
FromLang string `json:"from_lang"`
|
||||
ToLang string `json:"to_lang"`
|
||||
Model string `json:"model"`
|
||||
PromptName string `json:"prompt_name,omitempty"`
|
||||
PromptContent string `json:"prompt_content,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
LastUsedAt time.Time `json:"last_used_at"`
|
||||
}
|
||||
|
||||
// CacheStats 缓存统计信息
|
||||
type CacheStats struct {
|
||||
TotalRecords int `json:"total_records"`
|
||||
TotalSizeBytes int64 `json:"total_size_bytes"`
|
||||
OldestRecord time.Time `json:"oldest_record"`
|
||||
NewestRecord time.Time `json:"newest_record"`
|
||||
AvgTokensPerRecord float64 `json:"avg_tokens_per_record"`
|
||||
}
|
||||
|
||||
// CacheConfig 缓存配置
|
||||
type CacheConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
MaxRecords int `yaml:"max_records"`
|
||||
ExpireDays int `yaml:"expire_days"`
|
||||
DBPath string `yaml:"db_path"`
|
||||
}
|
||||
|
||||
// NewCacheConfig 创建默认缓存配置
|
||||
func NewCacheConfig() *CacheConfig {
|
||||
return &CacheConfig{
|
||||
Enabled: true,
|
||||
MaxRecords: 10000,
|
||||
ExpireDays: 30,
|
||||
DBPath: "~/.config/yoyo/cache.db",
|
||||
}
|
||||
}
|
||||
325
internal/cache/cache_test.go
vendored
Normal file
325
internal/cache/cache_test.go
vendored
Normal file
@@ -0,0 +1,325 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerateCacheKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
fromLang string
|
||||
toLang string
|
||||
wantSame bool
|
||||
}{
|
||||
{
|
||||
name: "相同输入应生成相同键",
|
||||
text: "Hello world",
|
||||
fromLang: "en",
|
||||
toLang: "zh-CN",
|
||||
wantSame: true,
|
||||
},
|
||||
{
|
||||
name: "不同文本应生成不同键",
|
||||
text: "Hello universe",
|
||||
fromLang: "en",
|
||||
toLang: "zh-CN",
|
||||
wantSame: false,
|
||||
},
|
||||
{
|
||||
name: "不同语言对应生成不同键",
|
||||
text: "Hello world",
|
||||
fromLang: "en",
|
||||
toLang: "zh-TW",
|
||||
wantSame: false,
|
||||
},
|
||||
{
|
||||
name: "大小写不敏感的语言代码",
|
||||
text: "Hello world",
|
||||
fromLang: "EN",
|
||||
toLang: "zh-cn",
|
||||
wantSame: true,
|
||||
},
|
||||
{
|
||||
name: "多余空白字符应规范化",
|
||||
text: " Hello world ",
|
||||
fromLang: "en",
|
||||
toLang: "zh-CN",
|
||||
wantSame: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
key1 := GenerateCacheKey(tt.text, tt.fromLang, tt.toLang)
|
||||
key2 := GenerateCacheKey(tt.text, tt.fromLang, tt.toLang)
|
||||
|
||||
if key1 != key2 {
|
||||
t.Errorf("相同输入生成了不同的键: %s != %s", key1, key2)
|
||||
}
|
||||
|
||||
// 检查键的长度(SHA256哈希应为64个字符)
|
||||
if len(key1) != 64 {
|
||||
t.Errorf("缓存键长度不正确: got %d, want 64", len(key1))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteCache(t *testing.T) {
|
||||
// 创建临时目录
|
||||
tmpDir, err := os.MkdirTemp("", "cache_test")
|
||||
if err != nil {
|
||||
t.Fatalf("创建临时目录失败: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test_cache.db")
|
||||
|
||||
// 创建缓存配置
|
||||
config := &CacheConfig{
|
||||
Enabled: true,
|
||||
MaxRecords: 100,
|
||||
ExpireDays: 1,
|
||||
DBPath: dbPath,
|
||||
}
|
||||
|
||||
// 创建缓存实例
|
||||
cache, err := NewSQLiteCache(config)
|
||||
if err != nil {
|
||||
t.Fatalf("创建缓存实例失败: %v", err)
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 测试设置缓存
|
||||
entry := &CacheEntry{
|
||||
CacheKey: "test_key_1",
|
||||
OriginalText: "Hello world",
|
||||
TranslatedText: "你好世界",
|
||||
FromLang: "en",
|
||||
ToLang: "zh-CN",
|
||||
Model: "gpt-3.5-turbo",
|
||||
PromptName: "simple",
|
||||
PromptContent: "请用简单易懂的语言翻译以下内容。",
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 5,
|
||||
TotalTokens: 15,
|
||||
}
|
||||
|
||||
err = cache.Set(ctx, entry)
|
||||
if err != nil {
|
||||
t.Fatalf("设置缓存失败: %v", err)
|
||||
}
|
||||
|
||||
// 测试获取缓存
|
||||
cachedEntry, err := cache.Get(ctx, "test_key_1")
|
||||
if err != nil {
|
||||
t.Fatalf("获取缓存失败: %v", err)
|
||||
}
|
||||
if cachedEntry == nil {
|
||||
t.Fatal("缓存未命中")
|
||||
}
|
||||
|
||||
if cachedEntry.TranslatedText != "你好世界" {
|
||||
t.Errorf("缓存翻译结果不正确: got %s, want 你好世界", cachedEntry.TranslatedText)
|
||||
}
|
||||
|
||||
// 测试缓存未命中
|
||||
missingEntry, err := cache.Get(ctx, "non_existent_key")
|
||||
if err != nil {
|
||||
t.Fatalf("查询不存在的缓存失败: %v", err)
|
||||
}
|
||||
if missingEntry != nil {
|
||||
t.Error("不存在的缓存应该返回nil")
|
||||
}
|
||||
|
||||
// 测试统计信息
|
||||
stats, err := cache.Stats(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("获取统计信息失败: %v", err)
|
||||
}
|
||||
if stats.TotalRecords != 1 {
|
||||
t.Errorf("统计记录数不正确: got %d, want 1", stats.TotalRecords)
|
||||
}
|
||||
|
||||
// 测试删除缓存
|
||||
err = cache.Delete(ctx, "test_key_1")
|
||||
if err != nil {
|
||||
t.Fatalf("删除缓存失败: %v", err)
|
||||
}
|
||||
|
||||
// 验证删除
|
||||
deletedEntry, err := cache.Get(ctx, "test_key_1")
|
||||
if err != nil {
|
||||
t.Fatalf("查询已删除的缓存失败: %v", err)
|
||||
}
|
||||
if deletedEntry != nil {
|
||||
t.Error("已删除的缓存应该返回nil")
|
||||
}
|
||||
|
||||
// 测试清空缓存
|
||||
err = cache.Set(ctx, entry)
|
||||
if err != nil {
|
||||
t.Fatalf("设置缓存失败: %v", err)
|
||||
}
|
||||
|
||||
err = cache.Clear(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("清空缓存失败: %v", err)
|
||||
}
|
||||
|
||||
stats, err = cache.Stats(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("获取统计信息失败: %v", err)
|
||||
}
|
||||
if stats.TotalRecords != 0 {
|
||||
t.Errorf("清空后记录数不正确: got %d, want 0", stats.TotalRecords)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheExpiration(t *testing.T) {
|
||||
// 创建临时目录
|
||||
tmpDir, err := os.MkdirTemp("", "cache_test")
|
||||
if err != nil {
|
||||
t.Fatalf("创建临时目录失败: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test_cache.db")
|
||||
|
||||
// 创建缓存配置,设置很短的过期时间
|
||||
config := &CacheConfig{
|
||||
Enabled: true,
|
||||
MaxRecords: 100,
|
||||
ExpireDays: 0, // 0天表示立即过期
|
||||
DBPath: dbPath,
|
||||
}
|
||||
|
||||
// 创建缓存实例
|
||||
cache, err := NewSQLiteCache(config)
|
||||
if err != nil {
|
||||
t.Fatalf("创建缓存实例失败: %v", err)
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 设置缓存
|
||||
entry := &CacheEntry{
|
||||
CacheKey: "test_key_1",
|
||||
OriginalText: "Hello world",
|
||||
TranslatedText: "你好世界",
|
||||
FromLang: "en",
|
||||
ToLang: "zh-CN",
|
||||
Model: "gpt-3.5-turbo",
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 5,
|
||||
TotalTokens: 15,
|
||||
}
|
||||
|
||||
err = cache.Set(ctx, entry)
|
||||
if err != nil {
|
||||
t.Fatalf("设置缓存失败: %v", err)
|
||||
}
|
||||
|
||||
// 立即清理(应该删除所有记录,因为过期时间为0)
|
||||
err = cache.Cleanup(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("清理缓存失败: %v", err)
|
||||
}
|
||||
|
||||
// 检查统计信息
|
||||
stats, err := cache.Stats(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("获取统计信息失败: %v", err)
|
||||
}
|
||||
if stats.TotalRecords != 0 {
|
||||
t.Errorf("清理后记录数不正确: got %d, want 0", stats.TotalRecords)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeText(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "正常文本",
|
||||
input: "Hello world",
|
||||
expected: "Hello world",
|
||||
},
|
||||
{
|
||||
name: "多余空白字符",
|
||||
input: " Hello world ",
|
||||
expected: "Hello world",
|
||||
},
|
||||
{
|
||||
name: "制表符和换行符",
|
||||
input: "Hello\tworld\n",
|
||||
expected: "Hello world",
|
||||
},
|
||||
{
|
||||
name: "空字符串",
|
||||
input: "",
|
||||
expected: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := normalizeText(tt.input)
|
||||
if result != tt.expected {
|
||||
t.Errorf("normalizeText(%q) = %q, want %q", tt.input, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLanguageCode(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "正常语言代码",
|
||||
input: "zh-CN",
|
||||
expected: "zh-cn",
|
||||
},
|
||||
{
|
||||
name: "大写语言代码",
|
||||
input: "EN-US",
|
||||
expected: "en-us",
|
||||
},
|
||||
{
|
||||
name: "空字符串",
|
||||
input: "",
|
||||
expected: "auto",
|
||||
},
|
||||
{
|
||||
name: "auto",
|
||||
input: "auto",
|
||||
expected: "auto",
|
||||
},
|
||||
{
|
||||
name: "前后空白",
|
||||
input: " zh-CN ",
|
||||
expected: "zh-cn",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := normalizeLanguageCode(tt.input)
|
||||
if result != tt.expected {
|
||||
t.Errorf("normalizeLanguageCode(%q) = %q, want %q", tt.input, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
53
internal/cache/cleanup.go
vendored
Normal file
53
internal/cache/cleanup.go
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
)
|
||||
|
||||
// CleanupManager 缓存清理管理器
|
||||
type CleanupManager struct {
|
||||
cache Cache
|
||||
}
|
||||
|
||||
// NewCleanupManager 创建清理管理器
|
||||
func NewCleanupManager(cache Cache) *CleanupManager {
|
||||
return &CleanupManager{
|
||||
cache: cache,
|
||||
}
|
||||
}
|
||||
|
||||
// ClearAll 清空所有缓存
|
||||
func (m *CleanupManager) ClearAll(ctx context.Context) error {
|
||||
if err := m.cache.Clear(ctx); err != nil {
|
||||
return fmt.Errorf("清空缓存失败: %w", err)
|
||||
}
|
||||
log.Println("缓存已清空")
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearByLanguage 清空指定语言对的缓存
|
||||
func (m *CleanupManager) ClearByLanguage(ctx context.Context, fromLang, toLang string) error {
|
||||
// 这个功能需要在SQLite实现中添加查询功能
|
||||
// 目前先返回一个提示信息
|
||||
return fmt.Errorf("按语言清理功能尚未实现")
|
||||
}
|
||||
|
||||
// GetStats 获取缓存统计信息
|
||||
func (m *CleanupManager) GetStats(ctx context.Context) (*CacheStats, error) {
|
||||
stats, err := m.cache.Stats(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取缓存统计失败: %w", err)
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// CleanupManual 手动清理
|
||||
func (m *CleanupManager) CleanupManual(ctx context.Context) error {
|
||||
if err := m.cache.Cleanup(ctx); err != nil {
|
||||
return fmt.Errorf("手动清理失败: %w", err)
|
||||
}
|
||||
log.Println("缓存清理完成")
|
||||
return nil
|
||||
}
|
||||
63
internal/cache/key.go
vendored
Normal file
63
internal/cache/key.go
vendored
Normal file
@@ -0,0 +1,63 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// GenerateCacheKey 生成缓存键
|
||||
// 使用原文+语言对进行SHA256哈希
|
||||
func GenerateCacheKey(originalText, fromLang, toLang string) string {
|
||||
// 规范化语言代码
|
||||
fromLang = normalizeLanguageCode(fromLang)
|
||||
toLang = normalizeLanguageCode(toLang)
|
||||
|
||||
// 规范化原文
|
||||
normalizedText := normalizeText(originalText)
|
||||
|
||||
// 生成缓存键
|
||||
data := fmt.Sprintf("%s|%s|%s", normalizedText, fromLang, toLang)
|
||||
hash := sha256.Sum256([]byte(data))
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
// normalizeLanguageCode 规范化语言代码
|
||||
func normalizeLanguageCode(lang string) string {
|
||||
if lang == "" || lang == "auto" {
|
||||
return "auto"
|
||||
}
|
||||
return strings.ToLower(strings.TrimSpace(lang))
|
||||
}
|
||||
|
||||
// normalizeText 规范化文本
|
||||
// 移除多余的空白字符,确保相同的文本生成相同的哈希
|
||||
func normalizeText(text string) string {
|
||||
// 移除首尾空白
|
||||
text = strings.TrimSpace(text)
|
||||
|
||||
// 将多个连续空白字符替换为单个空格
|
||||
text = strings.Join(strings.Fields(text), " ")
|
||||
|
||||
return text
|
||||
}
|
||||
|
||||
// GenerateCacheKeyWithModel 生成包含模型信息的缓存键
|
||||
// 如果需要更精确的缓存,可以使用这个函数
|
||||
func GenerateCacheKeyWithModel(originalText, fromLang, toLang, model string) string {
|
||||
// 规范化语言代码
|
||||
fromLang = normalizeLanguageCode(fromLang)
|
||||
toLang = normalizeLanguageCode(toLang)
|
||||
|
||||
// 规范化原文
|
||||
normalizedText := normalizeText(originalText)
|
||||
|
||||
// 规范化模型名称
|
||||
model = strings.ToLower(strings.TrimSpace(model))
|
||||
|
||||
// 生成缓存键
|
||||
data := fmt.Sprintf("%s|%s|%s|%s", normalizedText, fromLang, toLang, model)
|
||||
hash := sha256.Sum256([]byte(data))
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
337
internal/cache/sqlite.go
vendored
Normal file
337
internal/cache/sqlite.go
vendored
Normal file
@@ -0,0 +1,337 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// SQLiteCache SQLite缓存实现
|
||||
type SQLiteCache struct {
|
||||
db *sql.DB
|
||||
config *CacheConfig
|
||||
cleanupTTL time.Duration
|
||||
}
|
||||
|
||||
// NewSQLiteCache 创建SQLite缓存实例
|
||||
func NewSQLiteCache(config *CacheConfig) (*SQLiteCache, error) {
|
||||
if config == nil {
|
||||
config = NewCacheConfig()
|
||||
}
|
||||
|
||||
// 展开路径中的~符号
|
||||
dbPath, err := expandPath(config.DBPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("无效的数据库路径: %w", err)
|
||||
}
|
||||
|
||||
// 确保目录存在
|
||||
dir := filepath.Dir(dbPath)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("创建缓存目录失败: %w", err)
|
||||
}
|
||||
|
||||
// 打开数据库连接
|
||||
db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_synchronous=NORMAL")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("打开数据库失败: %w", err)
|
||||
}
|
||||
|
||||
// 设置连接池参数
|
||||
db.SetMaxOpenConns(1) // SQLite只支持单个写入连接
|
||||
db.SetMaxIdleConns(1)
|
||||
|
||||
cache := &SQLiteCache{
|
||||
db: db,
|
||||
config: config,
|
||||
cleanupTTL: time.Duration(config.ExpireDays) * 24 * time.Hour,
|
||||
}
|
||||
|
||||
// 初始化数据库表
|
||||
if err := cache.initTable(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("初始化缓存表失败: %w", err)
|
||||
}
|
||||
|
||||
// 设置清理定时器
|
||||
go cache.startCleanupTimer()
|
||||
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
// initTable 初始化缓存表
|
||||
func (c *SQLiteCache) initTable() error {
|
||||
query := `
|
||||
CREATE TABLE IF NOT EXISTS translation_cache (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
cache_key TEXT NOT NULL UNIQUE,
|
||||
original_text TEXT NOT NULL,
|
||||
translated_text TEXT NOT NULL,
|
||||
from_lang TEXT NOT NULL,
|
||||
to_lang TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
prompt_name TEXT,
|
||||
prompt_content TEXT,
|
||||
prompt_tokens INTEGER DEFAULT 0,
|
||||
completion_tokens INTEGER DEFAULT 0,
|
||||
total_tokens INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_used_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cache_key ON translation_cache(cache_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_original_text ON translation_cache(original_text);
|
||||
CREATE INDEX IF NOT EXISTS idx_created_at ON translation_cache(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_last_used_at ON translation_cache(last_used_at);
|
||||
`
|
||||
|
||||
_, err := c.db.Exec(query)
|
||||
return err
|
||||
}
|
||||
|
||||
// Get 获取缓存
|
||||
func (c *SQLiteCache) Get(ctx context.Context, key string) (*CacheEntry, error) {
|
||||
query := `
|
||||
SELECT
|
||||
id, cache_key, original_text, translated_text, from_lang, to_lang,
|
||||
model, prompt_name, prompt_content, prompt_tokens, completion_tokens,
|
||||
total_tokens, created_at, last_used_at
|
||||
FROM translation_cache
|
||||
WHERE cache_key = ?
|
||||
`
|
||||
|
||||
entry := &CacheEntry{}
|
||||
var promptName, promptContent sql.NullString
|
||||
var createdAt, lastUsedAt string
|
||||
|
||||
err := c.db.QueryRowContext(ctx, query, key).Scan(
|
||||
&entry.ID, &entry.CacheKey, &entry.OriginalText, &entry.TranslatedText,
|
||||
&entry.FromLang, &entry.ToLang, &entry.Model, &promptName, &promptContent,
|
||||
&entry.PromptTokens, &entry.CompletionTokens, &entry.TotalTokens,
|
||||
&createdAt, &lastUsedAt,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil // 缓存未命中
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询缓存失败: %w", err)
|
||||
}
|
||||
|
||||
// 处理可空字段
|
||||
if promptName.Valid {
|
||||
entry.PromptName = promptName.String
|
||||
}
|
||||
if promptContent.Valid {
|
||||
entry.PromptContent = promptContent.String
|
||||
}
|
||||
|
||||
// 解析时间
|
||||
entry.CreatedAt, _ = time.Parse("2006-01-02 15:04:05", createdAt)
|
||||
entry.LastUsedAt, _ = time.Parse("2006-01-02 15:04:05", lastUsedAt)
|
||||
|
||||
// 更新最后使用时间
|
||||
go c.updateLastUsed(context.Background(), key)
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
// Set 设置缓存
|
||||
func (c *SQLiteCache) Set(ctx context.Context, entry *CacheEntry) error {
|
||||
// 开始事务
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("开始事务失败: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// 插入或替换缓存
|
||||
query := `
|
||||
INSERT OR REPLACE INTO translation_cache
|
||||
(cache_key, original_text, translated_text, from_lang, to_lang,
|
||||
model, prompt_name, prompt_content, prompt_tokens, completion_tokens,
|
||||
total_tokens, created_at, last_used_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
now := time.Now().Format("2006-01-02 15:04:05")
|
||||
_, err = tx.ExecContext(ctx, query,
|
||||
entry.CacheKey, entry.OriginalText, entry.TranslatedText,
|
||||
entry.FromLang, entry.ToLang, entry.Model, entry.PromptName,
|
||||
entry.PromptContent, entry.PromptTokens, entry.CompletionTokens,
|
||||
entry.TotalTokens, now, now,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("插入缓存失败: %w", err)
|
||||
}
|
||||
|
||||
// 提交事务
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("提交事务失败: %w", err)
|
||||
}
|
||||
|
||||
// 触发清理(异步)
|
||||
go c.Cleanup(context.Background())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete 删除缓存
|
||||
func (c *SQLiteCache) Delete(ctx context.Context, key string) error {
|
||||
query := `DELETE FROM translation_cache WHERE cache_key = ?`
|
||||
_, err := c.db.ExecContext(ctx, query, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除缓存失败: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clear 清空缓存
|
||||
func (c *SQLiteCache) Clear(ctx context.Context) error {
|
||||
// 先删除所有记录
|
||||
_, err := c.db.ExecContext(ctx, `DELETE FROM translation_cache`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("清空缓存失败: %w", err)
|
||||
}
|
||||
|
||||
// 然后执行VACUUM(不能在事务中执行)
|
||||
_, err = c.db.ExecContext(ctx, `VACUUM`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("清理数据库失败: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stats 获取缓存统计信息
|
||||
func (c *SQLiteCache) Stats(ctx context.Context) (*CacheStats, error) {
|
||||
stats := &CacheStats{}
|
||||
|
||||
// 获取总记录数
|
||||
err := c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM translation_cache`).Scan(&stats.TotalRecords)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询缓存统计失败: %w", err)
|
||||
}
|
||||
|
||||
// 如果没有记录,直接返回
|
||||
if stats.TotalRecords == 0 {
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// 获取时间范围和平均tokens
|
||||
var oldestStr, newestStr sql.NullString
|
||||
var avgTokens sql.NullFloat64
|
||||
err = c.db.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
MIN(created_at),
|
||||
MAX(created_at),
|
||||
AVG(total_tokens)
|
||||
FROM translation_cache
|
||||
`).Scan(&oldestStr, &newestStr, &avgTokens)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("查询缓存时间范围失败: %w", err)
|
||||
}
|
||||
|
||||
// 解析时间字符串
|
||||
if oldestStr.Valid {
|
||||
stats.OldestRecord, _ = time.Parse("2006-01-02 15:04:05", oldestStr.String)
|
||||
}
|
||||
if newestStr.Valid {
|
||||
stats.NewestRecord, _ = time.Parse("2006-01-02 15:04:05", newestStr.String)
|
||||
}
|
||||
if avgTokens.Valid {
|
||||
stats.AvgTokensPerRecord = avgTokens.Float64
|
||||
}
|
||||
|
||||
// 计算数据库文件大小
|
||||
dbPath, _ := expandPath(c.config.DBPath)
|
||||
if info, err := os.Stat(dbPath); err == nil {
|
||||
stats.TotalSizeBytes = info.Size()
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// Cleanup 清理过期缓存
|
||||
func (c *SQLiteCache) Cleanup(ctx context.Context) error {
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("开始事务失败: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// 清理过期缓存
|
||||
if c.cleanupTTL > 0 {
|
||||
expiredTime := time.Now().Add(-c.cleanupTTL).Format("2006-01-02 15:04:05")
|
||||
_, err = tx.ExecContext(ctx, `DELETE FROM translation_cache WHERE last_used_at < ?`, expiredTime)
|
||||
if err != nil {
|
||||
return fmt.Errorf("清理过期缓存失败: %w", err)
|
||||
}
|
||||
} else if c.cleanupTTL == 0 {
|
||||
// 如果过期时间为0,清理所有记录
|
||||
_, err = tx.ExecContext(ctx, `DELETE FROM translation_cache`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("清理所有缓存失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 清理超出数量限制的缓存
|
||||
if c.config.MaxRecords > 0 {
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
DELETE FROM translation_cache
|
||||
WHERE id NOT IN (
|
||||
SELECT id FROM translation_cache
|
||||
ORDER BY last_used_at DESC
|
||||
LIMIT ?
|
||||
)
|
||||
`, c.config.MaxRecords)
|
||||
if err != nil {
|
||||
return fmt.Errorf("清理超出数量限制的缓存失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// Close 关闭缓存
|
||||
func (c *SQLiteCache) Close() error {
|
||||
if c.db != nil {
|
||||
return c.db.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateLastUsed 更新最后使用时间
|
||||
func (c *SQLiteCache) updateLastUsed(ctx context.Context, key string) {
|
||||
query := `UPDATE translation_cache SET last_used_at = ? WHERE cache_key = ?`
|
||||
now := time.Now().Format("2006-01-02 15:04:05")
|
||||
c.db.ExecContext(ctx, query, now, key)
|
||||
}
|
||||
|
||||
// startCleanupTimer 启动清理定时器
|
||||
func (c *SQLiteCache) startCleanupTimer() {
|
||||
ticker := time.NewTicker(1 * time.Hour) // 每小时清理一次
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
c.Cleanup(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
// expandPath 展开路径中的~符号
|
||||
func expandPath(path string) (string, error) {
|
||||
if strings.HasPrefix(path, "~") {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
path = filepath.Join(home, path[1:])
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
@@ -26,6 +26,9 @@ type Config struct {
|
||||
|
||||
// 内容过滤配置
|
||||
SkipKeywords []string `yaml:"skip_keywords"` // 不翻译的关键词
|
||||
|
||||
// 缓存配置
|
||||
Cache CacheConfig `yaml:"cache"`
|
||||
}
|
||||
|
||||
// ProviderConfig 厂商配置
|
||||
@@ -36,6 +39,14 @@ type ProviderConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// CacheConfig 缓存配置
|
||||
type CacheConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
MaxRecords int `yaml:"max_records"`
|
||||
ExpireDays int `yaml:"expire_days"`
|
||||
DBPath string `yaml:"db_path"`
|
||||
}
|
||||
|
||||
// ConfigLoader 配置加载器接口
|
||||
type ConfigLoader interface {
|
||||
Load(path string) (*Config, error)
|
||||
@@ -134,6 +145,17 @@ func (c *Config) setDefaults() {
|
||||
"BUG:", "WARN:", "IMPORTANT:",
|
||||
}
|
||||
}
|
||||
|
||||
// 设置缓存配置默认值
|
||||
if c.Cache.MaxRecords <= 0 {
|
||||
c.Cache.MaxRecords = 10000
|
||||
}
|
||||
if c.Cache.ExpireDays <= 0 {
|
||||
c.Cache.ExpireDays = 30
|
||||
}
|
||||
if c.Cache.DBPath == "" {
|
||||
c.Cache.DBPath = "~/.config/yoyo/cache.db"
|
||||
}
|
||||
}
|
||||
|
||||
// GetProviderConfig 获取指定厂商的配置
|
||||
@@ -221,5 +243,10 @@ func (c *Config) String() string {
|
||||
for name := range c.Prompts {
|
||||
builder.WriteString(fmt.Sprintf(" %s\n", name))
|
||||
}
|
||||
builder.WriteString("Cache:\n")
|
||||
builder.WriteString(fmt.Sprintf(" enabled: %v\n", c.Cache.Enabled))
|
||||
builder.WriteString(fmt.Sprintf(" max_records: %d\n", c.Cache.MaxRecords))
|
||||
builder.WriteString(fmt.Sprintf(" expire_days: %d\n", c.Cache.ExpireDays))
|
||||
builder.WriteString(fmt.Sprintf(" db_path: %s\n", c.Cache.DBPath))
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package content
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
@@ -46,7 +47,7 @@ func truncateConsecutiveSymbols(text string, maxCount int) string {
|
||||
symbols := []string{"=", "-", "_", "*", "#", "~", "`", "."}
|
||||
|
||||
for _, symbol := range symbols {
|
||||
pattern := regexp.MustCompile(`(?` + `(` + symbol + `){` + string(rune(maxCount+1)) + `,})`)
|
||||
pattern := regexp.MustCompile(regexp.QuoteMeta(symbol) + `{` + fmt.Sprintf("%d", maxCount+1) + `,}`)
|
||||
replacement := strings.Repeat(symbol, maxCount)
|
||||
text = pattern.ReplaceAllString(text, replacement)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/titor/fanyi/internal/cache"
|
||||
"github.com/titor/fanyi/internal/config"
|
||||
"github.com/titor/fanyi/internal/content"
|
||||
"github.com/titor/fanyi/internal/provider"
|
||||
@@ -16,16 +17,32 @@ type Translator struct {
|
||||
provider provider.Provider
|
||||
prompt *PromptManager
|
||||
contentParser *content.Parser
|
||||
cache cache.Cache
|
||||
}
|
||||
|
||||
// NewTranslator 创建翻译器实例
|
||||
func NewTranslator(config *config.Config, provider provider.Provider) *Translator {
|
||||
return &Translator{
|
||||
translator := &Translator{
|
||||
config: config,
|
||||
provider: provider,
|
||||
prompt: NewPromptManager(config.Prompts),
|
||||
contentParser: content.NewParser(config.SkipKeywords),
|
||||
}
|
||||
|
||||
// 初始化缓存(如果启用)
|
||||
if config.Cache.Enabled {
|
||||
cacheConfig := &cache.CacheConfig{
|
||||
Enabled: config.Cache.Enabled,
|
||||
MaxRecords: config.Cache.MaxRecords,
|
||||
ExpireDays: config.Cache.ExpireDays,
|
||||
DBPath: config.Cache.DBPath,
|
||||
}
|
||||
if cacheInstance, err := cache.NewSQLiteCache(cacheConfig); err == nil {
|
||||
translator.cache = cacheInstance
|
||||
}
|
||||
}
|
||||
|
||||
return translator
|
||||
}
|
||||
|
||||
// Translate 执行翻译
|
||||
@@ -68,6 +85,26 @@ func (t *Translator) Translate(ctx context.Context, text string, options *Transl
|
||||
Options: options.ExtraOptions,
|
||||
}
|
||||
|
||||
// 检查缓存
|
||||
if t.cache != nil {
|
||||
cacheKey := cache.GenerateCacheKey(filteredText, options.FromLang, options.ToLang)
|
||||
if cachedEntry, err := t.cache.Get(ctx, cacheKey); err == nil && cachedEntry != nil {
|
||||
// 缓存命中
|
||||
return &TranslateResult{
|
||||
Original: text,
|
||||
Translated: cachedEntry.TranslatedText,
|
||||
FromLang: cachedEntry.FromLang,
|
||||
ToLang: cachedEntry.ToLang,
|
||||
Model: cachedEntry.Model,
|
||||
Usage: &provider.Usage{
|
||||
PromptTokens: cachedEntry.PromptTokens,
|
||||
CompletionTokens: cachedEntry.CompletionTokens,
|
||||
TotalTokens: cachedEntry.TotalTokens,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 调用厂商API
|
||||
resp, err := t.provider.Translate(timeoutCtx, req)
|
||||
if err != nil {
|
||||
@@ -81,6 +118,26 @@ func (t *Translator) Translate(ctx context.Context, text string, options *Transl
|
||||
translatedText = t.contentParser.Reconstruct(parseResult, resp.Text)
|
||||
}
|
||||
|
||||
// 保存到缓存
|
||||
if t.cache != nil {
|
||||
cacheKey := cache.GenerateCacheKey(filteredText, options.FromLang, options.ToLang)
|
||||
cacheEntry := &cache.CacheEntry{
|
||||
CacheKey: cacheKey,
|
||||
OriginalText: filteredText,
|
||||
TranslatedText: translatedText,
|
||||
FromLang: resp.FromLang,
|
||||
ToLang: resp.ToLang,
|
||||
Model: resp.Model,
|
||||
PromptName: options.PromptName,
|
||||
PromptContent: prompt,
|
||||
PromptTokens: resp.Usage.PromptTokens,
|
||||
CompletionTokens: resp.Usage.CompletionTokens,
|
||||
TotalTokens: resp.Usage.TotalTokens,
|
||||
}
|
||||
// 异步保存缓存,不阻塞翻译结果返回
|
||||
go t.cache.Set(context.Background(), cacheEntry)
|
||||
}
|
||||
|
||||
// 构建结果
|
||||
return &TranslateResult{
|
||||
Original: text,
|
||||
|
||||
Reference in New Issue
Block a user