feat: add content filter and code processing module (v0.3.0)

- Add content filter module (internal/content/)
- Implement basic character filtering (control chars, line breaks, symbols)
- Implement code block and inline code detection
- Implement comment detection for 30+ languages (JS/Python/Go/HTML/etc)
- Add go-enry dependency for intelligent language detection
- Add SkipKeywords config option (default: TODO/FIXME/HACK/XXX/etc)
- Integrate content processing into Translator
- Update config.yaml with skip_keywords
This commit is contained in:
2026-03-29 18:41:25 +08:00
parent 1bce2d9c7a
commit 6807371c5e
8 changed files with 625 additions and 8 deletions

View File

@@ -6,22 +6,25 @@ import (
"time"
"github.com/titor/fanyi/internal/config"
"github.com/titor/fanyi/internal/content"
"github.com/titor/fanyi/internal/provider"
)
// Translator 核心翻译类
type Translator struct {
config *config.Config
provider provider.Provider
prompt *PromptManager
config *config.Config
provider provider.Provider
prompt *PromptManager
contentParser *content.Parser
}
// NewTranslator 创建翻译器实例
func NewTranslator(config *config.Config, provider provider.Provider) *Translator {
return &Translator{
config: config,
provider: provider,
prompt: NewPromptManager(config.Prompts),
config: config,
provider: provider,
prompt: NewPromptManager(config.Prompts),
contentParser: content.NewParser(config.SkipKeywords),
}
}
@@ -31,15 +34,33 @@ func (t *Translator) Translate(ctx context.Context, text string, options *Transl
timeoutCtx, cancel := context.WithTimeout(ctx, time.Duration(t.config.Timeout)*time.Second)
defer cancel()
// 基础字符过滤
filteredText := content.FilterBasic(text, nil)
// 内容解析(包含代码检测)
parseResult, parseErr := t.contentParser.Parse(filteredText)
// 选择Prompt
prompt := ""
if options.PromptName != "" {
prompt = t.prompt.GetPrompt(options.PromptName)
}
// 如果包含代码且解析成功使用增强的Prompt
if parseErr == nil && parseResult.HasCode {
enhancedPrompt := t.contentParser.BuildPrompt(parseResult)
if enhancedPrompt != "" {
if prompt != "" {
prompt = prompt + "\n\n" + enhancedPrompt
} else {
prompt = enhancedPrompt
}
}
}
// 构建请求
req := &provider.TranslateRequest{
Text: text,
Text: filteredText,
FromLang: options.FromLang,
ToLang: options.ToLang,
Prompt: prompt,
@@ -53,10 +74,17 @@ func (t *Translator) Translate(ctx context.Context, text string, options *Transl
return nil, fmt.Errorf("翻译失败: %w", err)
}
translatedText := resp.Text
// 如果包含代码且解析成功,重构结果
if parseErr == nil && parseResult.HasCode {
translatedText = t.contentParser.Reconstruct(parseResult, resp.Text)
}
// 构建结果
return &TranslateResult{
Original: text,
Translated: resp.Text,
Translated: translatedText,
FromLang: resp.FromLang,
ToLang: resp.ToLang,
Model: resp.Model,