feat: add content filter and code processing module (v0.3.0)
- Add content filter module (internal/content/) - Implement basic character filtering (control chars, line breaks, symbols) - Implement code block and inline code detection - Implement comment detection for 30+ languages (JS/Python/Go/HTML/etc) - Add go-enry dependency for intelligent language detection - Add SkipKeywords config option (default: TODO/FIXME/HACK/XXX/etc) - Integrate content processing into Translator - Update config.yaml with skip_keywords
This commit is contained in:
@@ -6,22 +6,25 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/titor/fanyi/internal/config"
|
||||
"github.com/titor/fanyi/internal/content"
|
||||
"github.com/titor/fanyi/internal/provider"
|
||||
)
|
||||
|
||||
// Translator 核心翻译类
|
||||
type Translator struct {
|
||||
config *config.Config
|
||||
provider provider.Provider
|
||||
prompt *PromptManager
|
||||
config *config.Config
|
||||
provider provider.Provider
|
||||
prompt *PromptManager
|
||||
contentParser *content.Parser
|
||||
}
|
||||
|
||||
// NewTranslator 创建翻译器实例
|
||||
func NewTranslator(config *config.Config, provider provider.Provider) *Translator {
|
||||
return &Translator{
|
||||
config: config,
|
||||
provider: provider,
|
||||
prompt: NewPromptManager(config.Prompts),
|
||||
config: config,
|
||||
provider: provider,
|
||||
prompt: NewPromptManager(config.Prompts),
|
||||
contentParser: content.NewParser(config.SkipKeywords),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,15 +34,33 @@ func (t *Translator) Translate(ctx context.Context, text string, options *Transl
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, time.Duration(t.config.Timeout)*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// 基础字符过滤
|
||||
filteredText := content.FilterBasic(text, nil)
|
||||
|
||||
// 内容解析(包含代码检测)
|
||||
parseResult, parseErr := t.contentParser.Parse(filteredText)
|
||||
|
||||
// 选择Prompt
|
||||
prompt := ""
|
||||
if options.PromptName != "" {
|
||||
prompt = t.prompt.GetPrompt(options.PromptName)
|
||||
}
|
||||
|
||||
// 如果包含代码且解析成功,使用增强的Prompt
|
||||
if parseErr == nil && parseResult.HasCode {
|
||||
enhancedPrompt := t.contentParser.BuildPrompt(parseResult)
|
||||
if enhancedPrompt != "" {
|
||||
if prompt != "" {
|
||||
prompt = prompt + "\n\n" + enhancedPrompt
|
||||
} else {
|
||||
prompt = enhancedPrompt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 构建请求
|
||||
req := &provider.TranslateRequest{
|
||||
Text: text,
|
||||
Text: filteredText,
|
||||
FromLang: options.FromLang,
|
||||
ToLang: options.ToLang,
|
||||
Prompt: prompt,
|
||||
@@ -53,10 +74,17 @@ func (t *Translator) Translate(ctx context.Context, text string, options *Transl
|
||||
return nil, fmt.Errorf("翻译失败: %w", err)
|
||||
}
|
||||
|
||||
translatedText := resp.Text
|
||||
|
||||
// 如果包含代码且解析成功,重构结果
|
||||
if parseErr == nil && parseResult.HasCode {
|
||||
translatedText = t.contentParser.Reconstruct(parseResult, resp.Text)
|
||||
}
|
||||
|
||||
// 构建结果
|
||||
return &TranslateResult{
|
||||
Original: text,
|
||||
Translated: resp.Text,
|
||||
Translated: translatedText,
|
||||
FromLang: resp.FromLang,
|
||||
ToLang: resp.ToLang,
|
||||
Model: resp.Model,
|
||||
|
||||
Reference in New Issue
Block a user