feat: add content filter and code processing module (v0.3.0)
- Add content filter module (internal/content/) - Implement basic character filtering (control chars, line breaks, symbols) - Implement code block and inline code detection - Implement comment detection for 30+ languages (JS/Python/Go/HTML/etc) - Add go-enry dependency for intelligent language detection - Add SkipKeywords config option (default: TODO/FIXME/HACK/XXX/etc) - Integrate content processing into Translator - Update config.yaml with skip_keywords
This commit is contained in:
17
internal/content/content.go
Normal file
17
internal/content/content.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package content
|
||||
|
||||
import (
|
||||
"github.com/go-enry/go-enry/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
Version = "1.0.0"
|
||||
)
|
||||
|
||||
func DetectLanguage(text string) string {
|
||||
return enry.GetLanguage("", []byte(text))
|
||||
}
|
||||
|
||||
func Filter(text string) string {
|
||||
return FilterBasic(text, nil)
|
||||
}
|
||||
Reference in New Issue
Block a user