fix: 将 libsql-client-go 作为普通目录提交，而非 gitlink
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
package sqliteparserutils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
)
|
||||
|
||||
// bufferedTokenizer preload few next tokens and put them in the tokens buffer to support look-ahead access with limited distance
|
||||
// Tokenizer maintains loaded tokens in ring buffer and use constant memory to support all requests (which is important for streaming mode)
|
||||
type bufferedTokenizer struct {
|
||||
source antlr.TokenSource
|
||||
tokens []antlr.Token
|
||||
index int
|
||||
}
|
||||
|
||||
// createBufferedTokenizer initialize tokenizer which will store current token and bufferSize-1 next tokens for look-ahead accesses
|
||||
func createBufferedTokenizer(source antlr.TokenSource, bufferSize int) *bufferedTokenizer {
|
||||
tokens := make([]antlr.Token, bufferSize)
|
||||
stream := bufferedTokenizer{source: source, tokens: tokens, index: 0}
|
||||
stream.load(bufferSize)
|
||||
return &stream
|
||||
}
|
||||
|
||||
func (stream *bufferedTokenizer) load(n int) {
|
||||
i := 0
|
||||
for i < n {
|
||||
token := stream.source.NextToken()
|
||||
if token.GetChannel() != antlr.TokenDefaultChannel {
|
||||
continue
|
||||
}
|
||||
stream.tokens[(stream.index+i)%len(stream.tokens)] = token
|
||||
i += 1
|
||||
}
|
||||
stream.index = (stream.index + i) % len(stream.tokens)
|
||||
}
|
||||
|
||||
func (stream *bufferedTokenizer) Consume() {
|
||||
stream.load(1)
|
||||
}
|
||||
|
||||
func (stream *bufferedTokenizer) Get(k int) antlr.Token {
|
||||
if k >= len(stream.tokens) {
|
||||
panic(fmt.Errorf("out of buffer read attempts: %d >= %d", k, len(stream.tokens)))
|
||||
}
|
||||
return stream.tokens[(stream.index+k)%len(stream.tokens)]
|
||||
}
|
||||
|
||||
func (stream *bufferedTokenizer) IsEOF() bool {
|
||||
return stream.Get(0).GetTokenType() == antlr.TokenEOF
|
||||
}
|
||||
122
go_modules/libsql-client-go/sqliteparserutils/utils.go
Normal file
122
go_modules/libsql-client-go/sqliteparserutils/utils.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package sqliteparserutils
|
||||
|
||||
import (
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
|
||||
"github.com/tursodatabase/libsql-client-go/sqliteparser"
|
||||
)
|
||||
|
||||
// TODO: Shell test begin transaction on shell
|
||||
|
||||
type SplitStatementExtraInfo struct {
|
||||
IncompleteCreateTriggerStatement bool
|
||||
IncompleteMultilineComment bool
|
||||
LastTokenType int
|
||||
}
|
||||
|
||||
type StatementIterator struct {
|
||||
tokenizer *bufferedTokenizer
|
||||
currentToken antlr.Token
|
||||
}
|
||||
|
||||
func CreateStatementIterator(statement string) *StatementIterator {
|
||||
return &StatementIterator{tokenizer: createStringTokenizer(statement)}
|
||||
}
|
||||
|
||||
func (iterator *StatementIterator) Next() (statement string, extraInfo SplitStatementExtraInfo, isEOF bool) {
|
||||
var (
|
||||
insideCreateTriggerStmt = false
|
||||
insideMultilineComment = false
|
||||
startPosition = -1
|
||||
previousToken = iterator.currentToken
|
||||
)
|
||||
for !iterator.tokenizer.IsEOF() {
|
||||
// We break loop here because we're sure multiline comment didn't finished, otherwise lexer would have just ignored it
|
||||
if atIncompleteMultilineCommentStart(iterator.tokenizer) {
|
||||
insideMultilineComment = true
|
||||
break
|
||||
}
|
||||
iterator.currentToken = iterator.tokenizer.Get(0)
|
||||
// skip empty statements (e.g. ... ; /* some comment */ ; ... )
|
||||
if startPosition == -1 && iterator.currentToken.GetTokenType() == sqliteparser.SQLiteLexerSCOL {
|
||||
iterator.tokenizer.Consume()
|
||||
continue
|
||||
}
|
||||
if startPosition == -1 {
|
||||
// initialize current statement start position
|
||||
insideCreateTriggerStmt = atCreateTriggerStart(iterator.tokenizer)
|
||||
startPosition = iterator.currentToken.GetStart()
|
||||
} else if insideCreateTriggerStmt {
|
||||
// extend trigger creation statement to include END token after last semicolon
|
||||
if iterator.currentToken.GetTokenType() == sqliteparser.SQLiteLexerEND_ {
|
||||
insideCreateTriggerStmt = false
|
||||
}
|
||||
} else if iterator.currentToken.GetTokenType() == sqliteparser.SQLiteLexerSCOL {
|
||||
// finish current statement (don't forget to consume as we are breaking here)
|
||||
iterator.tokenizer.Consume()
|
||||
break
|
||||
}
|
||||
previousToken = iterator.currentToken
|
||||
iterator.tokenizer.Consume()
|
||||
}
|
||||
lastTokenType := antlr.TokenInvalidType
|
||||
if iterator.currentToken != nil {
|
||||
lastTokenType = iterator.currentToken.GetTokenType()
|
||||
}
|
||||
extraInfo = SplitStatementExtraInfo{
|
||||
IncompleteCreateTriggerStatement: insideCreateTriggerStmt,
|
||||
IncompleteMultilineComment: insideMultilineComment,
|
||||
LastTokenType: lastTokenType,
|
||||
}
|
||||
statement = ""
|
||||
if startPosition != -1 {
|
||||
statement = iterator.tokenizer.source.GetInputStream().GetText(startPosition, previousToken.GetStop())
|
||||
}
|
||||
return statement, extraInfo, iterator.tokenizer.IsEOF() || insideMultilineComment
|
||||
}
|
||||
|
||||
func SplitStatement(statement string) (statements []string, extraInfo SplitStatementExtraInfo) {
|
||||
iterator := CreateStatementIterator(statement)
|
||||
|
||||
statements = make([]string, 0)
|
||||
for {
|
||||
statement, extraInfo, isEOF := iterator.Next()
|
||||
if statement != "" {
|
||||
statements = append(statements, statement)
|
||||
}
|
||||
if isEOF {
|
||||
return statements, extraInfo
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func atCreateTriggerStart(tokenStream *bufferedTokenizer) bool {
|
||||
token1 := tokenStream.Get(0).GetTokenType()
|
||||
token2 := tokenStream.Get(1).GetTokenType()
|
||||
token3 := tokenStream.Get(2).GetTokenType()
|
||||
|
||||
if token1 == sqliteparser.SQLiteLexerCREATE_ && token2 == sqliteparser.SQLiteLexerTRIGGER_ {
|
||||
return true
|
||||
}
|
||||
if token1 == sqliteparser.SQLiteLexerCREATE_ &&
|
||||
(token2 == sqliteparser.SQLiteLexerTEMP_ || token2 == sqliteparser.SQLiteLexerTEMPORARY_) &&
|
||||
token3 == sqliteparser.SQLiteLexerTRIGGER_ {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Note: Only starts for incomplete multiline comments will be detected cause lexer automatically ignores complete
|
||||
// multiline comments
|
||||
func atIncompleteMultilineCommentStart(stream *bufferedTokenizer) bool {
|
||||
token1 := stream.Get(0).GetTokenType()
|
||||
token2 := stream.Get(1).GetTokenType()
|
||||
return token1 == sqliteparser.SQLiteLexerDIV && token2 == sqliteparser.SQLiteLexerSTAR
|
||||
}
|
||||
|
||||
func createStringTokenizer(statement string) *bufferedTokenizer {
|
||||
statementStream := antlr.NewInputStream(statement)
|
||||
|
||||
lexer := sqliteparser.NewSQLiteLexer(statementStream)
|
||||
return createBufferedTokenizer(lexer, 3)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package sqliteparserutils
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkSplitStatement(b *testing.B) {
|
||||
var (
|
||||
mem runtime.MemStats
|
||||
count = 8192
|
||||
)
|
||||
|
||||
hugeStatement := strings.Repeat("INSERT INTO t VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10);", count)
|
||||
for i := 0; i < b.N; i++ {
|
||||
statements, _ := SplitStatement(hugeStatement)
|
||||
if len(statements) != count {
|
||||
b.Fail()
|
||||
}
|
||||
// these intermediate logs can help to estimate memory working set (instead of total allocations)
|
||||
runtime.ReadMemStats(&mem)
|
||||
b.Logf("heap in use: %v", mem.HeapInuse)
|
||||
}
|
||||
}
|
||||
209
go_modules/libsql-client-go/sqliteparserutils/utils_test.go
Normal file
209
go_modules/libsql-client-go/sqliteparserutils/utils_test.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package sqliteparserutils_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
|
||||
"github.com/tursodatabase/libsql-client-go/sqliteparser"
|
||||
"github.com/tursodatabase/libsql-client-go/sqliteparserutils"
|
||||
)
|
||||
|
||||
func generateSplitStatementExtraInfo(lastTokenType int, incompleteCreateTriggerStatement bool, incompleteMultilineComment bool) sqliteparserutils.SplitStatementExtraInfo {
|
||||
return sqliteparserutils.SplitStatementExtraInfo{
|
||||
IncompleteCreateTriggerStatement: incompleteCreateTriggerStatement,
|
||||
IncompleteMultilineComment: incompleteMultilineComment,
|
||||
LastTokenType: lastTokenType,
|
||||
}
|
||||
}
|
||||
|
||||
func generateSimpleSplitStatementExtraInfo(lastTokenType int) sqliteparserutils.SplitStatementExtraInfo {
|
||||
return sqliteparserutils.SplitStatementExtraInfo{
|
||||
LastTokenType: lastTokenType,
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitStatement(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
value string
|
||||
stmts []string
|
||||
extraInfo sqliteparserutils.SplitStatementExtraInfo
|
||||
}{
|
||||
{
|
||||
name: "EmptyStatement",
|
||||
value: "",
|
||||
stmts: []string{},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(antlr.TokenInvalidType),
|
||||
},
|
||||
{
|
||||
name: "OnlySemicolon",
|
||||
value: ";;;;",
|
||||
stmts: []string{},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithoutSemicolon",
|
||||
value: "select 1",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerNUMERIC_LITERAL),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithSemicolon",
|
||||
value: "select 1;",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementEndingWithWhitespaceAndSemicolon",
|
||||
value: "select 1 ;",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "OnlyWithSingleLineComment",
|
||||
value: "-- a simple comment",
|
||||
stmts: []string{},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(antlr.TokenInvalidType),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithSingleLineCommentWithoutSemicolon",
|
||||
value: "select 1; -- a simple comment",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithSingleLineCommentWithSemicolon",
|
||||
value: "select 1; -- ops; a comment with semicolon",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "OnlyMultilineComment",
|
||||
value: "/* a simple comment \n*/",
|
||||
stmts: []string{},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(antlr.TokenInvalidType),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithCompleteMultilineCommentWithoutSemicolon",
|
||||
value: "select 1; /* \na simple comment \n*/",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithCompleteMultilineCommentWithSemicolon",
|
||||
value: "select 1; /* \nops;\n a comment with semicolon \n*/",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "OnlyWithIncompleteMultilineComment",
|
||||
value: "/* a simple comment\n",
|
||||
stmts: []string{},
|
||||
extraInfo: generateSplitStatementExtraInfo(antlr.TokenInvalidType, false, true),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithIncompleteMultilineCommentWithoutSemicolon",
|
||||
value: "select 1; /* \na simple comment\n",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL, false, true),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithoutSemicolonWithIncompleteMultilineCommentWithoutSemicolon",
|
||||
value: "select 1 /* \na simple comment\n",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSplitStatementExtraInfo(sqliteparser.SQLiteLexerNUMERIC_LITERAL, false, true),
|
||||
},
|
||||
{
|
||||
name: "SingleStatementWithIncompleteMultilineCommentWithSemicolon",
|
||||
value: "select 1; /* \na simple comment\n",
|
||||
stmts: []string{"select 1"},
|
||||
extraInfo: generateSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL, false, true),
|
||||
},
|
||||
{
|
||||
name: "MultipleStatementsWithMultilineCommentBetween",
|
||||
value: "select 1; /* \na simple comment;\n*/ select 2;",
|
||||
stmts: []string{"select 1", "select 2"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "MultipleCorrectStatements",
|
||||
value: "select 1; INSERT INTO counter(country, city, value) VALUES(?, ?, 1) ON CONFLICT DO UPDATE SET value = IFNULL(value, 0) + 1 WHERE country = ? AND city = ?; select 2",
|
||||
stmts: []string{"select 1", "INSERT INTO counter(country, city, value) VALUES(?, ?, 1) ON CONFLICT DO UPDATE SET value = IFNULL(value, 0) + 1 WHERE country = ? AND city = ?", "select 2"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerNUMERIC_LITERAL),
|
||||
},
|
||||
{
|
||||
name: "MultipleWrongStatements",
|
||||
value: "select from table; INSERT counter(country, city, value) VALUES(?, ?, 1) ON CONFLICT DO UPDATE SET value = IFNULL(value, 0) + 1 WHERE country = ? AND city = ?; create something",
|
||||
stmts: []string{"select from table", "INSERT counter(country, city, value) VALUES(?, ?, 1) ON CONFLICT DO UPDATE SET value = IFNULL(value, 0) + 1 WHERE country = ? AND city = ?", "create something"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerIDENTIFIER),
|
||||
},
|
||||
{
|
||||
name: "MultipleWrongTokens",
|
||||
value: "sdfasdfigosdfg sadfgsd ggsadgf; sdfasdfasd; 1230kfvcasd; 213 dsf s 0 fs229dt",
|
||||
stmts: []string{"sdfasdfigosdfg sadfgsd ggsadgf", "sdfasdfasd", "1230kfvcasd", "213 dsf s 0 fs229dt"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerIDENTIFIER),
|
||||
},
|
||||
{
|
||||
name: "MultipleSemicolonsBetweenStatements",
|
||||
value: "select 1;;;;;; ;;; ; ; ; ; select 2",
|
||||
stmts: []string{"select 1", "select 2"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerNUMERIC_LITERAL),
|
||||
},
|
||||
{
|
||||
name: "CompleteCreateTriggerStatement",
|
||||
value: "CREATE TRIGGER update_updated_at AFTER UPDATE ON users FOR EACH ROW BEGIN UPDATE users SET updated_at = 0 WHERE id = NEW.id; END",
|
||||
stmts: []string{"CREATE TRIGGER update_updated_at AFTER UPDATE ON users FOR EACH ROW BEGIN UPDATE users SET updated_at = 0 WHERE id = NEW.id; END"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerEND_),
|
||||
},
|
||||
{
|
||||
name: "CompleteCreateTempTriggerStatement",
|
||||
value: "CREATE TEMP TRIGGER update_updated_at AFTER UPDATE ON users FOR EACH ROW BEGIN UPDATE users SET updated_at = 0 WHERE id = NEW.id; END",
|
||||
stmts: []string{"CREATE TEMP TRIGGER update_updated_at AFTER UPDATE ON users FOR EACH ROW BEGIN UPDATE users SET updated_at = 0 WHERE id = NEW.id; END"},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerEND_),
|
||||
},
|
||||
{
|
||||
name: "IncompleteCreateTriggerStatement",
|
||||
value: "CREATE TRIGGER update_updated_at AFTER UPDATE ON users FOR EACH ROW BEGIN UPDATE users SET updated_at = 0 WHERE id = NEW.id;",
|
||||
stmts: []string{"CREATE TRIGGER update_updated_at AFTER UPDATE ON users FOR EACH ROW BEGIN UPDATE users SET updated_at = 0 WHERE id = NEW.id;"},
|
||||
extraInfo: generateSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL, true, false),
|
||||
},
|
||||
{
|
||||
name: "IncompleteCreateTriggerStatementWithIncompleteMultilineComment",
|
||||
value: "CREATE TRIGGER update_updated_at AFTER UPDATE ON users FOR EACH ROW BEGIN UPDATE users SET updated_at = 0 WHERE id = NEW.id; /* this is the trigger;",
|
||||
stmts: []string{"CREATE TRIGGER update_updated_at AFTER UPDATE ON users FOR EACH ROW BEGIN UPDATE users SET updated_at = 0 WHERE id = NEW.id;"},
|
||||
extraInfo: generateSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL, true, true),
|
||||
},
|
||||
{
|
||||
name: "CompleteTransactionStatement",
|
||||
value: "BEGIN TRANSACTION; CREATE TABLE test_table (id INTEGER PRIMARY KEY, value TEXT); INSERT INTO test_table (value) VALUES ('Value 1'); COMMIT;",
|
||||
stmts: []string{"BEGIN TRANSACTION",
|
||||
"CREATE TABLE test_table (id INTEGER PRIMARY KEY, value TEXT)",
|
||||
"INSERT INTO test_table (value) VALUES ('Value 1')",
|
||||
"COMMIT",
|
||||
},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
{
|
||||
name: "IncompleteTransactionStatement",
|
||||
value: "BEGIN TRANSACTION; CREATE TABLE test_table (id INTEGER PRIMARY KEY, value TEXT); INSERT INTO test_table (value) VALUES ('Value 1');",
|
||||
stmts: []string{"BEGIN TRANSACTION",
|
||||
"CREATE TABLE test_table (id INTEGER PRIMARY KEY, value TEXT)",
|
||||
"INSERT INTO test_table (value) VALUES ('Value 1')",
|
||||
},
|
||||
extraInfo: generateSimpleSplitStatementExtraInfo(sqliteparser.SQLiteLexerSCOL),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotStmts, gotExtraInfo := sqliteparserutils.SplitStatement(tt.value)
|
||||
if !reflect.DeepEqual(gotStmts, tt.stmts) {
|
||||
t.Errorf("got %#v, want %#v", gotStmts, tt.stmts)
|
||||
}
|
||||
if !reflect.DeepEqual(gotExtraInfo, tt.extraInfo) {
|
||||
t.Errorf("got %#v, want %#v", gotExtraInfo, tt.extraInfo)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user