From f6119ae5fd4032e00a621a2cc280972390d0cfa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AF=89=E8=AF=B4=E6=99=9A=E7=A7=8B?= <81519560+yeliheng-010@users.noreply.github.com> Date: Thu, 8 Oct 2026 01:22:11 +0800 Subject: [PATCH] =?UTF-8?q?fix(agent):=20=E4=BF=9D=E7=95=99=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=88=86=E6=9E=90=20SQL=20=E5=AD=97=E7=AC=A6=E4=B8=B2?= =?UTF-8?q?=E5=AD=97=E9=9D=A2=E9=87=8F=20(#3812)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/agent/tools/data_analysis.go | 29 +++++-- .../tools/data_analysis_reconcile_test.go | 76 +++++++++++++++++++ 2 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 internal/agent/tools/data_analysis_reconcile_test.go diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.go index 36b8dda4..39447947 100644 --- a/internal/agent/tools/data_analysis.go +++ b/internal/agent/tools/data_analysis.go @@ -16,6 +16,7 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/Tencent/WeKnora/internal/utils" + pg_query "github.com/pganalyze/pg_query_go/v6" ) // DataAnalysisTableName is the only table name the model needs to know when @@ -72,19 +73,33 @@ func reconcileSQLColumnsWithSchema(sqlText string, schema *TableSchema) (string, } } - quotedIdentifierPattern := regexp.MustCompile(`"([^"]+)"`) + // Use the same SQL lexer as validation so quoted text in literals and + // comments is never mistaken for an identifier. Leave invalid SQL to validation. + tokens, err := pg_query.Scan(sqlText) + if err != nil { + return sqlText, nil + } fixes := make([]string, 0) - rewritten := quotedIdentifierPattern.ReplaceAllStringFunc(sqlText, func(token string) string { - name := strings.Trim(token, "\"") + var rewritten strings.Builder + previous := 0 + for _, token := range tokens.Tokens { + raw := sqlText[token.Start:token.End] + if token.Token != pg_query.Token_IDENT || !strings.HasPrefix(raw, `"`) { + continue + } + name := strings.ReplaceAll(raw[1:len(raw)-1], `""`, `"`) canonical, ok := normalizedToCanonical[normalizeIdentifierForMatch(name)] if !ok || canonical == name { - return token + continue } fixes = append(fixes, fmt.Sprintf("%q -> %q", name, canonical)) - return fmt.Sprintf(`"%s"`, canonical) - }) + rewritten.WriteString(sqlText[previous:token.Start]) + fmt.Fprintf(&rewritten, `"%s"`, strings.ReplaceAll(canonical, `"`, `""`)) + previous = int(token.End) + } + rewritten.WriteString(sqlText[previous:]) - return rewritten, fixes + return rewritten.String(), fixes } func buildMissingColumnSuggestion(sqlErr error, schema *TableSchema) string { diff --git a/internal/agent/tools/data_analysis_reconcile_test.go b/internal/agent/tools/data_analysis_reconcile_test.go new file mode 100644 index 00000000..c1dbbe58 --- /dev/null +++ b/internal/agent/tools/data_analysis_reconcile_test.go @@ -0,0 +1,76 @@ +package tools + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReconcileSQLPreservesNonIdentifiers(t *testing.T) { + schema := &TableSchema{Columns: []ColumnInfo{{Name: "Order Status"}}} + for _, literal := range []string{ + `'"orderstatus"'`, + `'customer''s "orderstatus"'`, + `$$"orderstatus"$$`, + `$value$"orderstatus"$value$`, + `E'customer\'s "orderstatus"'`, + } { + t.Run(literal, func(t *testing.T) { + query := `SELECT "orderstatus" FROM dataset WHERE "orderstatus" = ` + literal + want := `SELECT "Order Status" FROM dataset WHERE "Order Status" = ` + literal + got, fixes := reconcileSQLColumnsWithSchema(query, schema) + require.Equal(t, want, got) + require.Len(t, fixes, 2) + }) + } + for _, comment := range []string{ + "-- \"orderstatus\"\n", + `/* "orderstatus" */`, + `/* outer /* "orderstatus" */ end */`, + } { + t.Run(comment, func(t *testing.T) { + query := `SELECT ` + comment + ` "orderstatus" FROM dataset` + got, fixes := reconcileSQLColumnsWithSchema(query, schema) + require.Equal(t, `SELECT `+comment+` "Order Status" FROM dataset`, got) + require.Len(t, fixes, 1) + }) + } +} + +func TestReconcileSQLPreservesQuotedIdentifierEscapes(t *testing.T) { + schema := &TableSchema{Columns: []ColumnInfo{{Name: `Order "Status"`}, {Name: "订单 状态"}}} + query := `SELECT "order""status""","订单状态" FROM dataset` + got, fixes := reconcileSQLColumnsWithSchema(query, schema) + require.Equal(t, `SELECT "Order ""Status""","订单 状态" FROM dataset`, got) + require.Len(t, fixes, 2) +} + +func TestReconcileSQLLeavesUnscannableInputUnchanged(t *testing.T) { + schema := &TableSchema{Columns: []ColumnInfo{{Name: "Order Status"}}} + for _, query := range []string{ + `SELECT "orderstatus" FROM dataset WHERE note = 'unfinished`, + `SELECT "orderstatus`, + } { + got, fixes := reconcileSQLColumnsWithSchema(query, schema) + require.Equal(t, query, got) + require.Empty(t, fixes) + } +} + +func TestReconcileSQLPreservesDuckDBFilterResults(t *testing.T) { + db := newPlainDuckDB(t) + tool := &DataAnalysisTool{BaseTool: dataAnalysisTool, db: db, sessionID: "literal-filter"} + ctx := context.Background() + t.Cleanup(func() { tool.Cleanup(ctx) }) + path := writeCSV(t, "orders.csv", "Order Status,note", + `paid,"""orderstatus"""`, `unpaid,"""Order Status"""`) + schema, err := tool.LoadFromCSV(ctx, path, "k_literal_filter") + require.NoError(t, err) + query := `SELECT "orderstatus" AS status FROM dataset WHERE note = '"orderstatus"'` + rewritten, _ := reconcileSQLColumnsWithSchema(query, schema) + require.NoError(t, validateDataAnalysisSQL(rewritten, schema)) + rows, err := tool.executeSingleQuery(ctx, rewritten, schema.TableName) + require.NoError(t, err) + require.Equal(t, []map[string]string{{"status": "paid"}}, rows) +}