diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.go index fab508a3..374847d8 100644 --- a/internal/agent/tools/data_analysis.go +++ b/internal/agent/tools/data_analysis.go @@ -1,6 +1,7 @@ package tools import ( + "bytes" "context" "database/sql" "encoding/json" @@ -36,8 +37,9 @@ var dataAnalysisTool = BaseTool{ "the single table \"" + DataAnalysisTableName + "\"; write SQL against that table name and never put " + "the document ID inside the SQL. For Excel files with multiple sheets, non-empty sheets are loaded into " + "the same table and the source sheet name is exposed as a '__sheet_name' column so you can " + - "filter/aggregate per sheet. If the user's question requires data statistics, convert the " + - "question into SQL and execute it.", + "filter/aggregate per sheet. Only the .xlsx container is readable: a legacy binary .xls must be " + + "re-saved as .xlsx (or exported as .csv) before it can be analysed. If the user's question requires " + + "data statistics, convert the question into SQL and execute it.", schema: utils.GenerateSchema[DataAnalysisInput](), } @@ -45,6 +47,48 @@ var dataAnalysisTool = BaseTool{ // which Excel sheet a row came from when multiple sheets are unioned together. const excelSheetNameColumn = "__sheet_name" +// legacyXLSMagic is the OLE2/CFB compound-document signature that binary BIFF +// .xls workbooks start with ("D0 CF 11 E0 A1 B1 1A E1"). +// +// DuckDB's excel extension only reads the ZIP-based OOXML container +// (.xlsx/.xlsm), so a genuine .xls has to be rejected up front: handed to +// read_xlsx it fails with the opaque "No xl/workbook.xml found in xlsx file", +// which says nothing about the real cause or what to do about it. +var legacyXLSMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1} + +// hasLegacyXLSContainer reports whether path holds an OLE2 compound document, +// i.e. a binary BIFF .xls workbook. +// +// The sniff is by content, not by extension: a .xls that is really a renamed +// .xlsx is a ZIP archive and returns false, so it keeps loading through the +// normal read_xlsx path. Unreadable or truncated files also return false and +// are left to DuckDB to report. +func hasLegacyXLSContainer(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer func() { _ = f.Close() }() + + head := make([]byte, len(legacyXLSMagic)) + if _, err := io.ReadFull(f, head); err != nil { + return false + } + return bytes.Equal(head, legacyXLSMagic) +} + +// errLegacyBinaryXLS is the actionable failure returned for a binary .xls +// workbook. The data analysis tool cannot read it, and silently reporting an +// empty table (or leaking DuckDB's xlsx-internals error) leaves the model +// guessing; naming the conversion to perform is what the caller can act on. +func errLegacyBinaryXLS() error { + return fmt.Errorf( + "legacy binary .xls workbooks are not supported by data analysis: " + + "only the ZIP-based .xlsx format can be read; " + + "re-save the workbook as .xlsx (or export the sheet as .csv) and upload it again", + ) +} + // sqlSingleQuoteEscape escapes single quotes in a string so it can be safely // embedded inside a single-quoted SQL literal. func sqlSingleQuoteEscape(s string) string { @@ -547,9 +591,23 @@ func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tab // // Note: requires the DuckDB 'excel' extension (for read_xlsx) and the // 'spatial' extension (for st_read_meta used to enumerate sheets). +// +// Binary BIFF .xls workbooks are rejected with an actionable error before +// DuckDB is touched: the excel extension only reads the ZIP-based OOXML +// container, so a real .xls would otherwise surface as "No xl/workbook.xml +// found in xlsx file". func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, tableName string) (*TableSchema, error) { logger.Infof(ctx, "[Tool][DataAnalysis] Loading Excel file '%s' into table '%s' for session %s", filename, tableName, t.sessionID) + if hasLegacyXLSContainer(filename) { + logger.Warnf(ctx, + "[Tool][DataAnalysis] Rejecting legacy binary .xls workbook '%s' (session=%s): "+ + "DuckDB can only read the ZIP-based .xlsx container", + filename, t.sessionID, + ) + return nil, errLegacyBinaryXLS() + } + if !t.loadedTables[tableName] { t.recordCreatedTable(tableName) sheetNames, enumErr := t.listExcelSheets(ctx, filename) diff --git a/internal/agent/tools/data_analysis_legacy_xls_test.go b/internal/agent/tools/data_analysis_legacy_xls_test.go new file mode 100644 index 00000000..3868d0dc --- /dev/null +++ b/internal/agent/tools/data_analysis_legacy_xls_test.go @@ -0,0 +1,91 @@ +package tools + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// ole2Header is the compound-document signature a binary BIFF .xls starts with. +var ole2Header = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1} + +// writeLegacyXLS writes a file that looks like a binary BIFF .xls: the OLE2 +// signature followed by opaque payload bytes. DuckDB's read_xlsx cannot open +// this (there is no xl/workbook.xml inside), which is exactly the case #4239 +// reports. +func writeLegacyXLS(t *testing.T, path string) { + t.Helper() + payload := append([]byte{}, ole2Header...) + payload = append(payload, make([]byte, 512)...) + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatalf("write legacy xls fixture: %v", err) + } +} + +func TestHasLegacyXLSContainerDetectsOLE2Workbook(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.xls") + writeLegacyXLS(t, path) + + if !hasLegacyXLSContainer(path) { + t.Fatalf("expected %s to be detected as a binary BIFF workbook", path) + } +} + +func TestHasLegacyXLSContainerIgnoresZipBasedWorkbooks(t *testing.T) { + path := filepath.Join(t.TempDir(), "modern.xlsx") + writeWorkbook(t, path, map[string][][]any{ + "Data": {{"id", "amount"}, {1, 100}}, + }, []string{"Data"}) + + // A .xls that is really a renamed .xlsx is a ZIP archive and must keep + // loading through the normal read_xlsx path. + if hasLegacyXLSContainer(path) { + t.Fatalf("expected the ZIP-based workbook %s not to be flagged as legacy", path) + } +} + +func TestHasLegacyXLSContainerLeavesShortAndMissingFilesAlone(t *testing.T) { + dir := t.TempDir() + + empty := filepath.Join(dir, "empty.xls") + if err := os.WriteFile(empty, nil, 0o600); err != nil { + t.Fatalf("write empty fixture: %v", err) + } + if hasLegacyXLSContainer(empty) { + t.Fatalf("an empty file is not a legacy workbook") + } + + if hasLegacyXLSContainer(filepath.Join(dir, "missing.xls")) { + t.Fatalf("a missing file must not be reported as a legacy workbook") + } +} + +func TestLoadFromExcelRejectsLegacyBinaryXLSWithActionableError(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.xls") + writeLegacyXLS(t, path) + + // No DuckDB handle: the rejection has to happen before the tool touches + // the database, otherwise a nil db would panic instead of failing cleanly. + tool := &DataAnalysisTool{sessionID: "legacy-xls"} + + _, err := tool.LoadFromExcel(context.Background(), path, "dataset") + if err == nil { + t.Fatal("expected a binary .xls workbook to be rejected") + } + + message := err.Error() + for _, want := range []string{"legacy binary .xls", ".xlsx", ".csv"} { + if !strings.Contains(message, want) { + t.Errorf("error %q does not mention %q", message, want) + } + } +} + +func TestDataAnalysisToolDescriptionNamesTheXLSConversion(t *testing.T) { + description := dataAnalysisTool.description + if !strings.Contains(description, ".xlsx") || !strings.Contains(description, "binary .xls") { + t.Fatalf("tool description must tell the model how to handle a legacy .xls, got: %s", description) + } +}