fix(tools): reject legacy binary .xls with an actionable error (#4374)
* fix(tools): reject legacy binary .xls with an actionable error DuckDB's excel extension only reads the ZIP-based OOXML container (.xlsx/.xlsm). A genuine binary BIFF .xls therefore fails deep inside read_xlsx with "No xl/workbook.xml found in xlsx file", which neither names the real cause nor says what to do; the multi-sheet .xls in #4239 surfaced exactly this way and left the agent with an empty table. Sniff the OLE2/CFB signature by content (not by extension, so a renamed .xlsx keeps working) and fail up front with a message that names the conversion to perform. The tool description is updated so the model is told the same thing before it calls the tool. Refs #4239 * fix(tools): wrap the legacy-xls warning under the line limit --------- Co-authored-by: Jay0130-a <Jay0130-a@users.noreply.github.com>main
parent
f0adfac334
commit
e07a190219
|
|
@ -1,6 +1,7 @@
|
|||
package tools
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
|
|
@ -36,8 +37,9 @@ var dataAnalysisTool = BaseTool{
|
|||
"the single table \"" + DataAnalysisTableName + "\"; write SQL against that table name and never put " +
|
||||
"the document ID inside the SQL. For Excel files with multiple sheets, non-empty sheets are loaded into " +
|
||||
"the same table and the source sheet name is exposed as a '__sheet_name' column so you can " +
|
||||
"filter/aggregate per sheet. If the user's question requires data statistics, convert the " +
|
||||
"question into SQL and execute it.",
|
||||
"filter/aggregate per sheet. Only the .xlsx container is readable: a legacy binary .xls must be " +
|
||||
"re-saved as .xlsx (or exported as .csv) before it can be analysed. If the user's question requires " +
|
||||
"data statistics, convert the question into SQL and execute it.",
|
||||
schema: utils.GenerateSchema[DataAnalysisInput](),
|
||||
}
|
||||
|
||||
|
|
@ -45,6 +47,48 @@ var dataAnalysisTool = BaseTool{
|
|||
// which Excel sheet a row came from when multiple sheets are unioned together.
|
||||
const excelSheetNameColumn = "__sheet_name"
|
||||
|
||||
// legacyXLSMagic is the OLE2/CFB compound-document signature that binary BIFF
|
||||
// .xls workbooks start with ("D0 CF 11 E0 A1 B1 1A E1").
|
||||
//
|
||||
// DuckDB's excel extension only reads the ZIP-based OOXML container
|
||||
// (.xlsx/.xlsm), so a genuine .xls has to be rejected up front: handed to
|
||||
// read_xlsx it fails with the opaque "No xl/workbook.xml found in xlsx file",
|
||||
// which says nothing about the real cause or what to do about it.
|
||||
var legacyXLSMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
|
||||
|
||||
// hasLegacyXLSContainer reports whether path holds an OLE2 compound document,
|
||||
// i.e. a binary BIFF .xls workbook.
|
||||
//
|
||||
// The sniff is by content, not by extension: a .xls that is really a renamed
|
||||
// .xlsx is a ZIP archive and returns false, so it keeps loading through the
|
||||
// normal read_xlsx path. Unreadable or truncated files also return false and
|
||||
// are left to DuckDB to report.
|
||||
func hasLegacyXLSContainer(path string) bool {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
head := make([]byte, len(legacyXLSMagic))
|
||||
if _, err := io.ReadFull(f, head); err != nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Equal(head, legacyXLSMagic)
|
||||
}
|
||||
|
||||
// errLegacyBinaryXLS is the actionable failure returned for a binary .xls
|
||||
// workbook. The data analysis tool cannot read it, and silently reporting an
|
||||
// empty table (or leaking DuckDB's xlsx-internals error) leaves the model
|
||||
// guessing; naming the conversion to perform is what the caller can act on.
|
||||
func errLegacyBinaryXLS() error {
|
||||
return fmt.Errorf(
|
||||
"legacy binary .xls workbooks are not supported by data analysis: " +
|
||||
"only the ZIP-based .xlsx format can be read; " +
|
||||
"re-save the workbook as .xlsx (or export the sheet as .csv) and upload it again",
|
||||
)
|
||||
}
|
||||
|
||||
// sqlSingleQuoteEscape escapes single quotes in a string so it can be safely
|
||||
// embedded inside a single-quoted SQL literal.
|
||||
func sqlSingleQuoteEscape(s string) string {
|
||||
|
|
@ -547,9 +591,23 @@ func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tab
|
|||
//
|
||||
// Note: requires the DuckDB 'excel' extension (for read_xlsx) and the
|
||||
// 'spatial' extension (for st_read_meta used to enumerate sheets).
|
||||
//
|
||||
// Binary BIFF .xls workbooks are rejected with an actionable error before
|
||||
// DuckDB is touched: the excel extension only reads the ZIP-based OOXML
|
||||
// container, so a real .xls would otherwise surface as "No xl/workbook.xml
|
||||
// found in xlsx file".
|
||||
func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, tableName string) (*TableSchema, error) {
|
||||
logger.Infof(ctx, "[Tool][DataAnalysis] Loading Excel file '%s' into table '%s' for session %s", filename, tableName, t.sessionID)
|
||||
|
||||
if hasLegacyXLSContainer(filename) {
|
||||
logger.Warnf(ctx,
|
||||
"[Tool][DataAnalysis] Rejecting legacy binary .xls workbook '%s' (session=%s): "+
|
||||
"DuckDB can only read the ZIP-based .xlsx container",
|
||||
filename, t.sessionID,
|
||||
)
|
||||
return nil, errLegacyBinaryXLS()
|
||||
}
|
||||
|
||||
if !t.loadedTables[tableName] {
|
||||
t.recordCreatedTable(tableName)
|
||||
sheetNames, enumErr := t.listExcelSheets(ctx, filename)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ole2Header is the compound-document signature a binary BIFF .xls starts with.
|
||||
var ole2Header = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
|
||||
|
||||
// writeLegacyXLS writes a file that looks like a binary BIFF .xls: the OLE2
|
||||
// signature followed by opaque payload bytes. DuckDB's read_xlsx cannot open
|
||||
// this (there is no xl/workbook.xml inside), which is exactly the case #4239
|
||||
// reports.
|
||||
func writeLegacyXLS(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
payload := append([]byte{}, ole2Header...)
|
||||
payload = append(payload, make([]byte, 512)...)
|
||||
if err := os.WriteFile(path, payload, 0o600); err != nil {
|
||||
t.Fatalf("write legacy xls fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasLegacyXLSContainerDetectsOLE2Workbook(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "legacy.xls")
|
||||
writeLegacyXLS(t, path)
|
||||
|
||||
if !hasLegacyXLSContainer(path) {
|
||||
t.Fatalf("expected %s to be detected as a binary BIFF workbook", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasLegacyXLSContainerIgnoresZipBasedWorkbooks(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "modern.xlsx")
|
||||
writeWorkbook(t, path, map[string][][]any{
|
||||
"Data": {{"id", "amount"}, {1, 100}},
|
||||
}, []string{"Data"})
|
||||
|
||||
// A .xls that is really a renamed .xlsx is a ZIP archive and must keep
|
||||
// loading through the normal read_xlsx path.
|
||||
if hasLegacyXLSContainer(path) {
|
||||
t.Fatalf("expected the ZIP-based workbook %s not to be flagged as legacy", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasLegacyXLSContainerLeavesShortAndMissingFilesAlone(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
empty := filepath.Join(dir, "empty.xls")
|
||||
if err := os.WriteFile(empty, nil, 0o600); err != nil {
|
||||
t.Fatalf("write empty fixture: %v", err)
|
||||
}
|
||||
if hasLegacyXLSContainer(empty) {
|
||||
t.Fatalf("an empty file is not a legacy workbook")
|
||||
}
|
||||
|
||||
if hasLegacyXLSContainer(filepath.Join(dir, "missing.xls")) {
|
||||
t.Fatalf("a missing file must not be reported as a legacy workbook")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromExcelRejectsLegacyBinaryXLSWithActionableError(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "legacy.xls")
|
||||
writeLegacyXLS(t, path)
|
||||
|
||||
// No DuckDB handle: the rejection has to happen before the tool touches
|
||||
// the database, otherwise a nil db would panic instead of failing cleanly.
|
||||
tool := &DataAnalysisTool{sessionID: "legacy-xls"}
|
||||
|
||||
_, err := tool.LoadFromExcel(context.Background(), path, "dataset")
|
||||
if err == nil {
|
||||
t.Fatal("expected a binary .xls workbook to be rejected")
|
||||
}
|
||||
|
||||
message := err.Error()
|
||||
for _, want := range []string{"legacy binary .xls", ".xlsx", ".csv"} {
|
||||
if !strings.Contains(message, want) {
|
||||
t.Errorf("error %q does not mention %q", message, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataAnalysisToolDescriptionNamesTheXLSConversion(t *testing.T) {
|
||||
description := dataAnalysisTool.description
|
||||
if !strings.Contains(description, ".xlsx") || !strings.Contains(description, "binary .xls") {
|
||||
t.Fatalf("tool description must tell the model how to handle a legacy .xls, got: %s", description)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue