diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index 843ca46a..b6c45e05 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -24,8 +24,7 @@ on: - "THIRD_PARTY_NOTICES.md" - "licenses/**" - "internal/textconv/data/**" - - "scripts/*licenses.sh" - - "scripts/check-license-bundle.sh" + - "scripts/*license*.sh" - "cmd/desktop/build/windows/installer/project.nsi" - ".github/workflows/app.yml" pull_request: @@ -47,8 +46,7 @@ on: - "THIRD_PARTY_NOTICES.md" - "licenses/**" - "internal/textconv/data/**" - - "scripts/*licenses.sh" - - "scripts/check-license-bundle.sh" + - "scripts/*license*.sh" - "cmd/desktop/build/windows/installer/project.nsi" - ".github/workflows/app.yml" workflow_dispatch: @@ -87,7 +85,9 @@ jobs: run: go mod download - name: Verify third-party notices and source bundle - run: bash ./scripts/check-license-bundle.sh + run: | + bash ./scripts/check-license-bundle.sh + bash ./scripts/test-license-bundle.sh - name: Check formatting shell: bash diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 50d57a67..0084b169 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -3,7 +3,9 @@ This file supplements the third-party notices in `LICENSE`. The MIT license for WeKnora's own code does not replace the licenses of third-party components. Keep this file, `LICENSE`, and the `licenses/` directory with redistributed -backend and desktop packages. Container distributions include them in `/app`; +backend and desktop packages. `scripts/copy-licenses.sh` adds checksum-verified +source archives at packaging time; they are not stored in this Git repository. +Container distributions include the complete bundle in `/app`; macOS applications include them in `Contents/Resources`. Windows installers place them next to the installed executable. @@ -16,9 +18,8 @@ Windows installers place them next to the installed executable. - Copyright: The Go-MySQL-Driver Authors; the original per-file notices and `AUTHORS` are preserved in the accompanying source archive. - Modifications by WeKnora: none. Dialer configuration is in separate WeKnora files. -- Corresponding source, available under MPL-2.0: - [`licenses/sources/mysql-v1.10.0.zip`](licenses/sources/mysql-v1.10.0.zip). - The same version is available from the +- Corresponding source, available under MPL-2.0, is included in binary releases + as `licenses/sources/mysql-v1.10.0.zip`. Source repository users can obtain it from the [Go module proxy](https://proxy.golang.org/github.com/go-sql-driver/mysql/@v/v1.10.0.zip) and [upstream repository](https://github.com/go-sql-driver/mysql/tree/v1.10.0). @@ -33,9 +34,8 @@ Windows installers place them next to the installed executable. - Attribution: the go-m1cpu project and its contributors. The complete original source and notices are preserved in the accompanying archive. - Modifications by WeKnora: none. -- Corresponding source, available under MPL-2.0: - [`licenses/sources/go-m1cpu-v0.1.6.zip`](licenses/sources/go-m1cpu-v0.1.6.zip). - The same version is available from the +- Corresponding source, available under MPL-2.0, is included in binary releases + as `licenses/sources/go-m1cpu-v0.1.6.zip`. Source repository users can obtain it from the [Go module proxy](https://proxy.golang.org/github.com/shoenig/go-m1cpu/@v/v0.1.6.zip) and [upstream repository](https://github.com/shoenig/go-m1cpu/tree/v0.1.6). @@ -67,8 +67,13 @@ the tool itself, retain its license and provide its corresponding source too. ## Maintaining this bundle -When upgrading either MPL module, update the version, license, and matching -source archive together. Archives are unmodified Go module proxy ZIPs, including -their original copyright notices. Verify them with `go mod download -json` and -the module's `go.sum` entry before updating. `scripts/check-license-bundle.sh` -checks the pins and packaging inputs without downloading dependencies. +When upgrading either MPL module, update the version, license, and SHA-256 pin in +`licenses/sources/modules.tsv` together. Archives are unmodified Go module proxy +ZIPs, including their original copyright notices. Verify them with +`go mod download -json` and the module's `go.sum` entry before updating. +`scripts/check-license-bundle.sh` checks the pins and notices without downloading +dependencies. `scripts/copy-licenses.sh DESTINATION` obtains the pinned archives +through Go's configured `GOPROXY` and module cache, verifies their SHA-256 hashes, +and includes them in the destination bundle. Offline packaging requires those +module archives to be present in the Go module cache beforehand. To verify a +packaged source directory, run `scripts/check-license-bundle.sh PATH/TO/licenses/sources`. diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 44af7738..e79cb0c0 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -45,7 +45,7 @@ RUN --mount=type=cache,target=/go/pkg/mod go mod download COPY cmd/download cmd/download RUN go run cmd/download/duckdb/duckdb.go COPY . . -RUN bash ./scripts/check-license-bundle.sh +RUN --mount=type=cache,target=/go/pkg/mod bash ./scripts/copy-licenses.sh /license-bundle # Get version and commit info for build injection ARG VERSION_ARG @@ -139,8 +139,7 @@ COPY --from=builder /app/migrations ./migrations COPY --from=builder /app/dataset/samples ./dataset/samples COPY --from=builder /root/.duckdb /home/appuser/.duckdb COPY --from=builder /app/WeKnora . -COPY LICENSE THIRD_PARTY_NOTICES.md ./ -COPY licenses ./licenses +COPY --from=builder /license-bundle/ ./ # Copy and make entrypoint script executable COPY --from=builder /app/scripts/docker-entrypoint.sh ./scripts/docker-entrypoint.sh diff --git a/docreader/utils/__init__.py b/docreader/utils/__init__.py index f1c77023..e69de29b 100644 --- a/docreader/utils/__init__.py +++ b/docreader/utils/__init__.py @@ -1,83 +0,0 @@ -# -# Copyright 2024 The InfiniFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import re -import logging - -# 配置日志 -logger = logging.getLogger(__name__) - - -def singleton(cls, *args, **kw): - instances = {} - - def _singleton(): - key = str(cls) + str(os.getpid()) - if key not in instances: - logger.info(f"Creating new singleton instance with key: {key}") - instances[key] = cls(*args, **kw) - else: - logger.info(f"Returning existing singleton instance with key: {key}") - return instances[key] - - return _singleton - - -def rmSpace(txt): - logger.info(f"Removing spaces from text of length: {len(txt)}") - txt = re.sub(r"([^a-z0-9.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE) - return re.sub(r"([^ ]) +([^a-z0-9.,\(<])", r"\1\2", txt, flags=re.IGNORECASE) - - -def findMaxDt(fnm): - m = "1970-01-01 00:00:00" - logger.info(f"Finding maximum date in file: {fnm}") - try: - with open(fnm, "r") as f: - while True: - l = f.readline() - if not l: - break - l = l.strip("\n") - if l == "nan": - continue - if l > m: - m = l - logger.info(f"Maximum date found: {m}") - except Exception as e: - logger.error(f"Error reading file {fnm} for max date: {str(e)}") - return m - - -def findMaxTm(fnm): - m = 0 - logger.info(f"Finding maximum time in file: {fnm}") - try: - with open(fnm, "r") as f: - while True: - l = f.readline() - if not l: - break - l = l.strip("\n") - if l == "nan": - continue - if int(l) > m: - m = int(l) - logger.info(f"Maximum time found: {m}") - except Exception as e: - logger.error(f"Error reading file {fnm} for max time: {str(e)}") - return m diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 342ba7e3..9e82f011 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -34,9 +34,8 @@ ENV VITE_IS_DOCKER=true # Vite production builds of this SPA routinely exceed Node's default heap. ENV NODE_OPTIONS="--max-old-space-size=${NODE_MAX_OLD_SPACE_SIZE}" -# package-lock references file:packages/xlsx-*.tgz — copy before npm ci. +# The lockfile pins the remote SheetJS archive and its integrity hash. COPY package.json package-lock.json ./ -COPY packages ./packages RUN if [ -n "$NPM_REGISTRY" ]; then \ npm ci --registry="$NPM_REGISTRY"; \ else \ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 973152eb..0b30ad81 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -36,7 +36,7 @@ "vue-router": "^4.5.0", "vue-virtual-scroller": "^2.0.0-beta.8", "webpack": "^5.106.2", - "xlsx": "file:./packages/xlsx-0.20.2.tgz" + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" }, "devDependencies": { "@tsconfig/node22": "^22.0.5", @@ -5302,7 +5302,7 @@ }, "node_modules/xlsx": { "version": "0.20.2", - "resolved": "file:packages/xlsx-0.20.2.tgz", + "resolved": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz", "integrity": "sha512-+nKZ39+nvK7Qq6i0PvWWRA4j/EkfWOtkP/YhMtupm+lJIiHxUrgTr1CcKv1nBk1rHtkRRQ3O2+Ih/q/sA+FXZA==", "license": "Apache-2.0", "bin": { diff --git a/frontend/package.json b/frontend/package.json index a0e45a58..a4c9e19e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -44,7 +44,7 @@ "vue-router": "^4.5.0", "vue-virtual-scroller": "^2.0.0-beta.8", "webpack": "^5.106.2", - "xlsx": "file:./packages/xlsx-0.20.2.tgz" + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" }, "devDependencies": { "@tsconfig/node22": "^22.0.5", diff --git a/frontend/packages/.gitkeep b/frontend/packages/.gitkeep deleted file mode 100644 index 33e963e4..00000000 --- a/frontend/packages/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# 临时创建的 packages 目录 diff --git a/frontend/packages/xlsx-0.20.2.tgz b/frontend/packages/xlsx-0.20.2.tgz deleted file mode 100644 index d3ee3a23..00000000 Binary files a/frontend/packages/xlsx-0.20.2.tgz and /dev/null differ diff --git a/licenses/sources/SHA256SUMS b/licenses/sources/SHA256SUMS deleted file mode 100644 index 31b69a02..00000000 --- a/licenses/sources/SHA256SUMS +++ /dev/null @@ -1,2 +0,0 @@ -0ceab2ec73ef7d1291bd7663dd39203ee7037ee9dccb6fc3381ad819bd8550d1 go-m1cpu-v0.1.6.zip -dc93f5770556406e82bf750a980d2316f882a19d883a3689eadb820708c2b651 mysql-v1.10.0.zip diff --git a/licenses/sources/go-m1cpu-v0.1.6.zip b/licenses/sources/go-m1cpu-v0.1.6.zip deleted file mode 100644 index 15727b25..00000000 Binary files a/licenses/sources/go-m1cpu-v0.1.6.zip and /dev/null differ diff --git a/licenses/sources/modules.tsv b/licenses/sources/modules.tsv new file mode 100644 index 00000000..638cc01a --- /dev/null +++ b/licenses/sources/modules.tsv @@ -0,0 +1,2 @@ +github.com/go-sql-driver/mysql v1.10.0 dc93f5770556406e82bf750a980d2316f882a19d883a3689eadb820708c2b651 +github.com/shoenig/go-m1cpu v0.1.6 0ceab2ec73ef7d1291bd7663dd39203ee7037ee9dccb6fc3381ad819bd8550d1 diff --git a/licenses/sources/mysql-v1.10.0.zip b/licenses/sources/mysql-v1.10.0.zip deleted file mode 100644 index a7ec58d1..00000000 Binary files a/licenses/sources/mysql-v1.10.0.zip and /dev/null differ diff --git a/scripts/check-license-bundle.sh b/scripts/check-license-bundle.sh index bd3637e4..fad9d1d7 100755 --- a/scripts/check-license-bundle.sh +++ b/scripts/check-license-bundle.sh @@ -2,6 +2,10 @@ set -euo pipefail license_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source_dir="${1:-}" +if [ -n "${source_dir}" ]; then + source_dir="$(cd "${source_dir}" && pwd)" +fi cd "${license_root}" # A retained build-graph dependency would bring the GPL converter back even if @@ -11,24 +15,29 @@ if grep -Eq 'github.com/(longbridgeapp/opencc|liuzl/(da|cedar-go)|adamzy/cedar-g exit 1 fi -for module_pin in 'github.com/go-sql-driver/mysql v1.10.0' 'github.com/shoenig/go-m1cpu v0.1.6'; do - read -r module_name module_version <<< "${module_pin}" +test -s licenses/sources/modules.tsv +while read -r module_name module_version checksum; do + [[ "${checksum}" =~ ^[0-9a-f]{64}$ ]] || { echo "Invalid source checksum for ${module_name}" >&2; exit 1; } actual_version="$(awk -v name="${module_name}" '$1 == name { print $2 }' go.mod)" if [ "${actual_version}" != "${module_version}" ]; then - echo "Update the license/source bundle for ${module_name}: expected ${module_version}, got ${actual_version}" >&2 + echo "Update the source manifest for ${module_name}: expected ${module_version}, got ${actual_version}" >&2 exit 1 fi -done + if [ -n "${source_dir}" ]; then + archive="${module_name##*/}-${module_version}.zip" + ( + cd "${source_dir}" + if command -v sha256sum >/dev/null 2>&1; then + printf '%s %s\n' "${checksum}" "${archive}" | sha256sum -c - + else + printf '%s %s\n' "${checksum}" "${archive}" | shasum -a 256 -c - + fi + ) + fi +done < licenses/sources/modules.tsv for file in LICENSE THIRD_PARTY_NOTICES.md licenses/OpenCC-Apache-2.0.txt \ licenses/go-sql-driver-mysql-MPL-2.0.txt licenses/go-m1cpu-MPL-2.0.txt \ licenses/Wails-MIT.txt; do test -s "${file}" done - -cd licenses/sources -if command -v sha256sum >/dev/null 2>&1; then - sha256sum -c SHA256SUMS -else - shasum -a 256 -c SHA256SUMS -fi diff --git a/scripts/copy-licenses.sh b/scripts/copy-licenses.sh index ddfe0328..2d37b6fe 100755 --- a/scripts/copy-licenses.sh +++ b/scripts/copy-licenses.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -# Include this directory unchanged alongside every backend/desktop distribution. +# Assemble notices and verified corresponding source for backend/desktop releases. if [ "$#" -ne 1 ] || [ -z "$1" ]; then echo "Usage: $0 DESTINATION_DIRECTORY" >&2 exit 1 @@ -10,6 +10,20 @@ fi license_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" license_dest="$1" bash "${license_root}/scripts/check-license-bundle.sh" + +# Use Go's module cache and configured GOPROXY, including offline cache hits. +# Archives are release inputs, not files vendored in the source repository. +source_stage="$(mktemp -d)" +trap 'rm -rf "${source_stage}"' EXIT +module_cache="$(cd "${license_root}" && go env GOMODCACHE)" +while read -r module_name module_version _checksum; do + (cd "${license_root}" && go mod download "${module_name}@${module_version}") + cp "${module_cache}/cache/download/${module_name}/@v/${module_version}.zip" \ + "${source_stage}/${module_name##*/}-${module_version}.zip" +done < "${license_root}/licenses/sources/modules.tsv" +bash "${license_root}/scripts/check-license-bundle.sh" "${source_stage}" + mkdir -p "${license_dest}/licenses" cp "${license_root}/LICENSE" "${license_root}/THIRD_PARTY_NOTICES.md" "${license_dest}/" cp -R "${license_root}/licenses/." "${license_dest}/licenses/" +cp "${source_stage}/"*.zip "${license_dest}/licenses/sources/" diff --git a/scripts/test-license-bundle.sh b/scripts/test-license-bundle.sh new file mode 100644 index 00000000..4eec82d0 --- /dev/null +++ b/scripts/test-license-bundle.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Exercise packaging without network access or a populated developer module cache. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fixture="$(mktemp -d)" +trap 'rm -rf "${fixture}"' EXIT +mkdir -p "${fixture}/scripts" "${fixture}/bin" "${fixture}/licenses/sources" +cp "${repo_root}/scripts/"{check-license-bundle,copy-licenses}.sh "${fixture}/scripts/" +cp "${repo_root}/LICENSE" "${repo_root}/THIRD_PARTY_NOTICES.md" "${fixture}/" +cp "${repo_root}/licenses/"*.txt "${fixture}/licenses/" +touch "${fixture}/go.sum" + +checksum() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +# Supply two deterministic stand-ins for downloaded archives. This test verifies +# the packager's integrity gate, not Go's module download implementation. +export LICENSE_TEST_CACHE="${fixture}/module cache" +for pin in 'github.com/go-sql-driver/mysql v1.10.0' 'github.com/shoenig/go-m1cpu v0.1.6'; do + read -r module version <<< "${pin}" + cache_dir="${LICENSE_TEST_CACHE}/cache/download/${module}/@v" + mkdir -p "${cache_dir}" + printf 'source for %s\n' "${pin}" > "${cache_dir}/${version}.zip" + printf '%s %s %s\n' "${module}" "${version}" "$(checksum "${cache_dir}/${version}.zip")" \ + >> "${fixture}/licenses/sources/modules.tsv" + printf '%s\n' "${pin}" >> "${fixture}/go.mod" +done +cat > "${fixture}/bin/go" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "$*" in + 'env GOMODCACHE') printf '%s\n' "${LICENSE_TEST_CACHE}" ;; + 'mod download '*) exit "${LICENSE_TEST_DOWNLOAD_FAILURE:-0}" ;; + *) echo "Unexpected go invocation: $*" >&2; exit 1 ;; +esac +EOF +chmod +x "${fixture}/bin/go" +export PATH="${fixture}/bin:${PATH}" + +bash "${fixture}/scripts/check-license-bundle.sh" +bash "${fixture}/scripts/copy-licenses.sh" "${fixture}/release with spaces" +for pin in 'mysql v1.10.0' 'go-m1cpu v0.1.6'; do + read -r name version <<< "${pin}" + test -s "${fixture}/release with spaces/licenses/sources/${name}-${version}.zip" +done +cmp "${fixture}/LICENSE" "${fixture}/release with spaces/LICENSE" +cmp "${fixture}/THIRD_PARTY_NOTICES.md" "${fixture}/release with spaces/THIRD_PARTY_NOTICES.md" +bash "${fixture}/scripts/check-license-bundle.sh" "${fixture}/release with spaces/licenses/sources" + +if LICENSE_TEST_DOWNLOAD_FAILURE=1 bash "${fixture}/scripts/copy-licenses.sh" "${fixture}/failed download"; then + echo 'Packaging accepted a failed source download' >&2; exit 1 +fi +test ! -e "${fixture}/failed download" + +printf 'tampered source\n' > "${LICENSE_TEST_CACHE}/cache/download/github.com/go-sql-driver/mysql/@v/v1.10.0.zip" +if bash "${fixture}/scripts/copy-licenses.sh" "${fixture}/corrupt release" > "${fixture}/corrupt.log" 2>&1; then + echo 'Packaging accepted a mismatched source checksum' >&2; exit 1 +fi +test ! -e "${fixture}/corrupt release" + +printf 'github.com/go-sql-driver/mysql v0.0.0\n' > "${fixture}/go.mod" +if bash "${fixture}/scripts/check-license-bundle.sh" > "${fixture}/version.log" 2>&1; then + echo 'Notice validation accepted a stale module version' >&2; exit 1 +fi +echo 'License bundle packaging tests passed' diff --git a/website-docs/05-clients/01-frontend.md b/website-docs/05-clients/01-frontend.md index e805993a..bfdeecd0 100644 --- a/website-docs/05-clients/01-frontend.md +++ b/website-docs/05-clients/01-frontend.md @@ -29,7 +29,7 @@ WeKnora 的 Web 前端是一个基于 **Vue 3 + TypeScript + Vite** 的单页应 值得注意的依赖细节: -- `xlsx` 不走 npm registry,而是安装本地 tarball:`"xlsx": "file:./packages/xlsx-0.20.2.tgz"`(即 `frontend/packages/` 目录的用途,锁定版本、离线可装); +- `xlsx` 从 SheetJS 官方固定地址安装 `0.20.2`,`package-lock.json` 保留完整性校验值;源码仓库不附带组件包,离线构建需提前准备 npm 缓存; - `frontend/pnpm-workspace.yaml` 并非声明子包 workspace,只包含 `allowBuilds` 白名单(允许 `@vue-office/pptx`、`esbuild`、`vue-demi` 执行构建脚本),用于 pnpm 的构建脚本安全策略; - `overrides` / `resolutions` 中禁用了 `lightningcss` 并统一 `esbuild`、`serialize-javascript` 版本。 @@ -94,7 +94,6 @@ flowchart TB | `frontend/src/wailsjs/` | Wails 桌面端自动生成绑定(勿手改) | | `frontend/src/directives/`、`frontend/src/types/`、`frontend/src/config/` | 自定义指令、类型定义、配置 | | `frontend/public/` | 静态资源:`weknora-widget.js`(第三方站点嵌入加载器)、`config.js`(运行时配置占位,容器启动时覆盖)、离线 TDesign 图标 | -| `frontend/packages/` | 本地依赖 tarball(`xlsx-0.20.2.tgz`) | ## 页面路由清单