diff --git a/.gitignore b/.gitignore index c085c06..553259d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,142 +1,7 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* +# 编译缓存与临时文件 +__pycache__/ +*.pyc +*.pyo -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files -.env -.env.* -!.env.example - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp and cache directory -.temp -.cache - -# Sveltekit cache directory -.svelte-kit/ - -# vitepress build output -**/.vitepress/dist - -# vitepress cache directory -**/.vitepress/cache - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# Firebase cache directory -.firebase/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# yarn v3 -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/sdks -!.yarn/versions - -# Vite logs files -vite.config.js.timestamp-* -vite.config.ts.timestamp-* - -.DS_Store -.obsidian +# 上游子仓库元数据(raw/llm_wiki 的来源仓库备份,恢复上游更新时改名为 .git) +raw/llm_wiki/.git.bak diff --git a/.obsidian/app.json b/.obsidian/app.json new file mode 100644 index 0000000..78be447 --- /dev/null +++ b/.obsidian/app.json @@ -0,0 +1,3 @@ +{ + "showUnsupportedFiles": true +} \ No newline at end of file diff --git a/.obsidian/appearance.json b/.obsidian/appearance.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.obsidian/appearance.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/.obsidian/core-plugins.json b/.obsidian/core-plugins.json new file mode 100644 index 0000000..639b90d --- /dev/null +++ b/.obsidian/core-plugins.json @@ -0,0 +1,33 @@ +{ + "file-explorer": true, + "global-search": true, + "switcher": true, + "graph": true, + "backlink": true, + "canvas": true, + "outgoing-link": true, + "tag-pane": true, + "footnotes": false, + "properties": true, + "page-preview": true, + "daily-notes": true, + "templates": true, + "note-composer": true, + "command-palette": true, + "slash-command": false, + "editor-status": true, + "bookmarks": true, + "markdown-importer": false, + "zk-prefixer": false, + "random-note": false, + "outline": true, + "word-count": true, + "slides": false, + "audio-recorder": false, + "workspaces": false, + "file-recovery": true, + "publish": false, + "sync": true, + "bases": true, + "webviewer": false +} \ No newline at end of file diff --git a/.obsidian/graph.json b/.obsidian/graph.json new file mode 100644 index 0000000..7c90dc1 --- /dev/null +++ b/.obsidian/graph.json @@ -0,0 +1,51 @@ +{ + "collapse-filter": false, + "search": "path:wiki", + "showTags": false, + "showAttachments": false, + "hideUnresolved": false, + "showOrphans": false, + "collapse-color-groups": false, + "colorGroups": [ + { + "query": "path:wiki/practices ", + "color": { + "a": 1, + "rgb": 14048348 + } + }, + { + "query": "path:raw ", + "color": { + "a": 1, + "rgb": 6056662 + } + }, + { + "query": "path:wiki/concepts ", + "color": { + "a": 1, + "rgb": 11392604 + } + }, + { + "query": "path:wiki/queries ", + "color": { + "a": 1, + "rgb": 0 + } + } + ], + "collapse-display": true, + "showArrow": false, + "textFadeMultiplier": 0, + "nodeSizeMultiplier": 1, + "lineSizeMultiplier": 1, + "collapse-forces": false, + "centerStrength": 0.458860759493671, + "repelStrength": 10.3574397339983, + "linkStrength": 0.848101265822785, + "linkDistance": 322, + "scale": 1.0890794994553836, + "close": false +} \ No newline at end of file diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json new file mode 100644 index 0000000..4d1d8d6 --- /dev/null +++ b/.obsidian/workspace.json @@ -0,0 +1,244 @@ +{ + "main": { + "id": "d1b946394494730d", + "type": "split", + "children": [ + { + "id": "bafcef641122da54", + "type": "tabs", + "children": [ + { + "id": "7e8c0fba09ad238c", + "type": "leaf", + "state": { + "type": "markdown", + "state": { + "file": "raw/期货/20-品种新闻/焦煤/日报/2026-08-06_焦煤日报.md", + "mode": "source", + "source": false + }, + "icon": "lucide-file", + "title": "2026-08-06_焦煤日报" + } + } + ] + } + ], + "direction": "vertical" + }, + "left": { + "id": "25233c1a01ae40dd", + "type": "split", + "children": [ + { + "id": "556d942fb29f95c8", + "type": "tabs", + "children": [ + { + "id": "6841af199caa58fd", + "type": "leaf", + "state": { + "type": "file-explorer", + "state": { + "sortOrder": "alphabetical", + "autoReveal": false, + "showSearch": false, + "searchQuery": "" + }, + "icon": "lucide-folder-closed", + "title": "文件列表" + } + }, + { + "id": "efb5da93abae8a42", + "type": "leaf", + "state": { + "type": "search", + "state": { + "query": "", + "matchingCase": false, + "explainSearch": false, + "collapseAll": false, + "extraContext": false, + "sortOrder": "alphabetical" + }, + "icon": "lucide-search", + "title": "搜索" + } + }, + { + "id": "d47b0191e9ea948e", + "type": "leaf", + "state": { + "type": "bookmarks", + "state": { + "showSearch": false, + "searchQuery": "" + }, + "icon": "lucide-bookmark", + "title": "书签" + } + } + ] + } + ], + "direction": "horizontal", + "width": 300 + }, + "right": { + "id": "2b042b53367f20f0", + "type": "split", + "children": [ + { + "id": "103e3ddcc6065b7a", + "type": "tabs", + "children": [ + { + "id": "709f6cac006d06bd", + "type": "leaf", + "state": { + "type": "backlink", + "state": { + "file": "raw/llm_wiki/mcp-server/README.md", + "collapseAll": false, + "extraContext": false, + "sortOrder": "alphabetical", + "showSearch": false, + "searchQuery": "", + "backlinkCollapsed": false, + "unlinkedCollapsed": true + }, + "icon": "links-coming-in", + "title": "反向链接" + } + }, + { + "id": "a885af2199869ca7", + "type": "leaf", + "state": { + "type": "outgoing-link", + "state": { + "file": "raw/llm_wiki/mcp-server/README.md", + "linksCollapsed": false, + "unlinkedCollapsed": true + }, + "icon": "links-going-out", + "title": "出链" + } + }, + { + "id": "bf9b9d9c7779db2e", + "type": "leaf", + "state": { + "type": "tag", + "state": { + "sortOrder": "frequency", + "useHierarchy": true, + "showSearch": false, + "searchQuery": "" + }, + "icon": "lucide-tags", + "title": "标签" + } + }, + { + "id": "0cc1103a1470f556", + "type": "leaf", + "state": { + "type": "all-properties", + "state": { + "sortOrder": "frequency", + "showSearch": false, + "searchQuery": "" + }, + "icon": "lucide-archive", + "title": "添加笔记属性" + } + }, + { + "id": "c828df96a07dc63e", + "type": "leaf", + "state": { + "type": "outline", + "state": { + "file": "raw/llm_wiki/mcp-server/README.md", + "followCursor": false, + "showSearch": false, + "searchQuery": "" + }, + "icon": "lucide-list", + "title": "大纲" + } + } + ] + } + ], + "direction": "horizontal", + "width": 300, + "collapsed": true + }, + "left-ribbon": { + "hiddenItems": { + "switcher:打开快速切换": false, + "graph:查看关系图谱": false, + "canvas:新建白板": false, + "daily-notes:打开/创建今天的日记": false, + "templates:插入模板": false, + "command-palette:打开命令面板": false, + "bases:新建数据库": false + } + }, + "active": "6841af199caa58fd", + "lastOpenFiles": [ + "wiki/concepts/焦煤2026年8月市场回顾.md", + "raw/期货/01-基本分析逻辑/焦煤日报/scripts/__pycache__/auto_daily_report.cpython-314.pyc", + "raw/期货/20-品种新闻/焦煤/日报/2026-07-24_焦煤日报.md", + "raw/期货/01-基本分析逻辑/焦煤日报/scripts/auto_daily_report.py", + "raw/期货/20-品种新闻/焦煤/事件/2026-08-06_山西焦煤西曲矿安全事故停产.md", + "raw/期货/20-品种新闻/焦煤/日报/2026-08-05_焦煤日报.md", + "raw/期货/20-品种新闻/焦煤/日报/2026-08-04_焦煤日报.md", + "raw/期货/20-品种新闻/焦煤/日报/2026-08-03_焦煤日报.md", + "raw/期货/20-品种新闻/焦煤/日报/2026-07-31_焦煤日报.pdf", + "verify_tushare_jm.py", + "资本小论.md", + "wiki/queries/What-is-Harness-Engineering-in-Simple-Terms.md", + "wiki/practices/玻璃期货研究框架.md", + "wiki/practices/LangChain-Harness-Engineering.md", + "wiki/sources.md", + "raw/期货/10-品种相关/烧碱/烧碱期货品种手册-郑商所-2025-12.md", + "raw/工程技术:在智能体优先的世界中利用 Codex.md", + "raw/期货/10-品种相关/烧碱/烧碱期货投教材料.docx", + "raw/股票/01-投资方法/股票投资要义-胡斐-2015.md", + "raw/期货/10-品种相关/玻璃/README.md", + "raw/期货/10-品种相关/玻璃/玻璃周期、产能出清与研究问题.md", + "raw/期货/10-品种相关/玻璃/玻璃行业基础与供应链.md", + "raw/期货/10-品种相关/玻璃/玻璃数据字典与数据库设计.md", + "raw/期货/10-品种相关/玻璃/玻璃生产工艺与成本模型.md", + "raw/期货/10-品种相关/玻璃/玻璃期货投教材料-郑商所-2021-06.md", + "raw/期货/10-品种相关/玻璃/玻璃期货合约与研究方法.md", + "raw/期货/_meta/目录迁移记录.md", + "raw/期货/README.md", + "raw/期货/90-待核验/期货价格走势分析框架-原始剪藏.md", + "wiki/concepts/奇异型期限结构与期现套利分析.md", + "wiki/concepts/中国房地产市场制度演进与周期阶段.md", + "raw/期货/01-基本分析逻辑/焦煤分析技能", + "raw/期货/01-基本分析逻辑/事件驱动", + "raw/期货/01-基本分析逻辑/焦煤日报/scripts/Generate-JmDailyReport.ps1", + "raw/期货/01-基本分析逻辑/焦煤日报/scripts/extract_pdf_sections.py", + "raw/期货/01-基本分析逻辑/焦煤日报/scripts/extract_pdf_preview.py", + "未命名.canvas", + "images/webui.jpeg", + "images/skill.png", + "images/obsidian-index.png", + "未命名 3.canvas", + "未命名 1.canvas", + "未命名 2.canvas", + "raw/期货/01-基本分析逻辑/技术分析图.jpg", + "raw/llm_wiki/src/assets/logo.jpg", + "raw/llm_wiki/src-tauri/icons/icon.png", + "raw/llm_wiki/src-tauri/icons/32x32.png", + "raw/llm_wiki/src-tauri/icons/128x128@2x.png", + "raw/llm_wiki/src-tauri/icons/128x128.png", + "raw/images/4f67b1c10566552aec514a716ea43544ab330e0b-668x243.webp" + ] +} \ No newline at end of file diff --git a/raw/llm_wiki b/raw/llm_wiki deleted file mode 160000 index c92f82f..0000000 --- a/raw/llm_wiki +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c92f82ff42d6c0a855aa1305644370e8667bd824 diff --git a/raw/llm_wiki/.github/scripts/package-windows-portable.ps1 b/raw/llm_wiki/.github/scripts/package-windows-portable.ps1 new file mode 100644 index 0000000..e81742d --- /dev/null +++ b/raw/llm_wiki/.github/scripts/package-windows-portable.ps1 @@ -0,0 +1,60 @@ +param( + [Parameter(Mandatory = $true)] + [string]$Version +) + +$ErrorActionPreference = "Stop" + +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "../..") +$ExePath = Join-Path $RepoRoot "src-tauri/target/release/llm-wiki.exe" +$PdfiumPath = Join-Path $RepoRoot "src-tauri/pdfium/pdfium.dll" +$McpRoot = Join-Path $RepoRoot "mcp-server" +$DistRoot = Join-Path $RepoRoot "dist-portable" +$PortableRoot = Join-Path $DistRoot "LLM-Wiki-$Version-windows-x64-portable" +$ZipPath = Join-Path $DistRoot "LLM-Wiki-$Version-windows-x64-portable.zip" + +if (!(Test-Path $ExePath)) { + throw "Tauri executable was not found at $ExePath" +} +if (!(Test-Path $PdfiumPath)) { + throw "PDFium DLL was not found at $PdfiumPath" +} +foreach ($Path in @( + (Join-Path $McpRoot "package.json"), + (Join-Path $McpRoot "dist"), + (Join-Path $McpRoot "node_modules") +)) { + if (!(Test-Path $Path)) { + throw "Required MCP resource was not found at $Path. Run npm --prefix mcp-server ci and npm run mcp:build first." + } +} + +if (Test-Path $PortableRoot) { + Remove-Item -Recurse -Force $PortableRoot +} +if (Test-Path $ZipPath) { + Remove-Item -Force $ZipPath +} +New-Item -ItemType Directory -Force $PortableRoot | Out-Null + +Copy-Item $ExePath (Join-Path $PortableRoot "LLM Wiki.exe") + +New-Item -ItemType Directory -Force (Join-Path $PortableRoot "pdfium") | Out-Null +Copy-Item $PdfiumPath (Join-Path $PortableRoot "pdfium/pdfium.dll") + +$PortableMcpRoot = Join-Path $PortableRoot "mcp-server" +New-Item -ItemType Directory -Force $PortableMcpRoot | Out-Null +Copy-Item (Join-Path $McpRoot "package.json") (Join-Path $PortableMcpRoot "package.json") +Copy-Item -Recurse (Join-Path $McpRoot "dist") (Join-Path $PortableMcpRoot "dist") +Copy-Item -Recurse (Join-Path $McpRoot "node_modules") (Join-Path $PortableMcpRoot "node_modules") + +@" +LLM Wiki Windows Portable + +Run "LLM Wiki.exe" from this folder. Keep the pdfium/ and mcp-server/ folders next to the executable. + +This portable package does not install start-menu shortcuts or auto-update hooks. It still stores app data in the normal LLM Wiki application data directory. +"@ | Set-Content -Encoding UTF8 (Join-Path $PortableRoot "README-portable.txt") + +Compress-Archive -Path (Join-Path $PortableRoot "*") -DestinationPath $ZipPath -CompressionLevel Optimal +Write-Host "Created $ZipPath" diff --git a/raw/llm_wiki/.github/workflows/build.yml b/raw/llm_wiki/.github/workflows/build.yml new file mode 100644 index 0000000..3215677 --- /dev/null +++ b/raw/llm_wiki/.github/workflows/build.yml @@ -0,0 +1,204 @@ +name: Build & Release + +on: + push: + tags: + - 'v*' + workflow_dispatch: + +permissions: + contents: write + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - platform: macos-latest + args: '--target aarch64-apple-darwin' + rust_target: aarch64-apple-darwin + - platform: ubuntu-22.04 + args: '' + rust_target: '' + # GitHub-hosted ARM Linux runner (free for public repos + # since 2024). Native build — no cross-compile of webkit2gtk. + - platform: ubuntu-22.04-arm + args: '' + rust_target: '' + - platform: windows-latest + args: '' + rust_target: '' + + runs-on: ${{ matrix.platform }} + env: + # GitHub-hosted runners occasionally hit crates.io HTTP/2 stream + # resets while downloading large dependency graphs. Cargo retries plus + # HTTP/1.1 transport make release builds less flaky across the matrix. + CARGO_HTTP_MULTIPLEXING: "false" + CARGO_NET_RETRY: "5" + CARGO_HTTP_TIMEOUT: "60" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.rust_target }} + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: src-tauri + + - name: Install protoc (macOS) + if: startsWith(matrix.platform, 'macos-') + run: brew install protobuf + + - name: Install dependencies (Ubuntu) + if: startsWith(matrix.platform, 'ubuntu-22.04') + run: | + sudo apt-get update + # xdg-utils provides /usr/bin/xdg-open, which Tauri's + # AppImage bundler embeds into the produced AppImage. + # Pre-installed on the x86_64 runner image but NOT on + # the ARM64 image — list it explicitly so both arches + # bundle cleanly regardless of future image drift. + sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf protobuf-compiler xdg-utils + + # The repo ships pre-downloaded PDFium binaries under + # src-tauri/pdfium/ for every supported architecture + # (libpdfium.so = Linux x86_64, libpdfium-arm64.so = Linux aarch64, + # libpdfium.dylib = macOS arm64, pdfium.dll = Windows). + # For architectures whose bundled filename must + # stay `libpdfium.*`, swap the matching binary into place before cargo + # runs. We do NOT pull from bblanchon/pdfium-binaries during CI — that + # download has historically failed often enough that committing the + # binaries is the maintenance-friendly path. + - name: Verify PDFium binary checksums + if: matrix.platform != 'windows-latest' + run: shasum -a 256 -c src-tauri/pdfium/SHA256SUMS + + # rust_target is empty for both Ubuntu rows, so platform is the + # only stable discriminator for the Linux ARM swap. + - name: Use ARM64 pdfium binary (Ubuntu ARM only) + if: matrix.platform == 'ubuntu-22.04-arm' + run: | + cp src-tauri/pdfium/libpdfium-arm64.so src-tauri/pdfium/libpdfium.so + file src-tauri/pdfium/libpdfium.so + + - name: Install protoc (Windows) + if: matrix.platform == 'windows-latest' + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install frontend dependencies + run: npm install + + - name: Prepare MCP server resources + run: | + npm --prefix mcp-server ci + npm run mcp:build + + - name: Build Tauri app + uses: tauri-apps/tauri-action@v0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} + APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} + APPLE_ID: ${{ secrets.APPLE_ID }} + APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + with: + # Tag-triggered runs publish a real GitHub Release. + # Manual (workflow_dispatch) runs leave tagName/releaseName + # empty so tauri-action skips the release-upload step and + # just produces bundle artifacts — useful for testing a + # branch build without polluting the Releases page. + tagName: ${{ github.event_name == 'push' && github.ref_name || '' }} + releaseName: ${{ github.event_name == 'push' && format('LLM Wiki {0}', github.ref_name) || '' }} + releaseBody: 'See the assets below for download links.' + releaseDraft: false + prerelease: false + args: ${{ matrix.args }} + + - name: Package Windows portable zip + if: matrix.platform == 'windows-latest' + shell: pwsh + run: | + $version = node -p "require('./package.json').version" + .github/scripts/package-windows-portable.ps1 -Version $version + + - name: Attach Windows portable zip to release + if: github.event_name == 'push' && matrix.platform == 'windows-latest' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: pwsh + run: | + gh release upload "${{ github.ref_name }}" dist-portable/*.zip --clobber + + # On workflow_dispatch, no release is created, so the bundles + # would otherwise be discarded with the runner. Upload them as + # workflow artifacts so the maintainer can `gh run download` + # the .msi / .exe / .dmg / .deb to test locally. Skipped on + # tag pushes since the release page already has them. + - name: Upload bundles as workflow artifacts (manual runs only) + if: github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v4 + with: + name: bundle-${{ matrix.platform }} + # Glob covers both targeted (e.g. + # target/aarch64-apple-darwin/release/...) and default + # (target/release/...) build paths. + path: | + src-tauri/target/**/release/bundle/msi/*.msi + src-tauri/target/**/release/bundle/nsis/*.exe + src-tauri/target/**/release/bundle/dmg/*.dmg + src-tauri/target/**/release/bundle/deb/*.deb + src-tauri/target/**/release/bundle/appimage/*.AppImage + dist-portable/*.zip + if-no-files-found: warn + retention-days: 14 + + package-extension: + name: Package browser extension + needs: build + # Browser extension is only published as part of an actual + # tagged release; manual builds don't need it. + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Sync extension manifest version and zip + run: | + # Pull version from package.json so we have a single source of + # truth. Chrome's manifest requires numeric-only version + # (e.g. 0.3.5), which matches the repo's semver convention. + APP_VERSION=$(node -p "require('./package.json').version") + node -e " + const fs = require('fs'); + const p = 'extension/manifest.json'; + const m = JSON.parse(fs.readFileSync(p, 'utf-8')); + m.version = '${APP_VERSION}'; + fs.writeFileSync(p, JSON.stringify(m, null, 2) + '\n'); + " + mkdir -p dist-extension + (cd extension && zip -r "../dist-extension/llm-wiki-extension-${APP_VERSION}.zip" . -x "*.DS_Store") + ls -la dist-extension + + - name: Attach extension zip to release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "${{ github.ref_name }}" dist-extension/*.zip --clobber diff --git a/raw/llm_wiki/.github/workflows/ci.yml b/raw/llm_wiki/.github/workflows/ci.yml new file mode 100644 index 0000000..6ffa4c7 --- /dev/null +++ b/raw/llm_wiki/.github/workflows/ci.yml @@ -0,0 +1,62 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + check: + strategy: + fail-fast: false + matrix: + platform: [macos-latest, ubuntu-22.04, windows-latest] + + runs-on: ${{ matrix.platform }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc (macOS) + if: matrix.platform == 'macos-latest' + run: brew install protobuf + + - name: Install dependencies (Ubuntu) + if: matrix.platform == 'ubuntu-22.04' + run: | + sudo apt-get update + sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf protobuf-compiler + + - name: Install protoc (Windows) + if: matrix.platform == 'windows-latest' + run: choco install protoc -y + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: src-tauri + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install frontend dependencies + run: npm install + + - name: Check frontend build + run: npx vite build + + - name: Prepare MCP server resources + run: | + npm --prefix mcp-server ci + npm run mcp:build + + - name: Check Rust build + working-directory: src-tauri + run: cargo build diff --git a/raw/llm_wiki/.gitignore b/raw/llm_wiki/.gitignore new file mode 100644 index 0000000..9a9f924 --- /dev/null +++ b/raw/llm_wiki/.gitignore @@ -0,0 +1,51 @@ +# Dependencies +node_modules/ + +# Build outputs +dist/ +dist-ssr/ +# RC artifacts pulled from CI for local Windows / Intel Mac verification. +# These are pre-release binaries downloaded via `gh run download`, never +# meant for the repo. +dist-rc/ +*.tsbuildinfo + +# Rust build artifacts +src-tauri/target/ + +# Tauri generated files +src-tauri/gen/ + +# Environment files +.env +.env.local +.env.*.local + +# Internal docs (not shipped) +docs/ +AGENTS.md + +# Benchmark and test data (local only) +tests/ + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* + +# Brainstorm assets (not tracked in source control) +.superpowers/ +.claude/ diff --git a/raw/llm_wiki/LICENSE b/raw/llm_wiki/LICENSE new file mode 100644 index 0000000..a621fef --- /dev/null +++ b/raw/llm_wiki/LICENSE @@ -0,0 +1,676 @@ +LLM Wiki — Copyright (C) 2024-2026 Yong Su + + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/raw/llm_wiki/README.md b/raw/llm_wiki/README.md new file mode 100644 index 0000000..6335354 --- /dev/null +++ b/raw/llm_wiki/README.md @@ -0,0 +1,503 @@ +# LLM Wiki + +

+ LLM Wiki Logo +

+ +

+ A personal knowledge base that builds itself.
+ LLM reads your documents, builds a structured wiki, and keeps it current. +

+ +

+ What is this? • + Features • + Tech Stack • + Installation • + Credits • + License +

+ +

+ English | 中文 | 日本語 | 한국어 +

+ +--- + +

+ Overview +

+ +## Features + +- **Two-Step Chain-of-Thought Ingest** — LLM analyzes first, then generates wiki pages with source traceability and incremental cache +- **Multimodal Image Ingestion** — extract embedded images from PDFs, generate factual captions with a vision LLM, surface them in image-aware search results with lightbox preview and jump-to-source +- **Multi-format Document Parsing** — ingest PDF, Office documents, EPUB/MOBI, Org mode, images, media, web clips, and batches of URLs, with built-in, cloud, or local MinerU PDF processing +- **Flexible Model Configuration** — configure models per project, route Chat and Ingest independently, and manage custom providers, headers, and streaming output +- **Source-grounded Retrieval** — use Read Sources Only mode to answer exclusively from original imported material +- **Project Management & Migration** — export and import complete project archives across devices, and rebuild the Wiki index from existing pages +- **4-Signal Knowledge Graph** — relevance model with direct links, source overlap, Adamic-Adar, and type affinity +- **Louvain Community Detection** — automatic knowledge cluster discovery with cohesion scoring +- **Graph Insights** — surprising connections and knowledge gaps with one-click Deep Research +- **Vector Semantic Search** — optional embedding-based retrieval via LanceDB, supports any OpenAI-compatible endpoint +- **Persistent Ingest Queue** — serial processing with crash recovery, cancel, retry, and progress visualization +- **Folder Import** — recursive folder import preserving directory structure, folder context as LLM classification hint +- **Source Folder Auto-Watch** — detects external changes in `raw/sources/` and keeps ingest/delete cleanup in sync +- **Deep Research** — LLM-optimized search topics, multi-query web search via Tavily, SerpApi, or SearXNG, auto-ingest results into wiki +- **Rust Backend Chat Agent** — tool-using chat runtime with wiki/source/graph/web retrieval, workspace file generation, shell approval, cancellation, and streaming tool events +- **Agent Skills** — scan and enable local `SKILL.md` folders, select skills with `/skill`, and let the Agent read skill instructions on demand +- **Generated Outputs Preview** — Agent-created Markdown, HTML, images, and other workspace files appear as outputs with preview and quick folder access +- **Mermaid Diagram Rendering** — render Mermaid code blocks directly in chat and preview, with compact syntax-error cards instead of raw parser output +- **Async Review System** — LLM flags items for human judgment, predefined actions, pre-generated search queries +- **Chrome Web Clipper** — one-click web page capture with auto-ingest into knowledge base +- **Local HTTP API + MCP Server + AI Agent Skill** — built-in `127.0.0.1:19828` JSON API and bundled MCP server for hybrid search, file read, graph traversal, and source rescan; ready-made [agent skill](https://github.com/nashsu/llm_wiki_skill) installs into Claude Code / Codex with one command (`npx skills add …`) + +## What is this? + +LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM **incrementally builds and maintains a persistent wiki** from your sources. Knowledge is compiled once and kept current, not re-derived on every query. + +This project is based on [Karpathy's LLM Wiki pattern](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) — a methodology for building personal knowledge bases using LLMs. llm_wiki is created and maintained by [nash_su](https://x.com/nash_su), who implemented the core ideas as a full desktop application with significant enhancements. + +

+ LLM Wiki Architecture +

+ +## Credits + +The foundational methodology comes from **Andrej Karpathy**'s [llm-wiki.md](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f), which describes the pattern of using LLMs to incrementally build and maintain a personal wiki. The original document is an abstract design pattern; this project is a concrete implementation with substantial extensions. + +## What We Kept from the Original + +The core architecture follows Karpathy's design faithfully: + +- **Three-layer architecture**: Raw Sources (immutable) → Wiki (LLM-generated) → Schema (rules & config) +- **Three core operations**: Ingest, Query, Lint +- **index.md** as the content catalog and LLM navigation entry point +- **log.md** as the chronological operation record with parseable format +- **[[wikilink]]** syntax for cross-references +- **YAML frontmatter** on every wiki page +- **Obsidian compatibility** — the wiki directory works as an Obsidian vault +- **Human curates, LLM maintains** — the fundamental role division + +

+ Obsidian Compatibility +

+ +## What We Changed & Added + +### 1. From CLI to Desktop Application + +The original is an abstract pattern document designed to be copy-pasted to an LLM agent. We built it into a **full cross-platform desktop application** with: +- **Three-column layout**: Knowledge Tree / File Tree (left) + Chat (center) + Preview (right) +- **Icon sidebar** for switching between Wiki, Sources, Search, Graph, Lint, Review, Deep Research, Settings +- **Custom resizable panels** — drag-to-resize left and right panels with min/max constraints +- **Activity panel** — real-time processing status showing file-by-file ingest progress +- **All state persisted** — conversations, settings, review items, project config survive restarts +- **Scenario templates** — Research, Reading, Personal Growth, Business, General — each pre-configures purpose.md and schema.md + +### 2. Purpose.md — The Wiki's Soul + +The original has Schema (how the wiki works) but no formal place for **why** the wiki exists. We added `purpose.md`: +- Defines goals, key questions, research scope, evolving thesis +- LLM reads it during every ingest and query for context +- LLM can suggest updates based on usage patterns +- Different from schema — schema is structural rules, purpose is directional intent + +### 3. Two-Step Chain-of-Thought Ingest + +The original describes a single-step ingest where the LLM reads and writes simultaneously. We split it into **two sequential LLM calls** for significantly better quality: + +``` +Step 1 (Analysis): LLM reads source → structured analysis + - Key entities, concepts, arguments + - Connections to existing wiki content + - Contradictions & tensions with existing knowledge + - Recommendations for wiki structure + +Step 2 (Generation): LLM takes analysis → generates wiki files + - Source summary with frontmatter (type, title, sources[]) + - Entity pages, concept pages with cross-references + - Updated index.md, log.md, overview.md + - Review items for human judgment + - Search queries for Deep Research +``` + +Additional ingest enhancements beyond the original: +- **SHA256 incremental cache** — source file content is hashed before ingest; unchanged files are skipped automatically, saving LLM tokens and time +- **Persistent ingest queue** — serial processing prevents concurrent LLM calls; queue persisted to disk, survives app restart; failed tasks auto-retry up to 3 times +- **Folder import** — recursive folder import preserving directory structure; folder path passed to LLM as classification context (e.g., "papers > energy" helps categorize content) +- **Source folder auto-watch** — files added, edited, or deleted in `raw/sources/` outside the app are picked up automatically and reuse the same ingest/delete lifecycle as in-app actions +- **Queue visualization** — Activity Panel shows progress bar, pending/processing/failed tasks with cancel and retry buttons +- **Auto-embedding** — when vector search is enabled, new pages are automatically embedded after ingest +- **Source traceability** — every generated wiki page includes a `sources: []` field in YAML frontmatter, linking back to the raw source files that contributed to it +- **overview.md auto-update** — global summary page regenerated on every ingest to reflect the latest state of the wiki +- **Guaranteed source summary** — fallback ensures a source summary page is always created, even if the LLM omits it +- **Language-aware generation** — LLM responds in the user's configured language (English or Chinese) +- **Progressive Sources view** — large source folders render progressively while scrolling, keeping big source collections responsive + +### 4. Knowledge Graph with Relevance Model + +

+ Knowledge Graph +

+ +The original mentions `[[wikilinks]]` for cross-references but has no graph analysis. We built a **full knowledge graph visualization and relevance engine**: + +**4-Signal Relevance Model:** +| Signal | Weight | Description | +|--------|--------|-------------| +| Direct link | ×3.0 | Pages linked via `[[wikilinks]]` | +| Source overlap | ×4.0 | Pages sharing the same raw source (via frontmatter `sources[]`) | +| Adamic-Adar | ×1.5 | Pages sharing common neighbors (weighted by neighbor degree) | +| Type affinity | ×1.0 | Bonus for same page type (entity↔entity, concept↔concept) | + +**Graph Visualization (sigma.js + graphology + ForceAtlas2):** +- Node colors by page type or community, sizes scaled by link count (√ scaling) +- Edge thickness and color by relevance weight (green=strong, gray=weak) +- Hover interaction: neighbors stay visible, non-neighbors dim, edges highlight with relevance score label +- Zoom controls (ZoomIn, ZoomOut, Fit-to-screen) +- Position caching prevents layout jumps when data updates +- Legend switches between type counts and community info based on coloring mode + +### 5. Louvain Community Detection + +Not in the original. Automatic discovery of knowledge clusters using the **Louvain algorithm** (graphology-communities-louvain): + +- **Auto-clustering** — discovers which pages naturally group together based on link topology, independent of predefined page types +- **Type / Community toggle** — switch between coloring nodes by page type (entity, concept, source...) or by discovered knowledge cluster +- **Cohesion scoring** — each community scored by intra-edge density (actual edges / possible edges); low-cohesion clusters (< 0.15) flagged with warning +- **12-color palette** — distinct visual separation between clusters +- **Community legend** — shows top node label, member count, and cohesion per cluster + +

+ Louvain Community Detection +

+ +### 6. Graph Insights — Surprising Connections & Knowledge Gaps + +Not in the original. The system **automatically analyzes graph structure** to surface actionable insights: + +**Surprising Connections:** +- Detects unexpected relationships: cross-community edges, cross-type links, peripheral↔hub couplings +- Composite surprise score ranks the most noteworthy connections +- Dismissable — mark connections as reviewed so they don't reappear + +**Knowledge Gaps:** +- **Isolated pages** (degree ≤ 1) — pages with few or no connections to the rest of the wiki +- **Sparse communities** (cohesion < 0.15, ≥ 3 pages) — knowledge areas with weak internal cross-references +- **Bridge nodes** (connecting 3+ clusters) — critical junction pages that hold multiple knowledge areas together + +**Interactive:** +- Click any insight card to **highlight** corresponding nodes and edges in the graph; click again to deselect +- Knowledge gaps and bridge nodes have a **Deep Research button** — triggers LLM-optimized research with domain-aware topics (reads overview.md + purpose.md for context) +- Research topic shown in **editable confirmation dialog** before starting — user can refine topic and search queries + +

+ Graph Insights +

+ +### 7. Optimized Query Retrieval Pipeline + +The original describes a simple query where the LLM reads relevant pages. We built a **multi-phase retrieval pipeline** with optional vector search and budget control: + +``` +Phase 1: Tokenized Search + - English: word splitting + stop word removal + - Chinese: CJK bigram tokenization (每个 → [每个, 个…]) + - Title match bonus (+10 score) + - Searches both wiki/ and raw/sources/ + +Phase 1.5: Vector Semantic Search (optional) + - Embedding via any OpenAI-compatible /v1/embeddings endpoint + - Stored in LanceDB (Rust backend) for fast ANN retrieval + - Cosine similarity finds semantically related pages even without keyword overlap + - Results merged into search: boosts existing matches + adds new discoveries + +Phase 2: Graph Expansion + - Top search results used as seed nodes + - 4-signal relevance model finds related pages + - 2-hop traversal with decay for deeper connections + +Phase 3: Budget Control + - Configurable context window: 4K → 1M tokens + - Proportional allocation: 60% wiki pages, 20% chat history, 5% index, 15% system + - Pages prioritized by combined search + graph relevance score + +Phase 4: Context Assembly + - Numbered pages with full content (not just summaries) + - System prompt includes: purpose.md, language rules, citation format, index.md + - LLM instructed to cite pages by number: [1], [2], etc. +``` + +**Vector Search** is fully optional — disabled by default, enabled in Settings with independent endpoint, API key, and model configuration. When disabled, the pipeline falls back to tokenized search + graph expansion. Benchmark: overall recall improved from 58.2% to 71.4% with vector search enabled. + +### 8. Multi-Conversation Chat with Persistence + +The original has a single query interface. We built **full multi-conversation support**: + +- **Independent chat sessions** — create, rename, delete conversations +- **Conversation sidebar** — quick switching between topics +- **Per-conversation persistence** — each conversation saved to `.llm-wiki/chats/{id}.json` +- **Configurable history depth** — limit how many messages are sent as context (default: 10) +- **Cited references panel** — collapsible section on each response showing which wiki pages were used, grouped by type with icons +- **Reference persistence** — cited pages stored directly in message data, stable across restarts +- **Regenerate** — re-generate the last response with one click (removes last assistant + user message pair, re-sends) +- **Save to Wiki** — archive valuable answers to `wiki/queries/`, then auto-ingest to extract entities/concepts into the knowledge network + +### 9. Rust Backend Chat Agent & Skills + +Not in the original. Chat now runs through a Rust backend Agent runtime rather than a browser-only TypeScript loop: + +- **Tool-using Agent** — can choose wiki search, source search, graph search, web search, AnyTXT, workspace file tools, approved shell commands, and skill file reads +- **Skill management** — scan project and user skill folders, enable or disable skills, and pick a skill per conversation with `/skill` completion +- **Generated workspace outputs** — files produced by Agent tools are kept under `agent-workspace/`, shown as generated outputs, and can be previewed or opened from the chat +- **User interaction forms** — skills can ask for structured user input such as single choice, multiple choice, or free text without hardcoding skill-specific UI +- **Safer execution model** — project workspace commands can continue smoothly, while external shell commands still require explicit approval + +### 10. Thinking / Reasoning Display + +Not in the original. For LLMs that emit `` blocks (DeepSeek, QwQ, etc.): + +- **Streaming thinking** — rolling 5-line display with opacity fade during generation +- **Collapsed by default** — thinking blocks hidden after completion, click to expand +- **Visual separation** — thinking content shown in distinct style, separate from the main response + +### 11. Markdown Rendering: KaTeX Math & Mermaid Diagrams + +Not in the original. Rich Markdown rendering across chat and preview: + +- **KaTeX rendering** — inline `$...$` and block `$$...$$` formulas rendered via remark-math + rehype-katex +- **Milkdown math plugin** — preview editor renders math natively via @milkdown/plugin-math +- **Auto-detection** — bare `\begin{aligned}` and other LaTeX environments automatically wrapped with `$$` delimiters +- **Unicode fallback** — 100+ symbol mappings (α, ∑, →, ≤, etc.) for simple inline notation outside math blocks +- **Mermaid code blocks** — fenced `mermaid` diagrams render directly as flowcharts, sequence diagrams, and other Mermaid-supported visuals +- **Compact Mermaid errors** — syntax failures are captured inside a small error card instead of spilling raw parser output into the chat + +### 12. Review System (Async Human-in-the-Loop) + +The original suggests staying involved during ingest. We added an **asynchronous review queue**: + +- LLM flags items needing human judgment during ingest +- **Predefined action types**: Create Page, Deep Research, Skip — constrained to prevent LLM hallucination of arbitrary actions +- **Search queries generated at ingest time** — LLM pre-generates optimized web search queries for each review item +- User handles reviews at their convenience — doesn't block ingest + +### 13. Deep Research + +

+ Deep Research +

+ +Not in the original. When the LLM identifies knowledge gaps: + +- **Web search** via Tavily, SerpApi, or SearXNG finds relevant sources with full content extraction (no truncation) +- **Provider-specific configuration** — Tavily and SerpApi use independent API keys; SerpApi supports selectable engines, while SearXNG uses a configured instance URL and search categories +- **Multiple search queries** per topic — LLM-generated at ingest time, optimized for search engines +- **LLM-optimized research topics** — when triggered from Graph Insights, LLM reads overview.md + purpose.md to generate domain-specific topics and queries (not generic keywords) +- **User confirmation dialog** — editable topic and search queries shown for review before research starts +- **LLM synthesizes** findings into a wiki research page with cross-references to existing wiki +- **Thinking display** — `` blocks shown as collapsible sections during synthesis, auto-scroll to latest content +- **Auto-ingest** — research results automatically processed to extract entities/concepts into the wiki +- **Task queue** with 3 concurrent tasks +- **Research Panel** — dedicated sidebar panel with dynamic height, real-time streaming progress + +### 14. Browser Extension (Web Clipper) + +

+ Chrome Extension Web Clipper +

+ +The original mentions Obsidian Web Clipper. We built a **dedicated Chrome Extension** (Manifest V3): + +- **Mozilla Readability.js** for accurate article extraction (strips ads, nav, sidebars) +- **Turndown.js** for HTML → Markdown conversion with table support +- **Project picker** — choose which wiki to clip into (supports multi-project) +- **Local HTTP API** (port 19827, tiny_http) — Extension ↔ App communication +- **Auto-ingest** — clipped content automatically triggers the two-step ingest pipeline +- **Clip watcher** — polls every 3 seconds for new clips, processes automatically +- **Offline preview** — shows extracted content even when app is not running + +### 15. Multi-format Document Support + +The original focuses on text/markdown. We support structured extraction preserving document semantics: + +| Format | Method | +|--------|--------| +| PDF | Built-in pdf-extract (Rust) with file caching; optional MinerU Cloud, Local API, or Pipeline parsing for complex layouts | +| DOCX | docx-rs — headings, bold/italic, lists, tables → structured Markdown | +| PPTX | ZIP + XML — slide-by-slide extraction with heading/list structure | +| XLSX/XLS/ODS | calamine — proper cell types, multi-sheet support, Markdown tables | +| EPUB/MOBI | Electronic book metadata, chapters, and body text → ingest-ready content | +| Images | Native preview (png, jpg, gif, webp, svg, etc.) | +| Video/Audio | Built-in player | +| Web clips | Readability.js + Turndown.js → clean Markdown | + +> MinerU is optional. Use MinerU Cloud, an official Local API endpoint, or Local Pipeline mode for complex PDFs. Local modes keep processing on your machine, and extracted images are stored in the project-managed `wiki/media` directory. If MinerU fails, LLM Wiki falls back to the built-in parser. + +### 16. File Deletion with Cascade Cleanup + +The original has no deletion mechanism. We added **intelligent cascade deletion**: + +- Deleting a source file removes its wiki summary page +- **3-method matching** finds related wiki pages: frontmatter `sources[]` field, source summary page name, frontmatter section references +- **Shared entity preservation** — entity/concept pages linked to multiple sources only have the deleted source removed from their `sources[]` array, not deleted entirely +- **Index cleanup** — removed pages are purged from index.md +- **Wikilink cleanup** — dead `[[wikilinks]]` to deleted pages are removed from remaining wiki pages + +### 17. Configurable Context Window + +Not in the original. Users can configure how much context the LLM receives: + +- **Slider from 4K to 1M tokens** — adapts to different LLM capabilities +- **Proportional budget allocation** — larger windows get proportionally more wiki content +- **60/20/5/15 split** — wiki pages / chat history / index / system prompt + +### 18. Cross-Platform Compatibility + +The original is platform-agnostic (abstract pattern). We handle concrete cross-platform concerns: + +- **Path normalization** — unified `normalizePath()` used across 22+ files, backslash → forward slash +- **Unicode-safe string handling** — char-based slicing instead of byte-based (prevents crashes on CJK filenames) +- **macOS close-to-hide** — close button hides window (app stays running in background), click dock icon to restore, Cmd+Q to quit +- **Windows/Linux close confirmation** — confirmation dialog before quitting to prevent accidental data loss +- **Tauri v2** — native desktop on macOS, Windows, Linux +- **GitHub Actions CI/CD** — automated builds for macOS (ARM + Intel), Windows (.msi), Linux (.deb / .AppImage) + +### 19. Other Additions + +- **i18n** — English + Chinese interface (react-i18next) +- **Settings persistence** — LLM provider, API key, model, context size, language saved via Tauri Store +- **Obsidian config** — auto-generated `.obsidian/` directory with recommended settings +- **Markdown rendering** — GFM tables with borders, proper code blocks, wikilink processing in chat and preview +- **Multi-provider LLM support** — OpenAI, Anthropic, Google, Ollama, Custom — each with provider-specific streaming and headers +- **Configurable LLM timeout** — adjust request timeouts for slow local models and long-running operations +- **Configurable Firecrawl** — optional API key and custom Base URL for hosted or self-hosted services +- **Collapsible file sidebar** — collapse Knowledge/Files navigation while preserving its state +- **Project maintenance** — ZIP export/import for migration and deterministic `wiki/index.md` rebuilding +- **dataVersion signaling** — graph and UI automatically refresh when wiki content changes + +## Tech Stack + +| Layer | Technology | +|-------|-----------| +| Desktop | Tauri v2 (Rust backend) | +| Frontend | React 19 + TypeScript + Vite | +| UI | shadcn/ui + Tailwind CSS v4 | +| Editor | Milkdown (ProseMirror-based WYSIWYG) | +| Graph | sigma.js + graphology + ForceAtlas2 | +| Search | Tokenized search + graph relevance + optional vector (LanceDB) | +| Vector DB | LanceDB (Rust, embedded, optional) | +| Documents | pdf-extract + MinerU Cloud/Local + docx-rs + calamine + EPUB/MOBI extraction | +| i18n | react-i18next | +| State | Zustand | +| LLM | Streaming fetch (OpenAI, Anthropic, Google, Ollama, Custom) | +| Web Search | Tavily, SerpApi, SearXNG JSON API | + +## Installation + +### Pre-built Binaries + +Download from [Releases](https://github.com/nashsu/llm_wiki/releases): +- **macOS**: `.dmg` (Apple Silicon + Intel) +- **Windows**: `.msi` +- **Linux**: `.deb` / `.AppImage` + +### Build from Source + +```bash +# Prerequisites: Node.js 20+, Rust 1.70+ +git clone https://github.com/nashsu/llm_wiki.git +cd llm_wiki +npm install +npm run tauri dev # Development +npm run tauri build # Production build +``` + +### Chrome Extension + +1. Open `chrome://extensions` +2. Enable "Developer mode" +3. Click "Load unpacked" +4. Select the `extension/` directory +5. Clip the current page with `Alt+Shift+L` (`Command+Shift+L` on macOS). Customize it at `chrome://extensions/shortcuts`. + +## Quick Start + +1. Launch the app → Create a new project (choose a template) +2. Go to **Settings** → Configure your LLM provider (API key + model) +3. Optional: configure **Web Search** providers and source folder auto-watch in Settings +4. Go to **Sources** → Import documents (PDF, DOCX, MD, etc.) +5. Watch the **Activity Panel** — LLM automatically builds wiki pages +6. Use **Chat** to query your knowledge base +7. Browse the **Knowledge Graph** to see connections +8. Check **Review** for items needing your attention +9. Run **Lint** periodically to maintain wiki health + +## Local HTTP API + MCP Server + AI Agent Skill + +LLM Wiki ships a built-in local HTTP API at `http://127.0.0.1:19828` (token-protected, `127.0.0.1`-only) so external tools — including AI agents like **Claude Code**, **Codex**, or any HTTP-capable script — can query your wiki: + +- `GET /api/v1/health` — server status (no auth) +- `GET /api/v1/projects` — list projects +- `GET /api/v1/projects/{id}/files` / `files/content` — read files and content +- `GET /api/v1/projects/{id}/reviews?status=unresolved` — export Review tab items for wiki maintenance (`status`: `unresolved`, `resolved`, or `all`; optional `type` and `limit`) +- `PATCH /api/v1/projects/{id}/reviews/{reviewId}` — update one Review item (JSON body `{ "resolved": true, "action": "label" }`; `resolved` defaults to true, pass false to reopen) +- `POST /api/v1/projects/{id}/reviews/resolve` — bulk-resolve Review items (JSON body `{ "ids": [...], "action": "label" }`), returns `{ resolved, notFound, count }`; the Review tab's Refresh button re-reads the result from disk +- `POST /api/v1/projects/{id}/search` — **hybrid** retrieval (keyword + vector) returning `mode`, `tokenHits`, `vectorHits`, per-result `vectorScore` +- `POST /api/v1/projects/{id}/chat` — non-streaming backend Agent chat endpoint returning an assistant message, references, usage, and tool events for wiki/source/web/AnyTXT retrieval; `mode: "deep"` broadens evidence collection, while the full Deep Research workspace remains available in the desktop UI +- `GET /api/v1/projects/{id}/graph` — wikilinks graph +- `POST /api/v1/projects/{id}/sources/rescan` — trigger a backend rescan + +Enable the API, generate a token, and choose whether local unauthenticated access is allowed in **Settings → API + MCP**. + +For MCP-compatible clients, LLM Wiki also includes a local MCP server in `mcp-server/`. After building it with `npm run mcp:build`, **Settings → API + MCP** shows a copyable MCP client configuration with the correct local path for your machine. The MCP tools call the same API surface, so agent clients can list projects, read files, export unresolved Review items, run hybrid search, inspect the graph, trigger source rescans, and call the same Rust backend Agent chat endpoint without custom HTTP glue code. + +### Plug your AI agent in with one command + +A ready-made **agent skill** for LLM Wiki lives in its own repo. Install it into Claude Code / Codex / any skills-compatible runtime: + +```bash +npx skills add https://github.com/nashsu/llm_wiki_skill.git --skill llm-wiki +``` + +After install, the agent can answer prompts like "what does my LLM Wiki say about X", "search my 知识库 for Y", "show the neighborhood of node Z in my wiki graph", and "rescan my wiki sources" by talking to your locally-running app — read-only by default, citing wiki page paths so you can verify in-app. + +- **Skill repo**: +- **Trigger discipline**: it intentionally does **not** trigger on generic "search my notes" / "check my Obsidian / Notion / Logseq" — only when you explicitly name LLM Wiki / `my wiki` / `知识库`. + +## Project Structure + +``` +my-wiki/ +├── purpose.md # Goals, key questions, research scope +├── schema.md # Wiki structure rules, page types +├── raw/ +│ ├── sources/ # Uploaded documents (immutable) +│ └── assets/ # Local images +├── wiki/ +│ ├── index.md # Content catalog +│ ├── log.md # Operation history +│ ├── overview.md # Global summary (auto-updated) +│ ├── entities/ # People, organizations, products +│ ├── concepts/ # Theories, methods, techniques +│ ├── sources/ # Source summaries +│ ├── queries/ # Saved chat answers + research +│ ├── synthesis/ # Cross-source analysis +│ └── comparisons/ # Side-by-side comparisons +├── .obsidian/ # Obsidian vault config (auto-generated) +└── .llm-wiki/ # App config, chat history, review items +``` + +## Star History + + + + + + Star History Chart + + + +## License + +This project is licensed under the **GNU General Public License v3.0** — see [LICENSE](LICENSE) for details. diff --git a/raw/llm_wiki/README_CN.md b/raw/llm_wiki/README_CN.md new file mode 100644 index 0000000..9c4f7eb --- /dev/null +++ b/raw/llm_wiki/README_CN.md @@ -0,0 +1,500 @@ +# LLM Wiki + +

+ LLM Wiki Logo +

+ +

+ 一个能自我构建的个人知识库。
+ LLM 阅读你的文档,构建结构化 Wiki,并持续保持更新。 +

+ +

+ 这是什么? • + 功能特性 • + 技术栈 • + 安装 • + 致谢 • + 许可证 +

+ +

+ English | 中文 | 日本語 | 한국어 +

+ +--- + +

+ 概览 +

+ +## 功能亮点 + +- **两步思维链摄入** — LLM 先分析再生成 Wiki 页面,来源可追溯,支持增量缓存 +- **多模态图片摄入** — 自动提取 PDF 内嵌图片,调用视觉模型生成事实性描述,搜索结果按图文分区,支持 lightbox 预览与跳转到原始文档对应位置 +- **多格式文档解析** — 支持 PDF、Office 文档、EPUB/MOBI、Org mode、图片、音视频、网页剪藏和批量 URL 导入,并提供内置、云端或本地 MinerU PDF 处理 +- **灵活的模型配置** — 支持项目级模型配置、Chat/Ingest 独立模型路由,以及自定义 Provider、请求头和流式输出 +- **原始资料检索** — 可使用“只读原文”模式,仅依据导入的原始资料回答 +- **项目管理与迁移** — 支持完整项目归档的跨设备导入导出,并可根据现有 Wiki 页面重建索引 +- **四信号知识图谱** — 直接链接、来源重叠、Adamic-Adar、类型亲和四维关联度模型 +- **Louvain 社区检测** — 自动发现知识聚类,内聚度评分 +- **图谱洞察** — 惊奇连接与知识空白检测,一键触发 Deep Research +- **向量语义搜索** — 可选的 embedding 检索,基于 LanceDB,支持任意 OpenAI 兼容端点 +- **持久化摄入队列** — 串行处理,崩溃恢复,取消/重试,进度可视化 +- **文件夹导入** — 递归导入保留目录结构,文件夹路径作为 LLM 分类上下文 +- **Source 文件夹自动监听** — 检测 `raw/sources/` 的外部变更,并同步触发摄入或删除清理 +- **深度研究** — LLM 智能生成搜索主题,通过 Tavily、SerpApi 或 SearXNG 进行多查询网络搜索,研究结果自动摄入 Wiki +- **Rust 后端 Chat Agent** — 支持工具调用的聊天运行时,可进行 Wiki/Source/Graph/Web 检索、workspace 文件生成、shell 审批、取消和流式工具事件展示 +- **Agent Skills** — 扫描并启用本地 `SKILL.md` 目录,在聊天中用 `/skill` 选择,让 Agent 按需读取 Skill 指令 +- **生成物预览** — Agent 生成的 Markdown、HTML、图片等 workspace 文件会作为生成物展示,支持预览和快速打开目录 +- **Mermaid 流程图渲染** — 聊天和预览中可直接渲染 Mermaid 代码块,语法错误会显示为紧凑错误卡片 +- **异步审核系统** — LLM 在摄入时标记需人工判断的项,预定义操作,预生成搜索查询 +- **Chrome 网页剪藏** — 一键捕获网页内容,自动摄入知识库 +- **本地 HTTP API + MCP Server + AI Agent Skill** — 内置 `127.0.0.1:19828` JSON API 和随包提供的 MCP Server,支持 Hybrid 检索、文件读取、知识图谱遍历、源资料重新扫描;配套 [agent skill](https://github.com/nashsu/llm_wiki_skill) 一行命令接入 Claude Code / Codex(`npx skills add …`) + +## 这是什么? + +LLM Wiki 是一个跨平台桌面应用,能将你的文档自动转化为有组织、相互关联的知识库。与传统 RAG(每次查询都从头检索和回答)不同,LLM 会从你的资料中**增量构建并维护一个持久化的 Wiki**。知识只编译一次并持续更新,而非每次查询都重新推导。 + +本项目基于 [Karpathy 的 LLM Wiki 方法论](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) —— 一套使用 LLM 构建个人知识库的方法论。llm_wiki 由 [nash_su](https://x.com/nash_su) 创建和维护,并在保留核心理念的基础上,将其实现为完整桌面应用,加入了大量增强。 + +

+ LLM Wiki 架构图 +

+ +## 致谢 + +基础方法论来自 **Andrej Karpathy** 的 [llm-wiki.md](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f),描述了使用 LLM 增量构建和维护个人 Wiki 的设计模式。原始文档是一个抽象的设计范式;本项目是一个具体的实现,并有大量扩展。 + +## 保留的原始设计 + +核心架构忠实遵循 Karpathy 的方法论: + +- **三层架构**:原始资料(不可变)→ Wiki(LLM 生成)→ Schema(规则和配置) +- **三个核心操作**:Ingest(摄入)、Query(查询)、Lint(检查) +- **index.md** 作为内容目录和 LLM 导航入口 +- **log.md** 作为可解析格式的时序操作记录 +- **[[wikilink]]** 语法用于交叉引用 +- **YAML frontmatter** 存在于每个 Wiki 页面 +- **Obsidian 兼容** —— Wiki 目录可直接作为 Obsidian 仓库使用 +- **人类策展,LLM 维护** —— 基本角色分工 + +

+ Obsidian 兼容 +

+ +## 我们的修改与新增 + +### 1. 从命令行到桌面应用 + +原始设计是一个抽象的模式文档,设计上是复制粘贴给 LLM agent 使用的。我们将其构建为**完整的跨平台桌面应用**: +- **三栏布局**:知识树 / 文件树(左)+ 聊天(中)+ 预览(右) +- **图标侧边栏** —— 在 Wiki、资料源、搜索、图谱、Lint、审核、深度研究、设置之间快速切换 +- **自定义可调面板** —— 左右面板支持拖拽调整大小,带最小/最大约束 +- **活动面板** —— 实时处理状态,逐文件显示摄入进度 +- **全状态持久化** —— 对话、设置、审核项、项目配置在重启后保持 +- **场景模板** —— 研究、阅读、个人成长、商业、通用 —— 每个模板预配置 purpose.md 和 schema.md + +### 2. Purpose.md —— Wiki 的灵魂 + +原始设计有 Schema(Wiki 如何运作),但没有正式定义 **为什么** 这个 Wiki 存在。我们新增了 `purpose.md`: +- 定义目标、关键问题、研究范围、演进中的论点 +- LLM 在每次摄入和查询时都会读取它以获取上下文 +- LLM 可以根据使用模式建议更新 +- 与 schema 不同 —— schema 是结构规则,purpose 是方向意图 + +### 3. 两步思维链摄入 + +原始设计描述的是 LLM 同时阅读和写入的单步摄入。我们将其拆分为**两次顺序 LLM 调用**,显著提升质量: + +``` +第一步(分析):LLM 阅读资料 → 结构化分析 + - 关键实体、概念、论点 + - 与现有 Wiki 内容的关联 + - 与现有知识的矛盾和张力 + - Wiki 结构建议 + +第二步(生成):LLM 基于分析 → 生成 Wiki 文件 + - 带 frontmatter 的资料摘要(type, title, sources[]) + - 实体页面、概念页面及交叉引用 + - 更新 index.md、log.md、overview.md + - 需要人工判断的审核项 + - 深度研究的搜索查询 +``` + +超越原始设计的摄入增强: +- **SHA256 增量缓存** —— 摄入前检查源文件内容哈希,未变更则自动跳过,节省 LLM token 和时间 +- **持久化摄入队列** —— 串行处理防止并发 LLM 调用;队列持久化到磁盘,应用重启后自动恢复;失败任务自动重试最多 3 次 +- **文件夹导入** —— 递归导入保留目录结构;文件夹路径作为分类上下文传给 LLM(如 "papers > energy" 帮助分类) +- **Source 文件夹自动监听** —— 在应用外新增、修改或删除 `raw/sources/` 文件时会被自动检测,并复用应用内相同的摄入/删除生命周期 +- **队列可视化** —— 活动面板显示进度条、排队/处理中/失败任务,支持取消和重试 +- **自动 Embedding** —— 开启向量搜索时,新页面摄入后自动生成 embedding +- **来源可追溯** —— 每个生成的 Wiki 页面在 YAML frontmatter 中包含 `sources: []` 字段,链接回贡献的原始资料文件 +- **overview.md 自动更新** —— 全局概要页面在每次摄入后重新生成,反映 Wiki 最新状态 +- **保证资料摘要生成** —— 兜底机制确保资料摘要页面始终被创建,即使 LLM 遗漏 +- **语言感知生成** —— LLM 按用户配置的语言(中文或英文)响应 +- **资料源渐进渲染** —— 大型资料目录会随滚动分批渲染,保持 Sources 页面流畅 + +### 4. 知识图谱与关联度模型 + +

+ 知识图谱 +

+ +原始设计提到了 `[[wikilinks]]` 用于交叉引用,但没有图分析。我们构建了**完整的知识图谱可视化和关联度引擎**: + +**四信号关联度模型:** +| 信号 | 权重 | 描述 | +|------|------|------| +| 直接链接 | ×3.0 | 通过 `[[wikilinks]]` 链接的页面 | +| 来源重叠 | ×4.0 | 共享同一原始资料的页面(通过 frontmatter `sources[]`) | +| Adamic-Adar | ×1.5 | 共享共同邻居的页面(按邻居度数加权) | +| 类型亲和 | ×1.0 | 相同页面类型的加分(实体↔实体,概念↔概念) | + +**图谱可视化(sigma.js + graphology + ForceAtlas2):** +- 按页面类型或社区着色节点,按链接数缩放节点大小(√ 缩放) +- 边的粗细和颜色按关联权重变化(绿色=强,灰色=弱) +- 悬停交互:邻居节点保持可见,非邻居变暗,边高亮并显示关联度分数 +- 缩放控件(放大、缩小、适应屏幕) +- 位置缓存防止数据更新时布局跳动 +- 图例根据着色模式自动切换类型计数或社区信息 + +### 5. Louvain 社区检测 + +原始设计中没有。基于 **Louvain 算法**(graphology-communities-louvain)自动发现知识聚类: + +- **自动聚类** —— 根据链接拓扑发现哪些页面自然归为一组,独立于预定义的页面类型 +- **类型 / 社区 一键切换** —— 按页面类型(实体、概念、资料...)或按发现的知识集群着色 +- **内聚度评分** —— 每个社区按内部边密度(实际边数 / 可能边数)评分;低内聚社区(< 0.15)标警告 +- **12 色调色板** —— 集群之间视觉区分清晰 +- **社区图例** —— 显示核心节点标签、成员数和内聚度 + +

+ Louvain 社区检测 +

+ +### 6. 图谱洞察 —— 惊奇连接与知识空白 + +原始设计中没有。系统**自动分析图谱结构**,呈现可操作的洞察: + +**惊奇连接:** +- 检测意外关联:跨社区边、跨类型链接、边缘↔核心耦合 +- 复合惊奇度评分排序最值得关注的连接 +- 可消除 —— 标记为已查看后不再重复出现 + +**知识空白:** +- **孤立页面**(度 ≤ 1)—— 与 Wiki 其余部分缺少连接的页面 +- **稀疏社区**(cohesion < 0.15,≥ 3 页)—— 内部交叉引用薄弱的知识领域 +- **桥接节点**(连接 3+ 个集群)—— 维系多个知识领域的关键枢纽页面 + +**交互:** +- 点击洞察卡片**高亮**图谱中对应节点和边;再次点击取消 +- 知识空白和桥接节点附带 **Deep Research 按钮** —— 触发 LLM 智能主题生成(读取 overview.md + purpose.md 获取领域上下文) +- 研究主题在**可编辑确认对话框**中展示 —— 用户可修改主题和搜索查询后再启动 + +

+ 图谱洞察 +

+ +### 7. 优化的查询检索管线 + +原始设计描述了 LLM 读取相关页面的简单查询。我们构建了支持可选向量搜索的**多阶段检索管线**: + +``` +阶段 1:分词搜索 + - 英文:分词 + 停用词过滤 + - 中文:CJK 二元组分词(每个 → [每个, 个…]) + - 标题匹配加分(+10 分) + - 同时搜索 wiki/ 和 raw/sources/ + +阶段 1.5:向量语义搜索(可选) + - 通过任意 OpenAI 兼容的 /v1/embeddings 端点生成 embedding + - 存储在 LanceDB(Rust 后端)中进行快速 ANN 检索 + - 余弦相似度发现即使没有关键词重叠也语义相关的页面 + - 结果合并:增强已有匹配 + 添加新发现 + +阶段 2:图谱扩展 + - 搜索结果作为种子节点 + - 四信号关联度模型发现相关页面 + - 2 跳遍历带衰减,发现更深层关联 + +阶段 3:预算控制 + - 可配置上下文窗口:4K → 1M tokens + - 比例分配:60% Wiki 页面,20% 聊天历史,5% 索引,15% 系统提示 + - 页面按搜索 + 图谱关联度综合分数排序 + +阶段 4:上下文组装 + - 编号页面附完整内容(非仅摘要) + - 系统提示包含:purpose.md、语言规则、引用格式、index.md + - LLM 被指示按编号引用页面:[1]、[2] 等 +``` + +**向量搜索**完全可选 —— 默认关闭,在设置中开启,有独立的端点、API Key 和模型配置。关闭时管线 fallback 到分词搜索 + 图谱扩展。基准测试:开启向量搜索后整体召回率从 58.2% 提升至 71.4%。 + +### 8. 多对话聊天与持久化 + +原始设计只有单一查询接口。我们构建了**完整的多对话支持**: + +- **独立聊天会话** —— 创建、重命名、删除对话 +- **对话侧边栏** —— 快速切换不同主题 +- **逐对话持久化** —— 每个对话保存到 `.llm-wiki/chats/{id}.json` +- **可配置历史深度** —— 限制作为上下文发送的消息数量(默认:10) +- **引用参考面板** —— 每条回复上可折叠的区域,显示使用了哪些 Wiki 页面,按类型分组并附图标 +- **引用持久化** —— 引用的页面直接存储在消息数据中,重启后稳定不变 +- **重新生成** —— 一键重新生成最后一条回复(移除最后的助手+用户消息对,重新发送) +- **保存到 Wiki** —— 将有价值的回答归档到 `wiki/queries/`,然后自动摄入提取实体/概念到知识网络 + +### 9. Rust 后端 Chat Agent 与 Skills + +原始设计中没有。聊天现在由 Rust 后端 Agent runtime 驱动,而不是只在浏览器端运行 TypeScript 循环: + +- **工具型 Agent** —— 可自主选择 Wiki 检索、Source 检索、图谱检索、网页搜索、AnyTXT、workspace 文件工具、已批准的 shell 命令和 Skill 文件读取 +- **Skill 管理** —— 扫描项目级和用户级 Skill 目录,启用或禁用 Skill,并在每个会话中通过 `/skill` 补全选择 Skill +- **生成物管理** —— Agent 工具生成的文件统一放在 `agent-workspace/` 下,并作为生成物在聊天中展示、预览或打开目录 +- **用户交互表单** —— Skill 可以请求单选、多选或自由文本等结构化用户输入,不需要为每个 Skill 硬编码专用界面 +- **更安全的执行模型** —— 项目 workspace 内的命令可以顺畅继续执行,外部 shell 命令仍需要明确批准 + +### 10. 思维链 / 推理过程展示 + +原始设计中没有。针对会输出 `` 块的 LLM(DeepSeek、QwQ 等): + +- **流式思维展示** —— 生成中滚动显示 5 行,带透明度渐变 +- **默认折叠** —— 生成完成后思维块隐藏,点击展开 +- **视觉分离** —— 思维内容以独特样式显示,与主回复分开 + +### 11. Markdown 渲染:KaTeX 数学公式与 Mermaid 图表 + +原始设计中没有。聊天和预览支持更丰富的 Markdown 渲染: + +- **KaTeX 渲染** —— 行内 `$...$` 和块级 `$$...$$` 公式通过 remark-math + rehype-katex 渲染 +- **Milkdown 数学插件** —— 预览编辑器通过 @milkdown/plugin-math 原生渲染数学公式 +- **自动检测** —— 裸 `\begin{aligned}` 等 LaTeX 环境自动补上 `$$` 定界符 +- **Unicode 降级** —— 100+ 符号映射(α, ∑, →, ≤ 等)用于数学块外的简单行内符号 +- **Mermaid 代码块** —— fenced `mermaid` 代码块可直接渲染为流程图、时序图等 Mermaid 支持的图表 +- **紧凑 Mermaid 错误** —— 语法错误会被收敛到小型错误卡片中,不会把原始解析器输出铺满聊天界面 + +### 12. 审核系统(异步人机协作) + +原始设计建议在摄入时全程参与。我们新增了**异步审核队列**: + +- LLM 在摄入过程中标记需要人工判断的项目 +- **预定义操作类型**:创建页面、深度研究、跳过 —— 约束操作防止 LLM 凭空生成任意操作 +- **摄入时生成搜索查询** —— LLM 预先为每个审核项生成优化的网络搜索查询 +- 用户可在方便时处理审核 —— 不阻塞摄入流程 + +### 13. 深度研究 + +

+ 深度研究 +

+ +原始设计中没有。当 LLM 识别出知识空白时: + +- **网络搜索** 支持 Tavily、SerpApi 或 SearXNG,查找相关资料并返回完整内容(非截断摘要) +- **Provider 独立配置** —— Tavily 和 SerpApi 使用各自 API Key;SerpApi 支持选择搜索引擎,SearXNG 使用实例 URL 和搜索分类 +- **多条搜索查询** —— 摄入时由 LLM 生成,针对搜索引擎优化 +- **LLM 智能主题生成** —— 从图谱洞察触发时,LLM 读取 overview.md + purpose.md 生成领域精准的研究主题和查询(非泛泛关键词) +- **用户确认对话框** —— 研究主题和搜索查询可编辑,确认后才开始研究 +- **LLM 综合** 搜索结果生成 Wiki 研究页面,并交叉引用现有 Wiki +- **思维链展示** —— 综合过程中 `` 块显示为可折叠区域,自动滚动到最新内容 +- **自动摄入** —— 研究结果自动进入两步摄入流程,提取实体/概念到 Wiki +- **任务队列** —— 最多 3 个并发任务 +- **研究面板** —— 专用侧边面板,动态高度,实时流式进度 + +### 14. 浏览器扩展(网页剪藏) + +

+ Chrome 扩展网页剪藏 +

+ +原始设计提到了 Obsidian Web Clipper。我们构建了**专用 Chrome 扩展**(Manifest V3): + +- **Mozilla Readability.js** 精确提取文章内容(去除广告、导航、侧边栏) +- **Turndown.js** 将 HTML 转换为 Markdown,支持表格 +- **项目选择器** —— 选择剪藏到哪个 Wiki(支持多项目) +- **本地 HTTP API**(端口 19827,tiny_http)—— 扩展 ↔ 应用通信 +- **自动摄入** —— 剪藏内容自动触发两步摄入流程 +- **剪藏监听** —— 每 3 秒轮询新剪藏,自动处理 +- **离线预览** —— 即使应用未运行也能显示提取的内容 + +### 15. 多格式文档支持 + +原始设计聚焦于纯文本/Markdown。我们支持保留文档语义的结构化提取: + +| 格式 | 方法 | +|------|------| +| PDF | 内置 pdf-extract(Rust)+ 文件缓存;可选 MinerU 云端、Local API 或 Pipeline 模式解析复杂排版 | +| DOCX | docx-rs —— 标题、加粗/斜体、列表、表格 → 结构化 Markdown | +| PPTX | ZIP + XML —— 逐页提取,保留标题/列表结构 | +| XLSX/XLS/ODS | calamine —— 正确的单元格类型、多工作表支持、Markdown 表格 | +| EPUB/MOBI | 提取电子书元数据、章节和正文,转换为可摄取内容 | +| 图片 | 原生预览(png, jpg, gif, webp, svg 等) | +| 视频/音频 | 内置播放器 | +| 网页剪藏 | Readability.js + Turndown.js → 干净的 Markdown | + +> MinerU 是可选功能。复杂 PDF 可使用 MinerU 云端、官方 Local API 或本地 Pipeline 模式;本地模式无需上传文件,提取的图片会保存到项目管理的 `wiki/media` 目录。若 MinerU 失败,LLM Wiki 会回退到内置解析器。 + +### 16. 文件删除级联清理 + +原始设计没有删除机制。我们新增了**智能级联删除**: + +- 删除资料文件时同时移除其 Wiki 摘要页面 +- **三重匹配** 查找相关 Wiki 页面:frontmatter `sources[]` 字段、资料摘要页面名称、frontmatter 章节引用 +- **共享实体保护** —— 链接到多个资料的实体/概念页面仅从其 `sources[]` 数组中移除被删除的资料,而非删除整个页面 +- **索引清理** —— 被移除的页面从 index.md 中清除 +- **Wiki 链接清理** —— 指向已删除页面的失效 `[[wikilinks]]` 从其余 Wiki 页面中移除 + +### 17. 可配置上下文窗口 + +原始设计中没有。用户可配置 LLM 接收多少上下文: + +- **4K 到 1M tokens 滑块** —— 适配不同 LLM 的能力 +- **比例预算分配** —— 更大的窗口按比例获得更多 Wiki 内容 +- **60/20/5/15 分配** —— Wiki 页面 / 聊天历史 / 索引 / 系统提示 + +### 18. 跨平台兼容 + +原始设计与平台无关(抽象模式)。我们处理了具体的跨平台问题: + +- **路径规范化** —— 统一的 `normalizePath()` 在 22+ 个文件中使用,反斜杠 → 正斜杠 +- **Unicode 安全字符串处理** —— 基于字符而非字节的切片(防止中文文件名导致崩溃) +- **macOS 关闭隐藏** —— 关闭按钮隐藏窗口(程序后台运行),点击 Dock 图标恢复,Cmd+Q 退出 +- **Windows/Linux 关闭确认** —— 关闭时弹出确认对话框,防止误操作导致数据丢失 +- **Tauri v2** —— macOS、Windows、Linux 原生桌面 +- **GitHub Actions CI/CD** —— 自动构建 macOS(ARM + Intel)、Windows(.msi)、Linux(.deb / .AppImage) + +### 19. 其他新增 + +- **国际化** —— 中英文界面(react-i18next) +- **设置持久化** —— LLM 提供商、API 密钥、模型、上下文大小、语言通过 Tauri Store 保存 +- **Obsidian 配置** —— 自动生成 `.obsidian/` 目录及推荐设置 +- **Markdown 渲染** —— 带边框的 GFM 表格、代码块、聊天和预览中的 wikilink 处理 +- **多 LLM 提供商** —— OpenAI、Anthropic、Google、Ollama、自定义 —— 各有特定的流式传输和请求头 +- **可配置 LLM 超时** —— 可针对较慢的本地模型和长任务调整请求超时 +- **可配置 Firecrawl** —— 支持可选 API Key 和自定义 Base URL,可连接托管或自部署服务 +- **可折叠文件侧栏** —— 可收起 Knowledge/Files 导航并保存折叠状态 +- **项目维护** —— 支持 ZIP 导入导出迁移和确定性重建 `wiki/index.md` +- **dataVersion 信号** —— 图谱和 UI 在 Wiki 内容变更时自动刷新 + +## 技术栈 + +| 层级 | 技术 | +|------|------| +| 桌面 | Tauri v2(Rust 后端) | +| 前端 | React 19 + TypeScript + Vite | +| UI | shadcn/ui + Tailwind CSS v4 | +| 编辑器 | Milkdown(基于 ProseMirror 的所见即所得) | +| 图谱 | sigma.js + graphology + ForceAtlas2 | +| 搜索 | 分词搜索 + 图谱关联度 + 可选向量(LanceDB) | +| 向量数据库 | LanceDB(Rust,嵌入式,可选) | +| 文档解析 | pdf-extract + MinerU 云端/本地 + docx-rs + calamine + EPUB/MOBI 提取 | +| 国际化 | react-i18next | +| 状态管理 | Zustand | +| LLM | 流式 fetch(OpenAI、Anthropic、Google、Ollama、自定义) | +| 网络搜索 | Tavily、SerpApi、SearXNG JSON API | + +## 安装 + +### 预编译二进制文件 + +从 [Releases](https://github.com/nashsu/llm_wiki/releases) 下载: +- **macOS**:`.dmg`(Apple Silicon + Intel) +- **Windows**:`.msi` +- **Linux**:`.deb` / `.AppImage` + +### 从源码构建 + +```bash +# 前置条件:Node.js 20+, Rust 1.70+ +git clone https://github.com/nashsu/llm_wiki.git +cd llm_wiki +npm install +npm run tauri dev # 开发模式 +npm run tauri build # 生产构建 +``` + +### Chrome 扩展 + +1. 打开 `chrome://extensions` +2. 启用「开发者模式」 +3. 点击「加载已解压的扩展程序」 +4. 选择 `extension/` 目录 +5. 使用 `Alt+Shift+L`(macOS 为 `Command+Shift+L`)直接剪藏当前页面;可在 `chrome://extensions/shortcuts` 中自定义快捷键 + +## 快速开始 + +1. 启动应用 → 创建新项目(选择模板) +2. 进入 **设置** → 配置 LLM 提供商(API 密钥 + 模型) +3. 可选:在 **设置** 中配置网络搜索 Provider 和 source 文件夹自动监听 +4. 进入 **资料源** → 导入文档(PDF、DOCX、MD 等) +5. 观察 **活动面板** —— LLM 自动构建 Wiki 页面 +6. 使用 **聊天** 查询你的知识库 +7. 浏览 **知识图谱** 查看关联 +8. 查看 **审核** 处理需要你关注的项目 +9. 定期运行 **Lint** 维护 Wiki 健康度 + +## 本地 HTTP API + MCP Server + AI Agent Skill + +LLM Wiki 内置一个本地 HTTP API(监听 `http://127.0.0.1:19828`,Token 鉴权,仅本机可达),任何外部工具——包括 **Claude Code**、**Codex** 这类 AI Agent,或者任意能发 HTTP 请求的脚本——都可以直接查询你的知识库: + +- `GET /api/v1/health` —— 服务状态(无需鉴权) +- `GET /api/v1/projects` —— 项目列表 +- `GET /api/v1/projects/{id}/files` / `files/content` —— 读取文件树与内容 +- `POST /api/v1/projects/{id}/search` —— **Hybrid 混合检索**(关键词 + 向量),返回 `mode`、`tokenHits`、`vectorHits`,每条结果带 `vectorScore` +- `POST /api/v1/projects/{id}/chat` —— 非流式 Rust 后端 Agent 聊天接口,返回助手消息、引用、用量和工具事件;支持 Wiki/Source/Web/AnyTXT 检索,`mode: "deep"` 会扩展证据收集范围 +- `GET /api/v1/projects/{id}/graph` —— Wikilinks 知识图谱 +- `POST /api/v1/projects/{id}/sources/rescan` —— 触发后端重新扫描 + +在 **设置 → API + MCP** 中开启 API、生成 Token,并按需选择是否允许本机无鉴权访问。 + +对于兼容 MCP 的客户端,LLM Wiki 还内置了 `mcp-server/`。执行 `npm run mcp:build` 构建后,**设置 → API + MCP** 会展示一份可复制的 MCP 客户端配置,并自动填入当前机器上的真实入口路径。MCP 工具复用同一套 API 能力,因此 Agent 可以直接列出项目、读取文件、执行 Hybrid 检索、查看图谱、触发资料源重新扫描,并调用同一套 Rust 后端 Agent 聊天接口,不需要再手写 HTTP 调用。 + +### 一条命令把 AI Agent 接进你的知识库 + +LLM Wiki 配套的 **agent skill** 单独维护在另一个仓库。把它装进 Claude Code / Codex / 任意兼容 skills 的 runtime: + +```bash +npx skills add https://github.com/nashsu/llm_wiki_skill.git --skill llm-wiki +``` + +安装完成后,Agent 就能响应 "我的 LLM Wiki 里关于 X 是怎么说的"、"在我的知识库里搜 Y"、"展示我 wiki 图谱里 Z 的邻居"、"重新索引我的资料源" 等请求——直接调用本机运行的 App,默认只读,引用 wiki 页面路径方便你在 App 内核对。 + +- **Skill 仓库**: +- **触发约束**:刻意**不会**响应"搜我的笔记"/"看我的 Obsidian / Notion / Logseq"这类泛指的请求——只有你明确说 LLM Wiki / `我的 wiki` / `我的知识库` 时才会被调用。 + +## 项目结构 + +``` +my-wiki/ +├── purpose.md # 目标、关键问题、研究范围 +├── schema.md # Wiki 结构规则、页面类型 +├── raw/ +│ ├── sources/ # 上传的文档(不可变) +│ └── assets/ # 本地图片 +├── wiki/ +│ ├── index.md # 内容目录 +│ ├── log.md # 操作历史 +│ ├── overview.md # 全局概要(自动更新) +│ ├── entities/ # 人物、组织、产品 +│ ├── concepts/ # 理论、方法、技术 +│ ├── sources/ # 资料摘要 +│ ├── queries/ # 保存的聊天回答 + 研究 +│ ├── synthesis/ # 跨资料分析 +│ └── comparisons/ # 并列对比 +├── .obsidian/ # Obsidian 仓库配置(自动生成) +└── .llm-wiki/ # 应用配置、聊天历史、审核项 +``` + +## Star History + + + + + + Star History Chart + + + +## 许可证 + +本项目基于 **GNU 通用公共许可证 v3.0** 授权 —— 详见 [LICENSE](LICENSE)。 diff --git a/raw/llm_wiki/assets/1-deepresearch.jpg b/raw/llm_wiki/assets/1-deepresearch.jpg new file mode 100644 index 0000000..a6eea4a Binary files /dev/null and b/raw/llm_wiki/assets/1-deepresearch.jpg differ diff --git a/raw/llm_wiki/assets/2-ai_chat.jpg b/raw/llm_wiki/assets/2-ai_chat.jpg new file mode 100644 index 0000000..7a9577d Binary files /dev/null and b/raw/llm_wiki/assets/2-ai_chat.jpg differ diff --git a/raw/llm_wiki/assets/3-knowledge_graph.jpg b/raw/llm_wiki/assets/3-knowledge_graph.jpg new file mode 100644 index 0000000..d9d1182 Binary files /dev/null and b/raw/llm_wiki/assets/3-knowledge_graph.jpg differ diff --git a/raw/llm_wiki/assets/4-chrome_extension_webclipper.jpg b/raw/llm_wiki/assets/4-chrome_extension_webclipper.jpg new file mode 100644 index 0000000..2c5b660 Binary files /dev/null and b/raw/llm_wiki/assets/4-chrome_extension_webclipper.jpg differ diff --git a/raw/llm_wiki/assets/5-obsidian_compatibility.jpg b/raw/llm_wiki/assets/5-obsidian_compatibility.jpg new file mode 100644 index 0000000..f0eb210 Binary files /dev/null and b/raw/llm_wiki/assets/5-obsidian_compatibility.jpg differ diff --git a/raw/llm_wiki/assets/kg_community.jpg b/raw/llm_wiki/assets/kg_community.jpg new file mode 100644 index 0000000..bc5d8bd Binary files /dev/null and b/raw/llm_wiki/assets/kg_community.jpg differ diff --git a/raw/llm_wiki/assets/kg_insights.jpg b/raw/llm_wiki/assets/kg_insights.jpg new file mode 100644 index 0000000..366fa3e Binary files /dev/null and b/raw/llm_wiki/assets/kg_insights.jpg differ diff --git a/raw/llm_wiki/assets/llm_wiki_arch.jpg b/raw/llm_wiki/assets/llm_wiki_arch.jpg new file mode 100644 index 0000000..4ad7686 Binary files /dev/null and b/raw/llm_wiki/assets/llm_wiki_arch.jpg differ diff --git a/raw/llm_wiki/assets/overview.jpg b/raw/llm_wiki/assets/overview.jpg new file mode 100644 index 0000000..4202e42 Binary files /dev/null and b/raw/llm_wiki/assets/overview.jpg differ diff --git a/raw/llm_wiki/components.json b/raw/llm_wiki/components.json new file mode 100644 index 0000000..15addee --- /dev/null +++ b/raw/llm_wiki/components.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "base-nova", + "rsc": false, + "tsx": true, + "tailwind": { + "config": "", + "css": "src/index.css", + "baseColor": "neutral", + "cssVariables": true, + "prefix": "" + }, + "iconLibrary": "lucide", + "rtl": false, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" + }, + "menuColor": "default", + "menuAccent": "subtle", + "registries": {} +} diff --git a/raw/llm_wiki/extension/Readability.js b/raw/llm_wiki/extension/Readability.js new file mode 100644 index 0000000..1fedd0e --- /dev/null +++ b/raw/llm_wiki/extension/Readability.js @@ -0,0 +1,2789 @@ +(function() { +/* + * Copyright (c) 2010 Arc90 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This code is heavily based on Arc90's readability.js (1.7.1) script + * available at: http://code.google.com/p/arc90labs-readability + */ + +/** + * Public constructor. + * @param {HTMLDocument} doc The document to parse. + * @param {Object} options The options object. + */ +function Readability(doc, options) { + // In some older versions, people passed a URI as the first argument. Cope: + if (options && options.documentElement) { + doc = options; + options = arguments[2]; + } else if (!doc || !doc.documentElement) { + throw new Error( + "First argument to Readability constructor should be a document object." + ); + } + options = options || {}; + + this._doc = doc; + this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__; + this._articleTitle = null; + this._articleByline = null; + this._articleDir = null; + this._articleSiteName = null; + this._attempts = []; + this._metadata = {}; + + // Configurable options + this._debug = !!options.debug; + this._maxElemsToParse = + options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE; + this._nbTopCandidates = + options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES; + this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD; + this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat( + options.classesToPreserve || [] + ); + this._keepClasses = !!options.keepClasses; + this._serializer = + options.serializer || + function (el) { + return el.innerHTML; + }; + this._disableJSONLD = !!options.disableJSONLD; + this._allowedVideoRegex = options.allowedVideoRegex || this.REGEXPS.videos; + this._linkDensityModifier = options.linkDensityModifier || 0; + + // Start with all flags set + this._flags = + this.FLAG_STRIP_UNLIKELYS | + this.FLAG_WEIGHT_CLASSES | + this.FLAG_CLEAN_CONDITIONALLY; + + // Control whether log messages are sent to the console + if (this._debug) { + let logNode = function (node) { + if (node.nodeType == node.TEXT_NODE) { + return `${node.nodeName} ("${node.textContent}")`; + } + let attrPairs = Array.from(node.attributes || [], function (attr) { + return `${attr.name}="${attr.value}"`; + }).join(" "); + return `<${node.localName} ${attrPairs}>`; + }; + this.log = function () { + if (typeof console !== "undefined") { + let args = Array.from(arguments, arg => { + if (arg && arg.nodeType == this.ELEMENT_NODE) { + return logNode(arg); + } + return arg; + }); + args.unshift("Reader: (Readability)"); + // eslint-disable-next-line no-console + console.log(...args); + } else if (typeof dump !== "undefined") { + /* global dump */ + var msg = Array.prototype.map + .call(arguments, function (x) { + return x && x.nodeName ? logNode(x) : x; + }) + .join(" "); + dump("Reader: (Readability) " + msg + "\n"); + } + }; + } else { + this.log = function () {}; + } +} + +Readability.prototype = { + FLAG_STRIP_UNLIKELYS: 0x1, + FLAG_WEIGHT_CLASSES: 0x2, + FLAG_CLEAN_CONDITIONALLY: 0x4, + + // https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType + ELEMENT_NODE: 1, + TEXT_NODE: 3, + + // Max number of nodes supported by this parser. Default: 0 (no limit) + DEFAULT_MAX_ELEMS_TO_PARSE: 0, + + // The number of top candidates to consider when analysing how + // tight the competition is among candidates. + DEFAULT_N_TOP_CANDIDATES: 5, + + // Element tags to score by default. + DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre" + .toUpperCase() + .split(","), + + // The default number of chars an article must have in order to return a result + DEFAULT_CHAR_THRESHOLD: 500, + + // All of the regular expressions in use within readability. + // Defined up here so we don't instantiate them repeatedly in loops. + REGEXPS: { + // NOTE: These two regular expressions are duplicated in + // Readability-readerable.js. Please keep both copies in sync. + unlikelyCandidates: + /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i, + okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i, + + positive: + /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i, + negative: + /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i, + extraneous: + /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i, + byline: /byline|author|dateline|writtenby|p-author/i, + replaceFonts: /<(\/?)font[^>]*>/gi, + normalize: /\s{2,}/g, + videos: + /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i, + shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i, + nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i, + prevLink: /(prev|earl|old|new|<|«)/i, + tokenize: /\W+/g, + whitespace: /^\s*$/, + hasContent: /\S$/, + hashUrl: /^#.+/, + srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g, + b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i, + // Commas as used in Latin, Sindhi, Chinese and various other scripts. + // see: https://en.wikipedia.org/wiki/Comma#Comma_variants + commas: /\u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C/g, + // See: https://schema.org/Article + jsonLdArticleTypes: + /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/, + // used to see if a node's content matches words commonly used for ad blocks or loading indicators + adWords: + /^(ad(vertising|vertisement)?|pub(licité)?|werb(ung)?|广告|Реклама|Anuncio)$/iu, + loadingWords: + /^((loading|正在加载|Загрузка|chargement|cargando)(…|\.\.\.)?)$/iu, + }, + + UNLIKELY_ROLES: [ + "menu", + "menubar", + "complementary", + "navigation", + "alert", + "alertdialog", + "dialog", + ], + + DIV_TO_P_ELEMS: new Set([ + "BLOCKQUOTE", + "DL", + "DIV", + "IMG", + "OL", + "P", + "PRE", + "TABLE", + "UL", + ]), + + ALTER_TO_DIV_EXCEPTIONS: ["DIV", "ARTICLE", "SECTION", "P", "OL", "UL"], + + PRESENTATIONAL_ATTRIBUTES: [ + "align", + "background", + "bgcolor", + "border", + "cellpadding", + "cellspacing", + "frame", + "hspace", + "rules", + "style", + "valign", + "vspace", + ], + + DEPRECATED_SIZE_ATTRIBUTE_ELEMS: ["TABLE", "TH", "TD", "HR", "PRE"], + + // The commented out elements qualify as phrasing content but tend to be + // removed by readability when put into paragraphs, so we ignore them here. + PHRASING_ELEMS: [ + // "CANVAS", "IFRAME", "SVG", "VIDEO", + "ABBR", + "AUDIO", + "B", + "BDO", + "BR", + "BUTTON", + "CITE", + "CODE", + "DATA", + "DATALIST", + "DFN", + "EM", + "EMBED", + "I", + "IMG", + "INPUT", + "KBD", + "LABEL", + "MARK", + "MATH", + "METER", + "NOSCRIPT", + "OBJECT", + "OUTPUT", + "PROGRESS", + "Q", + "RUBY", + "SAMP", + "SCRIPT", + "SELECT", + "SMALL", + "SPAN", + "STRONG", + "SUB", + "SUP", + "TEXTAREA", + "TIME", + "VAR", + "WBR", + ], + + // These are the classes that readability sets itself. + CLASSES_TO_PRESERVE: ["page"], + + // These are the list of HTML entities that need to be escaped. + HTML_ESCAPE_MAP: { + lt: "<", + gt: ">", + amp: "&", + quot: '"', + apos: "'", + }, + + /** + * Run any post-process modifications to article content as necessary. + * + * @param Element + * @return void + **/ + _postProcessContent(articleContent) { + // Readability cannot open relative uris so we convert them to absolute uris. + this._fixRelativeUris(articleContent); + + this._simplifyNestedElements(articleContent); + + if (!this._keepClasses) { + // Remove classes. + this._cleanClasses(articleContent); + } + }, + + /** + * Iterates over a NodeList, calls `filterFn` for each node and removes node + * if function returned `true`. + * + * If function is not passed, removes all the nodes in node list. + * + * @param NodeList nodeList The nodes to operate on + * @param Function filterFn the function to use as a filter + * @return void + */ + _removeNodes(nodeList, filterFn) { + // Avoid ever operating on live node lists. + if (this._docJSDOMParser && nodeList._isLiveNodeList) { + throw new Error("Do not pass live node lists to _removeNodes"); + } + for (var i = nodeList.length - 1; i >= 0; i--) { + var node = nodeList[i]; + var parentNode = node.parentNode; + if (parentNode) { + if (!filterFn || filterFn.call(this, node, i, nodeList)) { + parentNode.removeChild(node); + } + } + } + }, + + /** + * Iterates over a NodeList, and calls _setNodeTag for each node. + * + * @param NodeList nodeList The nodes to operate on + * @param String newTagName the new tag name to use + * @return void + */ + _replaceNodeTags(nodeList, newTagName) { + // Avoid ever operating on live node lists. + if (this._docJSDOMParser && nodeList._isLiveNodeList) { + throw new Error("Do not pass live node lists to _replaceNodeTags"); + } + for (const node of nodeList) { + this._setNodeTag(node, newTagName); + } + }, + + /** + * Iterate over a NodeList, which doesn't natively fully implement the Array + * interface. + * + * For convenience, the current object context is applied to the provided + * iterate function. + * + * @param NodeList nodeList The NodeList. + * @param Function fn The iterate function. + * @return void + */ + _forEachNode(nodeList, fn) { + Array.prototype.forEach.call(nodeList, fn, this); + }, + + /** + * Iterate over a NodeList, and return the first node that passes + * the supplied test function + * + * For convenience, the current object context is applied to the provided + * test function. + * + * @param NodeList nodeList The NodeList. + * @param Function fn The test function. + * @return void + */ + _findNode(nodeList, fn) { + return Array.prototype.find.call(nodeList, fn, this); + }, + + /** + * Iterate over a NodeList, return true if any of the provided iterate + * function calls returns true, false otherwise. + * + * For convenience, the current object context is applied to the + * provided iterate function. + * + * @param NodeList nodeList The NodeList. + * @param Function fn The iterate function. + * @return Boolean + */ + _someNode(nodeList, fn) { + return Array.prototype.some.call(nodeList, fn, this); + }, + + /** + * Iterate over a NodeList, return true if all of the provided iterate + * function calls return true, false otherwise. + * + * For convenience, the current object context is applied to the + * provided iterate function. + * + * @param NodeList nodeList The NodeList. + * @param Function fn The iterate function. + * @return Boolean + */ + _everyNode(nodeList, fn) { + return Array.prototype.every.call(nodeList, fn, this); + }, + + _getAllNodesWithTag(node, tagNames) { + if (node.querySelectorAll) { + return node.querySelectorAll(tagNames.join(",")); + } + return [].concat.apply( + [], + tagNames.map(function (tag) { + var collection = node.getElementsByTagName(tag); + return Array.isArray(collection) ? collection : Array.from(collection); + }) + ); + }, + + /** + * Removes the class="" attribute from every element in the given + * subtree, except those that match CLASSES_TO_PRESERVE and + * the classesToPreserve array from the options object. + * + * @param Element + * @return void + */ + _cleanClasses(node) { + var classesToPreserve = this._classesToPreserve; + var className = (node.getAttribute("class") || "") + .split(/\s+/) + .filter(cls => classesToPreserve.includes(cls)) + .join(" "); + + if (className) { + node.setAttribute("class", className); + } else { + node.removeAttribute("class"); + } + + for (node = node.firstElementChild; node; node = node.nextElementSibling) { + this._cleanClasses(node); + } + }, + + /** + * Tests whether a string is a URL or not. + * + * @param {string} str The string to test + * @return {boolean} true if str is a URL, false if not + */ + _isUrl(str) { + try { + new URL(str); + return true; + } catch { + return false; + } + }, + /** + * Converts each and uri in the given element to an absolute URI, + * ignoring #ref URIs. + * + * @param Element + * @return void + */ + _fixRelativeUris(articleContent) { + var baseURI = this._doc.baseURI; + var documentURI = this._doc.documentURI; + function toAbsoluteURI(uri) { + // Leave hash links alone if the base URI matches the document URI: + if (baseURI == documentURI && uri.charAt(0) == "#") { + return uri; + } + + // Otherwise, resolve against base URI: + try { + return new URL(uri, baseURI).href; + } catch (ex) { + // Something went wrong, just return the original: + } + return uri; + } + + var links = this._getAllNodesWithTag(articleContent, ["a"]); + this._forEachNode(links, function (link) { + var href = link.getAttribute("href"); + if (href) { + // Remove links with javascript: URIs, since + // they won't work after scripts have been removed from the page. + if (href.indexOf("javascript:") === 0) { + // if the link only contains simple text content, it can be converted to a text node + if ( + link.childNodes.length === 1 && + link.childNodes[0].nodeType === this.TEXT_NODE + ) { + var text = this._doc.createTextNode(link.textContent); + link.parentNode.replaceChild(text, link); + } else { + // if the link has multiple children, they should all be preserved + var container = this._doc.createElement("span"); + while (link.firstChild) { + container.appendChild(link.firstChild); + } + link.parentNode.replaceChild(container, link); + } + } else { + link.setAttribute("href", toAbsoluteURI(href)); + } + } + }); + + var medias = this._getAllNodesWithTag(articleContent, [ + "img", + "picture", + "figure", + "video", + "audio", + "source", + ]); + + this._forEachNode(medias, function (media) { + var src = media.getAttribute("src"); + var poster = media.getAttribute("poster"); + var srcset = media.getAttribute("srcset"); + + if (src) { + media.setAttribute("src", toAbsoluteURI(src)); + } + + if (poster) { + media.setAttribute("poster", toAbsoluteURI(poster)); + } + + if (srcset) { + var newSrcset = srcset.replace( + this.REGEXPS.srcsetUrl, + function (_, p1, p2, p3) { + return toAbsoluteURI(p1) + (p2 || "") + p3; + } + ); + + media.setAttribute("srcset", newSrcset); + } + }); + }, + + _simplifyNestedElements(articleContent) { + var node = articleContent; + + while (node) { + if ( + node.parentNode && + ["DIV", "SECTION"].includes(node.tagName) && + !(node.id && node.id.startsWith("readability")) + ) { + if (this._isElementWithoutContent(node)) { + node = this._removeAndGetNext(node); + continue; + } else if ( + this._hasSingleTagInsideElement(node, "DIV") || + this._hasSingleTagInsideElement(node, "SECTION") + ) { + var child = node.children[0]; + for (var i = 0; i < node.attributes.length; i++) { + child.setAttributeNode(node.attributes[i].cloneNode()); + } + node.parentNode.replaceChild(child, node); + node = child; + continue; + } + } + + node = this._getNextNode(node); + } + }, + + /** + * Get the article title as an H1. + * + * @return string + **/ + _getArticleTitle() { + var doc = this._doc; + var curTitle = ""; + var origTitle = ""; + + try { + curTitle = origTitle = doc.title.trim(); + + // If they had an element with id "title" in their HTML + if (typeof curTitle !== "string") { + curTitle = origTitle = this._getInnerText( + doc.getElementsByTagName("title")[0] + ); + } + } catch (e) { + /* ignore exceptions setting the title. */ + } + + var titleHadHierarchicalSeparators = false; + function wordCount(str) { + return str.split(/\s+/).length; + } + + // If there's a separator in the title, first remove the final part + if (/ [\|\-\\\/>»] /.test(curTitle)) { + titleHadHierarchicalSeparators = / [\\\/>»] /.test(curTitle); + let allSeparators = Array.from(origTitle.matchAll(/ [\|\-\\\/>»] /gi)); + curTitle = origTitle.substring(0, allSeparators.pop().index); + + // If the resulting title is too short, remove the first part instead: + if (wordCount(curTitle) < 3) { + curTitle = origTitle.replace(/^[^\|\-\\\/>»]*[\|\-\\\/>»]/gi, ""); + } + } else if (curTitle.includes(": ")) { + // Check if we have an heading containing this exact string, so we + // could assume it's the full title. + var headings = this._getAllNodesWithTag(doc, ["h1", "h2"]); + var trimmedTitle = curTitle.trim(); + var match = this._someNode(headings, function (heading) { + return heading.textContent.trim() === trimmedTitle; + }); + + // If we don't, let's extract the title out of the original title string. + if (!match) { + curTitle = origTitle.substring(origTitle.lastIndexOf(":") + 1); + + // If the title is now too short, try the first colon instead: + if (wordCount(curTitle) < 3) { + curTitle = origTitle.substring(origTitle.indexOf(":") + 1); + // But if we have too many words before the colon there's something weird + // with the titles and the H tags so let's just use the original title instead + } else if (wordCount(origTitle.substr(0, origTitle.indexOf(":"))) > 5) { + curTitle = origTitle; + } + } + } else if (curTitle.length > 150 || curTitle.length < 15) { + var hOnes = doc.getElementsByTagName("h1"); + + if (hOnes.length === 1) { + curTitle = this._getInnerText(hOnes[0]); + } + } + + curTitle = curTitle.trim().replace(this.REGEXPS.normalize, " "); + // If we now have 4 words or fewer as our title, and either no + // 'hierarchical' separators (\, /, > or ») were found in the original + // title or we decreased the number of words by more than 1 word, use + // the original title. + var curTitleWordCount = wordCount(curTitle); + if ( + curTitleWordCount <= 4 && + (!titleHadHierarchicalSeparators || + curTitleWordCount != + wordCount(origTitle.replace(/[\|\-\\\/>»]+/g, "")) - 1) + ) { + curTitle = origTitle; + } + + return curTitle; + }, + + /** + * Prepare the HTML document for readability to scrape it. + * This includes things like stripping javascript, CSS, and handling terrible markup. + * + * @return void + **/ + _prepDocument() { + var doc = this._doc; + + // Remove all style tags in head + this._removeNodes(this._getAllNodesWithTag(doc, ["style"])); + + if (doc.body) { + this._replaceBrs(doc.body); + } + + this._replaceNodeTags(this._getAllNodesWithTag(doc, ["font"]), "SPAN"); + }, + + /** + * Finds the next node, starting from the given node, and ignoring + * whitespace in between. If the given node is an element, the same node is + * returned. + */ + _nextNode(node) { + var next = node; + while ( + next && + next.nodeType != this.ELEMENT_NODE && + this.REGEXPS.whitespace.test(next.textContent) + ) { + next = next.nextSibling; + } + return next; + }, + + /** + * Replaces 2 or more successive
elements with a single

. + * Whitespace between
elements are ignored. For example: + *

foo
bar


abc
+ * will become: + *
foo
bar

abc

+ */ + _replaceBrs(elem) { + this._forEachNode(this._getAllNodesWithTag(elem, ["br"]), function (br) { + var next = br.nextSibling; + + // Whether 2 or more
elements have been found and replaced with a + //

block. + var replaced = false; + + // If we find a
chain, remove the
s until we hit another node + // or non-whitespace. This leaves behind the first
in the chain + // (which will be replaced with a

later). + while ((next = this._nextNode(next)) && next.tagName == "BR") { + replaced = true; + var brSibling = next.nextSibling; + next.remove(); + next = brSibling; + } + + // If we removed a
chain, replace the remaining
with a

. Add + // all sibling nodes as children of the

until we hit another
+ // chain. + if (replaced) { + var p = this._doc.createElement("p"); + br.parentNode.replaceChild(p, br); + + next = p.nextSibling; + while (next) { + // If we've hit another

, we're done adding children to this

. + if (next.tagName == "BR") { + var nextElem = this._nextNode(next.nextSibling); + if (nextElem && nextElem.tagName == "BR") { + break; + } + } + + if (!this._isPhrasingContent(next)) { + break; + } + + // Otherwise, make this node a child of the new

. + var sibling = next.nextSibling; + p.appendChild(next); + next = sibling; + } + + while (p.lastChild && this._isWhitespace(p.lastChild)) { + p.lastChild.remove(); + } + + if (p.parentNode.tagName === "P") { + this._setNodeTag(p.parentNode, "DIV"); + } + } + }); + }, + + _setNodeTag(node, tag) { + this.log("_setNodeTag", node, tag); + if (this._docJSDOMParser) { + node.localName = tag.toLowerCase(); + node.tagName = tag.toUpperCase(); + return node; + } + + var replacement = node.ownerDocument.createElement(tag); + while (node.firstChild) { + replacement.appendChild(node.firstChild); + } + node.parentNode.replaceChild(replacement, node); + if (node.readability) { + replacement.readability = node.readability; + } + + for (var i = 0; i < node.attributes.length; i++) { + replacement.setAttributeNode(node.attributes[i].cloneNode()); + } + return replacement; + }, + + /** + * Prepare the article node for display. Clean out any inline styles, + * iframes, forms, strip extraneous

tags, etc. + * + * @param Element + * @return void + **/ + _prepArticle(articleContent) { + this._cleanStyles(articleContent); + + // Check for data tables before we continue, to avoid removing items in + // those tables, which will often be isolated even though they're + // visually linked to other content-ful elements (text, images, etc.). + this._markDataTables(articleContent); + + this._fixLazyImages(articleContent); + + // Clean out junk from the article content + this._cleanConditionally(articleContent, "form"); + this._cleanConditionally(articleContent, "fieldset"); + this._clean(articleContent, "object"); + this._clean(articleContent, "embed"); + this._clean(articleContent, "footer"); + this._clean(articleContent, "link"); + this._clean(articleContent, "aside"); + + // Clean out elements with little content that have "share" in their id/class combinations from final top candidates, + // which means we don't remove the top candidates even they have "share". + + var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD; + + this._forEachNode(articleContent.children, function (topCandidate) { + this._cleanMatchedNodes(topCandidate, function (node, matchString) { + return ( + this.REGEXPS.shareElements.test(matchString) && + node.textContent.length < shareElementThreshold + ); + }); + }); + + this._clean(articleContent, "iframe"); + this._clean(articleContent, "input"); + this._clean(articleContent, "textarea"); + this._clean(articleContent, "select"); + this._clean(articleContent, "button"); + this._cleanHeaders(articleContent); + + // Do these last as the previous stuff may have removed junk + // that will affect these + this._cleanConditionally(articleContent, "table"); + this._cleanConditionally(articleContent, "ul"); + this._cleanConditionally(articleContent, "div"); + + // replace H1 with H2 as H1 should be only title that is displayed separately + this._replaceNodeTags( + this._getAllNodesWithTag(articleContent, ["h1"]), + "h2" + ); + + // Remove extra paragraphs + this._removeNodes( + this._getAllNodesWithTag(articleContent, ["p"]), + function (paragraph) { + // At this point, nasty iframes have been removed; only embedded video + // ones remain. + var contentElementCount = this._getAllNodesWithTag(paragraph, [ + "img", + "embed", + "object", + "iframe", + ]).length; + return ( + contentElementCount === 0 && !this._getInnerText(paragraph, false) + ); + } + ); + + this._forEachNode( + this._getAllNodesWithTag(articleContent, ["br"]), + function (br) { + var next = this._nextNode(br.nextSibling); + if (next && next.tagName == "P") { + br.remove(); + } + } + ); + + // Remove single-cell tables + this._forEachNode( + this._getAllNodesWithTag(articleContent, ["table"]), + function (table) { + var tbody = this._hasSingleTagInsideElement(table, "TBODY") + ? table.firstElementChild + : table; + if (this._hasSingleTagInsideElement(tbody, "TR")) { + var row = tbody.firstElementChild; + if (this._hasSingleTagInsideElement(row, "TD")) { + var cell = row.firstElementChild; + cell = this._setNodeTag( + cell, + this._everyNode(cell.childNodes, this._isPhrasingContent) + ? "P" + : "DIV" + ); + table.parentNode.replaceChild(cell, table); + } + } + } + ); + }, + + /** + * Initialize a node with the readability object. Also checks the + * className/id for special names to add to its score. + * + * @param Element + * @return void + **/ + _initializeNode(node) { + node.readability = { contentScore: 0 }; + + switch (node.tagName) { + case "DIV": + node.readability.contentScore += 5; + break; + + case "PRE": + case "TD": + case "BLOCKQUOTE": + node.readability.contentScore += 3; + break; + + case "ADDRESS": + case "OL": + case "UL": + case "DL": + case "DD": + case "DT": + case "LI": + case "FORM": + node.readability.contentScore -= 3; + break; + + case "H1": + case "H2": + case "H3": + case "H4": + case "H5": + case "H6": + case "TH": + node.readability.contentScore -= 5; + break; + } + + node.readability.contentScore += this._getClassWeight(node); + }, + + _removeAndGetNext(node) { + var nextNode = this._getNextNode(node, true); + node.remove(); + return nextNode; + }, + + /** + * Traverse the DOM from node to node, starting at the node passed in. + * Pass true for the second parameter to indicate this node itself + * (and its kids) are going away, and we want the next node over. + * + * Calling this in a loop will traverse the DOM depth-first. + * + * @param {Element} node + * @param {boolean} ignoreSelfAndKids + * @return {Element} + */ + _getNextNode(node, ignoreSelfAndKids) { + // First check for kids if those aren't being ignored + if (!ignoreSelfAndKids && node.firstElementChild) { + return node.firstElementChild; + } + // Then for siblings... + if (node.nextElementSibling) { + return node.nextElementSibling; + } + // And finally, move up the parent chain *and* find a sibling + // (because this is depth-first traversal, we will have already + // seen the parent nodes themselves). + do { + node = node.parentNode; + } while (node && !node.nextElementSibling); + return node && node.nextElementSibling; + }, + + // compares second text to first one + // 1 = same text, 0 = completely different text + // works the way that it splits both texts into words and then finds words that are unique in second text + // the result is given by the lower length of unique parts + _textSimilarity(textA, textB) { + var tokensA = textA + .toLowerCase() + .split(this.REGEXPS.tokenize) + .filter(Boolean); + var tokensB = textB + .toLowerCase() + .split(this.REGEXPS.tokenize) + .filter(Boolean); + if (!tokensA.length || !tokensB.length) { + return 0; + } + var uniqTokensB = tokensB.filter(token => !tokensA.includes(token)); + var distanceB = uniqTokensB.join(" ").length / tokensB.join(" ").length; + return 1 - distanceB; + }, + + /** + * Checks whether an element node contains a valid byline + * + * @param node {Element} + * @param matchString {string} + * @return boolean + */ + _isValidByline(node, matchString) { + var rel = node.getAttribute("rel"); + var itemprop = node.getAttribute("itemprop"); + var bylineLength = node.textContent.trim().length; + + return ( + (rel === "author" || + (itemprop && itemprop.includes("author")) || + this.REGEXPS.byline.test(matchString)) && + !!bylineLength && + bylineLength < 100 + ); + }, + + _getNodeAncestors(node, maxDepth) { + maxDepth = maxDepth || 0; + var i = 0, + ancestors = []; + while (node.parentNode) { + ancestors.push(node.parentNode); + if (maxDepth && ++i === maxDepth) { + break; + } + node = node.parentNode; + } + return ancestors; + }, + + /*** + * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is + * most likely to be the stuff a user wants to read. Then return it wrapped up in a div. + * + * @param page a document to run upon. Needs to be a full document, complete with body. + * @return Element + **/ + /* eslint-disable-next-line complexity */ + _grabArticle(page) { + this.log("**** grabArticle ****"); + var doc = this._doc; + var isPaging = page !== null; + page = page ? page : this._doc.body; + + // We can't grab an article if we don't have a page! + if (!page) { + this.log("No body found in document. Abort."); + return null; + } + + var pageCacheHtml = page.innerHTML; + + while (true) { + this.log("Starting grabArticle loop"); + var stripUnlikelyCandidates = this._flagIsActive( + this.FLAG_STRIP_UNLIKELYS + ); + + // First, node prepping. Trash nodes that look cruddy (like ones with the + // class name "comment", etc), and turn divs into P tags where they have been + // used inappropriately (as in, where they contain no other block level elements.) + var elementsToScore = []; + var node = this._doc.documentElement; + + let shouldRemoveTitleHeader = true; + + while (node) { + if (node.tagName === "HTML") { + this._articleLang = node.getAttribute("lang"); + } + + var matchString = node.className + " " + node.id; + + if (!this._isProbablyVisible(node)) { + this.log("Removing hidden node - " + matchString); + node = this._removeAndGetNext(node); + continue; + } + + // User is not able to see elements applied with both "aria-modal = true" and "role = dialog" + if ( + node.getAttribute("aria-modal") == "true" && + node.getAttribute("role") == "dialog" + ) { + node = this._removeAndGetNext(node); + continue; + } + + // If we don't have a byline yet check to see if this node is a byline; if it is store the byline and remove the node. + if ( + !this._articleByline && + !this._metadata.byline && + this._isValidByline(node, matchString) + ) { + // Find child node matching [itemprop="name"] and use that if it exists for a more accurate author name byline + var endOfSearchMarkerNode = this._getNextNode(node, true); + var next = this._getNextNode(node); + var itemPropNameNode = null; + while (next && next != endOfSearchMarkerNode) { + var itemprop = next.getAttribute("itemprop"); + if (itemprop && itemprop.includes("name")) { + itemPropNameNode = next; + break; + } else { + next = this._getNextNode(next); + } + } + this._articleByline = (itemPropNameNode ?? node).textContent.trim(); + node = this._removeAndGetNext(node); + continue; + } + + if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) { + this.log( + "Removing header: ", + node.textContent.trim(), + this._articleTitle.trim() + ); + shouldRemoveTitleHeader = false; + node = this._removeAndGetNext(node); + continue; + } + + // Remove unlikely candidates + if (stripUnlikelyCandidates) { + if ( + this.REGEXPS.unlikelyCandidates.test(matchString) && + !this.REGEXPS.okMaybeItsACandidate.test(matchString) && + !this._hasAncestorTag(node, "table") && + !this._hasAncestorTag(node, "code") && + node.tagName !== "BODY" && + node.tagName !== "A" + ) { + this.log("Removing unlikely candidate - " + matchString); + node = this._removeAndGetNext(node); + continue; + } + + if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) { + this.log( + "Removing content with role " + + node.getAttribute("role") + + " - " + + matchString + ); + node = this._removeAndGetNext(node); + continue; + } + } + + // Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe). + if ( + (node.tagName === "DIV" || + node.tagName === "SECTION" || + node.tagName === "HEADER" || + node.tagName === "H1" || + node.tagName === "H2" || + node.tagName === "H3" || + node.tagName === "H4" || + node.tagName === "H5" || + node.tagName === "H6") && + this._isElementWithoutContent(node) + ) { + node = this._removeAndGetNext(node); + continue; + } + + if (this.DEFAULT_TAGS_TO_SCORE.includes(node.tagName)) { + elementsToScore.push(node); + } + + // Turn all divs that don't have children block level elements into p's + if (node.tagName === "DIV") { + // Put phrasing content into paragraphs. + var p = null; + var childNode = node.firstChild; + while (childNode) { + var nextSibling = childNode.nextSibling; + if (this._isPhrasingContent(childNode)) { + if (p !== null) { + p.appendChild(childNode); + } else if (!this._isWhitespace(childNode)) { + p = doc.createElement("p"); + node.replaceChild(p, childNode); + p.appendChild(childNode); + } + } else if (p !== null) { + while (p.lastChild && this._isWhitespace(p.lastChild)) { + p.lastChild.remove(); + } + p = null; + } + childNode = nextSibling; + } + + // Sites like http://mobile.slate.com encloses each paragraph with a DIV + // element. DIVs with only a P element inside and no text content can be + // safely converted into plain P elements to avoid confusing the scoring + // algorithm with DIVs with are, in practice, paragraphs. + if ( + this._hasSingleTagInsideElement(node, "P") && + this._getLinkDensity(node) < 0.25 + ) { + var newNode = node.children[0]; + node.parentNode.replaceChild(newNode, node); + node = newNode; + elementsToScore.push(node); + } else if (!this._hasChildBlockElement(node)) { + node = this._setNodeTag(node, "P"); + elementsToScore.push(node); + } + } + node = this._getNextNode(node); + } + + /** + * Loop through all paragraphs, and assign a score to them based on how content-y they look. + * Then add their score to their parent node. + * + * A score is determined by things like number of commas, class names, etc. Maybe eventually link density. + **/ + var candidates = []; + this._forEachNode(elementsToScore, function (elementToScore) { + if ( + !elementToScore.parentNode || + typeof elementToScore.parentNode.tagName === "undefined" + ) { + return; + } + + // If this paragraph is less than 25 characters, don't even count it. + var innerText = this._getInnerText(elementToScore); + if (innerText.length < 25) { + return; + } + + // Exclude nodes with no ancestor. + var ancestors = this._getNodeAncestors(elementToScore, 5); + if (ancestors.length === 0) { + return; + } + + var contentScore = 0; + + // Add a point for the paragraph itself as a base. + contentScore += 1; + + // Add points for any commas within this paragraph. + contentScore += innerText.split(this.REGEXPS.commas).length; + + // For every 100 characters in this paragraph, add another point. Up to 3 points. + contentScore += Math.min(Math.floor(innerText.length / 100), 3); + + // Initialize and score ancestors. + this._forEachNode(ancestors, function (ancestor, level) { + if ( + !ancestor.tagName || + !ancestor.parentNode || + typeof ancestor.parentNode.tagName === "undefined" + ) { + return; + } + + if (typeof ancestor.readability === "undefined") { + this._initializeNode(ancestor); + candidates.push(ancestor); + } + + // Node score divider: + // - parent: 1 (no division) + // - grandparent: 2 + // - great grandparent+: ancestor level * 3 + if (level === 0) { + var scoreDivider = 1; + } else if (level === 1) { + scoreDivider = 2; + } else { + scoreDivider = level * 3; + } + ancestor.readability.contentScore += contentScore / scoreDivider; + }); + }); + + // After we've calculated scores, loop through all of the possible + // candidate nodes we found and find the one with the highest score. + var topCandidates = []; + for (var c = 0, cl = candidates.length; c < cl; c += 1) { + var candidate = candidates[c]; + + // Scale the final candidates score based on link density. Good content + // should have a relatively small link density (5% or less) and be mostly + // unaffected by this operation. + var candidateScore = + candidate.readability.contentScore * + (1 - this._getLinkDensity(candidate)); + candidate.readability.contentScore = candidateScore; + + this.log("Candidate:", candidate, "with score " + candidateScore); + + for (var t = 0; t < this._nbTopCandidates; t++) { + var aTopCandidate = topCandidates[t]; + + if ( + !aTopCandidate || + candidateScore > aTopCandidate.readability.contentScore + ) { + topCandidates.splice(t, 0, candidate); + if (topCandidates.length > this._nbTopCandidates) { + topCandidates.pop(); + } + break; + } + } + } + + var topCandidate = topCandidates[0] || null; + var neededToCreateTopCandidate = false; + var parentOfTopCandidate; + + // If we still have no top candidate, just use the body as a last resort. + // We also have to copy the body node so it is something we can modify. + if (topCandidate === null || topCandidate.tagName === "BODY") { + // Move all of the page's children into topCandidate + topCandidate = doc.createElement("DIV"); + neededToCreateTopCandidate = true; + // Move everything (not just elements, also text nodes etc.) into the container + // so we even include text directly in the body: + while (page.firstChild) { + this.log("Moving child out:", page.firstChild); + topCandidate.appendChild(page.firstChild); + } + + page.appendChild(topCandidate); + + this._initializeNode(topCandidate); + } else if (topCandidate) { + // Find a better top candidate node if it contains (at least three) nodes which belong to `topCandidates` array + // and whose scores are quite closed with current `topCandidate` node. + var alternativeCandidateAncestors = []; + for (var i = 1; i < topCandidates.length; i++) { + if ( + topCandidates[i].readability.contentScore / + topCandidate.readability.contentScore >= + 0.75 + ) { + alternativeCandidateAncestors.push( + this._getNodeAncestors(topCandidates[i]) + ); + } + } + var MINIMUM_TOPCANDIDATES = 3; + if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) { + parentOfTopCandidate = topCandidate.parentNode; + while (parentOfTopCandidate.tagName !== "BODY") { + var listsContainingThisAncestor = 0; + for ( + var ancestorIndex = 0; + ancestorIndex < alternativeCandidateAncestors.length && + listsContainingThisAncestor < MINIMUM_TOPCANDIDATES; + ancestorIndex++ + ) { + listsContainingThisAncestor += Number( + alternativeCandidateAncestors[ancestorIndex].includes( + parentOfTopCandidate + ) + ); + } + if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) { + topCandidate = parentOfTopCandidate; + break; + } + parentOfTopCandidate = parentOfTopCandidate.parentNode; + } + } + if (!topCandidate.readability) { + this._initializeNode(topCandidate); + } + + // Because of our bonus system, parents of candidates might have scores + // themselves. They get half of the node. There won't be nodes with higher + // scores than our topCandidate, but if we see the score going *up* in the first + // few steps up the tree, that's a decent sign that there might be more content + // lurking in other places that we want to unify in. The sibling stuff + // below does some of that - but only if we've looked high enough up the DOM + // tree. + parentOfTopCandidate = topCandidate.parentNode; + var lastScore = topCandidate.readability.contentScore; + // The scores shouldn't get too low. + var scoreThreshold = lastScore / 3; + while (parentOfTopCandidate.tagName !== "BODY") { + if (!parentOfTopCandidate.readability) { + parentOfTopCandidate = parentOfTopCandidate.parentNode; + continue; + } + var parentScore = parentOfTopCandidate.readability.contentScore; + if (parentScore < scoreThreshold) { + break; + } + if (parentScore > lastScore) { + // Alright! We found a better parent to use. + topCandidate = parentOfTopCandidate; + break; + } + lastScore = parentOfTopCandidate.readability.contentScore; + parentOfTopCandidate = parentOfTopCandidate.parentNode; + } + + // If the top candidate is the only child, use parent instead. This will help sibling + // joining logic when adjacent content is actually located in parent's sibling node. + parentOfTopCandidate = topCandidate.parentNode; + while ( + parentOfTopCandidate.tagName != "BODY" && + parentOfTopCandidate.children.length == 1 + ) { + topCandidate = parentOfTopCandidate; + parentOfTopCandidate = topCandidate.parentNode; + } + if (!topCandidate.readability) { + this._initializeNode(topCandidate); + } + } + + // Now that we have the top candidate, look through its siblings for content + // that might also be related. Things like preambles, content split by ads + // that we removed, etc. + var articleContent = doc.createElement("DIV"); + if (isPaging) { + articleContent.id = "readability-content"; + } + + var siblingScoreThreshold = Math.max( + 10, + topCandidate.readability.contentScore * 0.2 + ); + // Keep potential top candidate's parent node to try to get text direction of it later. + parentOfTopCandidate = topCandidate.parentNode; + var siblings = parentOfTopCandidate.children; + + for (var s = 0, sl = siblings.length; s < sl; s++) { + var sibling = siblings[s]; + var append = false; + + this.log( + "Looking at sibling node:", + sibling, + sibling.readability + ? "with score " + sibling.readability.contentScore + : "" + ); + this.log( + "Sibling has score", + sibling.readability ? sibling.readability.contentScore : "Unknown" + ); + + if (sibling === topCandidate) { + append = true; + } else { + var contentBonus = 0; + + // Give a bonus if sibling nodes and top candidates have the example same classname + if ( + sibling.className === topCandidate.className && + topCandidate.className !== "" + ) { + contentBonus += topCandidate.readability.contentScore * 0.2; + } + + if ( + sibling.readability && + sibling.readability.contentScore + contentBonus >= + siblingScoreThreshold + ) { + append = true; + } else if (sibling.nodeName === "P") { + var linkDensity = this._getLinkDensity(sibling); + var nodeContent = this._getInnerText(sibling); + var nodeLength = nodeContent.length; + + if (nodeLength > 80 && linkDensity < 0.25) { + append = true; + } else if ( + nodeLength < 80 && + nodeLength > 0 && + linkDensity === 0 && + nodeContent.search(/\.( |$)/) !== -1 + ) { + append = true; + } + } + } + + if (append) { + this.log("Appending node:", sibling); + + if (!this.ALTER_TO_DIV_EXCEPTIONS.includes(sibling.nodeName)) { + // We have a node that isn't a common block level element, like a form or td tag. + // Turn it into a div so it doesn't get filtered out later by accident. + this.log("Altering sibling:", sibling, "to div."); + + sibling = this._setNodeTag(sibling, "DIV"); + } + + articleContent.appendChild(sibling); + // Fetch children again to make it compatible + // with DOM parsers without live collection support. + siblings = parentOfTopCandidate.children; + // siblings is a reference to the children array, and + // sibling is removed from the array when we call appendChild(). + // As a result, we must revisit this index since the nodes + // have been shifted. + s -= 1; + sl -= 1; + } + } + + if (this._debug) { + this.log("Article content pre-prep: " + articleContent.innerHTML); + } + // So we have all of the content that we need. Now we clean it up for presentation. + this._prepArticle(articleContent); + if (this._debug) { + this.log("Article content post-prep: " + articleContent.innerHTML); + } + + if (neededToCreateTopCandidate) { + // We already created a fake div thing, and there wouldn't have been any siblings left + // for the previous loop, so there's no point trying to create a new div, and then + // move all the children over. Just assign IDs and class names here. No need to append + // because that already happened anyway. + topCandidate.id = "readability-page-1"; + topCandidate.className = "page"; + } else { + var div = doc.createElement("DIV"); + div.id = "readability-page-1"; + div.className = "page"; + while (articleContent.firstChild) { + div.appendChild(articleContent.firstChild); + } + articleContent.appendChild(div); + } + + if (this._debug) { + this.log("Article content after paging: " + articleContent.innerHTML); + } + + var parseSuccessful = true; + + // Now that we've gone through the full algorithm, check to see if + // we got any meaningful content. If we didn't, we may need to re-run + // grabArticle with different flags set. This gives us a higher likelihood of + // finding the content, and the sieve approach gives us a higher likelihood of + // finding the -right- content. + var textLength = this._getInnerText(articleContent, true).length; + if (textLength < this._charThreshold) { + parseSuccessful = false; + // eslint-disable-next-line no-unsanitized/property + page.innerHTML = pageCacheHtml; + + this._attempts.push({ + articleContent, + textLength, + }); + + if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) { + this._removeFlag(this.FLAG_STRIP_UNLIKELYS); + } else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) { + this._removeFlag(this.FLAG_WEIGHT_CLASSES); + } else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) { + this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY); + } else { + // No luck after removing flags, just return the longest text we found during the different loops + this._attempts.sort(function (a, b) { + return b.textLength - a.textLength; + }); + + // But first check if we actually have something + if (!this._attempts[0].textLength) { + return null; + } + + articleContent = this._attempts[0].articleContent; + parseSuccessful = true; + } + } + + if (parseSuccessful) { + // Find out text direction from ancestors of final top candidate. + var ancestors = [parentOfTopCandidate, topCandidate].concat( + this._getNodeAncestors(parentOfTopCandidate) + ); + this._someNode(ancestors, function (ancestor) { + if (!ancestor.tagName) { + return false; + } + var articleDir = ancestor.getAttribute("dir"); + if (articleDir) { + this._articleDir = articleDir; + return true; + } + return false; + }); + return articleContent; + } + } + }, + + /** + * Converts some of the common HTML entities in string to their corresponding characters. + * + * @param str {string} - a string to unescape. + * @return string without HTML entity. + */ + _unescapeHtmlEntities(str) { + if (!str) { + return str; + } + + var htmlEscapeMap = this.HTML_ESCAPE_MAP; + return str + .replace(/&(quot|amp|apos|lt|gt);/g, function (_, tag) { + return htmlEscapeMap[tag]; + }) + .replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, function (_, hex, numStr) { + var num = parseInt(hex || numStr, hex ? 16 : 10); + + // these character references are replaced by a conforming HTML parser + if (num == 0 || num > 0x10ffff || (num >= 0xd800 && num <= 0xdfff)) { + num = 0xfffd; + } + + return String.fromCodePoint(num); + }); + }, + + /** + * Try to extract metadata from JSON-LD object. + * For now, only Schema.org objects of type Article or its subtypes are supported. + * @return Object with any metadata that could be extracted (possibly none) + */ + _getJSONLD(doc) { + var scripts = this._getAllNodesWithTag(doc, ["script"]); + + var metadata; + + this._forEachNode(scripts, function (jsonLdElement) { + if ( + !metadata && + jsonLdElement.getAttribute("type") === "application/ld+json" + ) { + try { + // Strip CDATA markers if present + var content = jsonLdElement.textContent.replace( + /^\s*\s*$/g, + "" + ); + var parsed = JSON.parse(content); + + if (Array.isArray(parsed)) { + parsed = parsed.find(it => { + return ( + it["@type"] && + it["@type"].match(this.REGEXPS.jsonLdArticleTypes) + ); + }); + if (!parsed) { + return; + } + } + + var schemaDotOrgRegex = /^https?\:\/\/schema\.org\/?$/; + var matches = + (typeof parsed["@context"] === "string" && + parsed["@context"].match(schemaDotOrgRegex)) || + (typeof parsed["@context"] === "object" && + typeof parsed["@context"]["@vocab"] == "string" && + parsed["@context"]["@vocab"].match(schemaDotOrgRegex)); + + if (!matches) { + return; + } + + if (!parsed["@type"] && Array.isArray(parsed["@graph"])) { + parsed = parsed["@graph"].find(it => { + return (it["@type"] || "").match(this.REGEXPS.jsonLdArticleTypes); + }); + } + + if ( + !parsed || + !parsed["@type"] || + !parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes) + ) { + return; + } + + metadata = {}; + + if ( + typeof parsed.name === "string" && + typeof parsed.headline === "string" && + parsed.name !== parsed.headline + ) { + // we have both name and headline element in the JSON-LD. They should both be the same but some websites like aktualne.cz + // put their own name into "name" and the article title to "headline" which confuses Readability. So we try to check if either + // "name" or "headline" closely matches the html title, and if so, use that one. If not, then we use "name" by default. + + var title = this._getArticleTitle(); + var nameMatches = this._textSimilarity(parsed.name, title) > 0.75; + var headlineMatches = + this._textSimilarity(parsed.headline, title) > 0.75; + + if (headlineMatches && !nameMatches) { + metadata.title = parsed.headline; + } else { + metadata.title = parsed.name; + } + } else if (typeof parsed.name === "string") { + metadata.title = parsed.name.trim(); + } else if (typeof parsed.headline === "string") { + metadata.title = parsed.headline.trim(); + } + if (parsed.author) { + if (typeof parsed.author.name === "string") { + metadata.byline = parsed.author.name.trim(); + } else if ( + Array.isArray(parsed.author) && + parsed.author[0] && + typeof parsed.author[0].name === "string" + ) { + metadata.byline = parsed.author + .filter(function (author) { + return author && typeof author.name === "string"; + }) + .map(function (author) { + return author.name.trim(); + }) + .join(", "); + } + } + if (typeof parsed.description === "string") { + metadata.excerpt = parsed.description.trim(); + } + if (parsed.publisher && typeof parsed.publisher.name === "string") { + metadata.siteName = parsed.publisher.name.trim(); + } + if (typeof parsed.datePublished === "string") { + metadata.datePublished = parsed.datePublished.trim(); + } + } catch (err) { + this.log(err.message); + } + } + }); + return metadata ? metadata : {}; + }, + + /** + * Attempts to get excerpt and byline metadata for the article. + * + * @param {Object} jsonld — object containing any metadata that + * could be extracted from JSON-LD object. + * + * @return Object with optional "excerpt" and "byline" properties + */ + _getArticleMetadata(jsonld) { + var metadata = {}; + var values = {}; + var metaElements = this._doc.getElementsByTagName("meta"); + + // property is a space-separated list of values + var propertyPattern = + /\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi; + + // name is a single value + var namePattern = + /^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i; + + // Find description tags. + this._forEachNode(metaElements, function (element) { + var elementName = element.getAttribute("name"); + var elementProperty = element.getAttribute("property"); + var content = element.getAttribute("content"); + if (!content) { + return; + } + var matches = null; + var name = null; + + if (elementProperty) { + matches = elementProperty.match(propertyPattern); + if (matches) { + // Convert to lowercase, and remove any whitespace + // so we can match below. + name = matches[0].toLowerCase().replace(/\s/g, ""); + // multiple authors + values[name] = content.trim(); + } + } + if (!matches && elementName && namePattern.test(elementName)) { + name = elementName; + if (content) { + // Convert to lowercase, remove any whitespace, and convert dots + // to colons so we can match below. + name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":"); + values[name] = content.trim(); + } + } + }); + + // get title + metadata.title = + jsonld.title || + values["dc:title"] || + values["dcterm:title"] || + values["og:title"] || + values["weibo:article:title"] || + values["weibo:webpage:title"] || + values.title || + values["twitter:title"] || + values["parsely-title"]; + + if (!metadata.title) { + metadata.title = this._getArticleTitle(); + } + + const articleAuthor = + typeof values["article:author"] === "string" && + !this._isUrl(values["article:author"]) + ? values["article:author"] + : undefined; + + // get author + metadata.byline = + jsonld.byline || + values["dc:creator"] || + values["dcterm:creator"] || + values.author || + values["parsely-author"] || + articleAuthor; + + // get description + metadata.excerpt = + jsonld.excerpt || + values["dc:description"] || + values["dcterm:description"] || + values["og:description"] || + values["weibo:article:description"] || + values["weibo:webpage:description"] || + values.description || + values["twitter:description"]; + + // get site name + metadata.siteName = jsonld.siteName || values["og:site_name"]; + + // get article published time + metadata.publishedTime = + jsonld.datePublished || + values["article:published_time"] || + values["parsely-pub-date"] || + null; + + // in many sites the meta value is escaped with HTML entities, + // so here we need to unescape it + metadata.title = this._unescapeHtmlEntities(metadata.title); + metadata.byline = this._unescapeHtmlEntities(metadata.byline); + metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt); + metadata.siteName = this._unescapeHtmlEntities(metadata.siteName); + metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime); + + return metadata; + }, + + /** + * Check if node is image, or if node contains exactly only one image + * whether as a direct child or as its descendants. + * + * @param Element + **/ + _isSingleImage(node) { + while (node) { + if (node.tagName === "IMG") { + return true; + } + if (node.children.length !== 1 || node.textContent.trim() !== "") { + return false; + } + node = node.children[0]; + } + return false; + }, + + /** + * Find all