Skip to main content

max / balanced_breakfast

Replace html2text with pter for HTML-to-markdown conversion Use pter::convert() in html_to_text and extract_article Rhai host functions, producing readable markdown instead of fixed-width plain text.
Co-Authored-By
Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Author: Max J. <87768334+MaxJMath@users.noreply.github.com> · 2026-05-02 23:13 UTC
Commit: 41f9224d581ead9f455f278f0cb6be9c886a1c03
Parent: f0c72dd
4 files changed, +249 insertions, -70 deletions
M Cargo.lock +233 -62
@@ -96,7 +96,7 @@
96 96 "cssparser 0.35.0",
97 97 "html5ever 0.35.0",
98 98 "maplit",
99 - "tendril",
99 + "tendril 0.4.3",
100 100 "url",
101 101 ]
102 102
@@ -204,6 +204,7 @@
204 204 "open",
205 205 "parking_lot",
206 206 "rand 0.8.5",
207 + "regex",
207 208 "roxmltree",
208 209 "serde",
209 210 "serde_json",
@@ -257,9 +258,9 @@
257 258 "chrono",
258 259 "docengine",
259 260 "futures",
260 - "html2text",
261 261 "keyring",
262 262 "parking_lot",
263 + "pter",
263 264 "rand 0.8.5",
264 265 "readable-readability",
265 266 "regex",
@@ -273,6 +274,7 @@
273 274 "tracing",
274 275 "ureq",
275 276 "url",
277 + "zeroize",
276 278 ]
277 279
278 280 [[package]]
@@ -795,6 +797,19 @@
795 797 "smallvec",
796 798 ]
797 799
800 + [[package]]
801 + name = "cssparser"
802 + version = "0.36.0"
803 + source = "registry+https://github.com/rust-lang/crates.io-index"
804 + checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2"
805 + dependencies = [
806 + "cssparser-macros",
807 + "dtoa-short",
808 + "itoa 1.0.17",
809 + "phf 0.13.1",
810 + "smallvec",
811 + ]
812 +
798 813 [[package]]
799 814 name = "cssparser-macros"
800 815 version = "0.6.1"
@@ -922,6 +937,27 @@
922 937 "syn 2.0.114",
923 938 ]
924 939
940 + [[package]]
941 + name = "derive_more"
942 + version = "2.1.1"
943 + source = "registry+https://github.com/rust-lang/crates.io-index"
944 + checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
945 + dependencies = [
946 + "derive_more-impl",
947 + ]
948 +
949 + [[package]]
950 + name = "derive_more-impl"
951 + version = "2.1.1"
952 + source = "registry+https://github.com/rust-lang/crates.io-index"
953 + checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb"
954 + dependencies = [
955 + "proc-macro2",
956 + "quote",
957 + "rustc_version",
958 + "syn 2.0.114",
959 + ]
960 +
925 961 [[package]]
926 962 name = "digest"
927 963 version = "0.10.7"
@@ -1058,6 +1094,12 @@
1058 1094 source = "registry+https://github.com/rust-lang/crates.io-index"
1059 1095 checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
1060 1096
1097 + [[package]]
1098 + name = "ego-tree"
1099 + version = "0.11.0"
1100 + source = "registry+https://github.com/rust-lang/crates.io-index"
1101 + checksum = "b04dc5a38e4f151a79d9f2451ae6037fb6eaf5cba34771f44781f80e508498e3"
1102 +
1061 1103 [[package]]
1062 1104 name = "either"
1063 1105 version = "1.15.0"
@@ -1505,7 +1547,7 @@
1505 1547 source = "registry+https://github.com/rust-lang/crates.io-index"
1506 1548 checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
1507 1549 dependencies = [
1508 - "unicode-width 0.2.2",
1550 + "unicode-width",
1509 1551 ]
1510 1552
1511 1553 [[package]]
@@ -1802,19 +1844,6 @@
1802 1844 "windows-sys 0.61.2",
1803 1845 ]
1804 1846
1805 - [[package]]
1806 - name = "html2text"
1807 - version = "0.12.6"
1808 - source = "registry+https://github.com/rust-lang/crates.io-index"
1809 - checksum = "042a9677c258ac2952dd026bb0cd21972f00f644a5a38f5a215cb22cdaf6834e"
1810 - dependencies = [
1811 - "html5ever 0.27.0",
1812 - "markup5ever 0.12.1",
1813 - "tendril",
1814 - "thiserror 1.0.69",
1815 - "unicode-width 0.1.13",
1816 - ]
1817 -
1818 1847 [[package]]
1819 1848 name = "html5ever"
1820 1849 version = "0.25.2"
@@ -1829,20 +1858,6 @@
1829 1858 "syn 1.0.109",
1830 1859 ]
1831 1860
1832 - [[package]]
1833 - name = "html5ever"
1834 - version = "0.27.0"
1835 - source = "registry+https://github.com/rust-lang/crates.io-index"
1836 - checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4"
1837 - dependencies = [
1838 - "log",
1839 - "mac",
1840 - "markup5ever 0.12.1",
1841 - "proc-macro2",
1842 - "quote",
1843 - "syn 2.0.114",
1844 - ]
1845 -
1846 1861 [[package]]
1847 1862 name = "html5ever"
1848 1863 version = "0.29.1"
@@ -1866,6 +1881,16 @@
1866 1881 "match_token 0.35.0",
1867 1882 ]
1868 1883
1884 + [[package]]
1885 + name = "html5ever"
1886 + version = "0.39.0"
1887 + source = "registry+https://github.com/rust-lang/crates.io-index"
1888 + checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8"
1889 + dependencies = [
1890 + "log",
1891 + "markup5ever 0.39.0",
1892 + ]
1893 +
1869 1894 [[package]]
1870 1895 name = "http"
1871 1896 version = "1.4.0"
@@ -2468,23 +2493,9 @@
2468 2493 "log",
2469 2494 "phf 0.8.0",
2470 2495 "phf_codegen 0.8.0",
2471 - "string_cache",
2472 - "string_cache_codegen",
2473 - "tendril",
2474 - ]
2475 -
2476 - [[package]]
2477 - name = "markup5ever"
2478 - version = "0.12.1"
2479 - source = "registry+https://github.com/rust-lang/crates.io-index"
2480 - checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45"
2481 - dependencies = [
2482 - "log",
2483 - "phf 0.11.3",
2484 - "phf_codegen 0.11.3",
2485 - "string_cache",
2486 - "string_cache_codegen",
2487 - "tendril",
2496 + "string_cache 0.8.9",
2497 + "string_cache_codegen 0.5.4",
2498 + "tendril 0.4.3",
2488 2499 ]
2489 2500
2490 2501 [[package]]
@@ -2496,9 +2507,9 @@
2496 2507 "log",
2497 2508 "phf 0.11.3",
2498 2509 "phf_codegen 0.11.3",
2499 - "string_cache",
2500 - "string_cache_codegen",
2501 - "tendril",
2510 + "string_cache 0.8.9",
2511 + "string_cache_codegen 0.5.4",
2512 + "tendril 0.4.3",
2502 2513 ]
2503 2514
2504 2515 [[package]]
@@ -2508,8 +2519,19 @@
2508 2519 checksum = "311fe69c934650f8f19652b3946075f0fc41ad8757dbb68f1ca14e7900ecc1c3"
2509 2520 dependencies = [
2510 2521 "log",
2511 - "tendril",
2512 - "web_atoms",
2522 + "tendril 0.4.3",
2523 + "web_atoms 0.1.3",
2524 + ]
2525 +
2526 + [[package]]
2527 + name = "markup5ever"
2528 + version = "0.39.0"
2529 + source = "registry+https://github.com/rust-lang/crates.io-index"
2530 + checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de"
2531 + dependencies = [
2532 + "log",
2533 + "tendril 0.5.0",
2534 + "web_atoms 0.2.4",
2513 2535 ]
2514 2536
2515 2537 [[package]]
@@ -3232,6 +3254,17 @@
3232 3254 "phf_shared 0.11.3",
3233 3255 ]
3234 3256
3257 + [[package]]
3258 + name = "phf"
3259 + version = "0.13.1"
3260 + source = "registry+https://github.com/rust-lang/crates.io-index"
3261 + checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
3262 + dependencies = [
3263 + "phf_macros 0.13.1",
3264 + "phf_shared 0.13.1",
3265 + "serde",
3266 + ]
3267 +
3235 3268 [[package]]
3236 3269 name = "phf_codegen"
3237 3270 version = "0.8.0"
@@ -3252,6 +3285,16 @@
3252 3285 "phf_shared 0.11.3",
3253 3286 ]
3254 3287
3288 + [[package]]
3289 + name = "phf_codegen"
3290 + version = "0.13.1"
3291 + source = "registry+https://github.com/rust-lang/crates.io-index"
3292 + checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
3293 + dependencies = [
3294 + "phf_generator 0.13.1",
3295 + "phf_shared 0.13.1",
3296 + ]
3297 +
3255 3298 [[package]]
3256 3299 name = "phf_generator"
3257 3300 version = "0.8.0"
@@ -3282,6 +3325,16 @@
3282 3325 "rand 0.8.5",
3283 3326 ]
3284 3327
3328 + [[package]]
3329 + name = "phf_generator"
3330 + version = "0.13.1"
3331 + source = "registry+https://github.com/rust-lang/crates.io-index"
3332 + checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
3333 + dependencies = [
3334 + "fastrand",
3335 + "phf_shared 0.13.1",
3336 + ]
3337 +
3285 3338 [[package]]
3286 3339 name = "phf_macros"
3287 3340 version = "0.8.0"
@@ -3323,6 +3376,19 @@
3323 3376 "syn 2.0.114",
3324 3377 ]
3325 3378
3379 + [[package]]
3380 + name = "phf_macros"
3381 + version = "0.13.1"
3382 + source = "registry+https://github.com/rust-lang/crates.io-index"
3383 + checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
3384 + dependencies = [
3385 + "phf_generator 0.13.1",
3386 + "phf_shared 0.13.1",
3387 + "proc-macro2",
3388 + "quote",
3389 + "syn 2.0.114",
3390 + ]
3391 +
3326 3392 [[package]]
3327 3393 name = "phf_shared"
3328 3394 version = "0.8.0"
@@ -3350,6 +3416,15 @@
3350 3416 "siphasher 1.0.2",
3351 3417 ]
3352 3418
3419 + [[package]]
3420 + name = "phf_shared"
3421 + version = "0.13.1"
3422 + source = "registry+https://github.com/rust-lang/crates.io-index"
3423 + checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
3424 + dependencies = [
3425 + "siphasher 1.0.2",
3426 + ]
3427 +
3353 3428 [[package]]
3354 3429 name = "pin-project-lite"
3355 3430 version = "0.2.16"
@@ -3542,6 +3617,13 @@
3542 3617 "unicode-ident",
3543 3618 ]
3544 3619
3620 + [[package]]
3621 + name = "pter"
3622 + version = "0.1.0"
3623 + dependencies = [
3624 + "scraper",
3625 + ]
3626 +
3545 3627 [[package]]
3546 3628 name = "pulldown-cmark"
3547 3629 version = "0.12.2"
@@ -3937,6 +4019,12 @@
3937 4019 "zeroize",
3938 4020 ]
3939 4021
4022 + [[package]]
4023 + name = "rustc-hash"
4024 + version = "2.1.2"
4025 + source = "registry+https://github.com/rust-lang/crates.io-index"
4026 + checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
4027 +
3940 4028 [[package]]
3941 4029 name = "rustc_version"
3942 4030 version = "0.4.1"
@@ -4120,6 +4208,21 @@
4120 4208 source = "registry+https://github.com/rust-lang/crates.io-index"
4121 4209 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
4122 4210
4211 + [[package]]
4212 + name = "scraper"
4213 + version = "0.26.0"
4214 + source = "registry+https://github.com/rust-lang/crates.io-index"
4215 + checksum = "f0f5297102b8b62b4454ee8561601b2d551b4913148feb4241ca9d1a04bf4526"
4216 + dependencies = [
4217 + "cssparser 0.36.0",
4218 + "ego-tree",
4219 + "getopts",
4220 + "html5ever 0.39.0",
4221 + "precomputed-hash",
4222 + "selectors 0.36.1",
4223 + "tendril 0.5.0",
4224 + ]
4225 +
4123 4226 [[package]]
4124 4227 name = "security-framework"
4125 4228 version = "3.6.0"
@@ -4151,7 +4254,7 @@
4151 4254 dependencies = [
4152 4255 "bitflags 1.3.2",
4153 4256 "cssparser 0.27.2",
4154 - "derive_more",
4257 + "derive_more 0.99.20",
4155 4258 "fxhash",
4156 4259 "log",
4157 4260 "matches",
@@ -4171,7 +4274,7 @@
4171 4274 dependencies = [
4172 4275 "bitflags 1.3.2",
4173 4276 "cssparser 0.29.6",
4174 - "derive_more",
4277 + "derive_more 0.99.20",
4175 4278 "fxhash",
4176 4279 "log",
4177 4280 "phf 0.8.0",
@@ -4181,6 +4284,25 @@
4181 4284 "smallvec",
4182 4285 ]
4183 4286
4287 + [[package]]
4288 + name = "selectors"
4289 + version = "0.36.1"
4290 + source = "registry+https://github.com/rust-lang/crates.io-index"
4291 + checksum = "c5d9c0c92a92d33f08817311cf3f2c29a3538a8240e94a6a3c622ce652d7e00c"
4292 + dependencies = [
4293 + "bitflags 2.10.0",
4294 + "cssparser 0.36.0",
4295 + "derive_more 2.1.1",
4296 + "log",
4297 + "new_debug_unreachable",
4298 + "phf 0.13.1",
4299 + "phf_codegen 0.13.1",
4300 + "precomputed-hash",
4301 + "rustc-hash",
4302 + "servo_arc 0.4.3",
4303 + "smallvec",
4304 + ]
4305 +
4184 4306 [[package]]
4185 4307 name = "semver"
4186 4308 version = "1.0.27"
@@ -4371,6 +4493,15 @@
4371 4493 "stable_deref_trait",
4372 4494 ]
4373 4495
4496 + [[package]]
4497 + name = "servo_arc"
4498 + version = "0.4.3"
4499 + source = "registry+https://github.com/rust-lang/crates.io-index"
4500 + checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"
4501 + dependencies = [
4502 + "stable_deref_trait",
4503 + ]
4504 +
4374 4505 [[package]]
4375 4506 name = "sha1"
4376 4507 version = "0.10.6"
@@ -4809,6 +4940,18 @@
4809 4940 "serde",
4810 4941 ]
4811 4942
4943 + [[package]]
4944 + name = "string_cache"
4945 + version = "0.9.0"
4946 + source = "registry+https://github.com/rust-lang/crates.io-index"
4947 + checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
4948 + dependencies = [
4949 + "new_debug_unreachable",
4950 + "parking_lot",
4951 + "phf_shared 0.13.1",
4952 + "precomputed-hash",
4953 + ]
4954 +
4812 4955 [[package]]
4813 4956 name = "string_cache_codegen"
4814 4957 version = "0.5.4"
@@ -4821,6 +4964,18 @@
4821 4964 "quote",
4822 4965 ]
4823 4966
4967 + [[package]]
4968 + name = "string_cache_codegen"
4969 + version = "0.6.1"
4970 + source = "registry+https://github.com/rust-lang/crates.io-index"
4971 + checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
4972 + dependencies = [
4973 + "phf_generator 0.13.1",
4974 + "phf_shared 0.13.1",
4975 + "proc-macro2",
4976 + "quote",
4977 + ]
4978 +
4824 4979 [[package]]
4825 4980 name = "stringprep"
4826 4981 version = "0.1.5"
@@ -5393,6 +5548,16 @@
5393 5548 "utf-8",
5394 5549 ]
5395 5550
5551 + [[package]]
5552 + name = "tendril"
5553 + version = "0.5.0"
5554 + source = "registry+https://github.com/rust-lang/crates.io-index"
5555 + checksum = "c4790fc369d5a530f4b544b094e31388b9b3a37c0f4652ade4505945f5660d24"
5556 + dependencies = [
5557 + "new_debug_unreachable",
5558 + "utf-8",
5559 + ]
5560 +
5396 5561 [[package]]
5397 5562 name = "theme-common"
5398 5563 version = "0.3.1"
@@ -5923,12 +6088,6 @@
5923 6088 source = "registry+https://github.com/rust-lang/crates.io-index"
5924 6089 checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
5925 6090
5926 - [[package]]
5927 - name = "unicode-width"
5928 - version = "0.1.13"
5929 - source = "registry+https://github.com/rust-lang/crates.io-index"
5930 - checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
5931 -
5932 6091 [[package]]
5933 6092 name = "unicode-width"
5934 6093 version = "0.2.2"
@@ -6214,8 +6373,20 @@
6214 6373 dependencies = [
6215 6374 "phf 0.11.3",
6216 6375 "phf_codegen 0.11.3",
6217 - "string_cache",
6218 - "string_cache_codegen",
6376 + "string_cache 0.8.9",
6377 + "string_cache_codegen 0.5.4",
6378 + ]
6379 +
6380 + [[package]]
6381 + name = "web_atoms"
6382 + version = "0.2.4"
6383 + source = "registry+https://github.com/rust-lang/crates.io-index"
6384 + checksum = "d7cff6eef815df1834fd250e3a2ff436044d82a9f1bc1980ca1dbdf07effc538"
6385 + dependencies = [
6386 + "phf 0.13.1",
6387 + "phf_codegen 0.13.1",
6388 + "string_cache 0.9.0",
6389 + "string_cache_codegen 0.6.1",
6219 6390 ]
6220 6391
6221 6392 [[package]]
M Cargo.toml +2
@@ -61,4 +61,6 @@
61 61 tauri = "2.10.2"
62 62 tauri-plugin-updater = "2"
63 63 tagtree = { path = "../../MNW/shared/tagtree" }
64 + regex = "1"
65 + zeroize = "1"
64 66 wiremock = "0.6"
M crates/bb-core/Cargo.toml +5 -2
@@ -24,6 +24,9 @@
24 24 # Keychain access for encryption key
25 25 keyring.workspace = true
26 26
27 + # Zeroing key material on drop
28 + zeroize.workspace = true
29 +
27 30 # Lock primitives (no poisoning)
28 31 parking_lot.workspace = true
29 32
@@ -36,8 +39,8 @@
36 39 # XML parsing for RSS/Atom feeds in plugins
37 40 roxmltree.workspace = true
38 41
39 - # HTML to text conversion
40 - html2text = "0.12.6"
42 + # HTML to markdown conversion
43 + pter = { path = "../../../../pter" }
41 44
42 45 # URL parsing for tracker stripping
43 46 url = "2"
M crates/bb-core/src/rhai_plugin/host_functions.rs +9 -6
@@ -50,6 +50,8 @@
50 50 host_and_port.split(':').next().unwrap_or("").to_string() // strip port
51 51 };
52 52 let host = host.as_str();
53 + // Strip userinfo (user@) from host to prevent blocklist bypass via http://user@localhost/
54 + let host = host.split('@').last().unwrap_or(host);
53 55 if host == "localhost"
54 56 || host == "127.0.0.1"
55 57 || host == "[::1]"
@@ -57,6 +59,7 @@
57 59 || host.starts_with("10.")
58 60 || host.starts_with("192.168.")
59 61 || host.starts_with("169.254.")
62 + || host.starts_with("100.")
60 63 {
61 64 return Err(format!("Blocked request to internal address: {}", url));
62 65 }
@@ -260,9 +263,9 @@
260 263 // Current UTC timestamp as Unix epoch seconds (i64).
261 264 engine.register_fn("timestamp_now", || -> i64 { chrono::Utc::now().timestamp() });
262 265
263 - // Strip HTML tags and render to plain text (80-char line width).
266 + // Convert HTML to readable markdown.
264 267 engine.register_fn("html_to_text", |html: &str| -> String {
265 - html2text::from_read(html.as_bytes(), 80)
268 + pter::convert(html)
266 269 });
267 270
268 271 // Truncate text with ellipsis (character-count aware for multibyte UTF-8)
@@ -370,8 +373,8 @@
370 373 node.serialize(&mut content).unwrap_or_default();
371 374 let content_html = String::from_utf8_lossy(&content).to_string();
372 375
373 - // Plain text via html2text
374 - let text = html2text::from_read(content_html.as_bytes(), 80);
376 + // Readable markdown via pter
377 + let text = pter::convert(&content_html);
375 378
376 379 map.insert(
377 380 "title".into(),
@@ -426,9 +429,9 @@
426 429 .map_err(|e| format!("Date parse error: {}", e))
427 430 }
428 431
429 - /// Strip HTML tags and render to plain text (mirrors the Rhai closure).
432 + /// Convert HTML to readable markdown (mirrors the Rhai closure).
430 433 fn html_to_text(html: &str) -> String {
431 - html2text::from_read(html.as_bytes(), 80)
434 + pter::convert(html)
432 435 }
433 436
434 437 /// String contains check (mirrors the Rhai closure).