From 29aceb5476073b19e8bd33183cd19c0cc7f78771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Qian=20Qian=20=22Cubik=22=E2=80=8E?= Date: Fri, 12 Jul 2024 20:25:40 -0400 Subject: [PATCH 1/3] refactor(core): Use `Vec<&str>` instead of `Vec` for HTML filter rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Qian Qian "Cubik"‎ --- src-rust/toolkit-core/src/models/html_filter_rule.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src-rust/toolkit-core/src/models/html_filter_rule.rs b/src-rust/toolkit-core/src/models/html_filter_rule.rs index 00927b0..c4fc61e 100644 --- a/src-rust/toolkit-core/src/models/html_filter_rule.rs +++ b/src-rust/toolkit-core/src/models/html_filter_rule.rs @@ -1,11 +1,10 @@ -#[derive(PartialEq, Eq, Debug, serde::Deserialize)] pub struct HTMLFilterRule { - pub tags: Vec, - pub classes: Vec, + pub tags: Vec<&str>, + pub classes: Vec<&str>, } impl HTMLFilterRule { - pub fn new(tags: Vec, classes: Vec) -> Self { + pub fn new(tags: Vec<&str>, classes: Vec<&str>) -> Self { Self { tags, classes, From 489dc49ac6295413fdbaa8f101d27f70cd117299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Qian=20Qian=20=22Cubik=22=E2=80=8E?= Date: Fri, 12 Jul 2024 20:27:30 -0400 Subject: [PATCH 2/3] fix(core): Fix lifetime specifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Qian Qian "Cubik"‎ --- src-rust/toolkit-core/src/models/html_filter_rule.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src-rust/toolkit-core/src/models/html_filter_rule.rs b/src-rust/toolkit-core/src/models/html_filter_rule.rs index c4fc61e..500ab9e 100644 --- a/src-rust/toolkit-core/src/models/html_filter_rule.rs +++ b/src-rust/toolkit-core/src/models/html_filter_rule.rs @@ -1,6 +1,6 @@ -pub struct HTMLFilterRule { - pub tags: Vec<&str>, - pub classes: Vec<&str>, +pub struct HTMLFilterRule<'a> { + pub tags: Vec<&'a str>, + pub classes: Vec<&'a str>, } impl HTMLFilterRule { From db518aa14e5fc85939d4e3310a3b3dea4c6210ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Qian=20Qian=20=22Cubik=22=E2=80=8E?= Date: Fri, 12 Jul 2024 20:37:02 -0400 Subject: [PATCH 3/3] refactor: Hardcode HTML Filter Rule in Code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Qian Qian "Cubik"‎ --- .../rsc/html_filter_rules/itsfoss.com.toml | 10 -------- .../src/models/html_filter_rule.rs | 25 +++++++++++++++++-- .../src/workflow/translate/select.rs | 25 +++---------------- 3 files changed, 26 insertions(+), 34 deletions(-) delete mode 100644 src-rust/rsc/html_filter_rules/itsfoss.com.toml diff --git a/src-rust/rsc/html_filter_rules/itsfoss.com.toml b/src-rust/rsc/html_filter_rules/itsfoss.com.toml deleted file mode 100644 index 5fff284..0000000 --- a/src-rust/rsc/html_filter_rules/itsfoss.com.toml +++ /dev/null @@ -1,10 +0,0 @@ -tags = [ - "script", "style", "link", "meta", "li", "desc", "title", "svg", "path", "dialog", "select", "head", "header", - "foot", "footer", "ul", "nav", "button", "form", "input", "picture", "time", "h2", "h3", "h4", "i", "aside", - "FreeStarVideoAdContainer", "freestar-video-parent", "reestar-video-child" -] - -classes = [ - "progress-bar", "js-menu", "social-share", "post-info__readtime", "cta__description", "cta__inner", "cta__content", - "hide-mobile", "js-toc", "author-card", "related-posts" -] diff --git a/src-rust/toolkit-core/src/models/html_filter_rule.rs b/src-rust/toolkit-core/src/models/html_filter_rule.rs index 500ab9e..3514b21 100644 --- a/src-rust/toolkit-core/src/models/html_filter_rule.rs +++ b/src-rust/toolkit-core/src/models/html_filter_rule.rs @@ -3,11 +3,32 @@ pub struct HTMLFilterRule<'a> { pub classes: Vec<&'a str>, } -impl HTMLFilterRule { - pub fn new(tags: Vec<&str>, classes: Vec<&str>) -> Self { +impl<'a> HTMLFilterRule<'a> { + fn new(tags: Vec<&'a str>, classes: Vec<&'a str>) -> Self { Self { tags, classes, } } + + pub fn get_filter_rule(url: &str) -> Self { + match url { + "itsfoss.com" | "news.itsfoss.com" => { + Self::new( + vec![ + "script", "style", "link", "meta", "li", "desc", "title", "svg", "path", + "dialog", "select", "head", "header", "foot", "footer", "ul", "nav", "button", + "form", "input", "picture", "time", "h2", "h3", "h4", "i", "aside", + "FreeStarVideoAdContainer", "freestar-video-parent", "reestar-video-child", + ], + vec![ + "progress-bar", "js-menu", "social-share", "post-info__readtime", + "cta__description", "cta__inner", "cta__content", "hide-mobile", "js-toc", + "author-card", "related-posts", + ], + ) + } + _ => Self::new(vec![], vec![]) + } + } } diff --git a/src-rust/toolkit-core/src/workflow/translate/select.rs b/src-rust/toolkit-core/src/workflow/translate/select.rs index fbd1506..b9eb796 100644 --- a/src-rust/toolkit-core/src/workflow/translate/select.rs +++ b/src-rust/toolkit-core/src/workflow/translate/select.rs @@ -23,32 +23,13 @@ pub fn get_content(url: &str) -> Result { host.unwrap() }; - let html_filter_rule_path = format!("rsc/html_filter_rules/{}.toml", host); - let html_filter_rule_str = std::fs::read_to_string(html_filter_rule_path); - let html_filter_rule = match html_filter_rule_str { - Ok(html_filter_rule_str) => { - let html_filter_rule: Result = toml::from_str(&html_filter_rule_str); - match html_filter_rule { - Ok(html_filter_rule) => html_filter_rule, - Err(_) => { - let error_msg = format!( - "Failed to parse the HTML filter rule for the website: {}", host - ); - return Err(error_msg); - } - } - }, - Err(_) => { - // Use the default HTML filter rule (no tags and classes to filter) - HTMLFilterRule::new(Vec::new(), Vec::new()) - } - }; + let html_filter_rule = HTMLFilterRule::get_filter_rule(host); // Filter the HTML content let filtered_html = libhtmlfilter::get_filtered_html_fullurl_removeref( url, - html_filter_rule.tags.iter().map(|s| s.as_str()).collect::>().as_slice(), - html_filter_rule.classes.iter().map(|s| s.as_str()).collect::>().as_slice() + &*html_filter_rule.tags, + &*html_filter_rule.classes ); // Parse HTML to markdown