feat(core&cli): Integrate HTML filter & Add article selection

src-rust/rsc/html_filter_rules/itsfoss.com.toml

@@ -0,0 +1,10 @@
+tags = [
+    "script", "style", "link", "meta", "li", "desc", "title", "svg", "path", "dialog", "select", "head", "header",
+    "foot", "footer", "ul", "nav", "button", "form", "input", "picture", "time", "h2", "h3", "h4", "i", "aside",
+    "FreeStarVideoAdContainer", "freestar-video-parent", "reestar-video-child"
+]
+
+classes = [
+    "progress-bar", "js-menu", "social-share", "post-info__readtime", "cta__description", "cta__inner", "cta__content",
+    "hide-mobile", "js-toc", "author-card", "related-posts"
+]

src-rust/rsc/translate_article_template.md

@@ -0,0 +1,32 @@
+---
+title: <标题>
+date: <发布时间>
+author:
+  - fosscope-translation-team
+  - <译者ID>
+  - <校对者ID>
+banner: <封面图片地址>
+cover: <封面图片地址（与 banner 一致）>
+categories:
+  - 翻译
+  - <类型>
+tags:
+  - <标签>
+authorInfo: |
+  via: <原文链接>
+
+  作者：[<原作者>](<原作者信息页>)
+  选题：[<选题者ID>](https://github.com/<选题者ID>)
+  译者：[<译者ID>](https://github.com/<译者ID>)
+  校对：[<校对者ID>](https://github.com/<校对者ID>)
+
+  本文由 [FOSScope翻译组](https://github.com/FOSScope/TranslateProject) 原创编译，[开源观察](https://fosscope.com/) 荣誉推出
+---
+
+<!-- 所有在被 `<>` 标记的地方都需要被替换成对应的内容 -->
+
+<摘要>
+
+<!-- more -->
+
+<正文>

src-rust/toolkit-cli/src/config/github/github_account.rs

@@ -1,5 +1,5 @@
 use octocrab::Octocrab;
-use fosscopetoolkit_core::apis::GitHubApi;
+use fosscopetoolkit_core::apis::github_api::GitHubApi;
 
 #[derive(serde::Serialize, serde::Deserialize)]
 pub struct GitHubAccount {

src-rust/toolkit-cli/src/config/github/mod.rs

@@ -1,3 +1 @@
-mod github_account;
-
-pub use github_account::GitHubAccount;
+pub mod github_account;

src-rust/toolkit-cli/src/main.rs

@@ -1,11 +1,11 @@
 use std::io::{stdin, stdout, Write};
-
-use fosscopetoolkit_core::apis::GitHubApi;
-use fosscopetoolkit_core::models::GitHubRepo;
+use tokio::runtime::Runtime;
 use fosscopetoolkit_core::{get_contributor_repo, set_contributor_repo};
-use config::github::GitHubAccount;
+use fosscopetoolkit_core::apis::github_api::GitHubApi;
+use fosscopetoolkit_core::models::github_repo::GitHubRepo;
+use crate::config::github::github_account::GitHubAccount;
 use crate::config::initial_configuration_process;
-use crate::workflow::select;
+use crate::workflow::select::select;
 
 mod workflow;
 mod config;
@@ -22,15 +22,15 @@ async fn fork_creation_process(github: &GitHubApi, upstream_repo: &GitHubRepo) -
     stdin().read_line(&mut user_input).unwrap_or(0);
     match user_input.to_lowercase().trim() {
         "y" | "yes" => {
-            let owner = github.get_user();
+            let owner = github.get_user().await.unwrap();
             println!("Please enter the name of the owner of the forked repository (default: {}):", owner);
             let mut fork_owner = String::new();
             stdin().read_line(&mut fork_owner).unwrap_or(0);
             fork_owner = fork_owner.trim().to_string();
             if fork_owner.is_empty() {
                 fork_owner = owner;
             }
-            let repo_name = upstream_repo.clone().name;
+            let repo_name = upstream_repo.name.clone();
             println!("Please enter the name of the forked repository (default: {}):", repo_name);
             let mut fork_repo_name = String::new();
             stdin().read_line(&mut fork_repo_name).unwrap_or(0);
@@ -201,11 +201,10 @@ async fn login() -> GitHubApi {
     }
 }
 
-#[tokio::main]
-async fn main() {
+fn main() {
     let _= initial_configuration_process();
 
-    let github = login().await;
+    let github = Runtime::new().unwrap().block_on(login());
 
     println!("Please select the upstream repository you want to work with:");
     println!("1. FOSScope/Articles - 开源观察原创文章与中文转载文章源文件");
@@ -239,7 +238,7 @@ async fn main() {
     }
 
     if get_contributor_repo(&upstream_repo).is_none() {
-        fork_check(&github, upstream_repo).await;
+        Runtime::new().unwrap().block_on(fork_check(&github, upstream_repo));
     }
 
     println!(
@@ -257,7 +256,7 @@ async fn main() {
         match user_input.trim() {
             "1" => {
                 valid_input = true;
-                select().await;
+                select();
             }
             "2" | "3" | "4" => {
                 eprintln!("Not implemented yet.");

src-rust/toolkit-cli/src/workflow/mod.rs

@@ -1,3 +1 @@
-mod select;
-
-pub use select::select;
+pub mod select;

src-rust/toolkit-cli/src/workflow/select.rs

@@ -1,12 +1,11 @@
 use std::io::{stdin, stdout, Write};
-
-use fosscopetoolkit_core::parser::html_to_markdown;
+use fosscopetoolkit_core::workflow::translate::select::get_content;
 use crate::config::get_config;
 
 /**
  * Workflow for selecting new articles to translate.
  */
-pub async fn select() {
+pub fn select() {
     let config = get_config();
 
     println!("欢迎参与开源观察翻译项目！");
@@ -17,8 +16,8 @@ pub async fn select() {
     let url = url.trim();
     println!("您选择的文章 URL 是：{}", url);
 
-    // Convert to Markdown
-    let markdown = html_to_markdown(url).await;
+    // Get markdown content
+    let markdown = get_content(url);
 
     match markdown {
         Ok(markdown) => {

src-rust/toolkit-core/Cargo.toml

@@ -19,10 +19,12 @@ serde = { version = "1", features = ["derive"] } # Serialization
 serde_json = "1.0.117" # JSON Serialization
 toml = "0.8.14" # TOML Serialization
 regex = "1.10.5" # Regular Expression
+url = "2.5.2" # URL Parsing
 reqwest = { version = "0.12", features = ["json"] } # HTTP Client
 octocrab = "0.38.0" # GitHub API
 html2md = { workspace = true } # HTML to Markdown
-libhtmlfilter = "0.1.0" # HTML Filter
+libhtmlfilter = "0.1.2" # HTML Filter
+openai_api_rust = "0.1.9" # OpenAI API
 
 [dev-dependencies]
 wiremock = "0.6.0"

src-rust/toolkit-core/src/apis/github_api.rs

@@ -1,7 +1,7 @@
 use octocrab::models::Repository;
 use serde_json;
 use octocrab::Octocrab;
-use super::super::models::GitHubRepo;
+use crate::models::github_repo::GitHubRepo;
 
 /// A wrapper around the GitHub API that provides a set of methods to interact with the GitHub API.
 ///

src-rust/toolkit-core/src/apis/mod.rs

@@ -1,3 +1 @@
-mod github_api;
-
-pub use github_api::*;
+pub mod github_api;

src-rust/toolkit-core/src/lib.rs

@@ -1,9 +1,10 @@
 use std::io::Write;
-use crate::models::{GitHubRepo, RepoRule};
+use crate::models::github_repo::GitHubRepo;
+use crate::models::repo_rule::RepoRule;
 
 pub mod apis;
 pub mod models;
-pub mod parser;
+pub mod workflow;
 
 /// Set the contributor repository of an upstream repository.
 /// The information of the contributor repository will be stored in a JSON file.

src-rust/toolkit-core/src/models/html_filter_rule.rs

@@ -0,0 +1,14 @@
+#[derive(PartialEq, Eq, Debug, serde::Deserialize)]
+pub struct HTMLFilterRule {
+    pub tags: Vec<String>,
+    pub classes: Vec<String>,
+}
+
+impl HTMLFilterRule {
+    pub fn new(tags: Vec<String>, classes: Vec<String>) -> Self {
+        Self {
+            tags,
+            classes,
+        }
+    }
+}

src-rust/toolkit-core/src/models/mod.rs

@@ -1,5 +1,3 @@
-mod github_repo;
-mod repo_rule;
-
-pub use github_repo::*;
-pub use repo_rule::*;
+pub mod github_repo;
+pub mod repo_rule;
+pub mod html_filter_rule;

src-rust/toolkit-core/src/workflow/mod.rs

@@ -0,0 +1 @@
+pub mod translate;

src-rust/toolkit-core/src/workflow/translate/mod.rs

@@ -0,0 +1 @@
+pub mod select;

src-rust/toolkit-core/src/workflow/translate/select.rs

@@ -0,0 +1,63 @@
+use crate::models::html_filter_rule::HTMLFilterRule;
+
+pub fn get_content(url: &str) -> Result<String, String> {
+    let article_template = std::fs::read_to_string("rsc/translate_article_template.md");
+    let article_template = match article_template {
+        Ok(article_template) => article_template,
+        Err(_) => {
+            return Err("Failed to read the translate article template.".to_string());
+        }
+    };
+
+    let website = url::Url::parse(url);
+    let website = match website {
+        Ok(website) => website,
+        Err(_) => {
+            return Err("Failed to parse the URL.".to_string());
+        }
+    };
+    let host = website.host_str();
+    let host = if host.is_none() {
+        return Err("Failed to get the host of the URL.".to_string());
+    } else {
+        host.unwrap()
+    };
+
+    let html_filter_rule_path = format!("rsc/html_filter_rules/{}.toml", host);
+    let html_filter_rule_str = std::fs::read_to_string(html_filter_rule_path);
+    let html_filter_rule = match html_filter_rule_str {
+        Ok(html_filter_rule_str) => {
+            let html_filter_rule: Result<HTMLFilterRule, _> = toml::from_str(&html_filter_rule_str);
+            match html_filter_rule {
+                Ok(html_filter_rule) => html_filter_rule,
+                Err(_) => {
+                    let error_msg = format!(
+                        "Failed to parse the HTML filter rule for the website: {}", host
+                    );
+                    return Err(error_msg);
+                }
+            }
+        },
+        Err(_) => {
+            // Use the default HTML filter rule (no tags and classes to filter)
+            HTMLFilterRule::new(Vec::new(), Vec::new())
+        }
+    };
+
+    // Filter the HTML content
+    let filtered_html = libhtmlfilter::get_filtered_html_fullurl_removeref(
+        url,
+        html_filter_rule.tags.iter().map(|s| s.as_str()).collect::<Vec<&str>>().as_slice(),
+        html_filter_rule.classes.iter().map(|s| s.as_str()).collect::<Vec<&str>>().as_slice()
+    );
+
+    // Parse HTML to markdown
+    let md = html2md::parse_html(&filtered_html).trim().to_string();
+
+    // Add header and footer
+    let mut content = String::new();
+    content.push_str(&article_template);
+    content.push_str(&md);
+
+    Ok(content)
+}