feat(actions): add an action to transform html to markdown

Author: joelwurtz

Cargo.toml

@@ -15,6 +15,7 @@ crate-type = ["staticlib", "cdylib", "rlib"]
 [build-dependencies]
 cbindgen = "0.29.2"
 glob = "^0.3.3"
+html-to-markdown-rs = "3.0.2"
 libtool = "0.1.1"
 linked_hash_set = { version = "0.1.6", features = ["serde"] }
 serde = { version = "1.0.228", features = ["derive"] }
@@ -45,6 +46,7 @@ cidr = { version = "0.3.2", features = ["serde"] }
 dot_graph = { version = "0.2.3", optional = true }
 flate2 = { version = "1.1.9", optional = true }
 heck = "0.5.0"
+html-to-markdown-rs = "3.0.2"
 http = "1.4.0"
 lazy_static = "1.5.0"
 linked_hash_set = { version = "0.1.6", features = ["serde"] }
@@ -63,6 +65,7 @@ url = "2.5.8"
 [dev-dependencies]
 pprof = { version = "0.15.0", features = ["flamegraph"] }
 criterion = { version = "0.8.2", default-features = false }
+tracing-subscriber = "0.3.17"
 
 [[bench]]
 name = "match_rule_benchmark"

src/action/mod.rs

@@ -226,6 +226,7 @@ impl Action {
                             id: text_body_filter.id.clone(),
                             target_hash: text_body_filter.target_hash.clone(),
                         }),
+                        BodyFilter::HTMLToMarkdown(html_to_markdown_filter) => BodyFilter::HTMLToMarkdown(html_to_markdown_filter.clone()),
                         BodyFilter::Other(_) => {
                             continue;
                         }

src/api/body_filter.rs

@@ -1,3 +1,4 @@
+use html_to_markdown_rs::ConversionOptions;
 use serde::{Deserialize, Serialize};
 
 use crate::{api::VariableValue, marker::StaticOrDynamic};
@@ -145,11 +146,24 @@ pub enum TextAction {
     Replace,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct HTMLToMarkdownFilter {
+    pub action: HTMLToMarkdownAction,
+    pub options: Option<ConversionOptions>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum HTMLToMarkdownAction {
+    #[serde(rename = "html_to_markdown")]
+    Filter,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(untagged)]
 pub enum BodyFilter {
     Text(TextBodyFilter),
     HTML(HTMLBodyFilter),
+    HTMLToMarkdown(HTMLToMarkdownFilter),
     #[serde(untagged)]
     Other(serde_json::Value),
 }

src/build.rs

@@ -11,10 +11,11 @@ use std::{
 };
 
 use glob::glob;
+use html_to_markdown_rs::ConversionOptions;
 use linked_hash_set::LinkedHashSet;
 use serde::{Deserialize, Serialize};
 use serde_yaml::from_str as yaml_decode;
-use tera::{Context, Tera};
+use tera::{Context, Tera, Value, try_get_value};
 #[derive(Serialize, Deserialize, Debug, Clone)]
 struct RuleSet {
     #[serde(default)]
@@ -180,6 +181,8 @@ struct BodyFilter {
     css_selector: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     ignore_css_selector: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    options: Option<ConversionOptions>,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -270,10 +273,23 @@ struct RuleTestHeader {
 #[derive(Serialize, Deserialize, Debug, Clone)]
 struct ShouldFilterBody {
     enable: bool,
+    #[serde(default = "default_false")]
+    is_binary: bool,
     original_body: String,
     expected_body: String,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum Body {
+    String(String),
+    Bytes(Vec<u8>),
+}
+
+fn default_false() -> bool {
+    false
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 struct ShouldFilterHeader {
     enable: bool,
@@ -347,11 +363,13 @@ fn make_router_tests() {
         return;
     }
 
-    let templating = match Tera::new("tests/templates/**/*") {
+    let mut templating = match Tera::new("tests/templates/**/*") {
         Ok(t) => t,
         Err(e) => panic!("{}", e),
     };
 
+    templating.register_filter("as_bytes", filter_as_bytes_str);
+
     let rule_sets_list = RuleSetList { rule_sets };
     let test_path = Path::new("tests/redirectionio_router_test.rs");
 
@@ -372,6 +390,18 @@ fn make_router_tests() {
     }
 }
 
+fn filter_as_bytes_str(value: &Value, _: &HashMap<String, Value>) -> tera::Result<Value> {
+    let body = try_get_value!("as_bytes", "value", String, value);
+
+    Ok(Value::String(
+        body.as_bytes()
+            .iter()
+            .map(|b| format!("\\x{b:02x}"))
+            .collect::<Vec<String>>()
+            .join(""),
+    ))
+}
+
 fn read_router_tests(path: &str) -> HashMap<String, RuleSet> {
     let mut rule_sets = HashMap::new();
 

src/filter/filter_body.rs

@@ -13,6 +13,7 @@ use crate::{
             HtmlBodyVisitor,
             body_capture::{BodyCapture, CaptureRegistry},
         },
+        html_to_markdown::HtmlToMarkdownFilter,
         text_filter_body::{TextFilterAction, TextFilterBodyAction},
     },
     http::Header,
@@ -27,6 +28,7 @@ pub struct FilterBodyAction {
 #[derive(Debug)]
 pub enum FilterBodyActionItem {
     Buffer(BufferFilterBody),
+    HtmlToMarkdown(Box<HtmlToMarkdownFilter>),
     Html(Box<HtmlFilterBodyAction>),
     Text(TextFilterBodyAction),
     #[cfg(feature = "compress")]
@@ -218,6 +220,24 @@ impl FilterBodyActionItem {
                 },
                 text_body_filter.content,
             ))),
+            BodyFilter::HTMLToMarkdown(html_to_md_filter) => match content_type {
+                Some(content_type) if content_type.contains("text/html") => {
+                    // @TODO Support charset
+                    Some(Self::HtmlToMarkdown(Box::new(HtmlToMarkdownFilter::new(html_to_md_filter.options))))
+                }
+                None => {
+                    // Assume HTML if no content type
+                    Some(Self::HtmlToMarkdown(Box::new(HtmlToMarkdownFilter::new(html_to_md_filter.options))))
+                }
+                _ => {
+                    log::warn!(
+                        "html to markdown is only supported for text/html content type, {} received",
+                        content_type.unwrap_or_default()
+                    );
+
+                    None
+                }
+            },
             BodyFilter::Other(_) => {
                 log::warn!("unsupported body filter: {filter:?}, you may need to update your agent or module");
                 None
@@ -229,6 +249,7 @@ impl FilterBodyActionItem {
         Ok(match self {
             FilterBodyActionItem::Buffer(buffer) => buffer.filter(data),
             FilterBodyActionItem::Html(html_body_filter) => html_body_filter.filter(data)?,
+            FilterBodyActionItem::HtmlToMarkdown(html_to_md_filter) => html_to_md_filter.filter(data),
             FilterBodyActionItem::Text(text_body_filter) => text_body_filter.filter(data, unit_trace),
             #[cfg(feature = "compress")]
             FilterBodyActionItem::Decode(decode_body_filter) => decode_body_filter.filter(data)?,
@@ -241,6 +262,7 @@ impl FilterBodyActionItem {
         Ok(match self {
             FilterBodyActionItem::Buffer(buffer) => buffer.end(),
             FilterBodyActionItem::Html(html_body_filter) => html_body_filter.end(),
+            FilterBodyActionItem::HtmlToMarkdown(html_to_md_filter) => html_to_md_filter.end(),
             FilterBodyActionItem::Text(text_body_filter) => text_body_filter.end(),
             #[cfg(feature = "compress")]
             FilterBodyActionItem::Decode(decode_body_filter) => decode_body_filter.end()?,

src/filter/html_to_markdown.rs

@@ -0,0 +1,41 @@
+use html_to_markdown_rs::{ConversionOptions, convert};
+
+#[derive(Debug)]
+pub struct HtmlToMarkdownFilter {
+    buffer: Vec<u8>,
+    options: Option<ConversionOptions>,
+}
+
+impl HtmlToMarkdownFilter {
+    pub fn new(options: Option<ConversionOptions>) -> Self {
+        Self {
+            buffer: Vec::new(),
+            options,
+        }
+    }
+
+    pub fn filter(&mut self, input: Vec<u8>) -> Vec<u8> {
+        self.buffer.extend_from_slice(&input);
+        Vec::new()
+    }
+
+    pub fn end(self) -> Vec<u8> {
+        let html = match String::from_utf8(self.buffer) {
+            Err(e) => {
+                log::error!("error while converting to utf8: {}", e);
+
+                return e.into_bytes();
+            }
+            Ok(html) => html,
+        };
+
+        match convert(html.as_str(), self.options) {
+            Ok(md) => md.content.unwrap_or_default().into_bytes(),
+            Err(e) => {
+                log::error!("error while converting html to markdown: {}", e);
+
+                html.into_bytes()
+            }
+        }
+    }
+}

src/filter/mod.rs

@@ -8,6 +8,7 @@ mod filter_header;
 mod header_action;
 mod html_body_action;
 mod html_filter_body;
+mod html_to_markdown;
 mod text_filter_body;
 
 pub use buffer::Buffer;