diff --git a/CHANGELOG.md b/CHANGELOG.md index 3597e68b..a91d9df1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [Unreleased] + +### Added +- Support `$to=` option on network filters. Destination hostnames are stored as hashes and checked at match time via mapped domain lookups (bucketing unchanged). + +### Changed +- DAT format version bumped to v6. + ## [0.13.0] - 2026-07-09 ### Added diff --git a/Cargo.toml b/Cargo.toml index 678383c1..35bdf899 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,8 @@ default = ["embedded-domain-resolver", "full-regex-handling", "single-thread"] full-regex-handling = [] single-thread = [] # disables `Send` and `Sync` on `Engine`. debug-info = [] +# Thread-local counters for network match funnel debugging (filters checked / reject stage). +match-debug-stats = [] css-validation = ["cssparser", "selectors"] content-blocking = [] embedded-domain-resolver = ["addr"] # Requires setting an external domain resolver if disabled. diff --git a/src/content_blocking.rs b/src/content_blocking.rs index 5ad01239..0c7db38d 100644 --- a/src/content_blocking.rs +++ b/src/content_blocking.rs @@ -199,6 +199,8 @@ pub enum CbRuleCreationFailure { NetworkCspUnsupported, /// Network rules with removeparam options cannot be supported in content blocking syntax. NetworkRemoveparamUnsupported, + /// Network rules with to= options cannot be supported in content blocking syntax. + NetworkToUnsupported, /// Content blocking syntax only supports a subset of regex features, namely: /// - Matching any character with “.”. /// - Matching ranges with the range syntax [a-b]. @@ -332,6 +334,9 @@ impl TryFrom> for CbRuleEquivalent { if v.is_removeparam() { return Err(CbRuleCreationFailure::NetworkRemoveparamUnsupported); } + if v.opt_to_domains.is_some() || v.opt_not_to_domains.is_some() { + return Err(CbRuleCreationFailure::NetworkToUnsupported); + } let load_type = if v .mask diff --git a/src/data_format/mod.rs b/src/data_format/mod.rs index 37202b05..f1e68ced 100644 --- a/src/data_format/mod.rs +++ b/src/data_format/mod.rs @@ -17,7 +17,7 @@ const ADBLOCK_RUST_DAT_MAGIC: [u8; 4] = [0xd1, 0xd9, 0x3a, 0xaf]; /// The version of the data format. /// If the data format version is incremented, the data is considered as incompatible. -const ADBLOCK_RUST_DAT_VERSION: u8 = 5; +const ADBLOCK_RUST_DAT_VERSION: u8 = 6; /// The total length of the header prefix (magic + version + seahash) const HEADER_PREFIX_LENGTH: usize = 4 + 1 + 8; diff --git a/src/filters/abstract_network.rs b/src/filters/abstract_network.rs index 970f4328..cf294326 100644 --- a/src/filters/abstract_network.rs +++ b/src/filters/abstract_network.rs @@ -44,6 +44,7 @@ pub(crate) enum HttpMethod { #[derive(Clone)] pub(crate) enum NetworkFilterOption<'a> { Domain(Vec<(bool, &'a str)>), + To(Vec<(bool, &'a str)>), Badfilter, Important, MatchCase, @@ -198,6 +199,23 @@ fn parse_filter_options<'a>( } NetworkFilterOption::Domain(domains) } + ("to", _) => { + let domains: Vec<(bool, &'a str)> = value + .split('|') + .map(|domain| { + if let Some(negated_domain) = domain.strip_prefix('~') { + (false, negated_domain) + } else { + (true, domain) + } + }) + .filter(|(_, d)| !(d.starts_with('/') && d.ends_with('/'))) + .collect(); + if domains.is_empty() { + return Err(NetworkFilterError::NoSupportedDomains); + } + NetworkFilterOption::To(domains) + } ("badfilter", true) => return Err(NetworkFilterError::NegatedBadFilter), ("badfilter", false) => NetworkFilterOption::Badfilter, ("important", true) => return Err(NetworkFilterError::NegatedImportant), diff --git a/src/filters/fb_network.rs b/src/filters/fb_network.rs index 66153e42..8a094ef8 100644 --- a/src/filters/fb_network.rs +++ b/src/filters/fb_network.rs @@ -143,6 +143,20 @@ impl<'a> FlatNetworkFilter<'a> { .map(|data| fb_vector_to_slice(data)) } + #[inline(always)] + pub fn include_to_domains(&self) -> Option<&[u32]> { + self.fb_filter + .opt_to_domains() + .map(|data| fb_vector_to_slice(data)) + } + + #[inline(always)] + pub fn exclude_to_domains(&self) -> Option<&[u32]> { + self.fb_filter + .opt_not_to_domains() + .map(|data| fb_vector_to_slice(data)) + } + #[inline(always)] pub fn hostname(&self) -> Option<&'a str> { if self.mask.is_hostname_anchor() { @@ -252,10 +266,16 @@ impl NetworkFilterMaskHelper for FlatNetworkFilter<'_> { impl NetworkMatchable for FlatNetworkFilter<'_> { fn matches(&self, request: &Request, regex_manager: &mut RegexManager) -> bool { use crate::filters::network_matchers::{ - check_excluded_domains_mapped, check_included_domains_mapped, check_options, + check_excluded_domains_mapped, check_excluded_to_domains_mapped, + check_included_domains_mapped, check_included_to_domains_mapped, check_options, check_pattern, }; + #[cfg(feature = "match-debug-stats")] + crate::match_debug_stats::record_checked(); + if !check_options(self.mask, request) { + #[cfg(feature = "match-debug-stats")] + crate::match_debug_stats::record_reject(crate::match_debug_stats::MatchStage::Options); return false; } if !check_included_domains_mapped( @@ -263,6 +283,10 @@ impl NetworkMatchable for FlatNetworkFilter<'_> { request, &self.filter_data_context.unique_domains_hashes_map, ) { + #[cfg(feature = "match-debug-stats")] + crate::match_debug_stats::record_reject( + crate::match_debug_stats::MatchStage::IncludedDomains, + ); return false; } if !check_excluded_domains_mapped( @@ -270,15 +294,54 @@ impl NetworkMatchable for FlatNetworkFilter<'_> { request, &self.filter_data_context.unique_domains_hashes_map, ) { + #[cfg(feature = "match-debug-stats")] + crate::match_debug_stats::record_reject( + crate::match_debug_stats::MatchStage::ExcludedDomains, + ); return false; } - check_pattern( + if !check_included_to_domains_mapped( + self.include_to_domains(), + request, + &self.filter_data_context.unique_domains_hashes_map, + ) { + #[cfg(feature = "match-debug-stats")] + crate::match_debug_stats::record_reject( + crate::match_debug_stats::MatchStage::IncludedToDomains, + ); + return false; + } + if !check_excluded_to_domains_mapped( + self.exclude_to_domains(), + request, + &self.filter_data_context.unique_domains_hashes_map, + ) { + #[cfg(feature = "match-debug-stats")] + crate::match_debug_stats::record_reject( + crate::match_debug_stats::MatchStage::ExcludedToDomains, + ); + return false; + } + let matched = check_pattern( self.mask, self.patterns().iter(), self.hostname(), self.key, request, regex_manager, - ) + ); + #[cfg(feature = "match-debug-stats")] + { + if matched { + crate::match_debug_stats::record_match( + crate::match_debug_stats::MatchStage::Pattern, + ); + } else { + crate::match_debug_stats::record_reject( + crate::match_debug_stats::MatchStage::Pattern, + ); + } + } + matched } } diff --git a/src/filters/fb_network_builder.rs b/src/filters/fb_network_builder.rs index 5a7987ad..0e305aac 100644 --- a/src/filters/fb_network_builder.rs +++ b/src/filters/fb_network_builder.rs @@ -92,6 +92,26 @@ impl<'a, 'f> FlatSerialize<'a, EngineFlatBuilder<'a>> FlatSerialize::serialize(o, builder) }); + let opt_to_domains = network_filter.opt_to_domains.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + + let opt_not_to_domains = network_filter.opt_not_to_domains.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + let modifier_option = network_filter .modifier_option .map(|s| builder.create_string(s)); @@ -135,6 +155,8 @@ impl<'a, 'f> FlatSerialize<'a, EngineFlatBuilder<'a>> modifier_option, opt_domains, opt_not_domains, + opt_to_domains, + opt_not_to_domains, hostname, tag, raw_line, diff --git a/src/filters/network.rs b/src/filters/network.rs index eddd7c2f..9d52cf73 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -408,6 +408,8 @@ pub struct NetworkFilter<'a> { pub filter: FilterPart<'a>, pub opt_domains: Option>, pub opt_not_domains: Option>, + pub opt_to_domains: Option>, + pub opt_not_to_domains: Option>, /// Used for `$redirect`, `$redirect-rule`, `$csp`, and `$removeparam` - only one of which is /// supported per-rule. pub modifier_option: Option<&'a str>, @@ -501,6 +503,8 @@ impl<'a> NetworkFilter<'a> { let mut opt_domains: Option> = None; let mut opt_not_domains: Option> = None; + let mut opt_to_domains: Option> = None; + let mut opt_not_to_domains: Option> = None; let mut modifier_option: Option<&'a str> = None; let mut tag: Option<&'a str> = None; @@ -560,6 +564,30 @@ impl<'a> NetworkFilter<'a> { opt_not_domains = Some(opt_not_domains_array); } } + NetworkFilterOption::To(domains) => { + let mut opt_to_domains_array: Vec = vec![]; + let mut opt_not_to_domains_array: Vec = vec![]; + + for (enabled, domain) in domains { + let domain_hash = utils::fast_hash(domain); + if !enabled { + opt_not_to_domains_array.push(domain_hash); + } else { + opt_to_domains_array.push(domain_hash); + } + } + + if !opt_to_domains_array.is_empty() { + opt_to_domains_array.sort_unstable(); + opt_to_domains_array.dedup(); + opt_to_domains = Some(opt_to_domains_array); + } + if !opt_not_to_domains_array.is_empty() { + opt_not_to_domains_array.sort_unstable(); + opt_not_to_domains_array.dedup(); + opt_not_to_domains = Some(opt_not_to_domains_array); + } + } NetworkFilterOption::Badfilter => { features_mask.set(NetworkFilterFeaturesMask::BAD_FILTER, true) } @@ -878,6 +906,8 @@ impl<'a> NetworkFilter<'a> { features_mask, opt_domains, opt_not_domains, + opt_to_domains, + opt_not_to_domains, tag, raw_line: if debug { Some(Cow::Borrowed(line)) @@ -931,6 +961,8 @@ impl<'a> NetworkFilter<'a> { features_mask: Default::default(), opt_domains: None, opt_not_domains: None, + opt_to_domains: None, + opt_not_to_domains: None, tag: None, raw_line: if debug { Some(Cow::Owned(rule)) } else { None }, modifier_option: None, @@ -947,6 +979,8 @@ impl<'a> NetworkFilter<'a> { self.hostname.as_deref(), self.opt_domains.as_ref(), self.opt_not_domains.as_ref(), + self.opt_to_domains.as_ref(), + self.opt_not_to_domains.as_ref(), ) } @@ -1122,6 +1156,8 @@ fn compute_filter_id( hostname: Option<&str>, opt_domains: Option<&Vec>, opt_not_domains: Option<&Vec>, + opt_to_domains: Option<&Vec>, + opt_not_to_domains: Option<&Vec>, ) -> Hash { let mut hasher = FxHasher::default(); @@ -1147,6 +1183,18 @@ fn compute_filter_id( } } + if let Some(domains) = opt_to_domains { + for d in domains { + hasher.write_u64(*d); + } + } + + if let Some(domains) = opt_not_to_domains { + for d in domains { + hasher.write_u64(*d); + } + } + match filter { FilterPart::Empty => {} FilterPart::Simple(s) => write_str_to_hasher(&mut hasher, s.as_ref()), diff --git a/src/filters/network_matchers.rs b/src/filters/network_matchers.rs index 830eb4f0..610820f8 100644 --- a/src/filters/network_matchers.rs +++ b/src/filters/network_matchers.rs @@ -467,6 +467,51 @@ pub fn check_excluded_domains_mapped( true } +#[inline] +pub fn check_included_to_domains_mapped( + opt_to_domains: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, +) -> bool { + if let Some(included_domains) = opt_to_domains.as_ref() { + if let Some(hostname_hashes) = request.hostname_hashes.as_ref() { + if hostname_hashes.iter().all(|h| { + mapping + .get(h) + .is_none_or(|index| !utils::bin_lookup(included_domains, *index)) + }) { + return false; + } + } else { + return false; + } + } + true +} + +#[inline] +pub fn check_excluded_to_domains_mapped( + opt_not_to_domains: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, +) -> bool { + if let Some(excluded_domains) = opt_not_to_domains.as_ref() { + if let Some(hostname_hashes) = request.hostname_hashes.as_ref() { + if hostname_hashes.iter().any(|h| { + mapping + .get(h) + .is_some_and(|index| utils::bin_lookup(excluded_domains, *index)) + }) { + return false; + } + } else { + return true; + } + } + + true +} + #[cfg(test)] #[path = "../../tests/unit/filters/network_matchers.rs"] mod unit_tests; diff --git a/src/flatbuffers/fb_network_filter.fbs b/src/flatbuffers/fb_network_filter.fbs index 1187c806..ee585d08 100644 --- a/src/flatbuffers/fb_network_filter.fbs +++ b/src/flatbuffers/fb_network_filter.fbs @@ -32,6 +32,10 @@ table NetworkFilter { raw_line: string; source_index: uint32 = 4294967295; // 0xFFFFFFFF line_number: uint32 = 4294967295; // 0xFFFFFFFF + + /// Same representation as |opt_domains|, for the `$to=` option. + opt_to_domains: [uint32]; + opt_not_to_domains: [uint32]; } table NetworkFilterList { diff --git a/src/flatbuffers/fb_network_filter_generated.rs b/src/flatbuffers/fb_network_filter_generated.rs index f0d622cd..fb7969e1 100644 --- a/src/flatbuffers/fb_network_filter_generated.rs +++ b/src/flatbuffers/fb_network_filter_generated.rs @@ -34,6 +34,8 @@ pub mod fb { pub const VT_RAW_LINE: ::flatbuffers::VOffsetT = 20; pub const VT_SOURCE_INDEX: ::flatbuffers::VOffsetT = 22; pub const VT_LINE_NUMBER: ::flatbuffers::VOffsetT = 24; + pub const VT_OPT_TO_DOMAINS: ::flatbuffers::VOffsetT = 26; + pub const VT_OPT_NOT_TO_DOMAINS: ::flatbuffers::VOffsetT = 28; #[inline] pub unsafe fn init_from_table(table: ::flatbuffers::Table<'a>) -> Self { @@ -50,6 +52,12 @@ pub mod fb { args: &'args NetworkFilterArgs<'args>, ) -> ::flatbuffers::WIPOffset> { let mut builder = NetworkFilterBuilder::new(_fbb); + if let Some(x) = args.opt_not_to_domains { + builder.add_opt_not_to_domains(x); + } + if let Some(x) = args.opt_to_domains { + builder.add_opt_to_domains(x); + } builder.add_line_number(args.line_number); builder.add_source_index(args.source_index); if let Some(x) = args.raw_line { @@ -104,6 +112,8 @@ pub mod fb { .map(|x| alloc::string::ToString::to_string(x)); let source_index = self.source_index(); let line_number = self.line_number(); + let opt_to_domains = self.opt_to_domains().map(|x| x.into_iter().collect()); + let opt_not_to_domains = self.opt_not_to_domains().map(|x| x.into_iter().collect()); NetworkFilterT { mask, opt_domains, @@ -116,6 +126,8 @@ pub mod fb { raw_line, source_index, line_number, + opt_to_domains, + opt_not_to_domains, } } @@ -248,6 +260,33 @@ pub mod fb { .unwrap() } } + /// Same representation as |opt_domains|, for the `$to=` option. + #[inline] + pub fn opt_to_domains(&self) -> Option<::flatbuffers::Vector<'a, u32>> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'a, u32>>>( + NetworkFilter::VT_OPT_TO_DOMAINS, + None, + ) + } + } + #[inline] + pub fn opt_not_to_domains(&self) -> Option<::flatbuffers::Vector<'a, u32>> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'a, u32>>>( + NetworkFilter::VT_OPT_NOT_TO_DOMAINS, + None, + ) + } + } } impl ::flatbuffers::Verifiable for NetworkFilter<'_> { @@ -294,6 +333,16 @@ pub mod fb { )? .visit_field::("source_index", Self::VT_SOURCE_INDEX, false)? .visit_field::("line_number", Self::VT_LINE_NUMBER, false)? + .visit_field::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'_, u32>>>( + "opt_to_domains", + Self::VT_OPT_TO_DOMAINS, + false, + )? + .visit_field::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'_, u32>>>( + "opt_not_to_domains", + Self::VT_OPT_NOT_TO_DOMAINS, + false, + )? .finish(); Ok(()) } @@ -314,6 +363,8 @@ pub mod fb { pub raw_line: Option<::flatbuffers::WIPOffset<&'a str>>, pub source_index: u32, pub line_number: u32, + pub opt_to_domains: Option<::flatbuffers::WIPOffset<::flatbuffers::Vector<'a, u32>>>, + pub opt_not_to_domains: Option<::flatbuffers::WIPOffset<::flatbuffers::Vector<'a, u32>>>, } impl<'a> Default for NetworkFilterArgs<'a> { #[inline] @@ -330,6 +381,8 @@ pub mod fb { raw_line: None, source_index: 4294967295, line_number: 4294967295, + opt_to_domains: None, + opt_not_to_domains: None, } } } @@ -420,6 +473,26 @@ pub mod fb { .push_slot::(NetworkFilter::VT_LINE_NUMBER, line_number, 4294967295); } #[inline] + pub fn add_opt_to_domains( + &mut self, + opt_to_domains: ::flatbuffers::WIPOffset<::flatbuffers::Vector<'b, u32>>, + ) { + self.fbb_.push_slot_always::<::flatbuffers::WIPOffset<_>>( + NetworkFilter::VT_OPT_TO_DOMAINS, + opt_to_domains, + ); + } + #[inline] + pub fn add_opt_not_to_domains( + &mut self, + opt_not_to_domains: ::flatbuffers::WIPOffset<::flatbuffers::Vector<'b, u32>>, + ) { + self.fbb_.push_slot_always::<::flatbuffers::WIPOffset<_>>( + NetworkFilter::VT_OPT_NOT_TO_DOMAINS, + opt_not_to_domains, + ); + } + #[inline] pub fn new( _fbb: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>, ) -> NetworkFilterBuilder<'a, 'b, A> { @@ -450,6 +523,8 @@ pub mod fb { ds.field("raw_line", &self.raw_line()); ds.field("source_index", &self.source_index()); ds.field("line_number", &self.line_number()); + ds.field("opt_to_domains", &self.opt_to_domains()); + ds.field("opt_not_to_domains", &self.opt_not_to_domains()); ds.finish() } } @@ -467,6 +542,8 @@ pub mod fb { pub raw_line: Option, pub source_index: u32, pub line_number: u32, + pub opt_to_domains: Option>, + pub opt_not_to_domains: Option>, } impl Default for NetworkFilterT { fn default() -> Self { @@ -482,6 +559,8 @@ pub mod fb { raw_line: None, source_index: 4294967295, line_number: 4294967295, + opt_to_domains: None, + opt_not_to_domains: None, } } } @@ -504,6 +583,11 @@ pub mod fb { let raw_line = self.raw_line.as_ref().map(|x| _fbb.create_string(x)); let source_index = self.source_index; let line_number = self.line_number; + let opt_to_domains = self.opt_to_domains.as_ref().map(|x| _fbb.create_vector(x)); + let opt_not_to_domains = self + .opt_not_to_domains + .as_ref() + .map(|x| _fbb.create_vector(x)); NetworkFilter::create( _fbb, &NetworkFilterArgs { @@ -518,6 +602,8 @@ pub mod fb { raw_line, source_index, line_number, + opt_to_domains, + opt_not_to_domains, }, ) } diff --git a/src/lib.rs b/src/lib.rs index 0658ee8e..3cd2b9f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,6 +29,8 @@ pub mod engine; pub mod filters; mod flatbuffers; pub mod lists; +#[cfg(feature = "match-debug-stats")] +pub mod match_debug_stats; mod network_filter_list; mod optimizer; pub mod regex_manager; diff --git a/src/match_debug_stats.rs b/src/match_debug_stats.rs new file mode 100644 index 00000000..0f8248fd --- /dev/null +++ b/src/match_debug_stats.rs @@ -0,0 +1,105 @@ +//! Optional instrumentation for network filter matching. +//! +//! Enable with the `match-debug-stats` feature, then call [`reset`] before a request +//! check and [`snapshot`] / [`take`] afterwards. + +use std::cell::RefCell; + +/// Which sequential check stage accepted or rejected a filter. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MatchStage { + Options, + IncludedDomains, + ExcludedDomains, + IncludedToDomains, + ExcludedToDomains, + Pattern, +} + +/// Counters collected while matching network filters. +#[derive(Clone, Debug, Default)] +pub struct MatchDebugStats { + /// Number of filters for which `matches()` was invoked. + pub filters_checked: u64, + /// Number of filters that returned a full match. + pub filters_matched: u64, + /// Stage of the last successful match (always [`MatchStage::Pattern`] today). + pub last_match_stage: Option, + /// Stage that rejected the most recently checked non-matching filter. + pub last_reject_stage: Option, + pub reject_options: u64, + pub reject_included_domains: u64, + pub reject_excluded_domains: u64, + pub reject_included_to_domains: u64, + pub reject_excluded_to_domains: u64, + pub reject_pattern: u64, +} + +thread_local! { + static STATS: RefCell = RefCell::new(MatchDebugStats::default()); +} + +/// Clears collected match statistics. +pub fn reset() { + STATS.with(|s| *s.borrow_mut() = MatchDebugStats::default()); +} + +/// Returns a copy of the current statistics. +pub fn snapshot() -> MatchDebugStats { + STATS.with(|s| s.borrow().clone()) +} + +/// Returns the current statistics and resets the collector. +pub fn take() -> MatchDebugStats { + STATS.with(|s| s.replace(MatchDebugStats::default())) +} + +#[inline] +pub(crate) fn record_checked() { + STATS.with(|s| s.borrow_mut().filters_checked += 1); +} + +#[inline] +pub(crate) fn record_reject(stage: MatchStage) { + STATS.with(|s| { + let mut stats = s.borrow_mut(); + stats.last_reject_stage = Some(stage); + match stage { + MatchStage::Options => stats.reject_options += 1, + MatchStage::IncludedDomains => stats.reject_included_domains += 1, + MatchStage::ExcludedDomains => stats.reject_excluded_domains += 1, + MatchStage::IncludedToDomains => stats.reject_included_to_domains += 1, + MatchStage::ExcludedToDomains => stats.reject_excluded_to_domains += 1, + MatchStage::Pattern => stats.reject_pattern += 1, + } + }); +} + +#[inline] +pub(crate) fn record_match(stage: MatchStage) { + STATS.with(|s| { + let mut stats = s.borrow_mut(); + stats.filters_matched += 1; + stats.last_match_stage = Some(stage); + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::request::Request; + use crate::Engine; + + #[test] + fn collects_filter_check_and_match_stage() { + reset(); + let engine = Engine::new_with_list_text("||example.com^$script"); + let request = + Request::new("https://example.com/ads.js", "https://foo.com/", "script", "").unwrap(); + assert!(engine.check_network_request(&request).should_block()); + let stats = take(); + assert!(stats.filters_checked >= 1); + assert_eq!(stats.filters_matched, 1); + assert_eq!(stats.last_match_stage, Some(MatchStage::Pattern)); + } +} diff --git a/src/optimizer.rs b/src/optimizer.rs index ed6522b0..f1b5ffdd 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -14,6 +14,8 @@ trait Optimization { pub fn is_filter_optimizable_by_patterns(filter: &NetworkFilter<'_>) -> bool { filter.opt_domains.is_none() && filter.opt_not_domains.is_none() + && filter.opt_to_domains.is_none() + && filter.opt_not_to_domains.is_none() && !filter.is_hostname_anchor() && !filter.is_redirect() && !filter.is_csp() diff --git a/src/request.rs b/src/request.rs index 47932a78..9f87d4e8 100644 --- a/src/request.rs +++ b/src/request.rs @@ -120,6 +120,7 @@ pub struct Request { pub url: String, pub hostname: String, pub source_hostname_hashes: Option>, + pub hostname_hashes: Option>, pub(crate) url_lower_cased: String, pub(crate) request_tokens: Vec, @@ -197,6 +198,19 @@ impl Request { None }; + let hostname_hashes = if !hostname.is_empty() { + let mut hashes = Vec::with_capacity(4); + hashes.push(utils::fast_hash(hostname)); + for (i, c) in hostname.char_indices() { + if c == '.' && i + 1 < hostname.len() { + hashes.push(utils::fast_hash(&hostname[i + 1..])); + } + } + Some(hashes) + } else { + None + }; + let url_lower_cased = url.to_ascii_lowercase(); Request { @@ -207,6 +221,7 @@ impl Request { hostname: hostname.to_owned(), request_tokens: calculate_tokens(&url_lower_cased), source_hostname_hashes, + hostname_hashes, is_third_party: third_party, is_http, is_https, diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index 9a290acc..83e412e7 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -195,7 +195,7 @@ mod tests { fn deserialization_generate_simple() { let mut engine = Engine::new_with_list_text("ad-banner"); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 10610779084220584492; + const EXPECTED_HASH: u64 = 5402083050694566055; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } @@ -206,7 +206,7 @@ mod tests { let mut engine = Engine::new_with_list_text("ad-banner$tag=abc"); engine.use_tags(&["abc"]); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 9111262975876879244; + const EXPECTED_HASH: u64 = 5609938949059438672; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } @@ -262,9 +262,9 @@ mod tests { assert_eq!(debug_info.source_info[0].cosmetic_filter_count, 42318); } let expected_hash: u64 = if cfg!(feature = "css-validation") { - 17859942404936466029 + 7965980321682907265 } else { - 11154142685266326628 + 11418517700884004113 }; assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}"); diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index ea5bfaa7..3657d1a1 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -730,6 +730,97 @@ mod parse_tests { } } + #[test] + fn parses_to() { + { + let filter = + NetworkFilter::parse("||foo.com$to=bar.com", true, Default::default()).unwrap(); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("bar.com")]) + ); + assert_eq!(filter.opt_not_to_domains, None); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=bar.com|baz.com", true, Default::default()) + .unwrap(); + let mut expected = vec![utils::fast_hash("bar.com"), utils::fast_hash("baz.com")]; + expected.sort_unstable(); + assert_eq!(filter.opt_to_domains, Some(expected)); + assert_eq!(filter.opt_not_to_domains, None); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=~bar.com", true, Default::default()).unwrap(); + assert_eq!(filter.opt_to_domains, None); + assert_eq!( + filter.opt_not_to_domains, + Some(vec![utils::fast_hash("bar.com")]) + ); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=~bar.com|~baz.com", true, Default::default()) + .unwrap(); + assert_eq!(filter.opt_to_domains, None); + let mut expected = vec![utils::fast_hash("bar.com"), utils::fast_hash("baz.com")]; + expected.sort_unstable(); + assert_eq!(filter.opt_not_to_domains, Some(expected)); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=~bar.com|baz.com", true, Default::default()) + .unwrap(); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("baz.com")]) + ); + assert_eq!( + filter.opt_not_to_domains, + Some(vec![utils::fast_hash("bar.com")]) + ); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=bar.com|~baz.com", true, Default::default()) + .unwrap(); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("bar.com")]) + ); + assert_eq!( + filter.opt_not_to_domains, + Some(vec![utils::fast_hash("baz.com")]) + ); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=foo|~bar|baz", true, Default::default()) + .unwrap(); + let mut expected = vec![utils::fast_hash("baz"), utils::fast_hash("foo")]; + expected.sort_unstable(); + assert_eq!(filter.opt_to_domains, Some(expected)); + assert_eq!( + filter.opt_not_to_domains, + Some(vec![utils::fast_hash("bar")]) + ); + } + { + let filter = NetworkFilter::parse("||foo.com", true, Default::default()).unwrap(); + assert_eq!(filter.opt_to_domains, None); + assert_eq!(filter.opt_not_to_domains, None); + } + { + let filter = NetworkFilter::parse( + r"||video.twimg.com/ext_tw_video/*/*.m3u8$to=/^i[a-z]*\.strmrdr[a-z]+\..*/", + true, + Default::default(), + ); + assert_eq!(filter.err(), Some(NetworkFilterError::NoSupportedDomains)); + } + } + #[test] fn parses_redirects() { // parses redirect diff --git a/tests/unit/filters/network_matchers.rs b/tests/unit/filters/network_matchers.rs index 4f95846e..99fc6ae1 100644 --- a/tests/unit/filters/network_matchers.rs +++ b/tests/unit/filters/network_matchers.rs @@ -382,6 +382,26 @@ mod match_tests { .collect::>() }); + let opt_to_domains = filter.opt_to_domains.clone().map(|domains| { + domains + .iter() + .map(|domain| { + mapping.insert(*domain, *domain as u32); + *domain as u32 + }) + .collect::>() + }); + + let opt_not_to_domains = filter.opt_not_to_domains.clone().map(|domains| { + domains + .iter() + .map(|domain| { + mapping.insert(*domain, *domain as u32); + *domain as u32 + }) + .collect::>() + }); + super::super::check_options(filter.mask, request) && super::super::check_included_domains_mapped( opt_domains.as_deref(), @@ -393,6 +413,16 @@ mod match_tests { request, &mapping, ) + && super::super::check_included_to_domains_mapped( + opt_to_domains.as_deref(), + request, + &mapping, + ) + && super::super::check_excluded_to_domains_mapped( + opt_not_to_domains.as_deref(), + request, + &mapping, + ) } #[test] @@ -886,6 +916,105 @@ mod match_tests { } } + #[test] + fn check_to_option_works() { + { + let network_filter = NetworkFilter::parse( + "adv$to=example.com|~foo.example.com", + true, + Default::default(), + ) + .unwrap(); + assert!(network_filter.matches_test( + &request::Request::new("http://example.com/adv", "http://other.com", "", "") + .unwrap() + )); + assert!(!network_filter.matches_test( + &request::Request::new("http://foo.example.com/adv", "http://other.com", "", "") + .unwrap() + )); + assert!(!network_filter.matches_test( + &request::Request::new( + "http://subfoo.foo.example.com/adv", + "http://other.com", + "", + "" + ) + .unwrap() + )); + assert!(network_filter.matches_test( + &request::Request::new("http://bar.example.com/adv", "http://other.com", "", "") + .unwrap() + )); + assert!(!network_filter.matches_test( + &request::Request::new("http://anotherexample.com/adv", "http://other.com", "", "") + .unwrap() + )); + } + { + let network_filter = NetworkFilter::parse( + "adv$to=~example.com|~foo.example.com", + true, + Default::default(), + ) + .unwrap(); + assert!(!network_filter.matches_test( + &request::Request::new("http://example.com/adv", "http://other.com", "", "") + .unwrap() + )); + assert!(!network_filter.matches_test( + &request::Request::new("http://foo.example.com/adv", "http://other.com", "", "") + .unwrap() + )); + assert!(!network_filter.matches_test( + &request::Request::new( + "http://subfoo.foo.example.com/adv", + "http://other.com", + "", + "" + ) + .unwrap() + )); + assert!(!network_filter.matches_test( + &request::Request::new("http://bar.example.com/adv", "http://other.com", "", "") + .unwrap() + )); + assert!(network_filter.matches_test( + &request::Request::new("http://anotherexample.com/adv", "http://other.com", "", "") + .unwrap() + )); + } + { + let network_filter = NetworkFilter::parse( + "adv$to=example.com|foo.example.com", + true, + Default::default(), + ) + .unwrap(); + assert!(network_filter.matches_test( + &request::Request::new("http://example.com/adv", "http://other.com", "", "") + .unwrap() + )); + assert!(network_filter.matches_test( + &request::Request::new("http://foo.example.com/adv", "http://other.com", "", "") + .unwrap() + )); + assert!(network_filter.matches_test( + &request::Request::new( + "http://subfoo.foo.example.com/adv", + "http://other.com", + "", + "" + ) + .unwrap() + )); + assert!(network_filter.matches_test( + &request::Request::new("http://bar.example.com/adv", "http://other.com", "", "") + .unwrap() + )); + } + } + #[test] fn check_get_url_after_hostname_handles_bad_input() { // The function requires the hostname to necessarily be there in the URL,