Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/blocker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@
pub fn check_generic_hide(&self, hostname_request: &Request) -> bool {
let mut regex_manager = self.borrow_regex_manager();
self.generic_hide()
.check(hostname_request, &HashSet::new(), &mut regex_manager)

Check failure on line 164 in src/blocker.rs

View workflow job for this annotation

GitHub Actions / sanity

no method named `check` found for struct `network_filter_list::NetworkFilterList<'a>` in the current scope
.is_some()
}

Expand Down Expand Up @@ -194,7 +194,7 @@
// Always check important filters
let important_filter = self
.importants()
.check(request, get_no_tags(), &mut regex_manager);

Check failure on line 197 in src/blocker.rs

View workflow job for this annotation

GitHub Actions / sanity

no method named `check` found for struct `network_filter_list::NetworkFilterList<'a>` in the current scope

// only check the rest of the rules if not previously matched
let filter = if important_filter.is_none() && !matched_rule {
Expand Down Expand Up @@ -529,6 +529,20 @@
let regex_manager = self.borrow_regex_manager();
regex_manager.get_debug_info()
}

#[cfg(feature = "debug-info")]
pub fn get_network_filter_index_debug_info(
&self,
) -> crate::network_filter_list::NetworkFilterIndexDebugInfo {
use crate::filters::fb_network_builder::NetworkFilterListId;
use crate::network_filter_list::NetworkFilterIndexDebugInfo;

NetworkFilterIndexDebugInfo {
lists: NetworkFilterListId::all()
.map(|id| self.get_list(id).index_stats(id.as_str()))
.collect(),
}
}
}

#[cfg(test)]
Expand Down
2 changes: 1 addition & 1 deletion src/data_format/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ const ADBLOCK_RUST_DAT_MAGIC: [u8; 4] = [0xd1, 0xd9, 0x3a, 0xaf];

/// The version of the data format.
/// If the data format version is incremented, the data is considered as incompatible.
const ADBLOCK_RUST_DAT_VERSION: u8 = 7;
const ADBLOCK_RUST_DAT_VERSION: u8 = 8;

/// The total length of the header prefix (magic + version + seahash)
const HEADER_PREFIX_LENGTH: usize = 4 + 1 + 8;
Expand Down
11 changes: 11 additions & 0 deletions src/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ pub struct Engine {
filter_data_context: FilterDataContextRef,
}

#[cfg(feature = "debug-info")]
pub use crate::network_filter_list::{
NetworkFilterIndexDebugInfo, NetworkFilterListIndexStats, NetworkFilterMapStats,
};

#[cfg(feature = "debug-info")]
pub struct SourceInfo {
pub title: Option<String>,
Expand Down Expand Up @@ -432,6 +437,12 @@ impl Engine {
}
}

/// Returns statistics about how network filters are stored in the engine indexes.
#[cfg(feature = "debug-info")]
pub fn get_network_filter_index_debug_info(&self) -> NetworkFilterIndexDebugInfo {
self.blocker.get_network_filter_index_debug_info()
}

/// Serializes the `Engine` into a binary format so that it can be quickly reloaded later.
pub fn serialize(&self) -> Vec<u8> {
let data = self.filter_data_context.memory.data();
Expand Down
32 changes: 30 additions & 2 deletions src/filters/fb_network.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,20 @@ impl<'a> FlatNetworkFilter<'a> {
.map(|data| fb_vector_to_slice(data))
}

#[inline(always)]
pub fn include_to_domains(&self) -> Option<&[u32]> {
self.fb_filter
.opt_to_domains()
.map(|data| fb_vector_to_slice(data))
}

#[inline(always)]
pub fn exclude_to_domains(&self) -> Option<&[u32]> {
self.fb_filter
.opt_to_not_domains()
.map(|data| fb_vector_to_slice(data))
}

#[inline(always)]
pub fn hostname(&self) -> Option<&'a str> {
if self.mask.is_hostname_anchor() {
Expand Down Expand Up @@ -260,14 +274,28 @@ impl NetworkMatchable for FlatNetworkFilter<'_> {
}
if !check_included_domains_mapped(
self.include_domains(),
request,
request.source_hostname_hashes.as_deref(),
&self.filter_data_context.unique_domains_hashes_map,
) {
return false;
}
if !check_excluded_domains_mapped(
self.exclude_domains(),
request,
request.source_hostname_hashes.as_deref(),
&self.filter_data_context.unique_domains_hashes_map,
) {
return false;
}
if !check_included_domains_mapped(
self.include_to_domains(),
request.hostname_hashes.as_deref(),
&self.filter_data_context.unique_domains_hashes_map,
) {
return false;
}
if !check_excluded_domains_mapped(
self.exclude_to_domains(),
request.hostname_hashes.as_deref(),
&self.filter_data_context.unique_domains_hashes_map,
) {
return false;
Expand Down
54 changes: 48 additions & 6 deletions src/filters/fb_network_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use crate::utils::{Hash, ShortHash, to_short_hash};

use super::flat::fb;

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum NetworkFilterListId {
Csp = 0,
Exceptions = 1,
Expand All @@ -30,6 +31,38 @@ pub(crate) enum NetworkFilterListId {
Size = 8,
}

impl NetworkFilterListId {
#[cfg(feature = "debug-info")]
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Csp => "csp",
Self::Exceptions => "exceptions",
Self::Importants => "importants",
Self::Redirects => "redirects",
Self::RemoveParam => "removeparam",
Self::Filters => "filters",
Self::GenericHide => "generichide",
Self::TaggedFiltersAll => "tagged_filters_all",
Self::Size => "size",
}
}

#[cfg(feature = "debug-info")]
pub(crate) fn all() -> impl Iterator<Item = Self> {
(0..Self::Size as usize).map(|id| match id {
0 => Self::Csp,
1 => Self::Exceptions,
2 => Self::Importants,
3 => Self::Redirects,
4 => Self::RemoveParam,
5 => Self::Filters,
6 => Self::GenericHide,
7 => Self::TaggedFiltersAll,
_ => unreachable!(),
})
}
}

struct NetworkFilterFlatEntry<'a> {
filter: WIPOffset<fb::NetworkFilter<'a>>,
id: Hash,
Expand All @@ -53,6 +86,7 @@ impl Default for NetworkFilterDebugData {
struct NetworkFilterListBuilder<'a, 'f> {
filter_map_builder: FlatMultiMapBuilder<ShortHash, NetworkFilterFlatEntry<'a>>,
opt_domains_map_builder: FlatMultiMapBuilder<ShortHash, NetworkFilterFlatEntry<'a>>,
opt_to_domains_map_builder: FlatMultiMapBuilder<ShortHash, NetworkFilterFlatEntry<'a>>,
token_frequencies: TokenSelector,
filters_to_optimize: HashMap<ShortHash, Vec<NetworkFilter<'f>>>,
tokens_buffer: TokensBuffer,
Expand Down Expand Up @@ -173,6 +207,7 @@ impl<'a, 'f> NetworkFilterListBuilder<'a, 'f> {
Self {
filter_map_builder: FlatMultiMapBuilder::with_capacity(1024),
opt_domains_map_builder: FlatMultiMapBuilder::with_capacity(256),
opt_to_domains_map_builder: FlatMultiMapBuilder::with_capacity(256),
token_frequencies: TokenSelector::new(1024),
filters_to_optimize: HashMap::new(),
tokens_buffer: TokensBuffer::default(),
Expand Down Expand Up @@ -216,6 +251,12 @@ impl<'a, 'f> NetworkFilterListBuilder<'a, 'f> {
.insert(to_short_hash(*token), NetworkFilterFlatEntry { filter, id });
}
}
FilterTokens::OptToDomains => {
for token in &self.tokens_buffer {
self.opt_to_domains_map_builder
.insert(to_short_hash(*token), NetworkFilterFlatEntry { filter, id });
}
}
}
} else {
// Defer serialization to the optimizer (pattern map only).
Expand Down Expand Up @@ -257,12 +298,6 @@ impl<'a, 'f> NetworkRulesBuilder<'a, 'f> {
return;
}

// For now, filters with $to options are parsed but ignored
// to preserve existing matching behavior.
if filter.has_to_option() {
return;
}

// Redirects are independent of blocking behavior.
if filter.is_redirect() {
self.add_filter_internal(
Expand Down Expand Up @@ -354,11 +389,16 @@ impl<'a, 'f> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder<'a
rule_list
.opt_domains_map_builder
.retain_by_value(|entry| !value.bad_filter_ids.contains(&entry.id));
rule_list
.opt_to_domains_map_builder
.retain_by_value(|entry| !value.bad_filter_ids.contains(&entry.id));

let flat_filter_map =
FlatMultiMapBuilder::finish(rule_list.filter_map_builder, builder);
let flat_opt_domains_map =
FlatMultiMapBuilder::finish(rule_list.opt_domains_map_builder, builder);
let flat_opt_to_domains_map =
FlatMultiMapBuilder::finish(rule_list.opt_to_domains_map_builder, builder);

serialized_lists.push(fb::NetworkFilterList::create(
builder.raw_builder(),
Expand All @@ -367,6 +407,8 @@ impl<'a, 'f> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder<'a
filter_map_values: Some(flat_filter_map.values),
opt_domains_map_index: Some(flat_opt_domains_map.keys),
opt_domains_map_values: Some(flat_opt_domains_map.values),
opt_to_domains_map_index: Some(flat_opt_to_domains_map.keys),
opt_to_domains_map_values: Some(flat_opt_to_domains_map.values),
},
));
}
Expand Down
35 changes: 35 additions & 0 deletions src/filters/network.rs
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ pub enum FilterPart<'a> {
pub(crate) enum FilterTokens {
Empty,
OptDomains,
OptToDomains,
Other,
}

Expand Down Expand Up @@ -974,6 +975,8 @@ impl<'a> NetworkFilter<'a> {
self.hostname.as_deref(),
self.opt_domains.as_ref(),
self.opt_not_domains.as_ref(),
self.opt_to_domains.as_ref(),
self.opt_to_not_domains.as_ref(),
)
}

Expand All @@ -992,6 +995,15 @@ impl<'a> NetworkFilter<'a> {
return FilterTokens::OptDomains;
}

// A single positive `$to=` is the next most selective key.
if self.opt_to_not_domains.is_none()
&& let Some(domains) = self.opt_to_domains.as_ref()
&& let [domain] = domains.as_slice()
{
tokens_buffer.push(*domain);
return FilterTokens::OptToDomains;
}

// Get tokens from filter
match &self.filter {
FilterPart::Simple(f) if !self.is_complete_regex() => {
Expand Down Expand Up @@ -1046,6 +1058,14 @@ impl<'a> NetworkFilter<'a> {
}
// Too many domains to bucket individually; fall back to the catch-all
// bucket (token 0).
} else if let Some(opt_to_domains) = self.opt_to_domains.as_ref()
&& !opt_to_domains.is_empty()
{
let cap = tokens_buffer.remaining_capacity();
if opt_to_domains.len() <= cap {
tokens_buffer.extend(opt_to_domains.iter().copied());
return FilterTokens::OptToDomains;
}
}
FilterTokens::Empty
} else {
Expand Down Expand Up @@ -1138,6 +1158,7 @@ fn write_str_to_hasher(hasher: &mut impl Hasher, s: &str) {
hasher.write(s.as_bytes());
}

#[allow(clippy::too_many_arguments)]
fn compute_filter_id(
modifier_option: Option<&str>,
mask: NetworkFilterMask,
Expand All @@ -1146,6 +1167,8 @@ fn compute_filter_id(
hostname: Option<&str>,
opt_domains: Option<&Vec<Hash>>,
opt_not_domains: Option<&Vec<Hash>>,
opt_to_domains: Option<&Vec<Hash>>,
opt_to_not_domains: Option<&Vec<Hash>>,
) -> Hash {
let mut hasher = FxHasher::default();

Expand All @@ -1171,6 +1194,18 @@ fn compute_filter_id(
}
}

if let Some(domains) = opt_to_domains {
for d in domains {
hasher.write_u64(*d);
}
}

if let Some(domains) = opt_to_not_domains {
for d in domains {
hasher.write_u64(*d);
}
}

match filter {
FilterPart::Empty => {}
FilterPart::Simple(s) => write_str_to_hasher(&mut hasher, s.as_ref()),
Expand Down
20 changes: 10 additions & 10 deletions src/filters/network_matchers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -421,21 +421,21 @@ pub fn check_options(mask: NetworkFilterMask, request: &request::Request) -> boo
#[inline]
pub fn check_included_domains_mapped(
opt_domains: Option<&[u32]>,
request: &request::Request,
hostname_hashes: Option<&[Hash]>,
mapping: &HashMap<Hash, u32>,
) -> bool {
// Source URL must be among these domains to match
// Hostname must be among these domains to match
if let Some(included_domains) = opt_domains.as_ref() {
if let Some(source_hashes) = request.source_hostname_hashes.as_ref() {
if source_hashes.iter().all(|h| {
if let Some(hashes) = hostname_hashes {
if hashes.iter().all(|h| {
mapping
.get(h)
.is_none_or(|index| !utils::bin_lookup(included_domains, *index))
}) {
return false;
}
} else {
// If there are domain restrictions but no source hostname, we can't apply the rule
// If there are domain restrictions but no hostname, we can't apply the rule
return false;
}
}
Expand All @@ -445,21 +445,21 @@ pub fn check_included_domains_mapped(
#[inline]
pub fn check_excluded_domains_mapped(
opt_not_domains: Option<&[u32]>,
request: &request::Request,
hostname_hashes: Option<&[Hash]>,
mapping: &HashMap<Hash, u32>,
) -> bool {
if let Some(excluded_domains) = opt_not_domains.as_ref() {
if let Some(source_hashes) = request.source_hostname_hashes.as_ref() {
if source_hashes.iter().any(|h| {
if let Some(hashes) = hostname_hashes {
if hashes.iter().any(|h| {
mapping
.get(h)
.is_some_and(|index| utils::bin_lookup(excluded_domains, *index))
}) {
return false;
}
} else {
// If there are domain restrictions but no source hostname
// (i.e. about:blank), apply the rule anyway.
// If there are domain restrictions but no hostname
// (i.e. about:blank for source), apply the rule anyway.
return true;
}
}
Expand Down
4 changes: 4 additions & 0 deletions src/flatbuffers/fb_network_filter.fbs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ table NetworkFilterList {
/// `$domain=` / `$from=` token buckets.
opt_domains_map_index: [uint32] (required);
opt_domains_map_values: [NetworkFilter] (required);

/// `$to=` token buckets.
opt_to_domains_map_index: [uint32] (required);
opt_to_domains_map_values: [NetworkFilter] (required);
}

/// A table to store the most host-specific cosmetic rules.
Expand Down
Loading
Loading