Skip to content

Words synchronization #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
WieszczY85 merged 3 commits into from
Mar 27, 2026
Merged

Words synchronization #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
285e476
Add optional remote banned-words synchronization
WieszczY85 Mar 27, 2026
5f83a4e
Massively expand default banned words with multilingual entries
WieszczY85 Mar 27, 2026
ff82e2d
Avoid YAML parse errors for plain remote word lists
WieszczY85 Mar 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions CleanerX-Paper/src/main/kotlin/pl/syntaxdevteam/cleanerx/base/WordFilter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ class WordFilter(private val plugin: CleanerX) {
whitelistWords = config.getStringList("whitelistWords")
.map { it.lowercase(Locale.ROOT) }
.toMutableList()
whitelistPatterns = whitelistWords.mapNotNull { toFilterPattern(it) }
whitelistPatterns = whitelistWords.mapNotNull { toFilterPattern(it, allowSuffix = false) }
}

/**
Expand Down Expand Up @@ -149,7 +149,7 @@ class WordFilter(private val plugin: CleanerX) {
val lowerWord = word.lowercase(Locale.ROOT)
if (!bannedWords.contains(lowerWord)) {
bannedWords.add(lowerWord)
bannedPatterns = bannedWords.mapNotNull { toFilterPattern(it) }
bannedPatterns = bannedWords.mapNotNull { toFilterPattern(it, allowSuffix = true) }
saveBannedWords()
plugin.logger.info("Word $word added to banned words list.")
}
Expand Down Expand Up @@ -182,7 +182,7 @@ class WordFilter(private val plugin: CleanerX) {
val lowerWord = word.lowercase(Locale.ROOT)
if (!whitelistWords.contains(lowerWord)) {
whitelistWords.add(lowerWord)
whitelistPatterns = whitelistWords.mapNotNull { toFilterPattern(it) }
whitelistPatterns = whitelistWords.mapNotNull { toFilterPattern(it, allowSuffix = false) }
saveWhitelistWords()
plugin.logger.info("Word $word added to whitelist.")
}
Expand Down Expand Up @@ -224,7 +224,7 @@ class WordFilter(private val plugin: CleanerX) {
bannedWords = words
.map { it.lowercase(Locale.ROOT) }
.toMutableList()
bannedPatterns = bannedWords.mapNotNull { toFilterPattern(it) }
bannedPatterns = bannedWords.mapNotNull { toFilterPattern(it, allowSuffix = true) }
}

/**
Expand Down Expand Up @@ -257,16 +257,16 @@ class WordFilter(private val plugin: CleanerX) {
}


private fun toFilterPattern(word: String): FilterPattern? {
private fun toFilterPattern(word: String, allowSuffix: Boolean): FilterPattern? {
val normalized = normalizeWord(word)
if (normalized.isEmpty()) {
return null
}

return FilterPattern(word, normalized, buildFuzzyRegex(normalized))
return FilterPattern(word, normalized, buildFuzzyRegex(normalized, allowSuffix))
}

private fun buildFuzzyRegex(normalizedWord: String): Regex {
private fun buildFuzzyRegex(normalizedWord: String, allowSuffix: Boolean): Regex {
val builder = StringBuilder()
builder.append("(?<![a-z])")
normalizedWord.forEachIndexed { index, character ->
Expand All @@ -275,6 +275,11 @@ class WordFilter(private val plugin: CleanerX) {
builder.append("[^a-z]*")
}
}
if (allowSuffix) {
builder.append("(?:[^a-z]*[a-z]{1,")
builder.append(MAX_SUFFIX_LENGTH)
builder.append("})?")
}
builder.append("(?![a-z])")
return Regex(builder.toString())
}
Expand Down Expand Up @@ -401,6 +406,7 @@ class WordFilter(private val plugin: CleanerX) {
}

companion object {
private const val MAX_SUFFIX_LENGTH = 3
private val NON_SPACING_MARKS_REGEX = "\\p{Mn}+".toRegex()
private val CHARACTER_REPLACEMENTS: Map<Char, String> = mapOf(
'4' to "a",
Expand Down
149 changes: 126 additions & 23 deletions CleanerX-Paper/src/main/kotlin/pl/syntaxdevteam/cleanerx/common/BannedWordsSynchronizer.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@ import pl.syntaxdevteam.cleanerx.CleanerX
import pl.syntaxdevteam.cleanerx.util.SchedulerAdapter
import java.io.File
import java.io.InputStreamReader
import java.io.StringReader
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.nio.charset.StandardCharsets
import java.time.Duration
import java.util.Locale

/**
Expand Down Expand Up @@ -33,41 +39,50 @@ class BannedWordsSynchronizer(private val plugin: CleanerX) {

val serverConfig = YamlConfiguration.loadConfiguration(serverFile)
val serverWords = serverConfig.getStringList(BANNED_WORDS_KEY).toMutableList()
val resourceWords = loadResourceWords()
val syncEnabled = plugin.config.getBoolean(REMOTE_SYNC_ENABLED_KEY, false)
val includeResourceWords = plugin.config.getBoolean(INCLUDE_RESOURCE_WORDS_KEY, true)
val timeoutMillis = plugin.config.getLong(REMOTE_SYNC_TIMEOUT_KEY, DEFAULT_REMOTE_TIMEOUT_MS).coerceAtLeast(500L)
val remoteSources = plugin.config.getStringList(REMOTE_SYNC_SOURCES_KEY).filter { it.isNotBlank() }

if (resourceWords.isEmpty()) {
plugin.logger.warning("The resource banned_words.yml file is empty or missing the bannedWords section.")
complete(onComplete, serverWords.toList())
return@runAsync
}

val normalizedExisting = serverWords
val mergedWords = serverWords.toMutableList()
val normalizedExisting = mergedWords
.map { it.lowercase(Locale.ROOT) }
.toMutableSet()

var modified = false
for (word in resourceWords) {
val normalized = word.lowercase(Locale.ROOT)
if (!normalizedExisting.contains(normalized)) {
serverWords.add(word)
normalizedExisting.add(normalized)
modified = true
}
if (includeResourceWords) {
mergeWords(mergedWords, normalizedExisting, loadResourceWords())
}

if (syncEnabled && remoteSources.isNotEmpty()) {
val remoteWords = loadRemoteWords(remoteSources, timeoutMillis)
mergeWords(mergedWords, normalizedExisting, remoteWords)
}

val modified = mergedWords.size != serverWords.size

if (modified) {
serverConfig.set(BANNED_WORDS_KEY, serverWords)
serverConfig.set(BANNED_WORDS_KEY, mergedWords)
serverConfig.save(serverFile)
plugin.logger.info("Synchronized banned_words.yml with plugin resources.")
plugin.logger.info("Synchronized banned_words.yml with configured sources.")
}

complete(onComplete, serverWords.toList())
complete(onComplete, mergedWords.toList())
} catch (exception: Exception) {
plugin.logger.err("Failed to synchronize banned words: ${exception.message}")
}
}
}

private fun mergeWords(target: MutableList<String>, normalized: MutableSet<String>, words: List<String>) {
for (word in words) {
val normalizedWord = word.lowercase(Locale.ROOT)
if (!normalized.contains(normalizedWord)) {
target.add(word)
normalized.add(normalizedWord)
}
}
}

private fun loadResourceWords(): List<String> {
val resourceStream = plugin.getResource(BANNED_WORDS_FILE)
if (resourceStream == null) {
Expand All @@ -78,9 +93,48 @@ class BannedWordsSynchronizer(private val plugin: CleanerX) {
resourceStream.use { stream ->
InputStreamReader(stream, StandardCharsets.UTF_8).use { reader ->
val config = YamlConfiguration.loadConfiguration(reader)
return config.getStringList(BANNED_WORDS_KEY)
return sanitizeWords(config.getStringList(BANNED_WORDS_KEY))
}
}
}

private fun loadRemoteWords(sources: List<String>, timeoutMillis: Long): List<String> {
val client = HttpClient.newBuilder()
.connectTimeout(Duration.ofMillis(timeoutMillis))
.build()

val words = mutableListOf<String>()
for (source in sources) {
runCatching {
val request = HttpRequest.newBuilder(URI.create(source))
.timeout(Duration.ofMillis(timeoutMillis))
.header("Accept", "text/plain, text/yaml, application/yaml, application/json")
.GET()
.build()

val response = client.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8))
if (response.statusCode() in 200..299) {
words.addAll(extractWordsFromPayload(response.body()))
} else {
plugin.logger.warning("Banned words sync skipped source $source due to HTTP ${response.statusCode()}.")
}
}.onFailure { exception ->
plugin.logger.warning("Banned words sync failed for $source: ${exception.message}")
}
}

return sanitizeWords(words)
}

private fun extractWordsFromPayload(payload: String): List<String> = parsePayloadToWords(payload)

private fun sanitizeWords(words: List<String>): List<String> {
return words.asSequence()
.map { it.trim().lowercase(Locale.ROOT) }
.filter { it.length >= MIN_WORD_LENGTH }
.filter { CANDIDATE_WORD_REGEX.matches(it) }
.distinct()
.toList()
}

private fun complete(onComplete: ((List<String>) -> Unit)?, result: List<String>) {
Expand All @@ -92,8 +146,57 @@ class BannedWordsSynchronizer(private val plugin: CleanerX) {
}
}

private companion object {
companion object {
private const val BANNED_WORDS_FILE = "banned_words.yml"
private const val BANNED_WORDS_KEY = "bannedWords"
internal const val BANNED_WORDS_KEY = "bannedWords"
private const val REMOTE_SYNC_ENABLED_KEY = "banned-words-sync.enabled"
private const val INCLUDE_RESOURCE_WORDS_KEY = "banned-words-sync.include-default-resource"
private const val REMOTE_SYNC_TIMEOUT_KEY = "banned-words-sync.timeout-ms"
private const val REMOTE_SYNC_SOURCES_KEY = "banned-words-sync.remote-sources"
private const val DEFAULT_REMOTE_TIMEOUT_MS = 2500L
private const val MIN_WORD_LENGTH = 3
private val CANDIDATE_WORD_REGEX = Regex("^[\\p{L}\\p{Nd}@!$+|]+$")

internal fun parsePayloadToWords(payload: String): List<String> {
val trimmed = payload.trim()
if (trimmed.isEmpty()) {
return emptyList()
}

val fromYaml = if (looksLikeYamlWithBannedWords(trimmed)) {
runCatching {
val config = YamlConfiguration.loadConfiguration(StringReader(trimmed))
config.getStringList(BANNED_WORDS_KEY)
}.getOrNull().orEmpty()
} else {
emptyList()
}

val candidates = if (fromYaml.isNotEmpty()) {
fromYaml
} else {
trimmed.lineSequence()
.map { line ->
line.substringBefore("#")
.substringBefore("//")
.trim()
}
.filter { it.isNotEmpty() }
.toList()
}

return candidates.asSequence()
.map { it.trim().lowercase(Locale.ROOT) }
.filter { it.length >= MIN_WORD_LENGTH }
.filter { CANDIDATE_WORD_REGEX.matches(it) }
.distinct()
.toList()
}

private fun looksLikeYamlWithBannedWords(payload: String): Boolean {
return payload.lineSequence()
.map { it.trimStart() }
.any { it.startsWith("$BANNED_WORDS_KEY:") }
}
}
}
}
Loading
Loading