From 29594ddaff1103564c90d50467ffa5f384a88919 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:01:45 +0000 Subject: [PATCH 01/22] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Sentinel:=20[MEDI?= =?UTF-8?q?UM]=20.html4ignore=20=ED=8C=8C=EC=9D=BC=20TOCTOU=20=EC=98=88?= =?UTF-8?q?=EC=99=B8=20=EC=B2=98=EB=A6=AC=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .jules/sentinel.md | 5 +++++ src/main/kotlin/html4tree/main.kt | 22 +++++++++++++--------- src/test/kotlin/html4tree/MainTest.kt | 23 +++++++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/.jules/sentinel.md b/.jules/sentinel.md index a885865d..437b9375 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -99,3 +99,8 @@ **Root cause:** The protected implementation added canonical names to the exclusion set but did not compare each observed directory entry through a locale-stable normalized key. **Prevention:** Build one `Locale.ROOT` lowercase set from the canonical sensitive names, compare every observed name against it, and add the original spelling to the exclusion set so downstream exact membership remains correct. **Evidence:** `testProcessIgnoreFileTreatsSensitiveNamesCaseInsensitively` failed on test-only commit `472b916cd40f70693c4e1eb48956042a25353feb` (CI run `31469596932`) and passed with the source fix at `bb113d858ccfc42ddaecf6729749b238e5ade2d0` (CI run `31469921661`). + +## 2024-09-04 - [html4tree] TOCTOU (Time-of-Check to Time-of-Use) 취약점 - .html4ignore 파일 읽기 +**Vulnerability:** `.html4ignore` 파일을 처리할 때 `ignore_file.isFile`, `canRead()` 등을 확인한 후 실제로 `useLines`로 파일을 여는 사이에(Time-of-Check to Time-of-Use) 파일이 삭제되거나 권한이 변경되면 `java.io.FileNotFoundException` 등 `IOException`이 발생하여 크롤링이 중단(DoS)되는 취약점. +**Learning:** 파일의 권한이나 존재 여부를 사전에 확인하더라도, 멀티스레드 환경이나 외부 요인으로 인해 파일을 실제로 여는 시점에는 상태가 변할 수 있음. 검사와 사용 사이의 시간차로 인한 예외를 적절히 처리하지 않으면 예상치 못한 서비스 중단(Fail Securely 위반)이 발생함. +**Prevention:** 파일 읽기 작업(`useLines` 등)을 `try-catch` 블록으로 감싸서 TOCTOU 문제로 인해 발생할 수 있는 `IOException`을 안전하게 처리(catch)하여 전체 프로세스가 충돌하지 않도록 방어해야 함. diff --git a/src/main/kotlin/html4tree/main.kt b/src/main/kotlin/html4tree/main.kt index 0972fa2c..fa50d4db 100644 --- a/src/main/kotlin/html4tree/main.kt +++ b/src/main/kotlin/html4tree/main.kt @@ -309,18 +309,22 @@ fun process_ignore_file(curr_dir: File, dirFilesNames: Array? = null): S if(ignore_file.isFile && !Files.isSymbolicLink(ignore_file.toPath()) && ignore_file.canRead() && ignore_file.length() <= 1048576){ val ignored_matchers = mutableListOf() - ignore_file.useLines { lines -> - for ((lineIndex, it) in lines.withIndex()) { - // 줄 수 제한이 패턴 수도 함께 상한(줄당 최대 1개 패턴)하므로 별도 패턴 카운터는 불필요 - if (lineIndex >= 1000) break - val pattern = it.trim() - if (pattern.isNotEmpty() && pattern.length <= 100) { - try { - ignored_matchers.add(java.nio.file.FileSystems.getDefault().getPathMatcher("glob:$pattern")) - } catch (_: IllegalArgumentException) { + try { + ignore_file.useLines { lines -> + for ((lineIndex, it) in lines.withIndex()) { + // 줄 수 제한이 패턴 수도 함께 상한(줄당 최대 1개 패턴)하므로 별도 패턴 카운터는 불필요 + if (lineIndex >= 1000) break + val pattern = it.trim() + if (pattern.isNotEmpty() && pattern.length <= 100) { + try { + ignored_matchers.add(java.nio.file.FileSystems.getDefault().getPathMatcher("glob:$pattern")) + } catch (_: IllegalArgumentException) { + } } } } + } catch (_: java.io.IOException) { + // Catch TOCTOU exceptions such as FileNotFoundException caused by race conditions. } // ⚡ Bolt Performance Optimization: 디렉토리 목록을 Set에 추가하기 위해 필터링만 할 때는 정렬이 불필요하므로 .sorted()를 제거하여 O(N log N) 오버헤드를 방지합니다. diff --git a/src/test/kotlin/html4tree/MainTest.kt b/src/test/kotlin/html4tree/MainTest.kt index 5b76cc5d..3e3ad979 100644 --- a/src/test/kotlin/html4tree/MainTest.kt +++ b/src/test/kotlin/html4tree/MainTest.kt @@ -12,6 +12,7 @@ import java.nio.file.attribute.BasicFileAttributes import java.nio.file.attribute.FileTime import kotlin.test.assertEquals import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull import kotlin.test.assertFalse import kotlin.test.assertNull import kotlin.test.assertTrue @@ -946,4 +947,26 @@ class MainTest { assertTrue(content.contains("

Root

")) } + @Test + fun testProcessIgnoreFileToctouExceptionHandling() { + val ignoreFile = File(tempDir, ".html4ignore") + ignoreFile.createNewFile() + + var t = kotlin.concurrent.thread { + while (!Thread.currentThread().isInterrupted) { + ignoreFile.setReadable(true) + ignoreFile.setReadable(false) + ignoreFile.delete() + ignoreFile.createNewFile() + } + } + + for (i in 1..100) { + val excluded = process_ignore_file(tempDir, null) + assertNotNull(excluded) + } + + t.interrupt() + t.join(1000) + } } From 470063a3da08bbaa1b9ed128ffd50d789b8ffe62 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Sat, 5 Sep 2026 02:42:46 +0000 Subject: [PATCH 02/22] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Sentinel:=20[MEDI?= =?UTF-8?q?UM]=20.html4ignore=20=ED=8C=8C=EC=9D=BC=20TOCTOU=20=EC=98=88?= =?UTF-8?q?=EC=99=B8=20=EC=B2=98=EB=A6=AC=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From b841fc3ad82befc17a57afda01258ee74310a1e4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 12:31:49 +0900 Subject: [PATCH 03/22] fix(ignore): fail closed on admitted policy read errors Replace the scheduler-dependent TOCTOU test with deterministic injected-I/O coverage. Propagate admitted .html4ignore read failures as a typed exception and have directory traversal suppress publication and child traversal for that directory. Restore repository-wide Sentinel doctrine to protected-base content and record the behavior in the changelog. --- .jules/sentinel.md | 5 -- CHANGELOG.md | 6 ++ src/main/kotlin/html4tree/main.kt | 26 +++++-- .../html4tree/IgnoreFileReadFailureTest.kt | 71 +++++++++++++++++++ src/test/kotlin/html4tree/MainTest.kt | 23 ------ 5 files changed, 98 insertions(+), 33 deletions(-) create mode 100644 src/test/kotlin/html4tree/IgnoreFileReadFailureTest.kt diff --git a/.jules/sentinel.md b/.jules/sentinel.md index 437b9375..a885865d 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -99,8 +99,3 @@ **Root cause:** The protected implementation added canonical names to the exclusion set but did not compare each observed directory entry through a locale-stable normalized key. **Prevention:** Build one `Locale.ROOT` lowercase set from the canonical sensitive names, compare every observed name against it, and add the original spelling to the exclusion set so downstream exact membership remains correct. **Evidence:** `testProcessIgnoreFileTreatsSensitiveNamesCaseInsensitively` failed on test-only commit `472b916cd40f70693c4e1eb48956042a25353feb` (CI run `31469596932`) and passed with the source fix at `bb113d858ccfc42ddaecf6729749b238e5ade2d0` (CI run `31469921661`). - -## 2024-09-04 - [html4tree] TOCTOU (Time-of-Check to Time-of-Use) 취약점 - .html4ignore 파일 읽기 -**Vulnerability:** `.html4ignore` 파일을 처리할 때 `ignore_file.isFile`, `canRead()` 등을 확인한 후 실제로 `useLines`로 파일을 여는 사이에(Time-of-Check to Time-of-Use) 파일이 삭제되거나 권한이 변경되면 `java.io.FileNotFoundException` 등 `IOException`이 발생하여 크롤링이 중단(DoS)되는 취약점. -**Learning:** 파일의 권한이나 존재 여부를 사전에 확인하더라도, 멀티스레드 환경이나 외부 요인으로 인해 파일을 실제로 여는 시점에는 상태가 변할 수 있음. 검사와 사용 사이의 시간차로 인한 예외를 적절히 처리하지 않으면 예상치 못한 서비스 중단(Fail Securely 위반)이 발생함. -**Prevention:** 파일 읽기 작업(`useLines` 등)을 `try-catch` 블록으로 감싸서 TOCTOU 문제로 인해 발생할 수 있는 `IOException`을 안전하게 처리(catch)하여 전체 프로세스가 충돌하지 않도록 방어해야 함. diff --git a/CHANGELOG.md b/CHANGELOG.md index c442b3b1..e7a0b5f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,12 +19,18 @@ All notable changes to this project are documented in this file. ### Fixed +- Fail closed for a directory when an admitted `.html4ignore` cannot be read, + instead of publishing entries with an empty custom-ignore policy after a + read-time filesystem race. - Generate the inline-style Content Security Policy SHA-256 source expression from the exact normalized UTF-8 stylesheet bytes emitted into each generated `index.html` file, preventing template whitespace from invalidating the policy. ### Tests +- Replace the scheduler- and permission-dependent `.html4ignore` race test with + deterministic injected-I/O regressions for typed failure propagation, + directory-publication suppression, and the normal readable glob path. - Add a real generated-file regression test that independently recomputes the declared style hash from the emitted `