Eviction

.clang-format

@@ -0,0 +1,10 @@
+BasedOnStyle: LLVM
+IndentWidth: 2
+TabWidth: 2
+UseTab: Never
+PointerAlignment: Left
+DerivePointerAlignment: false
+SortIncludes: false
+ReflowComments: false
+AlignConsecutiveMacros: true
+ColumnLimit: 0

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.vscode/settings.json

@@ -0,0 +1,14 @@
+{
+    "[c]": {
+        "editor.defaultFormatter": "ms-vscode.cpptools",
+        "editor.tabSize": 2,
+        "editor.insertSpaces": true
+    },
+    "[cpp]": {
+        "editor.defaultFormatter": "ms-vscode.cpptools",
+        "editor.tabSize": 2,
+        "editor.insertSpaces": true
+    },
+    "C_Cpp.clang_format_style": "file",
+    "C_Cpp.clang_format_fallbackStyle": "{ BasedOnStyle: LLVM, IndentWidth: 2, UseTab: Never, PointerAlignment: Left }"
+}

Makefile

@@ -16,7 +16,7 @@ SD_DMA_TICKS ?= 1 # number of emulator ticks per 4-byte SD DMA transfer
 
 # memory map
 TEXT_LOAD_ADDR := 0x10000
-DATA_LOAD_ADDR := 0x80000
+DATA_LOAD_ADDR := 0x90000
 RODATA_LOAD_ADDR := 0xD0000
 BSS_LOAD_ADDR := 0xE0000
 
@@ -502,6 +502,21 @@ define assemble_kernel_image
 	  echo "Kernel build error: section bases are not ordered text->rodata->data->bss->end." >&2; \
 	  exit 1; \
 	fi; \
+	text_zone_size=$$(( $(DATA_LOAD_ADDR) - $(TEXT_LOAD_ADDR) )); \
+	data_zone_size=$$(( $(RODATA_LOAD_ADDR) - $(DATA_LOAD_ADDR) )); \
+	rodata_zone_size=$$(( $(BSS_LOAD_ADDR) - $(RODATA_LOAD_ADDR) )); \
+	if [ $$((rodata_base - text_base)) -gt $$text_zone_size ]; then \
+	  echo "Kernel build error: text section exceeds its reserved zone $(TEXT_LOAD_ADDR)-$(DATA_LOAD_ADDR)." >&2; \
+	  exit 1; \
+	fi; \
+	if [ $$((data_base - rodata_base)) -gt $$rodata_zone_size ]; then \
+	  echo "Kernel build error: rodata section exceeds its reserved zone $(RODATA_LOAD_ADDR)-$(BSS_LOAD_ADDR)." >&2; \
+	  exit 1; \
+	fi; \
+	if [ $$((bss_base - data_base)) -gt $$data_zone_size ]; then \
+	  echo "Kernel build error: data section exceeds its reserved zone $(DATA_LOAD_ADDR)-$(RODATA_LOAD_ADDR)." >&2; \
+	  exit 1; \
+	fi; \
 	text_start_block=$$((text_base / $(KERNEL_BLOCK_SIZE))); \
 	rodata_start_block=$$((rodata_base / $(KERNEL_BLOCK_SIZE))); \
 	data_start_block=$$((data_base / $(KERNEL_BLOCK_SIZE))); \

docs/filesystem.md

@@ -62,7 +62,7 @@ The filesystem uses several lock layers:
 - rwx permission enforcement
 - uid / gid
 - atime / mtime / ctime updates
-- VFS layer or page cache
+- VFS layer
 
 ### Tests
 - `ext_read.c`

docs/kernel.md

@@ -53,7 +53,7 @@ See `filesystem.md` for more details.
 ## Virtual Memory
 Uses a 2-level table, similar to x86
 
-TLB is software managed, so any miss invoked the tlb handler
+TLB is software managed, so any miss invokes the tlb handler
 
 VMEM currently supports:
 - private anonymous

docs/kernel_mem_map.md

@@ -6,11 +6,11 @@ Interrupt Vector Table
 ### 0x400 - ...
 Where BIOS code is loaded (32KiB reserved). Can overwrite once kernel is entered. 
 
-### 0x10000 - 0x80000
-Kernel text (448KiB reserved for now)
+### 0x10000 - 0x90000
+Kernel text (512KiB reserved for now)
 
-### 0x80000 - 0xD0000
-Kernel data (320KiB)
+### 0x90000 - 0xD0000
+Kernel data (256KiB)
 
 ### 0xD0000 - 0xE0000
 Kernel rodata (64KiB)

docs/page_cache.md

@@ -0,0 +1,61 @@
+## Page Cache
+
+The kernel page cache serves as a canonicalizing mapping for accessing files, as well as a cache for file-backed pages to provide faster access.
+
+### Structure
+
+The cache is a simple hash-table mapping from `(inode pointer, page offset)` to a cached page entry, which contains a pointer to the physical page data and the number of bytes from the file that belong to this page. The cache is protected by a single global blocking lock and uses chaining for collision handling.
+
+The cache key is the tuple of the node's cached inode pointer and the supplied page offset. The hash function is a simple XOR of those values modulo the hash table size.
+
+### Supported Page Cache Features
+
+#### Initialization
+`page_cache_init()` allocates the hash table, stores the configured bucket count, initializes the cache lock, and clears all buckets to `NULL`.
+
+#### Lookup / Acquire
+`page_cache_acquire()` returns the cached page entry, locking the backing page. 
+
+If the entry is already cached. On a hit, it returns the existing entry. On a miss, it allocates a fresh physical page, inserts a new entry into the cache, locks the backing page, reads up to one frame from the backing node with `node_read_all()`, zero-fills any remaining bytes in the frame, and returns the newly created entry.
+- On a hit, the page cache lock must be released in order to claim the page lock; this creates a window for eviction and necessitates revalidation
+- On a miss, the freshly allocated frame is pinned, and thus not evictable
+
+The caller is expected to update the page's metadata (mostly the reverse mapping) and release the page lock when done.
+
+#### Release / Writeback
+Page cache entries are released when the backing page is evicted by `page_evict()`. Dirty pages are written back first with `node_write_all()` using the entry's recorded `file_bytes` value. Clean pages are discarded without writeback.
+
+#### Eviction
+`page_evict()` can be called on the metadata of a frame to free the frame and evict the cache entry. It may only be called if the backing frame is not pinned, and if the caller holds the page lock.
+
+Page eviction uses the reverse mapping (stored in `page->refs`) to find and invalidate all virtual mappings before reclaiming the physical frame. The pathway:
+- Lock the INode (prevents re-caching of the page while eviction writeback is in-flight)
+- Remove the entry from the cache
+- For each `PageRef` in `page->refs`, invalidate the corresponding PTE and perform a TLB shootdown (see **Reverse Mapping** in vmem.md)
+- If the page is dirty (checked via `page->flags`), write it back to the backing node with `node_write_all()` using the entry's recorded `file_bytes` value
+- Free the backing frame via `physmem_free()`
+
+Eviction is currently invoked only by explicit caller requests; there is no automatic reclamation policy. // TODO: add an eviction policy
+
+### Data Stored Per Entry
+
+- cached inode pointer and page offset key
+- pointer to the physical page data
+- `file_bytes`, the number of bytes from the file that belong to this page
+- next pointer for hash-chain collision handling
+
+### Locking
+
+- `page_cache_init()` initializes a single blocking lock for the whole cache
+- `page_cache_acquire()` acquires that lock while it inspects or mutates the hash table
+- `page_cache_lookup()` and `page_cache_insert()` do not lock on their own and are only safe to call while the cache lock is held
+
+#### Lock Ordering
+Locks must be acquired in order of decreasing granularity (i.e. you cannot hold a coarser lock while contesting for a finer lock). Specifically, locks must be acquired in the following order:
+- Page lock
+- INode lock
+- Global page cache lock
+
+### Not Yet Supported
+- Page replacement policy
+- Error handling for allocation or I/O failure

docs/physmem.md

@@ -25,6 +25,37 @@ one single top-level buddy tree. Instead, `physmem_init()` decomposes the arena
 into the largest aligned power-of-two blocks that fit and seeds one free list
 per order with that forest of top-level blocks.
 
+### Page Metadata
+
+Metadata for all physical frames is stored as a flat array in `physmem_map`. A frame's metadata is stored in a `struct Page`, retrievable via `get_page(frame_address)`. This metadata tracks the frame's current
+use across multiple subsystems:
+
+- `flags`: State bits for the page, drawn from `enum PageFlags`
+- `ref_cnt`: Reference count equal to the length of the `refs` list; indicates how
+  many virtual mappings currently point to this frame
+- `refs`: Head of a linked list of `struct PageRef` entries, each representing one
+  virtual mapping of this page (see **Reverse Mapping** in vmem.md)
+- `cache_entry`: Pointer to the page-cache entry if this frame is file-backed (see
+  **Data Stored Per Entry** in page_cache.md); `NULL` for anonymous pages
+- `lock`: Semaphore protecting this frame's metadata during modifications
+
+The metadata enables page reclamation.
+
+Currently defined `PageFlags` are:
+- `PG_DIRTY`: the page has been written to and needs to be written back before eviction
+- `PG_PINNED`: the page is pinned in memory and cannot be evicted until unpinned
+  - Note that only one thread may pin a page at a time, and the pinning thread is responsible for unpinning. 
+  - A page may not be reclaimed while it is pinned
+  - Currently, a page is pinned if it is part of the physical memory map, if it is free and thus being managed by the physical memory allocator, if it is backing a private mapping, or if it is being written to from disk (during `page_cache_acquire()`)
+    - Pinning during `page_cache_acquire()` signals that the page data is not stable yet
+- `PG_ACCESSED`: software managed accessed bit; not currently supported
+
+#### Concurrency / Invariants
+- A frame must be pinned as it is being freed; a frame will be pinned when it is allocated by the physmem allocator
+- A frame must not have any remaining references when it is freed
+- A frame's lock must be held during any modifications to its metadata. Flags may be examined when the lock is not held, but are not guaranteed to be stable
+- A frame's lock may still have waiters when it is freed, but they must release the lock in O(1) AND they must not modify the metadata. It is the responsibility of the waiter to detect that the page has been freed and fulfill this contract
+
 ### Supported Physmem Features
 
 #### Initialization

docs/vmem.md

@@ -84,6 +84,41 @@ the matching TLB value in `rA`.
 `vmem_core_init()` flushes the local core's TLB and clears the active PID to 0
 at boot.
 
+`tlb_shootdown()` performs cross-core TLB shootdown on a (pid, virtual address) pair, blocking until all cores have performed the shootdown.
+
+### Reverse Mapping
+
+The kernel maintains a reverse mapping from physical pages to their virtual mappings
+to support efficient invalidation during page eviction. This is currently only supported for file-backed pages in the page cache, but the same mechanism will be extended to anonymous pages once the kernel supports eviction for those as well.
+
+#### PageRef Structure
+
+Each virtual mapping of a physical page is represented by a `struct PageRef` entry
+containing:
+
+- `pid`: The page-directory ID (physical address of the page directory) of the mapping
+- `virtual_address`: The virtual address of the mapping
+- `next`: Pointer to the next `PageRef` in the list
+
+#### Reverse Mapping List
+
+When a thread installs a PTE pointing to a physical frame, it creates a new `PageRef`
+and inserts it at the head of `page->refs` (the linked list in the frame's `struct Page`
+metadata). The `page->ref_cnt` field maintains a count of these entries.
+
+When a thread later unmaps the page (via `munmap()` or during address-space teardown),
+it removes the corresponding `PageRef` from the list and decrements `page->ref_cnt`.
+
+#### Eviction Invalidation
+
+During page eviction (see **Eviction** in page_cache.md), the page-cache code walks
+the `page->refs` list to find all virtual mappings and invalidates them:
+
+- For each `PageRef`, construct a TLB shootdown request containing the `pid` and `virtual_address`
+- Submit the requests to other cores for TLB shootdown (via `tlb_shootdown()`)
+- Initiate the shootdown (via `send_ipi()`) and wait for acknowledgments from all cores
+- Once all cores have flushed the TLB entries, the physical page is safe to free
+
 ### Supported VM Features
 
 #### Global Initialization
@@ -234,15 +269,20 @@ Current behavior:
 The ISA provides TLB-miss vector at `0x82` / `0x208`, and the kernel
 registers it to `tlb_miss_handler()`.
 
-For a tlb miss, it:
-
-- finds the containing VME in the current thread's `vme_list`
-- allocates a page table if the enclosing PDE is still invalid
-- allocates or acquires the required backing page depending on the VME type
-- installs a PTE with the requested permissions
-- writes the resolved translation into the TLB
-
+For a tlb miss, it finds the PTE (allocates a page table if the enclosing PDE is invalid). If the PTE is "sufficient" to handle the fault (i.e. it maps the address and has the requested permissions), the translation is written to the TLB.
+
+Otherwise, the `page_fault_handler()` is called:
+- Finds the containing VME in the current thread's `vme_list`
+- If the page is mapped but there was a permission fault:
+  - Panics if the VME does not permisison for the attempted operation
+  - Otherwise (no write permission for VME allowing writes), updates the dirty bit and adds requested permission to the PTE
+    - This operation revalidates the PTE once the page lock is acquired (necessary to ensure eviction did not occur); if revalidation fails, falls thrugh to the "not mapped" case of the page fault handler
+  - Other permission faults may exist in the future but are not currently expected/supported
+- If the page is not mapped, allocates or acquires the required backing page depending on the VME type
+  - If the page is mapped file-backed and shared, it is mapped with read-only permissions so the dirty bit can be set on first write
+- Writes the resolved translation into the TLB
 If no containing VME exists, the kernel panics.
+The page fault handler is responsible for updating the reverse mapping of the backing page. So far this is only a concern for file-backed & shared mappings.
 
 ### Address-Space Teardown
 
@@ -269,10 +309,15 @@ Current VM code assumes:
 - one address space is active on only one core at a time
 
 That last point matters because `munmap()` invalidates TLB entries only on the
-current core. There is no cross-core TLB shootdown mechanism yet.
+current core. // TODO cross-core TLB shootdown exists, we should use it in munmap
 
-Shared file-backed page sharing is implemented with the global page cache, which
-has its own lock and reference counts.
+Shared file-backed page sharing is implemented with the global page cache.
+
+Current VM code assumes:
+- a thread's PTEs may be modified by another thread only during address space teardown or page reclaimation
+- these modifications will take the form of invalidation, fully zeroing the PTE
+When accessing metadata of a frame through the PTE, the PTE must be revalidated after the page lock is acquired to ensure that eviction hs not occurred.
+- Once the page lock is acquired, the PTE will be stable
 
 ### Current Limitations
 

kernel/countdown_latch.c

@@ -0,0 +1,21 @@
+#include "countdown_latch.h"
+#include "atomic.h"
+#include "threads.h"
+
+void countdownlatch_init(struct CountDownLatch* latch, unsigned count) {
+  latch->count = count;
+}
+
+void countdownlatch_sync(struct CountDownLatch* latch) {
+  while (latch->count > 0) {
+    yield();
+  }
+}
+
+void countdownlatch_down(struct CountDownLatch* latch) {
+  __atomic_fetch_add((int*)&latch->count, -1);
+}
+
+void countdownlatch_up(struct CountDownLatch* latch) {
+  __atomic_fetch_add((int*)&latch->count, 1);
+}

kernel/countdown_latch.h

@@ -0,0 +1,15 @@
+struct CountDownLatch {
+  unsigned count;
+};
+
+// initialize the barrier with the given count of threads
+void countdownlatch_init(struct CountDownLatch* latch, unsigned count);
+
+// block until the count reaches 0
+void countdownlatch_sync(struct CountDownLatch* latch);
+
+// Non blocking; increment or decrement the count. If the count reaches 0, all waiting threads will be woken.
+void countdownlatch_down(struct CountDownLatch* latch);
+void countdownlatch_up(struct CountDownLatch* latch);
+
+// Destroying the latch while threads are still waiting on it causes undefined behavior

kernel/ext.c

@@ -2414,19 +2414,14 @@ unsigned node_read_all(struct Node* node, unsigned offset, unsigned size, char*
   return cnt;
 }
 
-unsigned node_write_all(struct Node* node, unsigned offset, unsigned size, char* src){
+unsigned node_write_all_locked(struct Node* node, unsigned offset, unsigned size, char* src){
   if (size == 0) return 0;
 
   unsigned block_size = ext2_get_block_size(node->filesystem);
   unsigned start_block = offset / block_size;
   unsigned end_block = (offset + size - 1) / block_size;
   unsigned bytes_copied = 0;
 
-  // Serialize the full write path for one inode so block growth, inode writeback,
-  // and data writes observe one consistent per-file state without re-entering
-  // inode_lock through icache_set().
-  blocking_lock_acquire(&node->cached->lock);
-
   assert(node_is_file(node) || node_is_symlink(node), "node_write_all: can only write to regular files or symlinks.\n");
 
   // Host-built ext2 images may encode a trailing run of all-zero file blocks as
@@ -2451,11 +2446,20 @@ unsigned node_write_all(struct Node* node, unsigned offset, unsigned size, char*
 
     bytes_copied += copy_size;
   }
-  blocking_lock_release(&node->cached->lock);
 
   return size;
 }
 
+unsigned node_write_all(struct Node* node, unsigned offset, unsigned size, char* src){
+  // Serialize the full write path for one inode so block growth, inode writeback,
+  // and data writes observe one consistent per-file state without re-entering
+  // inode_lock through icache_set().
+  blocking_lock_acquire(&node->cached->lock);
+  size = node_write_all_locked(node, offset, size, src);
+  blocking_lock_release(&node->cached->lock);
+  return size;
+}
+
 bool node_shrink(struct Node* node, unsigned target_size){
   assert(node != NULL, "node_shrink: node is NULL.\n");
   assert(node_is_file(node), "node_shrink: can only shrink regular files.\n");

kernel/ext.h

@@ -197,6 +197,9 @@ void node_write_block(struct Node* node, unsigned block_num, char* src, unsigned
 // requested write size on success.
 unsigned node_write_all(struct Node* node, unsigned offset, unsigned size, char* src);
 
+// Like node_write_all but the caller must already hold the inode lock
+unsigned node_write_all_locked(struct Node* node, unsigned offset, unsigned size, char* src);
+
 // Shrinks a regular file to `target_size` bytes and writes the smaller inode
 // size back to disk. does not reclaim any blocks or clear truncated bytes.
 bool node_shrink(struct Node* node, unsigned target_size);