diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c80582..b527ce6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,31 +2,146 @@ name: CI on: push: + branches: [master] pull_request: + schedule: + - cron: '0 6 * * 1' + workflow_dispatch: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + VFLAGS: -no-parallel + V_REF: 316db75a07afe50a17c5e273a18d10af8e8068d1 + VC_REF: d31eceeb33b7356676c7fba0a80736d370923c34 + jobs: - test: - name: Test + checks: + name: Checks (ubuntu) runs-on: ubuntu-latest - + defaults: + run: + working-directory: leveldb steps: - name: Checkout V uses: actions/checkout@v7 with: repository: vlang/v - - - name: Checkout LevelDB + ref: ${{ env.V_REF }} + - name: Checkout VC + uses: actions/checkout@v7 + with: + repository: vlang/vc + ref: ${{ env.VC_REF }} + path: vc + - name: Checkout leveldb uses: actions/checkout@v7 with: path: leveldb + - name: Build V + working-directory: . + run: | + make latest_tcc + make local=1 + echo "$GITHUB_WORKSPACE" >> "$GITHUB_PATH" + - name: Link the module into V's module path + run: make link + + - name: Verify formatting + run: make fmt-check + - name: Vet + run: make vet + - name: Run the test suite + run: make test + - name: Build and run the example + run: make example + - name: Build and run the example, optimised + run: make example-prod + + test-macos: + name: Test (macos-latest) + runs-on: macos-latest + defaults: + run: + working-directory: leveldb + steps: + - name: Checkout V + uses: actions/checkout@v7 + with: + repository: vlang/v + ref: ${{ env.V_REF }} + - name: Checkout VC + uses: actions/checkout@v7 + with: + repository: vlang/vc + ref: ${{ env.VC_REF }} + path: vc + - name: Checkout leveldb + uses: actions/checkout@v7 + with: + path: leveldb - name: Build V - run: make + working-directory: . + run: | + make latest_tcc + make local=1 + echo "$GITHUB_WORKSPACE" >> "$GITHUB_PATH" + - name: Run the test suite + run: make test - - name: Test + test-windows: + name: Test (windows) + runs-on: windows-latest + defaults: + run: + working-directory: leveldb + continue-on-error: true + steps: + - name: Checkout V + uses: actions/checkout@v7 + with: + repository: vlang/v + ref: ${{ env.V_REF }} + - name: Checkout VC + uses: actions/checkout@v7 + with: + repository: vlang/vc + ref: ${{ env.VC_REF }} + path: vc + - name: Checkout leveldb + uses: actions/checkout@v7 + with: + path: leveldb + - name: Build V + shell: pwsh + working-directory: . run: | - cd leveldb - ../v test . + .\makev.bat --local -gcc + $env:GITHUB_WORKSPACE | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: Run the test suite + shell: pwsh + run: | + v test . + $code = $LASTEXITCODE + if ($code -eq 0) { + "Windows: the suite passed." | Tee-Object -Append $env:GITHUB_STEP_SUMMARY + } else { + "Windows: the suite failed (exit $code). This job is informational while the port is unverified. The log above has the detail." | + Tee-Object -Append $env:GITHUB_STEP_SUMMARY + } + # Reported, not enforced. + exit 0 + + actions: + name: Workflow lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Lint the workflows + uses: raven-actions/actionlint@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9aa3c39 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,136 @@ +name: Release + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+-*' + +permissions: + contents: read + +# Kept in step with ci.yml: bump both pins together. +env: + VFLAGS: -no-parallel + V_REF: 316db75a07afe50a17c5e273a18d10af8e8068d1 + VC_REF: d31eceeb33b7356676c7fba0a80736d370923c34 + +jobs: + verify: + name: Verify the tag + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + prerelease: ${{ steps.version.outputs.prerelease }} + steps: + - uses: actions/checkout@v7 + + - id: version + name: Read the version from the tag + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#v}" + echo "version=$version" >> "$GITHUB_OUTPUT" + if [[ "$version" == *-* ]]; then + echo "prerelease=true" >> "$GITHUB_OUTPUT" + else + echo "prerelease=false" >> "$GITHUB_OUTPUT" + fi + + # A tag that disagrees with v.mod produces a release that installs as a + # different version than it claims to be. Catching that here is cheaper + # than yanking it afterwards. + - name: Check that v.mod agrees with the tag + run: | + set -euo pipefail + declared="$(sed -n "s/^[[:space:]]*version:[[:space:]]*'\(.*\)'.*/\1/p" v.mod)" + expected="${{ steps.version.outputs.version }}" + if [ "$declared" != "$expected" ]; then + echo "::error::v.mod declares version '$declared' but the tag is '$expected'" + exit 1 + fi + + - name: Check that the changelog has an entry for this version + run: | + set -euo pipefail + expected="${{ steps.version.outputs.version }}" + if ! grep -q "^## \[$expected\]" CHANGELOG.md; then + echo "::error::CHANGELOG.md has no '## [$expected]' section" + exit 1 + fi + + test: + name: Test (${{ matrix.os }}) + needs: verify + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: leveldb + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - name: Checkout V + uses: actions/checkout@v7 + with: + repository: vlang/v + ref: ${{ env.V_REF }} + - name: Checkout VC + uses: actions/checkout@v7 + with: + repository: vlang/vc + ref: ${{ env.VC_REF }} + path: vc + - name: Checkout leveldb + uses: actions/checkout@v7 + with: + path: leveldb + - name: Build V + working-directory: . + run: | + make latest_tcc + make local=1 + echo "$GITHUB_WORKSPACE" >> "$GITHUB_PATH" + - name: Link the module into V's module path + run: make link + - name: Verify formatting + run: make fmt-check + - name: Run the test suite + run: make test + - name: Build and run the example + run: make example-prod + + publish: + name: Publish + needs: [verify, test] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Extract the changelog section for this release + run: | + set -euo pipefail + version="${{ needs.verify.outputs.version }}" + awk -v v="$version" ' + $0 ~ "^## \\[" v "\\]" { collecting = 1; next } + collecting && /^## \[/ { exit } + collecting { print } + ' CHANGELOG.md > release-notes.md + if [ ! -s release-notes.md ]; then + echo "::error::the changelog section for $version is empty" + exit 1 + fi + cat release-notes.md + + - name: Create the release + uses: softprops/action-gh-release@v3 + with: + body_path: release-notes.md + prerelease: ${{ needs.verify.outputs.prerelease == 'true' }} + generate_release_notes: true + fail_on_unmatched_files: true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ad2e062 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,60 @@ +# Changelog + +All notable changes to this project are recorded here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and +this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +While the major version is 0, the public API may change in any minor release; +breaking changes are listed under **Changed** with a migration note. + +## [Unreleased] + +Nothing yet. + +## [0.1.0] - 2026-09-18 + +The first tagged release. A database that survives the crashes and the +corruption it is supposed to survive and a version number to depend on. + +Everything below already existed in the repository; this release is where it +becomes something a dependent can pin. + +### Added + +- A LevelDB implementation in pure V: skiplist memtable, write-ahead journal, + SSTables with prefix compressed blocks and CRC32C checksums, bloom filters, + MANIFEST/CURRENT version tracking, atomic write batches, memtable flush to L0 + and size based level compaction and a snapshot iterator. Blocks are + compressed with zlib or raw deflate. +- **`db`**: `sync()` makes pending writes durable without forcing a compaction. +- CI builds the V it tests against from a pinned source checkout and a release + workflow publishes a tag whose changelog section becomes the release notes. + +### Fixed + +- **`journal`**: data recovered from the journal was lost on the next open. + Recovery left the records in the memtable and then removed the journal they + came from, writing them out only when they happened to exceed + `write_buffer_size`. A smaller recovery survived the crash and was then + dropped by an ordinary restart. The recovered memtable is now flushed before + its journal is removed. +- **`journal`**: a corrupt record was indistinguishable from the end of the + journal. Recovery stopped at the first bad checksum and reported success, so + corruption in the middle of a journal silently truncated the database to the + records before it. +- **`manifest`**: a truncated MANIFEST was accepted during recovery, opening a + database against a version edit that was never fully written. +- **`lock`**: the LOCK file was created but never locked, so two processes + could open the same database and write over each other. +- **`table`**: tables were published into a version before their contents were + durable. A crash between publication and the data reaching disk left the + MANIFEST pointing at a file whose blocks were not all there. +- **`journal`**, **`table`**: writes were unchecked. Records and blocks went + through a buffered file handle whose short writes were never detected. A + write that stored only part of a record still reported success. Journal + records, table blocks and the CURRENT file are now written through a loop + that writes every byte, retries on `EINTR` and turns a failure into an + error. + +[Unreleased]: https://github.com/bedrock-v/leveldb/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/bedrock-v/leveldb/releases/tag/v0.1.0 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1d7a487 --- /dev/null +++ b/Makefile @@ -0,0 +1,61 @@ +V ?= v +SOURCES := $(wildcard *.v) $(wildcard *.c.v) +EXAMPLE := examples/smoke + +.DEFAULT_GOAL := check + +.PHONY: help +help: + @printf ' %-12s %s\n' \ + link 'Link this checkout into V so `import leveldb` resolves' \ + check 'fmt-check, vet and test - what CI runs on a pull request' \ + test 'Run the test suite' \ + fmt 'Format in place' \ + fmt-check 'Fail if anything is unformatted' \ + vet "Run V's vet" \ + example 'Build and run the example that imports the module' \ + example-prod 'The same, with optimisation on' \ + docs 'Generate the API documentation into _docs/' \ + clean 'Remove build output' + +.PHONY: link +link: + @mkdir -p "$(HOME)/.vmodules" + @ln -sfn "$(CURDIR)" "$(HOME)/.vmodules/leveldb" + @echo "linked $(HOME)/.vmodules/leveldb -> $(CURDIR)" + +.PHONY: check +check: fmt-check vet test + +.PHONY: test +test: + $(V) test . + +.PHONY: fmt +fmt: + $(V) fmt -w $(SOURCES) $(EXAMPLE) + +.PHONY: fmt-check +fmt-check: + $(V) fmt -verify $(SOURCES) $(EXAMPLE) + +.PHONY: vet +vet: + $(V) vet . $(EXAMPLE) + +.PHONY: example +example: + $(V) run $(EXAMPLE) + +.PHONY: example-prod +example-prod: + $(V) -prod run $(EXAMPLE) + +.PHONY: docs +docs: + $(V) doc -f html -m -o _docs . + +.PHONY: clean +clean: + @rm -rf _docs + @rm -f *.so *.dll *.dylib diff --git a/db.v b/db.v index 3beaec8..9cfcc03 100644 --- a/db.v +++ b/db.v @@ -109,9 +109,7 @@ fn (mut db DB) replay_journals() ! { batch.each(fn [mut db, mut seq] (kt KeyType, key []u8, value []u8) ! { db.mem.put(make_internal_key(key, seq, kt), value.clone()) seq++ - }) or { - return error('leveldb: ${name}: malformed batch: ${err}') - } + }) or { return error('leveldb: ${name}: malformed batch: ${err}') } end_seq := batch.seq() + u64(batch.count) - 1 if end_seq > db.vs.last_seq { db.vs.last_seq = end_seq diff --git a/examples/smoke/main.v b/examples/smoke/main.v new file mode 100644 index 0000000..848e789 --- /dev/null +++ b/examples/smoke/main.v @@ -0,0 +1,27 @@ +module main + +import os +import leveldb + +fn main() { + dir := os.join_path(os.temp_dir(), 'leveldb-smoke') + os.rmdir_all(dir) or {} + defer { + os.rmdir_all(dir) or {} + } + + mut db := leveldb.open(dir, leveldb.Options{})! + db.put('key'.bytes(), 'value'.bytes(), leveldb.WriteOptions{ sync: true })! + db.close()! + + mut reopened := leveldb.open(dir, leveldb.Options{})! + value := reopened.get('key'.bytes(), leveldb.ReadOptions{}) or { + panic('the key written before the reopen is gone') + } + if value.bytestr() != 'value' { + panic('expected "value", got "${value.bytestr()}"') + } + reopened.close()! + + println('smoke ok') +} diff --git a/table_writer.v b/table_writer.v index e98e7c6..605ba5d 100644 --- a/table_writer.v +++ b/table_writer.v @@ -104,6 +104,7 @@ fn (mut w TableWriter) write_block(block []u8, compression Compression) !BlockHa } else {} } + handle := BlockHandle{ offset: w.offset size: u64(data.len) diff --git a/v.mod b/v.mod index a988849..0bfa7ad 100644 --- a/v.mod +++ b/v.mod @@ -1,7 +1,8 @@ Module { name: 'leveldb' description: 'LevelDB key/value database in V with flate compression.' - version: '0.0.1' + version: '0.1.0' license: 'BSD-2-Clause' + repo_url: 'https://github.com/bedrock-v/leveldb' dependencies: [] }