From df44033ef8ac3114b3deb9ff7b4aeee51468d408 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Thu, 9 Jul 2026 00:53:00 +0100 Subject: [PATCH 1/4] Fix the documentation and the metadata for the new plans --- .zenodo.json | 1 + NEWS.md | 7 ++++++ Project.toml | 9 +++++-- README.md | 24 ++++++++++++------ codemeta.json | 9 ++++--- test/GitUtils.jl | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 test/GitUtils.jl diff --git a/.zenodo.json b/.zenodo.json index e98042c..f724c0f 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,5 +1,6 @@ { "title": "ResearchSoftwareMetadata.jl", + "description": "Research Software metadata crosswalk between Project.toml, codemeta.json, .zenodo.json and LICENSE to provide consistent metadata", "upload_type": "software", "creators": [ { diff --git a/NEWS.md b/NEWS.md index 145ed00..8e09950 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,12 @@ # NEWS +- v0.2.0 + - Include authors without ORCIDs in codemeta.json and .zenodo.json + - Handle missing ORCID and ROR records without crashing + - Fail on connection errors and unexpected HTTP statuses, writing no files so everything is left in its original state + - Add optional top-level Project.toml keys description, keywords, category, development_status and publications as the source for codemeta.json and .zenodo.json, backfilling them from codemeta.json when absent + - Split the source into separate files and export crosswalk, increase_patch, increase_minor, increase_major + - Construct a missing author_details section in Project.toml from the authors field and codemeta.json or .zenodo.json when consistent with authors - v0.1.7 - Bump compat and fix workflows - v0.1.6 diff --git a/Project.toml b/Project.toml index dc2cea7..988ce3f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,8 +1,12 @@ name = "ResearchSoftwareMetadata" uuid = "58378933-4625-47fa-851e-05ee27d397bd" license = "BSD-2-Clause" +description = "Research Software metadata crosswalk between Project.toml, codemeta.json, .zenodo.json and LICENSE to provide consistent metadata" +keywords = ["RSMD", "julia", "metadata", "research software"] +category = "metadata" +development_status = "active" authors = ["Richard Reeve "] -version = "0.1.7" +version = "0.2.0" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" @@ -34,9 +38,10 @@ email = "richard.reeve@glasgow.ac.uk" [extras] Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "Git", "Logging", "TOML"] +test = ["Test", "Git", "JSON", "Logging", "TOML"] diff --git a/README.md b/README.md index 0f85c46..897a10a 100644 --- a/README.md +++ b/README.md @@ -18,13 +18,13 @@ The package is registered in the `General` registry so can be built and installed with `add`. For example: ```julia -(@v1.10) pkg> add ResearchSoftwareMetadata +(@v1.12) pkg> add ResearchSoftwareMetadata Resolving package versions... - Updating `~/.julia/environments/v1.10/Project.toml` - [aea672f4] + ResearchSoftwareMetadata v0.1.2 - Updating `~/.julia/environments/v1.10/Manifest.toml` + Updating `~/.julia/environments/v1.12/Project.toml` + [aea672f4] + ResearchSoftwareMetadata v0.2.0 + Updating `~/.julia/environments/v1.12/Manifest.toml` -(@v1.10) pkg> +(@v1.12) pkg> ``` ## Usage @@ -48,6 +48,16 @@ orcid = "0000-0003-2589-8091" ror = "00vtgdb53" ``` +You can also optionally add a `description` of the package, `keywords` associated with it, the software `category` it belongs to, its [repostatus.org](https://www.repostatus.org) `development_status`, and the DOIs of any `publications` associated with the package. All of these are propagated into `codemeta.json` and `.zenodo.json`, and any values already in `codemeta.json` but missing from `Project.toml` will be backfilled into it: + +```toml +description = "A package that does things" +keywords = ["julia", "metadata"] +category = "metadata" +development_status = "active" +publications = ["10.5281/zenodo.12789179"] +``` + Then, from the root of your package, you can just run a crosswalk: ```julia @@ -70,8 +80,8 @@ ResearchSoftwareMetadata.crosswalk(category = "metadata", keywords = ["julia", " # Increase version number during crosswalk ResearchSoftwareMetadata.increase_patch() # Bump patch version (e.g. 0.4.1 -> 0.4.2) -ResearchSoftwareMetadata.increase_minor() # Bump minor version (e.g. 0.4.1 -> 0.5.0) -ResearchSoftwareMetadata.increase_major() # Bump major version (e.g. 0.4.1 -> 1.0.0) +ResearchSoftwareMetadata.increase_minor() # Bump minor version (e.g. 0.4.2 -> 0.5.0) +ResearchSoftwareMetadata.increase_major() # Bump major version (e.g. 0.5.0 -> 1.0.0) ``` You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. diff --git a/codemeta.json b/codemeta.json index 98ccf16..db817d6 100644 --- a/codemeta.json +++ b/codemeta.json @@ -14,10 +14,10 @@ "Linux", "macOS" ], - "version": "v0.1.7", - "dateModified": "2026-07-07", + "version": "v0.2.0", + "dateModified": "2026-07-09", "datePublished": "2024-07-18", - "downloadUrl": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/archive/refs/tags/v0.1.7.tar.gz", + "downloadUrl": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/archive/refs/tags/v0.2.0.tar.gz", "license": "https://spdx.org/licenses/BSD-2-Clause", "author": [ { @@ -44,5 +44,6 @@ "julia", "metadata", "research software" - ] + ], + "description": "Research Software metadata crosswalk between Project.toml, codemeta.json, .zenodo.json and LICENSE to provide consistent metadata" } \ No newline at end of file diff --git a/test/GitUtils.jl b/test/GitUtils.jl new file mode 100644 index 0000000..8b1ae3c --- /dev/null +++ b/test/GitUtils.jl @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: BSD-2-Clause + +module GitUtils + +using Git + +export is_repo_clean + +""" + is_repo_clean(repo_path; strict = false) + +Determine whether the git repository at `repo_path` is clean, reporting on its state. + +The short-format status (`git status -s`) of the repository is split into three +categories using the two status columns of each entry — the first column records +changes staged in the index, the second records changes in the working tree: + + - **staged**: tracked files with changes recorded in the index; + - **unstaged**: tracked files with working-tree changes not yet staged (this + includes files that are both staged and further modified); + - **untracked**: files git is not tracking (shown as `??`). + +By default (`strict = false`) only *unstaged* changes make the repository dirty: +a repository whose only changes are already staged and/or untracked is treated as +clean. This suits a formatting check, which should fail only when reformatting +produced changes that have not yet been staged. Pass `strict = true` to require +the repository to be *genuinely* clean, with no staged, unstaged or untracked +changes at all. + +Regardless of the result, an informational message (`@info`) always reports how +many files are dirty (under the selected criterion) and lists any untracked files. +When the repository is considered dirty, the offending files are additionally +reported via `@error`. + +Return `true` if the repository is clean under the selected criterion, and `false` +otherwise. +""" +function is_repo_clean(repo_path; strict = false) + # Short-format porcelain status: each entry is "XY path", where column X is the + # index (staged) status and column Y is the working-tree status. + statuses = readlines(`$(Git.git()) status -s $repo_path`) + + untracked = filter(s -> startswith(s, "??"), statuses) + unstaged = filter(s -> !startswith(s, "??") && s[2] != ' ', statuses) + + # Relaxed (default): only unstaged changes to tracked files count as dirty. + # Strict: any staged, unstaged or untracked change counts. + dirty = strict ? statuses : unstaged + + # Always report the number of dirty files and any untracked files + msg = "Repository $repo_path: $(length(dirty)) file(s) dirty, " * + "$(length(untracked)) untracked" + isempty(untracked) || + (msg *= "\nUntracked files:\n" * join(untracked, "\n")) + @info msg + + is_clean = isempty(dirty) + is_clean || @error "Repository not clean:\n" * join(dirty, "\n") + + return is_clean +end + +end From 1e344755903b94d8a2e232d6ec3ffaa79dbdc2f2 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Thu, 9 Jul 2026 00:53:53 +0100 Subject: [PATCH 2/4] Improve crosswalk to move in both directions and add tests --- src/ResearchSoftwareMetadata.jl | 747 +------------------------------- src/crosswalk.jl | 606 ++++++++++++++++++++++++++ src/project.jl | 61 +++ src/remotequeries.jl | 127 ++++++ src/utils.jl | 219 ++++++++++ src/versioning.jl | 85 ++++ test/runtests.jl | 303 ++++++++++++- 7 files changed, 1414 insertions(+), 734 deletions(-) create mode 100644 src/crosswalk.jl create mode 100644 src/project.jl create mode 100644 src/remotequeries.jl create mode 100644 src/utils.jl create mode 100644 src/versioning.jl diff --git a/src/ResearchSoftwareMetadata.jl b/src/ResearchSoftwareMetadata.jl index 33270d5..cd83560 100644 --- a/src/ResearchSoftwareMetadata.jl +++ b/src/ResearchSoftwareMetadata.jl @@ -1,5 +1,22 @@ # SPDX-License-Identifier: BSD-2-Clause +""" + ResearchSoftwareMetadata + +`ResearchSoftwareMetadata` provides a crosswalk between `Project.toml`, +`codemeta.json`, `.zenodo.json` and the package `LICENSE` file, so that +consistent research software metadata can be picked up from a package by +Julia's General registry, GitHub and Zenodo, following the FAIR-IMPACT +Research Software MetaData (RSMD) guidelines. `Project.toml` is treated +as the authoritative source of metadata wherever possible. + +The exported entry points are [`ResearchSoftwareMetadatacrosswalk`](@ref), +which enforces consistency across the metadata files and the julia source +code, and [`ResearchSoftwareMetadata.increase_patch`](@ref), +[`ResearchSoftwareMetadata.increase_minor`](@ref) and +[`ResearchSoftwareMetadata.increase_major`](@ref), which bump the package +version and then re-run the crosswalk. +""" module ResearchSoftwareMetadata using Dates @@ -10,730 +27,10 @@ using DataStructures using HTTP using YAML -""" - ResearchSoftwareMetadata.read_project() - -Read a `Project.toml` file in and return it in its canonical order in -an OrderedDict. -""" -function read_project(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) - file = joinpath(git_dir, "Project.toml") - project_d = TOML.parsefile(file) - project = OrderedDict{String, Any}() - for key in [ - "name", - "uuid", - "license", - "authors", - "version", - "deps", - "weakdeps", - "extensions", - "compat", - "author_details", - "extras", - "targets" - ] - if haskey(project_d, key) - val = project_d[key] - if val isa AbstractDict - d = OrderedDict{String, Any}() - for k2 in sort(collect(keys(val))) - d[k2] = val[k2] - end - project[key] = d - else - project[key] = val - end - delete!(project_d, key) - end - end - for key in keys(project_d) - project[key] = project_d[key] - end - - return project -end - -""" - ResearchSoftwareMetadata.get_person_from_orcid(orcid::String) - -Take an `ORCID` from the user and query the orcid.org API to return -a Dict containing the relevant metadata or nothing if no such ORCID exists. -""" -function get_person_from_orcid(orcid::String) - url = "https://pub.orcid.org/v3.0/$orcid" - headers = ["Accept" => "application/json"] - response = HTTP.get(url, headers) - - if response.status == 200 - data = JSON.parse(String(response.body)) - name = data["person"]["name"] - given_names = name["given-names"]["value"] - family_name = name["family-name"]["value"] - d = Dict("orcid" => orcid, "pid" => data["orcid-identifier"]["uri"], - "givenName" => given_names, "familyName" => family_name, - "full_name" => "$given_names $family_name") - emails = data["person"]["emails"]["email"] - if length(emails) ≥ 1 - d["email"] = emails[1]["email"] - d["name_with_email"] = d["full_name"] * " <" * d["email"] * ">" - else - d["name_with_email"] = d["full_name"] - end - return d - else - return nothing - end -end - -""" - ResearchSoftwareMetadata.get_organisation_from_ror(ror::String) - -Take a `ROR` from the user and query the ror.org API to return -a Dict containing the relevant metadata or nothing if no such ROR exists. -""" -function get_organisation_from_ror(ror::String) - url = "https://api.ror.org/organizations/$ror" - - response = HTTP.get(url) - - if response.status == 200 - data = JSON.parse(String(response.body)) - if haskey(data, "name") - name = data["name"] - elseif haskey(data, "names") && length(data["names"]) ≥ 1 - names = [record["value"] for record in data["names"] if "ror_display" in record["types"]] - if length(names) ≥ 1 - name = names[1] - else - @warn "No display name found for ROR $ror" - names = [record["value"] for record in data["names"] if record["lang"] == "en"] - if length(names) ≥ 1 - name = names[1] - else - @info "No English name found for ROR $ror, using first available name" - name = data["names"][1]["value"] - end - end - else - @error "No name found for ROR $ror" - name = "Unknown" - end - d = Dict("name" => name, "ror" => ror, "pid" => data["id"]) - return d - else - return nothing - end -end - -""" - ResearchSoftwareMetadata.get_first_release_date() - -Returns the first release date of this package on Julia's `General` -Registry, or today's date if the package has not been registered yet. -""" -function get_first_release_date(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) - project = read_project(git_dir) - package = project["name"] - url = "https://raw.githubusercontent.com/JuliaRegistries/General/master/$(package[1])/$package/Versions.toml" - headers = ["Accept" => "application/toml"] - response = HTTP.get(url, headers, status_exception = false) - - if response.status == 200 - data = TOML.parse(String(response.body)) - version = minimum(VersionNumber.(keys(data))) - cd(git_dir) - date = readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/v$version`) - return date - elseif response.status == 404 - @info "No release yet on General, imputing first release will be today" - return string(today()) - end - - return nothing -end - -""" - ResearchSoftwareMetadata.get_os_from_workflows() - -Returns the operating systems that the GitHub workflows associated with this package -work on. This is presumed to represent the operating systems that the software runs on. -""" -function get_os_from_workflows(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) - workflow_folder = joinpath(git_dir, ".github", "workflows") - files = filter(isfile, readdir(workflow_folder, join = true)) - oses = Set{String}() - for file in files - jobs = YAML.load_file(file)["jobs"] - for job in keys(jobs) - os = jobs[job]["runs-on"] - if occursin(r"\${{.*}}", os) - k2s = split(replace(os, r"\${{ *([^ ]*) *}}" => s"\1"), ".") - os = jobs[job]["strategy"] - for k2 in k2s - os = os[k2] - end - if os isa String - push!(oses, os) - else - for val in os - push!(oses, val) - end - end - else - push!(oses, os) - end - end - end - - platforms = Set{String}() - for os in oses - push!(platforms, - replace(replace(replace(os, "ubuntu" => "Linux"), - "windows" => "Windows"), - r"-.*" => "")) - end - - return sort(collect(platforms)) -end - -""" - ResearchSoftwareMetadata.crosswalk(; category = nothing, keywords = nothing, build = false) - -Runs a crosswalk across `Project.toml`, `LICENSE`, `codemeta.json` and `.zenodo.json` as -well as the julia source files to enforce consistency between the different metadata formats. -It logs warnings and errors if it identifies inconsistencies while it is editing the files. -The software category can be set with the `category` argument, likewise the `keywords` argument -can contain a vector of keyword strings. The `build` argument sets the `buildInstructions` RSMD -field - `false` leaves the instructions as is, `true` sets it to the same as the README, -and a string sets it to that value. If `update` is true, mismatches between version numbers in -`codemeta.json` are accepted. -""" -function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`); - category = nothing, keywords = nothing, build = false, - update = false) - project = read_project(git_dir) - proj_version = VersionNumber(project["version"]) - - now = string(today()) - cd(git_dir) - init = readchomp(`$(Git.git()) log --max-parents=0 --format=%ad --date=short -n 1`) - tags = readlines(`$(Git.git()) tag -l --sort="version:refname"`) - tag = isempty(tags) ? proj_version : maximum(VersionNumber.(tags)) - tag_date = isempty(tags) ? now : - readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/v$tag`) - branch = readchomp(`$(Git.git()) branch --show-current`) - remotes = split(readchomp(`$(Git.git()) remote`), '\n') - urls = String[] - for remote in remotes - push!(urls, - replace(readchomp(`$(Git.git()) remote get-url $remote`), - r"\.git" => "")) - end - - repos = replace.(urls, r"^.*/([^/]+)$" => s"\1") - - file = joinpath(git_dir, "codemeta.json") - codemeta = isfile(file) ? - JSON.parsefile(file, dicttype = OrderedDict) : - OrderedDict{String, Any}() - - codemeta["@context"] = "https://w3id.org/codemeta/3.0" - codemeta["type"] = "SoftwareSourceCode" - if isnothing(category) - get!(codemeta, "applicationCategory", "ecology") - else - codemeta["applicationCategory"] = category - end - codemeta["programmingLanguage"] = "julia" - codemeta["developmentStatus"] = "active" - - repo_index = "origin" ∈ remotes ? - repo_index = findfirst(==("origin"), remotes) : 1 - - if haskey(codemeta, "codeRepository") - cm_url = codemeta["codeRepository"] - if cm_url ∉ urls - @error "codemeta has wrong repo URL – $cm_url not in $urls – using $(remotes[repo_index])" - else - repo_index = findfirst(==(cm_url), urls) - end - end - codemeta["codeRepository"] = urls[repo_index] - - if haskey(codemeta, "name") - cm_name = codemeta["name"] - if cm_name ≠ repos[repo_index] - @error "codemeta has wrong repo repo name – $cm_name not $(repos[repo_index]) – fixing" - end - end - codemeta["name"] = repos[repo_index] - - codemeta["issueTracker"] = urls[repo_index] * "/issues" - - readme = urls[repo_index] * "/blob/" * branch * "/README.md" - cm_readme = get!(codemeta, "readme", readme) - cm_readme == readme || - @info "README set to $cm_readme, not $readme" - - if build isa Bool - if build - codemeta["buildInstructions"] = cm_readme - elseif !haskey(codemeta, "buildInstructions") - @warn "No build instructions, set using `build` keyword argument" - end - else - codemeta["buildInstructions"] = build - end - - cm_created = get(codemeta, "dateCreated", "") - if cm_created ≠ init - @warn "Fixing creation date to first git commit: $init" - codemeta["dateCreated"] = init - end - - platforms = get_os_from_workflows() - cm_platforms = sort(string.(get(codemeta, "operatingSystem", String[]))) - if length(platforms) ≠ length(cm_platforms) || - any(platforms .≠ cm_platforms) - if isempty(cm_platforms) - @info "No platform info in codemeta.json, so filling from workflows ($platforms)" - else - @error "codemeta platforms do not match workflows ($cm_platforms ≠ $platforms), fixing" - end - codemeta["operatingSystem"] = platforms - end - - years = string(year(Date(init))) - - cm_version = VersionNumber(get!(codemeta, "version", string(proj_version))) - - if proj_version == tag - @debug "Still on latest release version: $tag" - codemeta["dateModified"] = tag_date - if cm_version ≠ tag - if !update - @warn "Correcting codemeta tag version ($cm_version) to release tag ($tag)" - end - cm_version = tag - tag_year = string(year(Date(tag_date))) - if tag_year ≠ years - years = years * "-" * tag_year - end - else - tag_year = string(year(Date(codemeta["dateModified"]))) - if tag_year ≠ years - years = years * "-" * tag_year - end - end - elseif proj_version > tag - @info "Preparing for new release" - codemeta["dateModified"] = now - if cm_version ≠ proj_version - @info "Updating codemeta tag version ($cm_version) to " * - "new release ($proj_version)" - cm_version = proj_version - this_year = string(year(Date(now))) - if this_year ≠ years - years = years * "-" * this_year - end - else - tag_year = string(year(Date(codemeta["dateModified"]))) - if tag_year ≠ years - years = years * "-" * tag_year - end - end - else # Project version is lower than latest release! - @error "Project.toml version is behind release ($proj_version < $tag), fixing." - proj_version = tag - cm_version = tag - codemeta["dateModified"] = tag_date - tag_year = string(year(Date(codemeta["dateModified"]))) - if tag_year ≠ years - years = years * "-" * tag_year - end - end - - first_release_date = get_first_release_date(git_dir) - if !isnothing(first_release_date) - if haskey(codemeta, "datePublished") - codemeta["datePublished"] == first_release_date || - @warn "codemeta.json publication date inconsistent with Julia's General registry, fixing ($(codemeta["datePublished"]) ≠ $first_release_date)" - end - codemeta["datePublished"] = first_release_date - end - project["version"] = string(proj_version) - codemeta["version"] = "v$cm_version" - - codemeta["downloadUrl"] = urls[repo_index] * "/archive/refs/tags/" * - codemeta["version"] * ".tar.gz" - - authors = String[] - author_data = [] - if haskey(project, "author_details") - for author in project["author_details"] - if haskey(author, "orcid") - person = get_person_from_orcid(author["orcid"]) - push!(authors, person["name_with_email"]) - if haskey(author, "name") - person["full_name"] == author["name"] || - @warn "Name mismatch between ORCID and Project.toml: " * - "$(person["full_name"]) ≠ $(author["name"])" - end - if haskey(author, "email") && haskey(person, "email") - person["email"] == author["email"] || - @warn "Email mismatch between ORCID and Project.toml: " * - "$(person["email"]) ≠ $(author["email"])" - end - elseif haskey(author, "name") - if haskey(author, "email") - push!(authors, - author["name"] * " <" * author["email"] * ">") - else - push!(authors, author["name"]) - end - else - @warn "Missing name and ORCID in authors block" - end - end - else - authors = project["authors"] - for author in authors - name = replace(author, r" *<.*> *$" => "") - if '<' ∈ author - email = replace(author, r"^.*<([^>]+)>.*$" => s"\1") - push!(author_data, Dict("name" => name, "email" => email)) - else - push!(author_data, Dict("name" => name)) - end - end - project["author_details"] = author_data - end - - replace_authors = true - if !isempty(authors) - proj_authors = project["authors"] - for author in authors - if author ∉ proj_authors - if lowercase(author) ∈ lowercase.(proj_authors) - @info "Changing case of $author in Project.toml" - elseif any(occursin.(author, proj_authors)) - @info "Completing $author in Project.toml" - else - @error "Author $author not in $proj_authors" - replace_authors = false - end - end - end - - if length(authors) < length(proj_authors) - @error "Author mismatch between $authors and $proj_authors" - replace_authors = false - end - end - - if replace_authors - project["authors"] = authors - end - - haslicense = false - license = nothing - - if haskey(project, "license") - proj_license = project["license"] - cm_license = "https://spdx.org/licenses/" * proj_license - if haskey(codemeta, "license") - if codemeta["license"] == cm_license - haslicense = true - license = proj_license - else - @error "License mismatch between Project.toml and codemeta.json: " * - "$(codemeta["license"]) ≠ $cm_license" - end - else - codemeta["license"] = cm_license - haslicense = true - license = proj_license - end - else - if haskey(codemeta, "license") - project["license"] = replace(codemeta["license"], - "https://spdx.org/licenses/" => "") - haslicense = true - license = project["license"] - else - @warn "No license metadata" - end - end - - open_license = nothing - if haslicense - url = "https://spdx.org/licenses/$license.json" - headers = ["Accept" => "application/json"] - response = HTTP.get(url, headers) - - if response.status == 200 - just_names = replace.(project["authors"], r" *<[^>]+> *" => "") - name_list = join(just_names, ", ", " and ") - json = JSON.parse(String(response.body)) - open_license = json["isOsiApproved"] - content = json["licenseText"] - replaces = [r""i => years, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list] - for r in replaces - content = replace(content, r) - end - file = joinpath(git_dir, "LICENSE") - open(file, "w") do file - return write(file, content) - end - file = joinpath(git_dir, "LICENSE.md") - rm(file, force = true) - end - end - - file = joinpath(git_dir, "Project.toml") - open(file, "w") do io - return TOML.print(io, project) - end - - # Repeat ro ensure correct order if there were elements missing - project = read_project(git_dir) - open(file, "w") do io - return TOML.print(io, project) - end - - cm_authors = get(codemeta, "author", OrderedDict{String, Any}[]) - proj_authors = project["author_details"] - cm_from_proj = OrderedDict{String, Any}[] - for author in proj_authors - if haskey(author, "orcid") - person = get_person_from_orcid(author["orcid"]) - dict = OrderedDict{String, Any}("type" => "Person") - if haskey(person, "givenName") - dict["givenName"] = person["givenName"] - end - if haskey(person, "familyName") - dict["familyName"] = person["familyName"] - end - if haskey(person, "email") - dict["email"] = person["email"] - end - if haskey(person, "orcid") - dict["id"] = person["pid"] - end - if haskey(author, "affiliation") - dict["affiliation"] = OrderedDict{String, String}[] - for org in author["affiliation"] - d = OrderedDict("type" => "Organization") - if haskey(org, "ror") - ror = org["ror"] - info = get_organisation_from_ror(ror) - d["name"] = info["name"] - d["identifier"] = info["pid"] - else - d["name"] = org["name"] - end - push!(dict["affiliation"], d) - end - end - push!(cm_from_proj, dict) - end - end - - if isempty(cm_authors) - @info "Filling codemeta authors from Project.toml" - codemeta["author"] = cm_from_proj - else - if length(cm_authors) == length(cm_from_proj) - ids = [dict["id"] for dict in cm_from_proj] - for dict in cm_authors - if get(dict, "id", nothing) ∉ ids - @error "$dict not found in Project.toml" - end - end - else - @error "Mismatch between Project.toml and codemeta.json authors: " * - "$(cm_authors) ≠ $(proj_authors)" - end - end - - if haskey(codemeta, "continuousIntegration") - if haskey(codemeta, "codemeta:contIntegration") - if codemeta["continuousIntegration"] ≠ - codemeta["codemeta:contIntegration"]["id"] - @error "Mismatch between continuousIntegration and codemeta:contIntegration, " * - "$(codemeta["continuousIntegration"]) ≠ $(codemeta["codemeta:contIntegration"]["id"])" - end - else - codemeta["codemeta:contIntegration"] = Dict("id" => codemeta["continuousIntegration"]) - end - else - if haskey(codemeta, "codemeta:contIntegration") - codemeta["continuousIntegration"] = codemeta["codemeta:contIntegration"]["id"] - elseif isfile(".github/workflows/testing.yaml") - @info "Using .github/workflows/testing.yaml for CI" - codemeta["continuousIntegration"] = urls[repo_index] * - "/actions/workflows/testing.yaml" - codemeta["codemeta:contIntegration"] = Dict("id" => codemeta["continuousIntegration"]) - elseif isfile(".github/workflows/CI.yaml") - @info "Using .github/workflows/CI.yaml for CI" - codemeta["continuousIntegration"] = urls[repo_index] * - "/actions/workflows/CI.yaml" - codemeta["codemeta:contIntegration"] = Dict("id" => codemeta["continuousIntegration"]) - elseif isdir(".github/workflows") - @warn "CI not found in codemeta.json, but .github/workflows exists" - end - end - - if isnothing(keywords) - keywords = get(codemeta, "keywords", ["julia"]) - end - codemeta["keywords"] = sort(keywords) - - file = joinpath(git_dir, "codemeta.json") - open(file, "w") do io - return JSON.print(io, codemeta, 4) - end - - crosswalk_d = OrderedDict{String, Any}() - crosswalk_d["title"] = codemeta["name"] - if haskey(codemeta, "description") - crosswalk_d["description"] = codemeta["description"] - end - crosswalk_d["upload_type"] = "software" - crosswalk_d["creators"] = [] - for author in codemeta["author"] - dict = OrderedDict{String, String}() - dict["name"] = "$(author["familyName"]), $(author["givenName"])" - if haskey(author, "id") - dict["orcid"] = replace(author["id"], "https://orcid.org/" => "") - end - if haskey(author, "affiliation") - affiliations = author["affiliation"] - affiliation = affiliations isa Vector ? first(affiliations) : - affiliations - if haskey(affiliation, "name") - dict["affiliation"] = affiliation["name"] - end - if haskey(affiliation, "identifier") - dict["ror"] = replace(affiliation["identifier"], - "https://ror.org/" => "") - end - end - push!(crosswalk_d["creators"], dict) - end - if !isnothing(open_license) - crosswalk_d["access_right"] = open_license ? "open" : "closed" - end - crosswalk_d["license"] = project["license"] - dict = OrderedDict{String, String}() - dict["scheme"] = "url" - dict["identifier"] = codemeta["codeRepository"] - dict["relation"] = "isOriginalFormOf" - crosswalk_d["related_identifiers"] = [dict] - crosswalk_d["keywords"] = codemeta["keywords"] - - file = joinpath(git_dir, ".zenodo.json") - open(file, "w") do io - return JSON.print(io, crosswalk_d, 4) - end - - # Recursively walk through the directory - path = joinpath(git_dir, ".git") - notpath = joinpath(git_dir, ".github") - for (root, _, files) in walkdir(git_dir) - if !startswith(root, path) || startswith(root, notpath) - for file in files - if endswith(file, ".jl") - jl_file = joinpath(root, file) - data = readlines(jl_file) - if startswith(data[1], "# SPDX-License-Identifier:") - data[1] = "# SPDX-License-Identifier: $(project["license"])" - elseif !startswith(data[1], "### A Pluto.jl notebook ###") - pushfirst!(data, "") - pushfirst!(data, - "# SPDX-License-Identifier: $(project["license"])") - end - open(jl_file, "w") do io - return println.(Ref(io), data) - end - end - end - end - end - - return nothing -end - -""" - ResearchSoftwareMetadata.increase_patch() - -Increases the `Project.toml` version number by a patch (e.g. 0.4.1 to 0.4.2), and then -runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. -""" -function increase_patch(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) - project = read_project(git_dir) - version = project["version"] - v = VersionNumber(version) - new_version = VersionNumber(v.major, v.minor, v.patch + 1) - @info "Bumping patch version from $version to $new_version" - project["version"] = string(new_version) - file = joinpath(git_dir, "Project.toml") - open(file, "w") do io - return TOML.print(io, project) - end - - return crosswalk(git_dir, update = true) -end - -""" - ResearchSoftwareMetadata.increase_minor() - -Increases the `Project.toml` version number by a minor number (e.g. 0.4.1 to 0.5.0), and then -runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. -""" -function increase_minor(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) - project = read_project(git_dir) - version = project["version"] - v = VersionNumber(version) - new_version = VersionNumber(v.major, v.minor + 1, 0) - @info "Bumping minor version from $version to $new_version" - project["version"] = string(new_version) - file = joinpath(git_dir, "Project.toml") - open(file, "w") do io - return TOML.print(io, project) - end - - return crosswalk(git_dir, update = true) -end - -""" - ResearchSoftwareMetadata.increase_major() - -Increases the `Project.toml` version number by a major number (e.g. 0.4.1 to 1.0.0), and then -runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. -""" -function increase_major(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) - project = read_project(git_dir) - version = project["version"] - v = VersionNumber(version) - new_version = VersionNumber(v.major + 1, 0, 0) - @info "Bumping major version from $version to $new_version" - project["version"] = string(new_version) - file = joinpath(git_dir, "Project.toml") - open(file, "w") do io - return TOML.print(io, project) - end - - return crosswalk(git_dir, update = true) -end +include("project.jl") +include("remotequeries.jl") +include("utils.jl") +include("crosswalk.jl") +include("versioning.jl") end diff --git a/src/crosswalk.jl b/src/crosswalk.jl new file mode 100644 index 0000000..a77e416 --- /dev/null +++ b/src/crosswalk.jl @@ -0,0 +1,606 @@ +# SPDX-License-Identifier: BSD-2-Clause + +""" + crosswalk(; category = nothing, keywords = nothing, build = false) + +Runs a crosswalk across `Project.toml`, `LICENSE`, `codemeta.json` and `.zenodo.json` as +well as the julia source files to enforce consistency between the different metadata formats. +It logs warnings and errors if it identifies inconsistencies while it is editing the files. +`Project.toml` is the authoritative source of metadata: as well as its standard fields, the +optional top-level keys `description`, `keywords`, `category`, `development_status` and +`publications` (a vector of DOIs of associated papers) are propagated into `codemeta.json` +and `.zenodo.json`; values found only in `codemeta.json` are backfilled into `Project.toml`. +A missing `author_details` section is likewise constructed from `codemeta.json` or +`.zenodo.json`, provided the information there is consistent with the definitive `authors` +field. New entries in `authors` are propagated into `author_details` (with a warning to add +their ORCID and ROR affiliation there), `codemeta.json` and `.zenodo.json`, while authors in +`codemeta.json` that are missing from `authors` are removed with an error. +The software category can be set with the `category` argument, likewise the `keywords` +argument can contain a vector of keyword strings; both take precedence over and are written +back into `Project.toml`. The `build` argument sets the `buildInstructions` RSMD +field - `false` leaves the instructions as is, `true` sets it to the same as the README, +and a string sets it to that value. If `update` is true, mismatches between version numbers in +`codemeta.json` are accepted. If any remote metadata query (orcid.org, ror.org, spdx.org, +doi.org or Julia's General registry) cannot be completed, the crosswalk throws an error and +all files are left in their original state. +""" +function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`); + category = nothing, keywords = nothing, build = false, + update = false) + project = read_project(git_dir) + proj_version = VersionNumber(project["version"]) + + now = string(today()) + cd(git_dir) + init = readchomp(`$(Git.git()) log --max-parents=0 --format=%ad --date=short -n 1`) + tags = readlines(`$(Git.git()) tag -l --sort="version:refname"`) + tag = isempty(tags) ? proj_version : maximum(VersionNumber.(tags)) + tag_date = isempty(tags) ? now : + readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/v$tag`) + branch = readchomp(`$(Git.git()) branch --show-current`) + remotes = split(readchomp(`$(Git.git()) remote`), '\n') + urls = String[] + for remote in remotes + push!(urls, + replace(readchomp(`$(Git.git()) remote get-url $remote`), + r"\.git" => "")) + end + + repos = replace.(urls, r"^.*/([^/]+)$" => s"\1") + + file = joinpath(git_dir, "codemeta.json") + codemeta = isfile(file) ? + JSON.parsefile(file, dicttype = OrderedDict) : + OrderedDict{String, Any}() + + codemeta["@context"] = "https://w3id.org/codemeta/3.0" + codemeta["type"] = "SoftwareSourceCode" + isnothing(reconcile!(project, codemeta, "category", "applicationCategory", + value = category)) && + @warn "No category metadata, add `category` to Project.toml" + codemeta["programmingLanguage"] = "julia" + reconcile!(project, codemeta, "development_status", "developmentStatus", + default = "active") + isnothing(reconcile!(project, codemeta, "description", "description")) && + @warn "No description metadata, add `description` to Project.toml" + + repo_index = "origin" ∈ remotes ? + repo_index = findfirst(==("origin"), remotes) : 1 + + if haskey(codemeta, "codeRepository") + cm_url = codemeta["codeRepository"] + if cm_url ∉ urls + @error "codemeta has wrong repo URL – $cm_url not in $urls – using $(remotes[repo_index])" + else + repo_index = findfirst(==(cm_url), urls) + end + end + codemeta["codeRepository"] = urls[repo_index] + + if haskey(codemeta, "name") + cm_name = codemeta["name"] + if cm_name ≠ repos[repo_index] + @error "codemeta has wrong repo repo name – $cm_name not $(repos[repo_index]) – fixing" + end + end + codemeta["name"] = repos[repo_index] + + codemeta["issueTracker"] = urls[repo_index] * "/issues" + + readme = urls[repo_index] * "/blob/" * branch * "/README.md" + cm_readme = get!(codemeta, "readme", readme) + cm_readme == readme || + @info "README set to $cm_readme, not $readme" + + if build isa Bool + if build + codemeta["buildInstructions"] = cm_readme + elseif !haskey(codemeta, "buildInstructions") + @warn "No build instructions, set using `build` keyword argument" + end + else + codemeta["buildInstructions"] = build + end + + cm_created = get(codemeta, "dateCreated", "") + if cm_created ≠ init + @warn "Fixing creation date to first git commit: $init" + codemeta["dateCreated"] = init + end + + platforms = get_os_from_workflows() + cm_platforms = sort(string.(get(codemeta, "operatingSystem", String[]))) + if length(platforms) ≠ length(cm_platforms) || + any(platforms .≠ cm_platforms) + if isempty(cm_platforms) + @info "No platform info in codemeta.json, so filling from workflows ($platforms)" + else + @error "codemeta platforms do not match workflows ($cm_platforms ≠ $platforms), fixing" + end + codemeta["operatingSystem"] = platforms + end + + years = string(year(Date(init))) + + cm_version = VersionNumber(get!(codemeta, "version", string(proj_version))) + + if proj_version == tag + @debug "Still on latest release version: $tag" + codemeta["dateModified"] = tag_date + if cm_version ≠ tag + if !update + @warn "Correcting codemeta tag version ($cm_version) to release tag ($tag)" + end + cm_version = tag + tag_year = string(year(Date(tag_date))) + if tag_year ≠ years + years = years * "-" * tag_year + end + else + tag_year = string(year(Date(codemeta["dateModified"]))) + if tag_year ≠ years + years = years * "-" * tag_year + end + end + elseif proj_version > tag + @info "Preparing for new release" + codemeta["dateModified"] = now + if cm_version ≠ proj_version + @info "Updating codemeta tag version ($cm_version) to " * + "new release ($proj_version)" + cm_version = proj_version + this_year = string(year(Date(now))) + if this_year ≠ years + years = years * "-" * this_year + end + else + tag_year = string(year(Date(codemeta["dateModified"]))) + if tag_year ≠ years + years = years * "-" * tag_year + end + end + else # Project version is lower than latest release! + @error "Project.toml version is behind release ($proj_version < $tag), fixing." + proj_version = tag + cm_version = tag + codemeta["dateModified"] = tag_date + tag_year = string(year(Date(codemeta["dateModified"]))) + if tag_year ≠ years + years = years * "-" * tag_year + end + end + + first_release_date = get_first_release_date(git_dir) + if !isnothing(first_release_date) + if haskey(codemeta, "datePublished") + codemeta["datePublished"] == first_release_date || + @warn "codemeta.json publication date inconsistent with Julia's General registry, fixing ($(codemeta["datePublished"]) ≠ $first_release_date)" + end + codemeta["datePublished"] = first_release_date + end + project["version"] = string(proj_version) + codemeta["version"] = "v$cm_version" + + codemeta["downloadUrl"] = urls[repo_index] * "/archive/refs/tags/" * + codemeta["version"] * ".tar.gz" + + if !haskey(project, "author_details") + details = nothing + source = nothing + if !isempty(get(codemeta, "author", [])) + details = author_details_from_codemeta(codemeta["author"]) + source = "codemeta.json" + elseif isfile(joinpath(git_dir, ".zenodo.json")) + zenodo = JSON.parsefile(joinpath(git_dir, ".zenodo.json"), + dicttype = OrderedDict) + if !isempty(get(zenodo, "creators", [])) + details = author_details_from_zenodo(zenodo["creators"]) + source = ".zenodo.json" + end + end + if !isnothing(details) + if author_details_consistent(details, project["authors"]) + @info "Reconstructing author_details in Project.toml " * + "from $source" + project["author_details"] = details + else + @error "Authors in $source are inconsistent with authors " * + "in Project.toml, rebuilding author_details from " * + "Project.toml alone" + end + end + end + + authors = String[] + author_data = [] + orcid_people = Dict{String, Any}() + if haskey(project, "author_details") + for author in project["author_details"] + person = nothing + if haskey(author, "orcid") + person = get_person_from_orcid(author["orcid"]) + orcid_people[author["orcid"]] = person + isnothing(person) && + @error "Could not retrieve ORCID $(author["orcid"]) " * + "from orcid.org, falling back to Project.toml data" + end + if !isnothing(person) + push!(authors, person["name_with_email"]) + if haskey(author, "name") + person["full_name"] == author["name"] || + @warn "Name mismatch between ORCID and Project.toml: " * + "$(person["full_name"]) ≠ $(author["name"])" + end + if haskey(author, "email") && haskey(person, "email") + person["email"] == author["email"] || + @warn "Email mismatch between ORCID and Project.toml: " * + "$(person["email"]) ≠ $(author["email"])" + end + elseif haskey(author, "name") + if haskey(author, "email") + push!(authors, + author["name"] * " <" * author["email"] * ">") + else + push!(authors, author["name"]) + end + else + @warn "Missing name and ORCID in authors block" + end + end + else + authors = project["authors"] + for author in authors + name, email = parse_author(author) + detail = OrderedDict{String, Any}("name" => name) + isnothing(email) || (detail["email"] = email) + push!(author_data, detail) + end + project["author_details"] = author_data + end + + # Add authors listed in `authors` but missing from author_details + for proj_author in project["authors"] + name, email = parse_author(proj_author) + covered = any(authors) do author + author == proj_author || + lowercase(author) == lowercase(proj_author) || + occursin(author, proj_author) || + parse_author(author) == (name, email) + end + if !covered + detail = OrderedDict{String, Any}("name" => name) + isnothing(email) || (detail["email"] = email) + push!(project["author_details"], detail) + push!(authors, isnothing(email) ? name : name * " <" * email * ">") + @warn "Added $name to author_details in Project.toml, please " * + "add their ORCID and ROR affiliation there if you can" + end + end + + replace_authors = true + if !isempty(authors) + proj_authors = project["authors"] + for author in authors + if author ∉ proj_authors + if lowercase(author) ∈ lowercase.(proj_authors) + @info "Changing case of $author in Project.toml" + elseif any(occursin.(author, proj_authors)) + @info "Completing $author in Project.toml" + elseif parse_author(author) ∈ parse_author.(proj_authors) + @info "Fixing format of $author in Project.toml" + else + @error "Author $author not in $proj_authors" + replace_authors = false + end + end + end + + if length(authors) < length(proj_authors) + @error "Author mismatch between $authors and $proj_authors" + replace_authors = false + end + end + + if replace_authors + project["authors"] = authors + end + + haslicense = false + license = nothing + + if haskey(project, "license") + proj_license = project["license"] + cm_license = "https://spdx.org/licenses/" * proj_license + if haskey(codemeta, "license") + if codemeta["license"] == cm_license + haslicense = true + license = proj_license + else + @error "License mismatch between Project.toml and codemeta.json: " * + "$(codemeta["license"]) ≠ $cm_license" + end + else + codemeta["license"] = cm_license + haslicense = true + license = proj_license + end + else + if haskey(codemeta, "license") + project["license"] = replace(codemeta["license"], + "https://spdx.org/licenses/" => "") + haslicense = true + license = project["license"] + else + @warn "No license metadata" + end + end + + open_license = nothing + license_content = nothing + if haslicense + url = "https://spdx.org/licenses/$license.json" + headers = ["Accept" => "application/json"] + response = HTTP.get(url, headers, status_exception = false) + + response.status == 200 || + error("Unable to fetch license text for $license from " * + "spdx.org, HTTP status $(response.status)") + + just_names = replace.(project["authors"], r" *<[^>]+> *" => "") + name_list = join(just_names, ", ", " and ") + json = JSON.parse(String(response.body)) + open_license = json["isOsiApproved"] + content = json["licenseText"] + replaces = [r""i => years, + r""i => name_list, + r""i => name_list, + r""i => name_list, + r""i => name_list, + r""i => name_list, + r""i => name_list, + r""i => name_list] + for r in replaces + content = replace(content, r) + end + license_content = content + end + + cm_authors = get(codemeta, "author", OrderedDict{String, Any}[]) + proj_authors = project["author_details"] + cm_from_proj = OrderedDict{String, Any}[] + for author in proj_authors + person = haskey(author, "orcid") ? + get(orcid_people, author["orcid"], nothing) : nothing + dict = OrderedDict{String, Any}("type" => "Person") + if isnothing(person) + if haskey(author, "name") + given, family = split_name(author["name"]) + if isnothing(given) + dict["name"] = family + else + dict["givenName"] = given + dict["familyName"] = family + end + end + if haskey(author, "email") + dict["email"] = author["email"] + end + if haskey(author, "orcid") + dict["id"] = "https://orcid.org/" * author["orcid"] + end + else + if haskey(person, "givenName") + dict["givenName"] = person["givenName"] + end + if haskey(person, "familyName") + dict["familyName"] = person["familyName"] + end + if haskey(person, "email") + dict["email"] = person["email"] + elseif haskey(author, "email") + dict["email"] = author["email"] + end + dict["id"] = person["pid"] + end + if haskey(author, "affiliation") + dict["affiliation"] = OrderedDict{String, String}[] + for org in author["affiliation"] + d = OrderedDict("type" => "Organization") + if haskey(org, "ror") + ror = org["ror"] + info = get_organisation_from_ror(ror) + if isnothing(info) + @error "Could not retrieve ROR $ror from ror.org, " * + "falling back to Project.toml data" + haskey(org, "name") && (d["name"] = org["name"]) + d["identifier"] = "https://ror.org/" * ror + else + d["name"] = info["name"] + d["identifier"] = info["pid"] + end + else + d["name"] = org["name"] + end + push!(dict["affiliation"], d) + end + end + push!(cm_from_proj, dict) + end + + if isempty(cm_authors) + @info "Filling codemeta authors from Project.toml" + else + author_key(d) = get(d, "id", + (get(d, "givenName", ""), + get(d, "familyName", get(d, "name", "")))) + keys_from_proj = author_key.(cm_from_proj) + keys_from_cm = author_key.(cm_authors) + for dict in cm_authors + if author_key(dict) ∉ keys_from_proj + @error "$dict not in Project.toml, removing from codemeta.json" + end + end + for dict in cm_from_proj + if author_key(dict) ∉ keys_from_cm + @info "Adding $(get(dict, "givenName", "")) " * + "$(get(dict, "familyName", get(dict, "name", ""))) " * + "to codemeta.json authors" + end + end + end + codemeta["author"] = cm_from_proj + + if haskey(codemeta, "continuousIntegration") + if haskey(codemeta, "codemeta:contIntegration") + if codemeta["continuousIntegration"] ≠ + codemeta["codemeta:contIntegration"]["id"] + @error "Mismatch between continuousIntegration and codemeta:contIntegration, " * + "$(codemeta["continuousIntegration"]) ≠ $(codemeta["codemeta:contIntegration"]["id"])" + end + else + codemeta["codemeta:contIntegration"] = Dict("id" => codemeta["continuousIntegration"]) + end + else + if haskey(codemeta, "codemeta:contIntegration") + codemeta["continuousIntegration"] = codemeta["codemeta:contIntegration"]["id"] + elseif isfile(".github/workflows/testing.yaml") + @info "Using .github/workflows/testing.yaml for CI" + codemeta["continuousIntegration"] = urls[repo_index] * + "/actions/workflows/testing.yaml" + codemeta["codemeta:contIntegration"] = Dict("id" => codemeta["continuousIntegration"]) + elseif isfile(".github/workflows/CI.yaml") + @info "Using .github/workflows/CI.yaml for CI" + codemeta["continuousIntegration"] = urls[repo_index] * + "/actions/workflows/CI.yaml" + codemeta["codemeta:contIntegration"] = Dict("id" => codemeta["continuousIntegration"]) + elseif isdir(".github/workflows") + @warn "CI not found in codemeta.json, but .github/workflows exists" + end + end + + reconcile!(project, codemeta, "keywords", "keywords", + value = keywords, default = ["julia"], to_cm = sort) + + publications = reconcile!(project, codemeta, "publications", + "referencePublication", + to_cm = dois -> "https://doi.org/" .* dois, + from_cm = refs -> replace.(refs isa Vector ? + refs : [refs], + "https://doi.org/" => "")) + if !isnothing(publications) + for doi in publications + check_doi(doi) || + @error "DOI $doi in Project.toml publications does not resolve" + end + end + + crosswalk_d = OrderedDict{String, Any}() + crosswalk_d["title"] = codemeta["name"] + if haskey(codemeta, "description") + crosswalk_d["description"] = codemeta["description"] + end + crosswalk_d["upload_type"] = "software" + crosswalk_d["creators"] = [] + for author in codemeta["author"] + dict = OrderedDict{String, String}() + if haskey(author, "familyName") && haskey(author, "givenName") + dict["name"] = "$(author["familyName"]), $(author["givenName"])" + elseif haskey(author, "name") + dict["name"] = author["name"] + elseif haskey(author, "familyName") + dict["name"] = author["familyName"] + else + @warn "Skipping codemeta author with no name in .zenodo.json: " * + "$author" + continue + end + if haskey(author, "id") + dict["orcid"] = replace(author["id"], "https://orcid.org/" => "") + end + if haskey(author, "affiliation") + affiliations = author["affiliation"] + affiliation = affiliations isa Vector ? first(affiliations) : + affiliations + if haskey(affiliation, "name") + dict["affiliation"] = affiliation["name"] + end + if haskey(affiliation, "identifier") + dict["ror"] = replace(affiliation["identifier"], + "https://ror.org/" => "") + end + end + push!(crosswalk_d["creators"], dict) + end + if !isnothing(open_license) + crosswalk_d["access_right"] = open_license ? "open" : "closed" + end + crosswalk_d["license"] = project["license"] + dict = OrderedDict{String, String}() + dict["scheme"] = "url" + dict["identifier"] = codemeta["codeRepository"] + dict["relation"] = "isOriginalFormOf" + crosswalk_d["related_identifiers"] = [dict] + if !isnothing(publications) + for doi in publications + dict = OrderedDict{String, String}() + dict["scheme"] = "doi" + dict["identifier"] = doi + dict["relation"] = "isSupplementTo" + push!(crosswalk_d["related_identifiers"], dict) + end + end + crosswalk_d["keywords"] = codemeta["keywords"] + + # Round-trip to ensure correct order if there were elements missing + project = order_project(TOML.parse(sprint(TOML.print, project))) + + # All remote queries have succeeded, so the files can now be written + if !isnothing(license_content) + file = joinpath(git_dir, "LICENSE") + open(file, "w") do io + return write(io, license_content) + end + rm(joinpath(git_dir, "LICENSE.md"), force = true) + end + + file = joinpath(git_dir, "Project.toml") + open(file, "w") do io + return TOML.print(io, project) + end + + file = joinpath(git_dir, "codemeta.json") + open(file, "w") do io + return JSON.print(io, codemeta, 4) + end + + file = joinpath(git_dir, ".zenodo.json") + open(file, "w") do io + return JSON.print(io, crosswalk_d, 4) + end + + # Recursively walk through the directory + path = joinpath(git_dir, ".git") + notpath = joinpath(git_dir, ".github") + for (root, _, files) in walkdir(git_dir) + if !startswith(root, path) || startswith(root, notpath) + for file in files + if endswith(file, ".jl") + jl_file = joinpath(root, file) + data = readlines(jl_file) + if startswith(data[1], "# SPDX-License-Identifier:") + data[1] = "# SPDX-License-Identifier: $(project["license"])" + elseif !startswith(data[1], "### A Pluto.jl notebook ###") + pushfirst!(data, "") + pushfirst!(data, + "# SPDX-License-Identifier: $(project["license"])") + end + open(jl_file, "w") do io + return println.(Ref(io), data) + end + end + end + end + end + + return nothing +end diff --git a/src/project.jl b/src/project.jl new file mode 100644 index 0000000..b581231 --- /dev/null +++ b/src/project.jl @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: BSD-2-Clause + +""" + ResearchSoftwareMetadata.order_project(project_d::AbstractDict) + +Take a parsed `Project.toml` dictionary and return it in its canonical +order in an OrderedDict. +""" +function order_project(project_d::AbstractDict) + project_d = copy(project_d) + project = OrderedDict{String, Any}() + for key in [ + "name", + "uuid", + "license", + "description", + "keywords", + "category", + "development_status", + "publications", + "authors", + "version", + "deps", + "weakdeps", + "extensions", + "compat", + "author_details", + "extras", + "targets" + ] + if haskey(project_d, key) + val = project_d[key] + if val isa AbstractDict + d = OrderedDict{String, Any}() + for k2 in sort(collect(keys(val))) + d[k2] = val[k2] + end + project[key] = d + else + project[key] = val + end + delete!(project_d, key) + end + end + for key in keys(project_d) + project[key] = project_d[key] + end + + return project +end + +""" + ResearchSoftwareMetadata.read_project() + +Read a `Project.toml` file in and return it in its canonical order in +an OrderedDict. +""" +function read_project(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) + file = joinpath(git_dir, "Project.toml") + return order_project(TOML.parsefile(file)) +end diff --git a/src/remotequeries.jl b/src/remotequeries.jl new file mode 100644 index 0000000..6ba7e6b --- /dev/null +++ b/src/remotequeries.jl @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: BSD-2-Clause + +""" + ResearchSoftwareMetadata.get_person_from_orcid(orcid::String) + +Take an `ORCID` from the user and query the orcid.org API to return +a Dict containing the relevant metadata or nothing if no such ORCID exists. +Throws an error if orcid.org cannot be reached or returns an unexpected +HTTP status. +""" +function get_person_from_orcid(orcid::String) + url = "https://pub.orcid.org/v3.0/$orcid" + headers = ["Accept" => "application/json"] + response = HTTP.get(url, headers, status_exception = false) + + if response.status == 200 + data = JSON.parse(String(response.body)) + name = data["person"]["name"] + given_names = name["given-names"]["value"] + family_name = name["family-name"]["value"] + d = Dict("orcid" => orcid, "pid" => data["orcid-identifier"]["uri"], + "givenName" => given_names, "familyName" => family_name, + "full_name" => "$given_names $family_name") + emails = data["person"]["emails"]["email"] + if length(emails) ≥ 1 + d["email"] = emails[1]["email"] + d["name_with_email"] = d["full_name"] * " <" * d["email"] * ">" + else + d["name_with_email"] = d["full_name"] + end + return d + else + response.status == 404 && return nothing + error("Unable to retrieve ORCID $orcid from orcid.org, " * + "HTTP status $(response.status)") + end +end + +""" + ResearchSoftwareMetadata.get_organisation_from_ror(ror::String) + +Take a `ROR` from the user and query the ror.org API to return +a Dict containing the relevant metadata or nothing if no such ROR exists. +Throws an error if ror.org cannot be reached or returns an unexpected +HTTP status. +""" +function get_organisation_from_ror(ror::String) + url = "https://api.ror.org/organizations/$ror" + + response = HTTP.get(url, status_exception = false) + + if response.status == 200 + data = JSON.parse(String(response.body)) + if haskey(data, "name") + name = data["name"] + elseif haskey(data, "names") && length(data["names"]) ≥ 1 + names = [record["value"] for record in data["names"] if "ror_display" in record["types"]] + if length(names) ≥ 1 + name = names[1] + else + @warn "No display name found for ROR $ror" + names = [record["value"] for record in data["names"] if record["lang"] == "en"] + if length(names) ≥ 1 + name = names[1] + else + @info "No English name found for ROR $ror, using first available name" + name = data["names"][1]["value"] + end + end + else + @error "No name found for ROR $ror" + name = "Unknown" + end + d = Dict("name" => name, "ror" => ror, "pid" => data["id"]) + return d + else + response.status == 404 && return nothing + error("Unable to retrieve ROR $ror from ror.org, " * + "HTTP status $(response.status)") + end +end + +""" + ResearchSoftwareMetadata.check_doi(doi::String) + +Check that a DOI resolves by querying the doi.org handle API. Returns +`true` if it resolves and `false` if it does not exist. Throws an error +if doi.org cannot be reached or returns an unexpected HTTP status. +""" +function check_doi(doi::String) + url = "https://doi.org/api/handles/$doi" + response = HTTP.get(url, status_exception = false) + response.status == 200 && return true + response.status == 404 && return false + return error("Unable to check DOI $doi on doi.org, " * + "HTTP status $(response.status)") +end + +""" + ResearchSoftwareMetadata.get_first_release_date() + +Returns the first release date of this package on Julia's `General` +Registry, or today's date if the package has not been registered yet. +Throws an error if the registry cannot be reached or returns an +unexpected HTTP status. +""" +function get_first_release_date(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) + project = read_project(git_dir) + package = project["name"] + url = "https://raw.githubusercontent.com/JuliaRegistries/General/master/$(package[1])/$package/Versions.toml" + headers = ["Accept" => "application/toml"] + response = HTTP.get(url, headers, status_exception = false) + + if response.status == 200 + data = TOML.parse(String(response.body)) + version = minimum(VersionNumber.(keys(data))) + cd(git_dir) + date = readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/v$version`) + return date + elseif response.status == 404 + @info "No release yet on General, imputing first release will be today" + return string(today()) + end + + return error("Unable to query Julia's General registry for $package, " * + "HTTP status $(response.status)") +end diff --git a/src/utils.jl b/src/utils.jl new file mode 100644 index 0000000..5de2d7d --- /dev/null +++ b/src/utils.jl @@ -0,0 +1,219 @@ +# SPDX-License-Identifier: BSD-2-Clause + +""" + ResearchSoftwareMetadata.split_name(full_name::AbstractString) + +Split a full name into given and family names, treating the last word as +the family name. Returns a `(givenName, familyName)` tuple, where +`givenName` is `nothing` if `full_name` contains only a single word. +""" +function split_name(full_name::AbstractString) + parts = split(strip(full_name)) + length(parts) < 2 && return nothing, String(strip(full_name)) + return join(parts[1:(end - 1)], " "), String(parts[end]) +end + +""" + ResearchSoftwareMetadata.parse_author(author::AbstractString) + +Parse a `Project.toml` authors entry of the form "Name " into a +`(name, email)` tuple, where `email` is `nothing` if the entry contains +no email address. Tolerates a missing closing bracket on the email. +""" +function parse_author(author::AbstractString) + m = match(r"^\s*([^<]*?)\s*<\s*([^<>\s]+?)\s*>?\s*$", author) + isnothing(m) && return String(strip(author)), nothing + return String(m.captures[1]), String(m.captures[2]) +end + +""" + ResearchSoftwareMetadata.reconcile!(project, codemeta, proj_key, cm_key; + value = nothing, default = nothing, + to_cm = identity, from_cm = identity) + +Reconcile a metadata field between `Project.toml` (authoritative) and +`codemeta.json`. An explicit `value` (e.g. from a keyword argument to +`crosswalk`) takes precedence and is written into both; otherwise the +`Project.toml` entry is used, fixing `codemeta.json` with a warning if it +disagrees. If the field is missing from `Project.toml` but present in +`codemeta.json`, it is backfilled into `Project.toml`. If it is absent +from both, `default` is used for `codemeta.json` (when provided) without +being backfilled. `to_cm` and `from_cm` convert values between the +`Project.toml` and `codemeta.json` representations. Returns the +`Project.toml`-side value, or `nothing` if the field is absent everywhere. +""" +function reconcile!(project, codemeta, proj_key, cm_key; + value = nothing, default = nothing, + to_cm = identity, from_cm = identity) + if !isnothing(value) + project[proj_key] = value + end + if haskey(project, proj_key) + val = project[proj_key] + cm_val = to_cm(val) + if haskey(codemeta, cm_key) && codemeta[cm_key] ≠ cm_val && + isnothing(value) + @warn "Fixing codemeta.json $cm_key to match Project.toml " * + "($(codemeta[cm_key]) ≠ $cm_val)" + end + codemeta[cm_key] = cm_val + return val + elseif haskey(codemeta, cm_key) + val = from_cm(codemeta[cm_key]) + @info "Backfilling $proj_key into Project.toml from codemeta.json" + project[proj_key] = val + codemeta[cm_key] = to_cm(val) + return val + elseif !isnothing(default) + codemeta[cm_key] = to_cm(default) + return default + end + + return nothing +end + +""" + ResearchSoftwareMetadata.get_os_from_workflows() + +Returns the operating systems that the GitHub workflows associated with this package +work on. This is presumed to represent the operating systems that the software runs on. +""" +function get_os_from_workflows(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) + workflow_folder = joinpath(git_dir, ".github", "workflows") + files = filter(isfile, readdir(workflow_folder, join = true)) + oses = Set{String}() + for file in files + jobs = YAML.load_file(file)["jobs"] + for job in keys(jobs) + os = jobs[job]["runs-on"] + if occursin(r"\${{.*}}", os) + k2s = split(replace(os, r"\${{ *([^ ]*) *}}" => s"\1"), ".") + os = jobs[job]["strategy"] + for k2 in k2s + os = os[k2] + end + if os isa String + push!(oses, os) + else + for val in os + push!(oses, val) + end + end + else + push!(oses, os) + end + end + end + + platforms = Set{String}() + for os in oses + push!(platforms, + replace(replace(replace(os, "ubuntu" => "Linux"), + "windows" => "Windows"), + r"-.*" => "")) + end + + return sort(collect(platforms)) +end + +""" + ResearchSoftwareMetadata.author_details_from_codemeta(cm_authors) + +Reconstruct a `Project.toml` `author_details` array from the `author` +array of a `codemeta.json` file. Each entry contains a `name`, plus an +`orcid`, an `email` and an `affiliation` array where available. +""" +function author_details_from_codemeta(cm_authors) + details = OrderedDict{String, Any}[] + for author in cm_authors + detail = OrderedDict{String, Any}() + if haskey(author, "givenName") && haskey(author, "familyName") + detail["name"] = author["givenName"] * " " * author["familyName"] + elseif haskey(author, "name") + detail["name"] = author["name"] + end + id = get(author, "id", "") + if startswith(id, "https://orcid.org/") + detail["orcid"] = replace(id, "https://orcid.org/" => "") + end + if haskey(author, "email") + detail["email"] = author["email"] + end + if haskey(author, "affiliation") + affiliations = author["affiliation"] + affiliations isa Vector || (affiliations = [affiliations]) + orgs = OrderedDict{String, Any}[] + for org in affiliations + d = OrderedDict{String, Any}() + identifier = get(org, "identifier", "") + if startswith(identifier, "https://ror.org/") + d["ror"] = replace(identifier, "https://ror.org/" => "") + elseif haskey(org, "name") + d["name"] = org["name"] + end + isempty(d) || push!(orgs, d) + end + isempty(orgs) || (detail["affiliation"] = orgs) + end + push!(details, detail) + end + + return details +end + +""" + ResearchSoftwareMetadata.author_details_from_zenodo(creators) + +Reconstruct a `Project.toml` `author_details` array from the `creators` +array of a `.zenodo.json` file. Each entry contains a `name` (reversing +Zenodo's "Family, Given" format), plus an `orcid` and an `affiliation` +array where available. Zenodo does not record email addresses. +""" +function author_details_from_zenodo(creators) + details = OrderedDict{String, Any}[] + for creator in creators + detail = OrderedDict{String, Any}() + if haskey(creator, "name") + parts = split(creator["name"], ", ") + detail["name"] = length(parts) == 2 ? + parts[2] * " " * parts[1] : creator["name"] + end + if haskey(creator, "orcid") + detail["orcid"] = creator["orcid"] + end + d = OrderedDict{String, Any}() + if haskey(creator, "ror") + d["ror"] = creator["ror"] + elseif haskey(creator, "affiliation") + d["name"] = creator["affiliation"] + end + isempty(d) || (detail["affiliation"] = [d]) + push!(details, detail) + end + + return details +end + +""" + ResearchSoftwareMetadata.author_details_consistent(details, proj_authors) + +Check whether a reconstructed `author_details` array is consistent with +the definitive `authors` entries in `Project.toml`. Every entry must match +an author string by name (and email when it has one), and the counts must +agree. +""" +function author_details_consistent(details, proj_authors) + length(details) == length(proj_authors) || return false + for detail in details + haskey(detail, "name") || return false + name = detail["name"] + candidates = haskey(detail, "email") ? + [name * " <" * detail["email"] * ">", name] : [name] + matches(author) = author ∈ candidates || + (!haskey(detail, "email") && + startswith(author, name * " <")) + any(matches, proj_authors) || return false + end + + return true +end diff --git a/src/versioning.jl b/src/versioning.jl new file mode 100644 index 0000000..e33e805 --- /dev/null +++ b/src/versioning.jl @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: BSD-2-Clause + +""" + increase_patch() + +Increases the `Project.toml` version number by a patch (e.g. 0.4.1 to 0.4.2), and then +runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. +""" +function increase_patch(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) + project = read_project(git_dir) + version = project["version"] + v = VersionNumber(version) + new_version = VersionNumber(v.major, v.minor, v.patch + 1) + @info "Bumping patch version from $version to $new_version" + project["version"] = string(new_version) + file = joinpath(git_dir, "Project.toml") + old_project = read(file, String) + open(file, "w") do io + return TOML.print(io, project) + end + + try + return crosswalk(git_dir, update = true) + catch + # Restore Project.toml so a failed crosswalk leaves files unchanged + write(file, old_project) + rethrow() + end +end + +""" + increase_minor() + +Increases the `Project.toml` version number by a minor number (e.g. 0.4.1 to 0.5.0), and then +runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. +""" +function increase_minor(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) + project = read_project(git_dir) + version = project["version"] + v = VersionNumber(version) + new_version = VersionNumber(v.major, v.minor + 1, 0) + @info "Bumping minor version from $version to $new_version" + project["version"] = string(new_version) + file = joinpath(git_dir, "Project.toml") + old_project = read(file, String) + open(file, "w") do io + return TOML.print(io, project) + end + + try + return crosswalk(git_dir, update = true) + catch + # Restore Project.toml so a failed crosswalk leaves files unchanged + write(file, old_project) + rethrow() + end +end + +""" + increase_major() + +Increases the `Project.toml` version number by a major number (e.g. 0.4.1 to 1.0.0), and then +runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. +""" +function increase_major(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) + project = read_project(git_dir) + version = project["version"] + v = VersionNumber(version) + new_version = VersionNumber(v.major + 1, 0, 0) + @info "Bumping major version from $version to $new_version" + project["version"] = string(new_version) + file = joinpath(git_dir, "Project.toml") + old_project = read(file, String) + open(file, "w") do io + return TOML.print(io, project) + end + + try + return crosswalk(git_dir, update = true) + catch + # Restore Project.toml so a failed crosswalk leaves files unchanged + write(file, old_project) + rethrow() + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 88bba05..3ba0fc5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,20 +1,14 @@ # SPDX-License-Identifier: BSD-2-Clause using Git +using JSON using Logging using ResearchSoftwareMetadata using TOML using Test -function is_repo_clean(repo_path) - # Get the status of the repository - statuses = readlines(`$(Git.git()) status -s $repo_path`) - - is_clean = isempty(statuses) - is_clean || @error "\n" * join(statuses, "\n") - - return is_clean -end +include("GitUtils.jl") +using .GitUtils @testset "ResearchSoftwareMetadata.jl" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) @@ -32,3 +26,294 @@ end @test_nowarn ResearchSoftwareMetadata.crosswalk(update = true) @test is_repo_clean(git_dir) end + +@testset "Failed metadata lookups" begin + @test isnothing(ResearchSoftwareMetadata.get_person_from_orcid("0000-0000-0000-0000")) + @test isnothing(ResearchSoftwareMetadata.get_organisation_from_ror("invalid")) +end + +@testset "split_name" begin + @test ResearchSoftwareMetadata.split_name("Ann B Smith") == + ("Ann B", "Smith") + @test ResearchSoftwareMetadata.split_name("Plato") == (nothing, "Plato") +end + +function make_fixture(dir; license = "MIT", extra = "", author_details = true) + project_content = """ + name = "RSMDFixture" + uuid = "d9a1c9c6-91f3-4f9a-8b4a-9b4c8d3a1e2f" + license = "$license" + authors = ["Ann B Smith "] + version = "0.1.0" + $extra + """ + if author_details + project_content *= """ + + [[author_details]] + name = "Ann B Smith" + email = "ann@example.com" + """ + end + open(joinpath(dir, "Project.toml"), "w") do io + return write(io, project_content) + end + src_content = """ + # SPDX-License-Identifier: $license + + module RSMDFixture + end + """ + mkpath(joinpath(dir, "src")) + open(joinpath(dir, "src", "RSMDFixture.jl"), "w") do io + return write(io, src_content) + end + mkpath(joinpath(dir, ".github", "workflows")) + open(joinpath(dir, ".github", "workflows", "testing.yaml"), "w") do io + return write(io, + """ + name: CI + on: push + jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + """) + end + run(`$(Git.git()) -C $dir init -q -b main`) + run(`$(Git.git()) -C $dir remote add origin + https://github.com/example/RSMDFixture.jl`) + run(`$(Git.git()) -C $dir add -A`) + run(`$(Git.git()) -C $dir -c user.name=Test + -c user.email=test@example.com commit -q -m Fixture`) + + return project_content, src_content +end + +@testset "Crosswalk without ORCIDs" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + mktempdir() do dir + make_fixture(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test length(codemeta["author"]) == 1 + author = codemeta["author"][1] + @test author["givenName"] == "Ann B" + @test author["familyName"] == "Smith" + @test author["email"] == "ann@example.com" + @test !haskey(author, "id") + zenodo = JSON.parsefile(joinpath(dir, ".zenodo.json")) + @test [c["name"] for c in zenodo["creators"]] == ["Smith, Ann B"] + end +end + +@testset "Project.toml as metadata source" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + doi = "10.5281/zenodo.12789179" + extra = """ + description = "A fixture package" + keywords = ["fixture", "metadata"] + category = "metadata" + development_status = "wip" + publications = ["$doi"] + """ + mktempdir() do dir + make_fixture(dir, extra = extra) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["description"] == "A fixture package" + @test codemeta["keywords"] == ["fixture", "metadata"] + @test codemeta["applicationCategory"] == "metadata" + @test codemeta["developmentStatus"] == "wip" + @test codemeta["referencePublication"] == ["https://doi.org/$doi"] + zenodo = JSON.parsefile(joinpath(dir, ".zenodo.json")) + @test zenodo["description"] == "A fixture package" + @test zenodo["keywords"] == ["fixture", "metadata"] + @test any(d -> get(d, "scheme", "") == "doi" && + d["identifier"] == doi && + d["relation"] == "isSupplementTo", + zenodo["related_identifiers"]) + end +end + +@testset "Reconstruct author_details" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + # From codemeta.json + mktempdir() do dir + make_fixture(dir, author_details = false) + open(joinpath(dir, "codemeta.json"), "w") do io + return write(io, + """ + { + "author": [ + { + "type": "Person", + "givenName": "Ann B", + "familyName": "Smith", + "email": "ann@example.com", + "affiliation": [ + { + "type": "Organization", + "name": "Example University" + } + ] + } + ] + } + """) + end + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + project = TOML.parsefile(joinpath(dir, "Project.toml")) + @test haskey(project, "author_details") + detail = project["author_details"][1] + @test detail["name"] == "Ann B Smith" + @test detail["email"] == "ann@example.com" + @test detail["affiliation"][1]["name"] == "Example University" + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test length(codemeta["author"]) == 1 + end + # From .zenodo.json when codemeta.json is missing + mktempdir() do dir + make_fixture(dir, author_details = false) + open(joinpath(dir, ".zenodo.json"), "w") do io + return write(io, + """ + { + "creators": [ + { + "name": "Smith, Ann B", + "affiliation": "Example University" + } + ] + } + """) + end + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + project = TOML.parsefile(joinpath(dir, "Project.toml")) + detail = project["author_details"][1] + @test detail["name"] == "Ann B Smith" + @test !haskey(detail, "email") + @test detail["affiliation"][1]["name"] == "Example University" + end + # Inconsistent with authors, so rebuilt from Project.toml alone + mktempdir() do dir + make_fixture(dir, author_details = false) + open(joinpath(dir, "codemeta.json"), "w") do io + return write(io, + """ + { + "author": [ + { + "type": "Person", + "givenName": "Someone", + "familyName": "Else" + } + ] + } + """) + end + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + project = TOML.parsefile(joinpath(dir, "Project.toml")) + @test project["author_details"] == + [Dict("name" => "Ann B Smith", "email" => "ann@example.com")] + # authors is definitive, so the inconsistent codemeta author goes + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test [a["familyName"] for a in codemeta["author"]] == ["Smith"] + end +end + +@testset "Add new author from authors" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + mktempdir() do dir + make_fixture(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + # Add an author to `authors` alone, with a missing closing bracket + # to check the entry gets normalised + toml = joinpath(dir, "Project.toml") + project = TOML.parsefile(toml) + push!(project["authors"], "Bob Jones ", "Bob Jones "] + details = project["author_details"] + @test length(details) == 2 + @test details[2]["name"] == "Bob Jones" + @test details[2]["email"] == "bob@example.com" + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test length(codemeta["author"]) == 2 + @test codemeta["author"][2]["givenName"] == "Bob" + @test codemeta["author"][2]["familyName"] == "Jones" + zenodo = JSON.parsefile(joinpath(dir, ".zenodo.json")) + @test [c["name"] for c in zenodo["creators"]] == + ["Smith, Ann B", "Jones, Bob"] + end +end + +@testset "Backfill Project.toml from codemeta.json" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + mktempdir() do dir + make_fixture(dir) + open(joinpath(dir, "codemeta.json"), "w") do io + return write(io, + """ + { + "description": "A fixture package", + "keywords": ["fixture", "metadata"], + "applicationCategory": "metadata" + } + """) + end + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + project = TOML.parsefile(joinpath(dir, "Project.toml")) + @test project["description"] == "A fixture package" + @test project["keywords"] == ["fixture", "metadata"] + @test project["category"] == "metadata" + # Defaults are not backfilled into Project.toml + @test !haskey(project, "development_status") + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["developmentStatus"] == "active" + end +end + +@testset "Failed crosswalk leaves files unchanged" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + mktempdir() do dir + # An invalid SPDX identifier makes the license lookup fail + project_content, src_content = make_fixture(dir, + license = "Not-A-License") + @test_throws ErrorException ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) # crosswalk leaves the working directory changed + @test read(joinpath(dir, "Project.toml"), String) == project_content + @test read(joinpath(dir, "src", "RSMDFixture.jl"), String) == + src_content + @test !isfile(joinpath(dir, "codemeta.json")) + @test !isfile(joinpath(dir, ".zenodo.json")) + @test !isfile(joinpath(dir, "LICENSE")) + end + + # Does not currently work on Windows runners on GitHub due to file writing issues + if !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" + @testset "RSMD" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + @test isnothing(ResearchSoftwareMetadata.crosswalk()) + global_logger(SimpleLogger(stderr, Logging.Warn)) + @test_nowarn ResearchSoftwareMetadata.crosswalk() + global_logger(SimpleLogger(stderr, Logging.Info)) + @test is_repo_clean(git_dir; strict = haskey(ENV, "RUNNER_OS")) + end + else + @test_broken !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" + end +end From 6c11e11cb9260195e0fdb65594e5c7c12245938f Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Thu, 9 Jul 2026 01:13:53 +0100 Subject: [PATCH 3/4] Time zone fix --- codemeta.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codemeta.json b/codemeta.json index db817d6..2bcb2b8 100644 --- a/codemeta.json +++ b/codemeta.json @@ -15,7 +15,7 @@ "macOS" ], "version": "v0.2.0", - "dateModified": "2026-07-09", + "dateModified": "2026-07-08", "datePublished": "2024-07-18", "downloadUrl": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/archive/refs/tags/v0.2.0.tar.gz", "license": "https://spdx.org/licenses/BSD-2-Clause", From d1279ebd53424d2b0754b776030d48dc831baec3 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Thu, 9 Jul 2026 01:24:29 +0100 Subject: [PATCH 4/4] Give more info is repo is dirty --- codemeta.json | 2 +- test/GitUtils.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codemeta.json b/codemeta.json index 2bcb2b8..db817d6 100644 --- a/codemeta.json +++ b/codemeta.json @@ -15,7 +15,7 @@ "macOS" ], "version": "v0.2.0", - "dateModified": "2026-07-08", + "dateModified": "2026-07-09", "datePublished": "2024-07-18", "downloadUrl": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/archive/refs/tags/v0.2.0.tar.gz", "license": "https://spdx.org/licenses/BSD-2-Clause", diff --git a/test/GitUtils.jl b/test/GitUtils.jl index 8b1ae3c..7ddeda0 100644 --- a/test/GitUtils.jl +++ b/test/GitUtils.jl @@ -55,7 +55,7 @@ function is_repo_clean(repo_path; strict = false) @info msg is_clean = isempty(dirty) - is_clean || @error "Repository not clean:\n" * join(dirty, "\n") + is_clean || @error "Repository not clean:\n" * join(dirty, "\n") * join(readlines(`$(Git.git()) diff $repo_path`), "\n") return is_clean end