From 1b694069329b8703bac201b2dedef2cdb7b64f0e Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Fri, 1 May 2020 12:46:56 -0400 Subject: [PATCH 01/50] USAGE: howto set citation style using pandoc defaults merges https://github.com/manubot/rootstock/pull/338 closes https://github.com/manubot/rootstock/issues/334 --- USAGE.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/USAGE.md b/USAGE.md index 4cf69ec..58f54d1 100644 --- a/USAGE.md +++ b/USAGE.md @@ -272,9 +272,21 @@ Modifying the manuscript formatting requires modifying the CSS in the file [`bui Common formatting changes, such as [font size](https://github.com/manubot/rootstock/issues/239) and [double spacing](https://github.com/manubot/rootstock/issues/244), can be found by searching the [Rootstock issues](https://github.com/manubot/rootstock/issues). Open a [new issue](https://github.com/manubot/rootstock/issues/new) if you have a new formatting question. -Changing the citation style or which interactive HTML plugins are loaded requires editing the build script [`build/build.sh`](build/build.sh). -The citation style is determined by the Citation Style Language file specified by `CSL_PATH`. -It can be changed to use other existing styles as [described here](https://github.com/manubot/rootstock/issues/242#issuecomment-507688339). +Changing the citation style or which interactive HTML plugins are loaded requires editing the options specified by Pandoc defaults files in [`build/pandoc-defaults`](build/pandoc-defaults). +The citation style is determined by the Citation Style Language file specified in [`common.yaml`](build/pandoc-defaults/common.yaml): + +```yaml +metadata: + csl: build/assets/style.csl +``` + +The value for `metadata.csl` can be a URL, allowing access to thousands of existing styles hosted by [Zotero](https://www.zotero.org/styles) or the [CSL GitHub](https://github.com/citation-style-language/styles). +For example, the following options replace the Manubot citation style with the _PeerJ_ style: + +```yaml +metadata: + csl: https://github.com/citation-style-language/styles/raw/906cd6d43d0c136190ecfbb12f6af0ca794e3c5b/peerj.csl +``` ## Spellchecking From 92f0285a37636017e0a3e231bb64e414b8a0cfb5 Mon Sep 17 00:00:00 2001 From: Anthony Gitter Date: Tue, 5 May 2020 11:21:58 -0500 Subject: [PATCH 02/50] Fix spellcheck bug and track unique spelling errors merges https://github.com/manubot/rootstock/pull/337 refs https://github.com/manubot/rootstock/issues/336 * Track unique spelling errors Fixes a bug in how unique words were calculated. Also stores the list of unique misspelled words in addition to the locations of the misspelled words. * Create separate spellcheck install script * Add spellcheck to GitHub Actions workflow * Search for expanded punctuation in misspelled words --- .appveyor.yml | 9 ++++++--- .github/workflows/manubot.yaml | 7 +++++++ .gitignore | 3 +++ USAGE.md | 2 +- build/build.sh | 20 ++++++++++++++++++-- ci/install-spellcheck.sh | 12 ++++++++++++ ci/install.sh | 7 +++---- 7 files changed, 50 insertions(+), 10 deletions(-) create mode 100755 ci/install-spellcheck.sh diff --git a/.appveyor.yml b/.appveyor.yml index bade8d0..d57e0cb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -44,8 +44,11 @@ test_script: - | if [ "${SPELLCHECK:-}" = "true" ]; then SPELLING_ERRORS_FILENAME=spelling-errors-$APPVEYOR_BUILD_VERSION-${TRIGGERING_COMMIT:0:7}.txt - cp output/spelling-errors.txt $SPELLING_ERRORS_FILENAME; + cp output/spelling-errors.txt $SPELLING_ERRORS_FILENAME appveyor PushArtifact $SPELLING_ERRORS_FILENAME + SPELLING_ERROR_LOCATIONS_FILENAME=spelling-error-locations-$APPVEYOR_BUILD_VERSION-${TRIGGERING_COMMIT:0:7}.txt + cp output/spelling-error-locations.txt $SPELLING_ERROR_LOCATIONS_FILENAME + appveyor PushArtifact $SPELLING_ERROR_LOCATIONS_FILENAME fi build: off @@ -59,8 +62,8 @@ on_success: - appveyor AddMessage "$JOB_MESSAGE is now complete." - | if [ "${SPELLCHECK:-}" = "true" ]; then - SPELLING_ERROR_COUNT=($(wc -l $SPELLING_ERRORS_FILENAME)) - appveyor AddMessage "
Found $SPELLING_ERROR_COUNT potential spelling error(s). Preview:$(cat $SPELLING_ERRORS_FILENAME)" + SPELLING_ERROR_COUNT=($(wc -l $SPELLING_ERROR_LOCATIONS_FILENAME)) + appveyor AddMessage "
Found $SPELLING_ERROR_COUNT potential spelling error(s). Preview:$(head -n 100 $SPELLING_ERROR_LOCATIONS_FILENAME)" appveyor AddMessage "...
" fi diff --git a/.github/workflows/manubot.yaml b/.github/workflows/manubot.yaml index 84c618d..42509c5 100644 --- a/.github/workflows/manubot.yaml +++ b/.github/workflows/manubot.yaml @@ -12,6 +12,7 @@ jobs: runs-on: ubuntu-latest env: GITHUB_PULL_REQUEST_SHA: ${{ github.event.pull_request.head.sha }} + SPELLCHECK: true steps: - name: Set Environment Variables run: | @@ -38,6 +39,12 @@ jobs: environment-file: build/environment.yml auto-activate-base: false miniconda-version: 'latest' + - name: Install Spellcheck + shell: bash --login {0} + run: | + if [ "${SPELLCHECK:-}" = "true" ]; then + bash ci/install-spellcheck.sh + fi - name: Build Manuscript shell: bash --login {0} run: bash build/build.sh diff --git a/.gitignore b/.gitignore index ac39263..e57fe35 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ webpage/v # Manubot cache directory ci/cache +# Pandoc filters downloaded during continuous integration setup +build/assets/spellcheck.lua + # Python __pycache__/ *.pyc diff --git a/USAGE.md b/USAGE.md index 58f54d1..efec927 100644 --- a/USAGE.md +++ b/USAGE.md @@ -293,7 +293,7 @@ metadata: When the `SPELLCHECK` environment variable is `true`, the pandoc [spellcheck filter](https://github.com/pandoc/lua-filters/tree/master/spellcheck) is run. Potential spelling errors will be printed in the continuous integration log along with the files and line numbers in which they appeared. Words in `build/assets/custom-dictionary.txt` are ignored during spellchecking. -Spellchecking is currently only supported for English language manuscripts and with Travis CI and AppVeyor continuous integration services. +Spellchecking is currently only supported for English language manuscripts. ## Manubot feedback diff --git a/build/build.sh b/build/build.sh index 8cde86b..3c35bf4 100755 --- a/build/build.sh +++ b/build/build.sh @@ -85,12 +85,28 @@ fi # Spellcheck if [ "${SPELLCHECK:-}" = "true" ]; then export ASPELL_CONF="add-extra-dicts $(pwd)/build/assets/custom-dictionary.txt; ignore-case true" + + # Identify and store spelling errors + pandoc --lua-filter build/assets/spellcheck.lua output/manuscript.md | sort -fu > output/spelling-errors.txt + echo >&2 "Potential spelling errors:" + cat output/spelling-errors.txt + + # Add additional forms of punctuation that Pandoc converts so that the + # locations can be detected + # Create a new expanded spelling errors file so that the saved artifact + # contains only the original misspelled words + cp output/spelling-errors.txt output/expanded-spelling-errors.txt + grep "’" output/spelling-errors.txt | sed "s/’/'/g" >> output/expanded-spelling-errors.txt || true + + # Find locations of spelling errors # Use "|| true" after grep because otherwise this step of the pipeline will # return exit code 1 if any of the markdown files do not contain a # misspelled word - pandoc --lua-filter spellcheck.lua output/manuscript.md | uniq | while read word; do grep -ion "\<$word\>" content/*.md; done | sort -h -t ":" -k 1b,1 -k2,2 > output/spelling-errors.txt || true + cat output/expanded-spelling-errors.txt | while read word; do grep -ion "\<$word\>" content/*.md; done | sort -h -t ":" -k 1b,1 -k2,2 > output/spelling-error-locations.txt || true echo >&2 "Filenames and line numbers with potential spelling errors:" - cat output/spelling-errors.txt + cat output/spelling-error-locations.txt + + rm output/expanded-spelling-errors.txt fi echo >&2 "Build complete" diff --git a/ci/install-spellcheck.sh b/ci/install-spellcheck.sh new file mode 100755 index 0000000..6d5ac88 --- /dev/null +++ b/ci/install-spellcheck.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +## install-spellcheck.sh: run during a CI build to install Pandoc spellcheck dependencies. + +# Set options for extra caution & debugging +set -o errexit \ + -o pipefail + +sudo apt-get update -y +sudo apt-get install -y aspell aspell-en +wget https://raw.githubusercontent.com/pandoc/lua-filters/13c3fa7e97206413609a48a82575cb43137e037f/spellcheck/spellcheck.lua +mv spellcheck.lua build/assets/spellcheck.lua diff --git a/ci/install.sh b/ci/install.sh index 7d61b21..61928d7 100755 --- a/ci/install.sh +++ b/ci/install.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -## install.sh: run during a Travis CI or AppVeyor build to install the conda environment. +## install.sh: run during a Travis CI or AppVeyor build to install the conda environment +## and the optional Pandoc spellcheck dependencies. # Set options for extra caution & debugging set -o errexit \ @@ -20,7 +21,5 @@ conda activate manubot # Install Spellcheck filter for Pandoc if [ "${SPELLCHECK:-}" = "true" ]; then - sudo apt-get update -y - sudo apt-get install -y aspell aspell-en - wget https://raw.githubusercontent.com/pandoc/lua-filters/1c553017ecc58914c22bf2372902dca4a456929b/spellcheck/spellcheck.lua + bash ci/install-spellcheck.sh fi From 8b9b5ced2c7c963bf3ea5afb8f31f9a4a54ab697 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Thu, 7 May 2020 10:59:18 -0400 Subject: [PATCH 03/50] move build/pandoc-defaults to build/pandoc/defaults (#340) merges https://github.com/manubot/rootstock/pull/340 * Move build/pandoc-defaults to build/pandoc/defaults Improves compatibility with Pandoc's directory naming expectations in case when setting --data-dir=build/assets in the future. Allows for other subdirectories like build/pandoc/filters * set pandoc --data-dir * update spellcheck filter to use build/pandoc/filters --- .gitignore | 2 +- USAGE.md | 4 +-- build/README.md | 2 +- build/build.sh | 27 ++++++++++++------- .../defaults}/common.yaml | 0 .../defaults}/docx.yaml | 0 .../defaults}/html.yaml | 0 .../defaults}/pdf-weasyprint.yaml | 0 ci/install-spellcheck.sh | 8 +++--- 9 files changed, 25 insertions(+), 18 deletions(-) rename build/{pandoc-defaults => pandoc/defaults}/common.yaml (100%) rename build/{pandoc-defaults => pandoc/defaults}/docx.yaml (100%) rename build/{pandoc-defaults => pandoc/defaults}/html.yaml (100%) rename build/{pandoc-defaults => pandoc/defaults}/pdf-weasyprint.yaml (100%) diff --git a/.gitignore b/.gitignore index e57fe35..e48c49e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ webpage/v ci/cache # Pandoc filters downloaded during continuous integration setup -build/assets/spellcheck.lua +build/pandoc/filters/spellcheck.lua # Python __pycache__/ diff --git a/USAGE.md b/USAGE.md index efec927..95f1b35 100644 --- a/USAGE.md +++ b/USAGE.md @@ -272,8 +272,8 @@ Modifying the manuscript formatting requires modifying the CSS in the file [`bui Common formatting changes, such as [font size](https://github.com/manubot/rootstock/issues/239) and [double spacing](https://github.com/manubot/rootstock/issues/244), can be found by searching the [Rootstock issues](https://github.com/manubot/rootstock/issues). Open a [new issue](https://github.com/manubot/rootstock/issues/new) if you have a new formatting question. -Changing the citation style or which interactive HTML plugins are loaded requires editing the options specified by Pandoc defaults files in [`build/pandoc-defaults`](build/pandoc-defaults). -The citation style is determined by the Citation Style Language file specified in [`common.yaml`](build/pandoc-defaults/common.yaml): +Changing the citation style or which interactive HTML plugins are loaded requires editing the options specified by Pandoc defaults files in [`build/pandoc/defaults`](build/pandoc/defaults). +The citation style is determined by the Citation Style Language file specified in [`common.yaml`](build/pandoc/defaults/common.yaml): ```yaml metadata: diff --git a/build/README.md b/build/README.md index 9516922..27bf76d 100644 --- a/build/README.md +++ b/build/README.md @@ -12,7 +12,7 @@ To export DOCX for all CI builds, set an environment variable (see docs for [Git Currently, equation numbers via `pandoc-eqnos` are not supported for DOCX output. Format conversion is done using [Pandoc](https://pandoc.org/MANUAL.html). -`build.sh` calls `pandoc` commands using the options specified in [`pandoc-defaults`](pandoc-defaults). +`build.sh` calls `pandoc` commands using the options specified in [`pandoc/defaults`](pandoc/defaults). Each file specifies a set of pandoc `--defaults` options for a given format. To change the options, either edit the YAML files directly or add additional `--defaults` files. diff --git a/build/build.sh b/build/build.sh index 3c35bf4..c3028c0 100755 --- a/build/build.sh +++ b/build/build.sh @@ -21,8 +21,8 @@ manubot process \ --log-level=INFO # Pandoc's configuration is specified via files of option defaults -# located in the PANDOC_DEFAULTS_DIR directory. -PANDOC_DEFAULTS_DIR="${PANDOC_DEFAULTS_DIR:-build/pandoc-defaults}" +# located in the $PANDOC_DATA_DIR/defaults directory. +PANDOC_DATA_DIR="${PANDOC_DATA_DIR:-build/pandoc}" # Make output directory mkdir -p output @@ -31,8 +31,9 @@ mkdir -p output # https://pandoc.org/MANUAL.html echo >&2 "Exporting HTML manuscript" pandoc --verbose \ - --defaults="$PANDOC_DEFAULTS_DIR/common.yaml" \ - --defaults="$PANDOC_DEFAULTS_DIR/html.yaml" + --data-dir="$PANDOC_DATA_DIR" \ + --defaults=common.yaml \ + --defaults=html.yaml # Return null if docker command is missing, otherwise return path to docker DOCKER_EXISTS="$(command -v docker || true)" @@ -44,9 +45,10 @@ if [ "${BUILD_PDF:-}" != "false" ] && [ -z "$DOCKER_EXISTS" ]; then if [ -L images ]; then rm images; fi # if images is a symlink, remove it ln -s content/images pandoc \ - --defaults="$PANDOC_DEFAULTS_DIR/common.yaml" \ - --defaults="$PANDOC_DEFAULTS_DIR/html.yaml" \ - --defaults="$PANDOC_DEFAULTS_DIR/pdf-weasyprint.yaml" + --data-dir="$PANDOC_DATA_DIR" \ + --defaults=common.yaml \ + --defaults=html.yaml \ + --defaults=pdf-weasyprint.yaml rm images fi @@ -78,8 +80,9 @@ fi if [ "${BUILD_DOCX:-}" = "true" ]; then echo >&2 "Exporting Word Docx manuscript" pandoc --verbose \ - --defaults="$PANDOC_DEFAULTS_DIR/common.yaml" \ - --defaults="$PANDOC_DEFAULTS_DIR/docx.yaml" + --data-dir="$PANDOC_DATA_DIR" \ + --defaults=common.yaml \ + --defaults=docx.yaml fi # Spellcheck @@ -87,7 +90,11 @@ if [ "${SPELLCHECK:-}" = "true" ]; then export ASPELL_CONF="add-extra-dicts $(pwd)/build/assets/custom-dictionary.txt; ignore-case true" # Identify and store spelling errors - pandoc --lua-filter build/assets/spellcheck.lua output/manuscript.md | sort -fu > output/spelling-errors.txt + pandoc \ + --data-dir="$PANDOC_DATA_DIR" \ + --lua-filter spellcheck.lua \ + output/manuscript.md \ + | sort -fu > output/spelling-errors.txt echo >&2 "Potential spelling errors:" cat output/spelling-errors.txt diff --git a/build/pandoc-defaults/common.yaml b/build/pandoc/defaults/common.yaml similarity index 100% rename from build/pandoc-defaults/common.yaml rename to build/pandoc/defaults/common.yaml diff --git a/build/pandoc-defaults/docx.yaml b/build/pandoc/defaults/docx.yaml similarity index 100% rename from build/pandoc-defaults/docx.yaml rename to build/pandoc/defaults/docx.yaml diff --git a/build/pandoc-defaults/html.yaml b/build/pandoc/defaults/html.yaml similarity index 100% rename from build/pandoc-defaults/html.yaml rename to build/pandoc/defaults/html.yaml diff --git a/build/pandoc-defaults/pdf-weasyprint.yaml b/build/pandoc/defaults/pdf-weasyprint.yaml similarity index 100% rename from build/pandoc-defaults/pdf-weasyprint.yaml rename to build/pandoc/defaults/pdf-weasyprint.yaml diff --git a/ci/install-spellcheck.sh b/ci/install-spellcheck.sh index 6d5ac88..e12661e 100755 --- a/ci/install-spellcheck.sh +++ b/ci/install-spellcheck.sh @@ -6,7 +6,7 @@ set -o errexit \ -o pipefail -sudo apt-get update -y -sudo apt-get install -y aspell aspell-en -wget https://raw.githubusercontent.com/pandoc/lua-filters/13c3fa7e97206413609a48a82575cb43137e037f/spellcheck/spellcheck.lua -mv spellcheck.lua build/assets/spellcheck.lua +sudo apt-get update --yes +sudo apt-get install --yes aspell aspell-en +wget --directory-prefix=build/pandoc/filters \ + https://github.com/pandoc/lua-filters/raw/13c3fa7e97206413609a48a82575cb43137e037f/spellcheck/spellcheck.lua From b7462701ff85854a67d878db4af89d0268bb11d2 Mon Sep 17 00:00:00 2001 From: Daniel Himmelstein Date: Sun, 24 May 2020 19:42:56 -0400 Subject: [PATCH 04/50] Update manubot for more flexible citation processing merges https://github.com/manubot/rootstock/pull/342 Includes major updates to how citations are processed by the pandoc-manubot-cite filter. See the following commit messages for more information: - https://github.com/manubot/manubot/commit/7055bcc6524fdf1ef97d838cf0158973e2061595 - https://github.com/manubot/manubot/commit/47b03e0c202e6a0561e6548ba08a4ad790f56eee Removes requirement to prefix certain citekeys with raw: or tag: Removes support for deprecated `content/citation-tags.tsv`. Switches from tag to alias terminology for citation aliases. closes https://github.com/manubot/manubot/issues/120 Removes pandas and jsonref dependencies, which are no longer needed. --- USAGE.md | 84 ++++++++++++++++++++--------------------- build/environment.yml | 4 +- content/02.delete-me.md | 14 +++---- 3 files changed, 50 insertions(+), 52 deletions(-) diff --git a/USAGE.md b/USAGE.md index 95f1b35..06b7eaf 100644 --- a/USAGE.md +++ b/USAGE.md @@ -76,15 +76,15 @@ We recommend always specifying the width of SVG images (even if just `width="100 ### Citations -Manubot supports Pandoc [citations](https://pandoc.org/MANUAL.html#citations). +Manubot supports [Pandoc citations](https://pandoc.org/MANUAL.html#citations), but with added support for citing persistent identifiers directly. Citations are processed in 3 stages: 1. Pandoc parses the input Markdown to locate citation keys. -2. The [`pandoc-manubot-cite`](https://github.com/manubot/manubot#pandoc-filter) filter automatically retreives the bibliographic metadata for citation keys. -3. The [`pandoc-citeproc`](https://github.com/jgm/pandoc-citeproc/blob/master/man/pandoc-citeproc.1.md) filter renders in-text citations and generates styled references. +2. The [`pandoc-manubot-cite` filter](https://github.com/manubot/manubot#pandoc-filter) automatically retrieves the bibliographic metadata for citation keys. +3. The [`pandoc-citeproc` filter](https://github.com/jgm/pandoc-citeproc/blob/master/man/pandoc-citeproc.1.md) renders in-text citations and generates styled references. -When using Manubot, citation keys should be formatted like `@source:identifier`, -where `source` is one of the options described below. +When citing persistent identifiers, citation keys should be formatted like `@prefix:accession`, +where `prefix` is one of the options described below. When choosing which source to use for a citation, we recommend the following order: 1. DOI (Digital Object Identifier), cite like `@doi:10.15363/thinklab.4`. @@ -92,35 +92,38 @@ When choosing which source to use for a citation, we recommend the following ord shortDOIs begin with `10/` rather than `10.` and can also be cited. For example, Manubot will expand `@doi:10/993` to the DOI above. We suggest using shortDOIs to cite DOIs containing forbidden characters, such as `(` or `)`. -2. PubMed Central ID, cite like `@pmcid:PMC4497619`. -3. PubMed ID, cite like `@pmid:26158728`. +2. PubMed Central ID, cite like `@pmc:PMC4497619`. +3. PubMed ID, cite like `@pubmed:26158728`. 4. _arXiv_ ID, cite like `@arxiv:1508.06576v2`. 5. ISBN (International Standard Book Number), cite like `@isbn:9781339919881`. -6. URL / webpage, cite like `@url:https://nyti.ms/1QUgAt1`. +6. URL / webpage, cite like `@https://nyti.ms/1QUgAt1`. URL citations can be helpful if the above methods return incorrect metadata. - For example, `@doi:10.1038/ng.3834` [incorrectly handles](https://github.com/manubot/manubot/issues/158) the consortium name resulting in a blank author, while `@url:https://doi.org/10.1038/ng.3834` succeeds. - Similarly, `@url:https://doi.org/10.1101/142760` is a [workaround](https://github.com/manubot/manubot/issues/16) to set the journal name of bioRxiv preprints to _bioRxiv_. + For example, `@doi:10.1038/ng.3834` [incorrectly handles](https://github.com/manubot/manubot/issues/158) the consortium name resulting in a blank author, while `@https://doi.org/10.1038/ng.3834` succeeds. + Similarly, `@https://doi.org/10.1101/142760` is a [workaround](https://github.com/manubot/manubot/issues/16) to set the journal name of bioRxiv preprints to _bioRxiv_. 7. Wikidata Items, cite like `@wikidata:Q50051684`. - Note that anyone can edit or add records on [Wikidata](https://www.wikidata.org), so users are encouraged to contribute metadata for hard-to-cite works to Wikidata as an alternative to using a `raw` citation. -8. For references that do not have any of the persistent identifiers above, use a raw citation like `@raw:old-manuscript`. - Metadata for raw citations must be provided manually. + Note that anyone can edit or add records on [Wikidata](https://www.wikidata.org), so users are encouraged to contribute metadata for hard-to-cite works to Wikidata. +8. Any other compact identifier supported by . + Manubot uses the Identifiers.org Resolution Service to support [hundreds](https://github.com/manubot/manubot/blob/7055bcc6524fdf1ef97d838cf0158973e2061595/manubot/cite/handlers.py#L122-L831 "Actual prefix support is determined by this manubot source code.") of [prefixes](https://registry.identifiers.org/registry "Identifiers.org prefix search"). + For example, citing `@clinicaltrials:NCT04280705` will produce the same bibliographic metadata as `@https://identifiers.org/clinicaltrials:NCT04280705` or `@https://clinicaltrials.gov/ct2/show/NCT04280705`. +9. For references that do not have any of the above persistent identifiers, the citation key does not need to include a prefix. + Citing `@old-manuscript` will work, but only if reference metadata is [provided manually](#reference-metadata). Cite multiple items at once like: ```md -Here is a sentence with several citations [@doi:10.15363/thinklab.4; @pmid:26158728; @arxiv:1508.06576; @isbn:9780394603988]. +Here is a sentence with several citations [@doi:10.15363/thinklab.4; @pubmed:26158728; @arxiv:1508.06576; @isbn:9780394603988]. ``` Note that multiple citations must be semicolon separated. Be careful not to cite the same study using identifiers from multiple sources. -For example, the following citations all refer to the same study, but will be treated as separate references: `[@doi:10.7717/peerj.705; @pmcid:PMC4304851; @pmid:25648772]`. +For example, the following citations all refer to the same study, but will be treated as separate references: `[@doi:10.7717/peerj.705; @pmc:PMC4304851; @pubmed:25648772]`. Citation keys must adhere to the syntax described in the [Pandoc manual](https://pandoc.org/MANUAL.html#citations): > The citation key must begin with a letter, digit, or `_`, and may contain alphanumerics, `_`, and internal punctuation characters (`:.#$%&-+?<>~/`). To evaluate whether a citation key fully matches this syntax, try [this online regex](https://regex101.com/r/mXZyY2/latest). -If the citation key is not valid, use the [citation tag](#citation-tag) workaround below. +If the citation key is not valid, use the [citation aliases](#citation-aliases) workaround below. This is required for citation keys that contain forbidden characters such as `;` or `=` or end with a non-alphanumeric character such as `/`. @@ -134,66 +137,63 @@ pandoc: manubot-fail-on-errors: True ``` -#### Citation tags +#### Citation aliases -The system also supports citation tags, which map from one citation key (an alias) to another. -Tags are recommended for the following applications: +The system also supports citation aliases, which map from one citation key (the "alias" or "tag") to another. +Aliases are recommended for the following applications: -1. A citation's identifier contains forbidden characters, you must use a tag. +1. A citation key contains forbidden characters. 2. A single reference is cited many times. - Therefore, it might make sense to define a tag, so if the citation updates (e.g. a newer version becomes available), only a single change is required. + Therefore, it might make sense to define an alias, so if the citation updates (e.g. a newer version becomes available), only a single change is required. -Tags can be defined using Markdown's [link reference syntax](https://spec.commonmark.org/0.29/#link-reference-definitions) as follows: +Aliases can be defined using Markdown's [link reference syntax](https://spec.commonmark.org/0.29/#link-reference-definitions) as follows: ```markdown -Citing a URL containing a `?` character [@tag:my-url]. -Citing a DOI containing parentheses [@doi:my-doi]. +Citing a URL containing a `?` character [@my-url]. +Citing a DOI containing parentheses [@my-doi]. -[@tag:my-url]: url:https://openreview.net/forum?id=HkwoSDPgg -[@tag:my-doi]: doi:10.1016/S0022-2836(05)80360-2 +[@my-url]: https://openreview.net/forum?id=HkwoSDPgg +[@my-doi]: doi:10.1016/S0022-2836(05)80360-2 ``` This syntax is also used by [`pandoc-url2cite`](https://github.com/phiresky/pandoc-url2cite). Make sure to place these link reference definitions in their own paragraphs. These paragraphs can be in any of the content Markdown files. -Another method for defining tags is to define `pandoc.citekey-aliases` in `metadata.yaml`: +Another method for defining aliases is to define `pandoc.citekey-aliases` in `metadata.yaml`: ```yaml pandoc: citekey-aliases: - tag:my-url: url:https://openreview.net/forum?id=HkwoSDPgg - tag:my-doi: doi:10.1016/S0022-2836(05)80360-2 + my-url: https://openreview.net/forum?id=HkwoSDPgg + my-doi: doi:10.1016/S0022-2836(05)80360-2 ``` -For backwards compatibility, tags can also be defined in `content/citation-tags.tsv`. -If `citation-tags.tsv` defines the tag `study-x`, then this study can be cited like `@tag:study-x`. -This method is deprecated. - ## Reference metadata Manubot stores the bibliographic details for references (the set of all cited works) as CSL JSON ([Citation Style Language Items](http://citeproc-js.readthedocs.io/en/latest/csl-json/markup.html#csl-json-items)). -For all citation sources besides `raw`, Manubot automatically generates CSL JSON. +Manubot automatically generates CSL JSON for most persistent identifiers (as described in [Citations](#citations) above). In some cases, automatic metadata retrieval fails or provides incorrect or incomplete information. -Errors are most common for `url` references. +Errors are most common for references generated from scraping HTML metadata from websites. +This occurs most frequently for `https`/`http`/`url` citations as well as identifiers.org prefixes without explicit support listed above. Therefore, Manubot supports user-provided metadata, which we refer to as "manual references". When a manual reference is provided, Manubot uses the supplied metadata and does not attempt to generate it. Manubot searches the `content` directory for files that match the glob pattern `manual-references*.*` and expects that these files contain manual references. [`content/manual-references.json`](content/manual-references.json) is the default file to specify custom CSL JSON metadata. Manual references are matched to citations using their "id" field. -For example, to manually specify the metadata for the citation `@url:https://github.com/manubot/rootstock`, add a CSL JSON Item to `manual-references.json` that contains the following excerpt: +For example, to manually specify the metadata for the citation `@https://github.com/manubot/rootstock`, add a CSL JSON Item to `manual-references.json` that contains the following excerpt: ```json -"id": "url:https://github.com/manubot/rootstock", +"id": "https://github.com/manubot/rootstock", ``` -The metadata for `raw` citations must be provided in a manual reference file (e.g. `manual-references.json`) or an error will occur. -For example, to cite `@raw:private-message` in a manuscript, a corresponding CSL JSON Item is required, such as: +The metadata for unhandled citations — any citation key that is a not a supported persistent ID — must be provided in a manual reference file (e.g. `manual-references.json`) or an error will occur. +For example, to cite `@private-message` in a manuscript, a corresponding CSL JSON Item is required, such as: ```json { - "id": "raw:private-message", + "id": "private-message", "type": "personal_communication", "title": "Personal communication with Doctor X" } @@ -204,10 +204,10 @@ For guidance on what CSL JSON should be like for different document types, refer Manubot offers some support for other bibliographic metadata formats besides CSL JSON, by delegating conversion to the `pandoc-citeproc --bib2json` [utility](https://github.com/jgm/pandoc-citeproc/blob/master/man/pandoc-citeproc.1.md#convert-mode). Formats are inferred from filename extensions. -So, for example, to provide metadata for `@url:https://github.com/manubot/rootstock` in BibTeX format, create the file `content/manual-references.bib` and create an item whose definition starts with the excerpt: +So, for example, to provide metadata for `@https://github.com/manubot/rootstock` in BibTeX format, create the file `content/manual-references.bib` and create an item whose definition starts with the excerpt: ```latex -@misc{url:https://github.com/manubot/rootstock, +@misc{https://github.com/manubot/rootstock, ``` Processed reference metadata in CSL JSON format, either generated by Manubot or specified via manual references, is exported to `references.json`. diff --git a/build/environment.yml b/build/environment.yml index 84bc456..b156d6c 100644 --- a/build/environment.yml +++ b/build/environment.yml @@ -8,7 +8,6 @@ dependencies: - ghp-import=0.5.5 - jinja2=2.11.2 - jsonschema=3.2.0 - - pandas=1.0.3 - pandoc=2.9.2 - pango=1.40.14 - pip=20.0 @@ -20,8 +19,7 @@ dependencies: - yamllint=1.21.0 - pip: - errorhandler==2.0.1 - - git+https://github.com/manubot/manubot@890b76891f139a26d36cd9a4aa652f7e019501f8 - - jsonref==0.2 + - git+https://github.com/manubot/manubot@31968197d1ccd96a46bf092cdba4b575764bb954 - opentimestamps-client==0.7.0 - opentimestamps==0.4.1 - pandoc-eqnos==2.1.1 diff --git a/content/02.delete-me.md b/content/02.delete-me.md index 557caeb..8f2ed07 100644 --- a/content/02.delete-me.md +++ b/content/02.delete-me.md @@ -95,24 +95,24 @@ Bare URL link: Citation by DOI [@doi:10.7554/eLife.32822]. -Citation by PubMed Central ID [@pmcid:PMC6103790]. +Citation by PubMed Central ID [@pmc:PMC6103790]. -Citation by PubMed ID [@pmid:30718888]. +Citation by PubMed ID [@pubmed:30718888]. Citation by Wikidata ID [@wikidata:Q56458321]. Citation by ISBN [@isbn:9780262517638]. -Citation by URL [@url:https://greenelab.github.io/meta-review/]. +Citation by URL [@https://greenelab.github.io/meta-review/]. -Citation by tag [@tag:deep-review]. +Citation by alias [@deep-review]. -Multiple citations can be put inside the same set of brackets [@doi:10.7554/eLife.32822; @tag:deep-review; @isbn:9780262517638]. -Manubot plugins provide easier, more convenient visualization of and navigation between citations [@doi:10.1371/journal.pcbi.1007128; @pmid:30718888; @pmcid:PMC6103790; @tag:deep-review]. +Multiple citations can be put inside the same set of brackets [@doi:10.7554/eLife.32822; @deep-review; @isbn:9780262517638]. +Manubot plugins provide easier, more convenient visualization of and navigation between citations [@doi:10.1371/journal.pcbi.1007128; @pubmed:30718888; @pmc:PMC6103790; @deep-review]. Citation tags (i.e. aliases) can be defined in their own paragraphs using Markdown's reference link syntax: -[@tag:deep-review]: doi:10.1098/rsif.2017.0387 +[@deep-review]: doi:10.1098/rsif.2017.0387 ## Referencing figures, tables, equations From d61267c8ff1d7bc16644f4e15b814c1b4ca2bd31 Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Fri, 5 Jun 2020 16:03:54 -0400 Subject: [PATCH 05/50] webpage: improve Hypothes.is plugin & CSS heading sizes merges https://github.com/manubot/rootstock/pull/345 Enhancements related to Hypothes.is annotations plugin at build/plugins/hypothesis.html. Adds button to load hypothesis plugin, rather than automatically loading it. closes https://github.com/manubot/rootstock/issues/281 PDF build / print page view will no longer include annotations by default, since hypothesis not loaded until activated. closes https://github.com/manubot/rootstock/issues/280 Fix overlap of hypothesis sidebar and table of contents plugin on small screens. closes https://github.com/manubot/rootstock/issues/306 Change default font sizes of h1 through h6 to be larger than base text size, breaking with browser precedent closes https://github.com/manubot/rootstock/issues/293 --- build/plugins/hypothesis.html | 95 +++++++++++++++++++++++++--- build/plugins/table-of-contents.html | 7 +- build/themes/default.html | 73 +++++++++++++++------ content/02.delete-me.md | 4 ++ 4 files changed, 147 insertions(+), 32 deletions(-) diff --git a/build/plugins/hypothesis.html b/build/plugins/hypothesis.html index b38888e..7bd1691 100644 --- a/build/plugins/hypothesis.html +++ b/build/plugins/hypothesis.html @@ -1,6 +1,16 @@ - + - + // hypothesis client script + const embed = 'https://hypothes.is/embed.js'; + // hypothesis annotation count query url + const query = 'https://api.hypothes.is/api/search?limit=0&url=' - + + + + diff --git a/build/plugins/table-of-contents.html b/build/plugins/table-of-contents.html index fc57f78..49d6e01 100644 --- a/build/plugins/table-of-contents.html +++ b/build/plugins/table-of-contents.html @@ -169,14 +169,13 @@ // create toc button const button = document.createElement('button'); button.id = 'toc_button'; - button.innerHTML = document.querySelector( - '.icon_th_list' - ).innerHTML; + button.innerHTML = document.querySelector('.icon_th_list').innerHTML; + button.title = 'Table of Contents'; button.classList.add('icon_button'); // create header text const text = document.createElement('h3'); - text.innerHTML = 'Table of Contents'; + text.innerHTML = 'View Table of Contents'; // create container for toc list const list = document.createElement('div'); diff --git a/build/themes/default.html b/build/themes/default.html index 26675bd..cfff898 100644 --- a/build/themes/default.html +++ b/build/themes/default.html @@ -80,6 +80,26 @@ border-bottom: solid 1px #bdbdbd; } + /* heading font sizes */ + h1 { + font-size: 2em; + } + h2 { + font-size: 1.5em; + } + h3{ + font-size: 1.35em; + } + h4 { + font-size: 1.25em; + } + h5 { + font-size: 1.15em; + } + h6 { + font-size: 1em; + } + /* -------------------------------------------------- */ /* manuscript header */ /* -------------------------------------------------- */ @@ -513,22 +533,6 @@ margin: 15px 0; } - /* heading 1 */ - h1 { - font-size: 1.75em; - } - - /* heading 2 */ - h2 { - font-size: 1.25em; - margin-top: 0; - } - - /* heading 3 */ - h3 { - font-size: 1.10em; - } - /* figures and tables */ figure, table { font-size: 0.85em; @@ -1182,10 +1186,39 @@ /* hypothesis (annotations) plugin */ /* -------------------------------------------------- */ + /* hypothesis activation button */ + #hypothesis_button { + box-sizing: border-box; + position: fixed; + top: 0; + right: 0; + width: 60px; + height: 60px; + background: #ffffff; + border-radius: 0; + border-left: solid 1px #bdbdbd; + border-bottom: solid 1px #bdbdbd; + box-shadow: 0 0 20px rgba(0, 0, 0, 0.05); + z-index: 2; + } + + /* hypothesis button svg */ + #hypothesis_button > svg { + position: relative; + top: -4px; + } + + /* hypothesis annotation count */ + #hypothesis_count { + position: absolute; + left: 0; + right: 0; + bottom: 5px; + } + /* side panel */ .annotator-frame { width: 280px !important; - z-index: 0 !important; } /* match highlight color to rest of theme */ @@ -1203,8 +1236,12 @@ background: #f5f5f5 !important; } - /* always hide toolbar and tooltip on print */ + /* always hide button, toolbar, and tooltip on print */ @media only print { + #hypothesis_button { + display: none; + } + .annotator-frame { display: none !important; } diff --git a/content/02.delete-me.md b/content/02.delete-me.md index 8f2ed07..78d9c7e 100644 --- a/content/02.delete-me.md +++ b/content/02.delete-me.md @@ -63,6 +63,10 @@ Document section headings: #### Heading 4 +##### Heading 5 + +###### Heading 6 + ### A heading centered on its own printed page{.center .page_center} From adc7b8bca102594d8e7d46d93071c4018f0c6944 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Thu, 11 Jun 2020 12:43:21 -0400 Subject: [PATCH 06/50] CSL: add space delimiter across the style merges https://github.com/manubot/rootstock/pull/346 closes https://github.com/manubot/rootstock/issues/155 supersedes https://github.com/manubot/rootstock/pull/158 tested via https://github.com/manubot/manubot/pull/110 refs https://twitter.com/csl_styles/status/1270049414200074243 Add a space delimiter across the style. Fixes some references where there is a missing space after the title, when authors are missing. Set second-field-align="flush" This option is not supported by pandoc yet, but it more closely resembles hanging-indent="true", which pandoc is currently using to layout the bibliography. Co-authored-by: Rintze M. Zelle --- build/assets/style.csl | 44 ++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/build/assets/style.csl b/build/assets/style.csl index 84daaa1..942a043 100644 --- a/build/assets/style.csl +++ b/build/assets/style.csl @@ -1,4 +1,4 @@ - + diff --git a/build/plugins/analytics.html b/build/plugins/analytics.html index 0e641a1..cbab532 100644 --- a/build/plugins/analytics.html +++ b/build/plugins/analytics.html @@ -1,3 +1,5 @@ - - - + diff --git a/build/plugins/anchors.html b/build/plugins/anchors.html index 4f2e6ba..68f1225 100644 --- a/build/plugins/anchors.html +++ b/build/plugins/anchors.html @@ -1,144 +1,127 @@ - - - + + diff --git a/build/plugins/attributes.html b/build/plugins/attributes.html index b968198..7009782 100644 --- a/build/plugins/attributes.html +++ b/build/plugins/attributes.html @@ -1,130 +1,84 @@ - - - diff --git a/build/plugins/core.html b/build/plugins/core.html new file mode 100644 index 0000000..4c680a4 --- /dev/null +++ b/build/plugins/core.html @@ -0,0 +1,129 @@ + + + diff --git a/build/plugins/d3.html b/build/plugins/d3.html index 48c16d8..d04226e 100644 --- a/build/plugins/d3.html +++ b/build/plugins/d3.html @@ -1,16 +1,12 @@ - + + src="https://cdnjs.cloudflare.com/ajax/libs/d3/6.2.0/d3.min.js" + integrity="sha512-C2RveGuPIWqkaLAluvoxyiaN1XYNe5ss11urhZWZYBUA9Ydgj+hfGKPcxCzTwut1/fmjEZR7Ac35f2aycT8Ogw==" + crossorigin="anonymous" +> diff --git a/build/plugins/hypothesis.html b/build/plugins/hypothesis.html index 7bd1691..1986d36 100644 --- a/build/plugins/hypothesis.html +++ b/build/plugins/hypothesis.html @@ -1,104 +1,166 @@ - + + + diff --git a/build/plugins/inline-svg.html b/build/plugins/inline-svg.html index be40242..619a3a1 100644 --- a/build/plugins/inline-svg.html +++ b/build/plugins/inline-svg.html @@ -1,62 +1,53 @@ - - - diff --git a/build/plugins/jump-to-first.html b/build/plugins/jump-to-first.html index e8a8088..d205093 100644 --- a/build/plugins/jump-to-first.html +++ b/build/plugins/jump-to-first.html @@ -1,272 +1,102 @@ - - - + + diff --git a/build/plugins/lightbox.html b/build/plugins/lightbox.html index 6b9f2ae..1fbdf77 100644 --- a/build/plugins/lightbox.html +++ b/build/plugins/lightbox.html @@ -1,620 +1,660 @@ - - - + + diff --git a/build/plugins/link-highlight.html b/build/plugins/link-highlight.html index 4462f50..04263a8 100644 --- a/build/plugins/link-highlight.html +++ b/build/plugins/link-highlight.html @@ -1,184 +1,157 @@ - - - + + diff --git a/build/plugins/math.html b/build/plugins/math.html deleted file mode 100644 index f47b24d..0000000 --- a/build/plugins/math.html +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - diff --git a/build/plugins/mathjax.html b/build/plugins/mathjax.html new file mode 100644 index 0000000..14ba331 --- /dev/null +++ b/build/plugins/mathjax.html @@ -0,0 +1,53 @@ + + + + + + + diff --git a/build/plugins/scite.html b/build/plugins/scite.html new file mode 100644 index 0000000..d01fe51 --- /dev/null +++ b/build/plugins/scite.html @@ -0,0 +1,61 @@ + + + + + + + + diff --git a/build/plugins/table-of-contents.html b/build/plugins/table-of-contents.html index e415945..ea24c97 100644 --- a/build/plugins/table-of-contents.html +++ b/build/plugins/table-of-contents.html @@ -1,315 +1,404 @@ - + + diff --git a/build/plugins/tooltips.html b/build/plugins/tooltips.html index f9cbc60..b6fc505 100644 --- a/build/plugins/tooltips.html +++ b/build/plugins/tooltips.html @@ -1,614 +1,507 @@ - - - + + diff --git a/build/themes/default.html b/build/themes/default.html index 4022f31..2396cc4 100644 --- a/build/themes/default.html +++ b/build/themes/default.html @@ -1,60 +1,525 @@ From 52ab9dc8bba4d0c6cee348fa9d7b0b764a89ffcf Mon Sep 17 00:00:00 2001 From: Vincent Rubinetti Date: Fri, 12 Feb 2021 10:49:07 -0500 Subject: [PATCH 21/50] Update scite plugin merges https://github.com/manubot/rootstock/pull/415 --- build/plugins/scite.html | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/build/plugins/scite.html b/build/plugins/scite.html index d01fe51..4d62049 100644 --- a/build/plugins/scite.html +++ b/build/plugins/scite.html @@ -5,11 +5,6 @@ See https://scite.ai/. --> - diff --git a/build/plugins/hypothesis.html b/build/plugins/hypothesis.html index 1986d36..3cd5e10 100644 --- a/build/plugins/hypothesis.html +++ b/build/plugins/hypothesis.html @@ -4,7 +4,7 @@ Allows public annotation of the manuscript. See https://web.hypothes.is/. --> - -