diff --git a/build.boot b/build.boot index 87c569fb..6625c82a 100644 --- a/build.boot +++ b/build.boot @@ -11,7 +11,8 @@ [gravatar "1.1.1" :scope "test"] [clj-time "0.12.0" :scope "test"] [mvxcvi/puget "1.0.0" :scope "test"] - [com.novemberain/pantomime "2.8.0" :scope "test"]]) + [com.novemberain/pantomime "2.8.0" :scope "test"] + [org.asciidoctor/asciidoctorj "1.5.4" :scope "test"]]) (require '[adzerk.bootlaces :refer :all]) diff --git a/src/io/perun.clj b/src/io/perun.clj index b6d29c67..23edc5ec 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -142,6 +142,59 @@ (reset! prev-meta final-metadata) (perun/set-meta fileset final-metadata))))) +(def ^:private asciidoctor-deps + '[[org.asciidoctor/asciidoctorj "1.5.4"] + [circleci/clj-yaml "0.5.5"]]) + +(def ^:private +asciidoctor-defaults+ + {:gempath "" ; no given gempath + :libraries ["asciidoctor-diagram"] ; asciidoctor-diagram incl. + :header_footer false ; no full HTML doc + :attributes {:generator "perun" ; context to document + :backend "html5" ; for HTML5 output + :skip-front-matter "" ; skip YAML frontmatter + :showtitle "" ; include

from header + :imagesdir "."}}) ; image dir relative to adoc file + +(deftask asciidoctor + "Parse asciidoc files + + This task will look for files ending with `adoc` (preferred), + `ad`, `asc`, `adoc` or `asciidoc` and add a `:content` key to + their metadata containing the HTML resulting from processing + asciidoc file's content" + [o options OPTS edn "options to be passed to the asciidoctor parser"] + + (let [options (merge +asciidoctor-defaults+ *opts*) + pod (create-pod asciidoctor-deps) + prev-meta (atom {}) + prev-fs (atom nil)] + (boot/with-pre-wrap fileset + (let [ad-files (->> fileset + (boot/fileset-diff @prev-fs) + boot/user-files + (boot/by-ext ["ad" "asc" "adoc" "asciidoc"]) + add-filedata) + ; process all removed asciidoc files + removed? (->> fileset + (boot/fileset-removed @prev-fs) + boot/user-files + (boot/by-ext ["ad" "asc" "adoc" "asciidoc"]) + (map #(boot/tmp-path %)) + set) + updated-files (pod/with-call-in @pod + (io.perun.contrib.asciidoctor/parse-asciidoc ~ad-files ~(merge +asciidoctor-defaults+ options))) + initial-metadata (perun/merge-meta* (perun/get-meta fileset) @prev-meta) + ; Pure merge instead of `merge-with merge` (meta-meta). + ; This is because updated metadata should replace previous metadata to + ; correctly handle cases where a metadata key is removed from post metadata. + final-metadata (vals (merge (perun/key-meta initial-metadata) (perun/key-meta updated-files))) + final-metadata (remove #(-> % :path removed?) final-metadata)] + (reset! prev-fs fileset) + (reset! prev-meta final-metadata) + (perun/set-meta fileset final-metadata))))) +;; TODO Support task option syntax + (deftask global-metadata "Read global metadata from `perun.base.edn` or configured file. diff --git a/src/io/perun/contrib/asciidoctor.clj b/src/io/perun/contrib/asciidoctor.clj new file mode 100644 index 00000000..24598919 --- /dev/null +++ b/src/io/perun/contrib/asciidoctor.clj @@ -0,0 +1,158 @@ +; Copyright (c) 2016 Nico Rikken nico@nicorikken.eu +; All rights reserved. +; The use and distribution terms for this software are covered by the +; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) +; which can be found in the file LICENSE at the root of this distribution. +; By using this software in any fashion, you are agreeing to be bound by +; the terms of this license. +; You must not remove this notice, or any other, from this software. + +(ns io.perun.contrib.asciidoctor + "AsciidoctorJ based converter from Asciidoc to HTML." + (:require [io.perun.core :as perun] + [clojure.java.io :as io] + [clojure.string :as str] + [clj-yaml.core :as yaml] + [io.perun.markdown :as md]) + (:import [org.asciidoctor Asciidoctor Asciidoctor$Factory])) + +(defn keywords->names + "Converts a map with keywords to a map with named keys. Only handles the top + level of any nested structure." + [m] + (reduce-kv #(assoc %1 (name %2) %3) {} m)) + +(defn names->keywords + "Converts a map with named keys to a map with keywords. Only handles the top + level of any nested structure." + [m] + (reduce-kv #(assoc %1 (keyword %2) %3) {} m)) + +(defn normalize-options + "Takes the options for the Asciidoctor parser and puts the in the format + appropriate for handling by the downstream functions. Mostly to better suit + the parsing by the AsciidoctorJ library." + [clj-opts] + (let [atr (-> (:attributes clj-opts) + (keywords->names) + (java.util.HashMap.)) + opts (assoc clj-opts :attributes atr)] + (keywords->names opts))) + +(defn base-dir + "Derive the `base_dir` from the meta-data, as a basis for links and inclusions + but also for image generation. The regex will filter out the last part of the + file path, after the last slash (`/`) to get back the base_dir." + [full-path] + (get (re-matches #"(.*\/)[^\/]+" full-path) 1)) + +(defn extract-meta + "Extract the above YAML metadata (front-matter) from the head of the file. + It returns a map with the `:meta` and the `:asciidoc` content. The `:meta` + key contains a map of the metadata, or a `nil` if the extraction or parsing + failed. The `:asciidoc` key contains a string of the remaining Asciidoc + content. + + This function prevents the need to rely on the `skip-front-matter` option in + the AsciidoctorJ conversion process." + [content] + (let [first-line (first (drop-while str/blank? (str/split-lines content))) + start? (= "---" first-line) + splitted (str/split content #"---\n" 3) + finish? (> (count splitted) 2)] + (if (and start? finish?) + ;; metadata was found, try to parse it + (let [;metadata-str (nth splitted 1) + ;adoc-content (nth splitted 2)] + metadata-str (get splitted 1) + adoc-content (get splitted 2)] + (if-let [parsed-yaml (md/normal-colls (yaml/parse-string metadata-str))] + ;; yaml parsing succeeded, return the map + {:meta (assoc parsed-yaml :original true) + :asciidoc adoc-content} + ;; yaml parsing failed, return only the adoc-content + {:meta nil + :asciidoc adoc-content})) + ;; no metadata found, return the original content + {:meta nil + :asciidoc content}))) + +(defn new-adoc-container + "Creates a new AsciidoctorJ (JRuby) container, based on the normalized options + provided." + [n-opts] + (let [acont (Asciidoctor$Factory/create (str (get n-opts "gempath")))] + (doto acont (.requireLibraries (into '() (get n-opts "libraries")))))) + +(defn perunize-meta + [meta] + "Add duplicate entries for the metadata keys gathered from the AsciidoctorJ + parsing using keys that adhere to the Perun specification of keys. The native + AsciidoctorJ keys are still available for reference and debugging." + (merge meta {:author-email (:email meta) + :name (:doctitle meta) + :date-build (:localdate meta) + :date-modified (:docdate meta)})) + +(defn parse-file-metadata + "Read the asciidoc content and derive relevant metadata for use in other Perun + tasks. The document is read in its entirety (.readDocumentStructure instead + of .readDocumentHeader) to have the results of the options reflected into the + resulting metadata. As the document is rendered again, the time-based + attributes will vary from the asciidoc-to-html convertion (doctime, + docdatetime, localdate, localdatetime, localtime)." + [container adoc-content frontmatter n-opts] + (let [attributes (->> (.readDocumentStructure container adoc-content n-opts) + (.getHeader) + (.getAttributes) + (into {}) + (names->keywords))] + (merge frontmatter (perunize-meta attributes)))) + +(defn asciidoc-to-html + "Converts a given string of asciidoc into HTML. The normalized options that + can be provided, influence the behavior of the conversion." + [container adoc-content n-opts] + (.convert container adoc-content n-opts)) + +(defn process-file + "Parses the content of a single file and associates the available metadata to + the resulting html string. The HTML conversion is dispatched." + [container file options] + (perun/report-debug "asciidoctor" "processing asciidoc" (:filename file)) + (let [basedir {:base_dir (base-dir (:full-path file))} + opts (merge-with options {:attributes {:base_dir (base-dir (:full-path file))}}) + n-opts (normalize-options opts) + file-content (-> file :full-path io/file slurp) + extraction (extract-meta file-content) + adoc-content (:asciidoc extraction) + frontmatter (:meta extraction) + ad-metadata (parse-file-metadata container adoc-content frontmatter n-opts) + html (asciidoc-to-html container adoc-content n-opts)] + (merge ad-metadata {:content html} file))) +;; TODO get 'skip-front-matter' attribute working to avoid the extract-meta call + +(defn parse-asciidoc + "The main function of `io.perun.contrib.asciidoctor`. Responsible for parsing + all provided asciidoc files. The actual parsing is dispatched. It accepts a + boot fileset and a map of options. + + The map of options typically includes an array of libraries and an array of + attributes: {:libraries [] :attributes {}}. Libraries are loaded from the + AsciidoctorJ project, and can be loaded specifically + (\"asciidoctor-diagram/ditaa\") or more broadly (\"asciidoctor-diagram\"). + Attributes can be set freely, although a large set has been predefined in the + Asciidoctor project to configure rendering options or set meta-data. + + This will create a new AsciidoctorJ (JRuby) container for parsing the given + set of files. All the downstream operations on the files will use this + container, preventing concurrent parsing. But the container creation and + computing overhead is such that having a couple of AsciidoctorJ containers + only makes sense for large or complex jobs, taking minutes rather than + seconds." + [asciidoc-files options] + (let [n-opts (normalize-options options) + container (new-adoc-container n-opts) + updated-files (doall (map #(process-file container % options ) asciidoc-files))] + (perun/report-info "asciidoctor" "parsed %s asciidoc files" (count asciidoc-files)) + updated-files)) diff --git a/test/io/perun/contrib/asciidoctor_test.clj b/test/io/perun/contrib/asciidoctor_test.clj new file mode 100644 index 00000000..1b4c8058 --- /dev/null +++ b/test/io/perun/contrib/asciidoctor_test.clj @@ -0,0 +1,183 @@ +(ns io.perun.contrib.asciidoctor-test + (:require [clojure.test :refer :all] + [clojure.set :as s] + [io.perun.contrib.asciidoctor :refer :all] + [io.perun])) + +(def sample-adoc " +--- +draft: +name: in my own image +--- += In my own image: Perun +:author: Zeus +:email: zeus@thunderdome.olympus +:revdate: 02-08-907 +:toc: +:description: Some posts are close to your heart... + +I Zeus would like to describe how the god Perun relates to my image. + +[quote, Perun, Having struck Veles] +\"Well, there is your place, remain there!\" + +.No power more godlike then the Clojure power of Perun +[source, clojure] +---- +(deftask build + \"Build blog.\" + [] + (comp (asciidoctor) + (render :renderer renderer))) +---- +") + +(def expected-html "

In my own image: Perun

+
+

I Zeus would like to describe how the god Perun relates to my image.

+
+
+
+\"Well, there is your place, remain there!\" +
+
+— Perun
+Having struck Veles +
+
+
+
No power more godlike then the Clojure power of Perun
+
+
(deftask build
+  \"Build blog.\"
+  []
+  (comp (asciidoctor)
+        (render :renderer renderer)))
+
+
") + +(def expected-meta + {:appendix-caption "Appendix", + :asciidoctor "", + :asciidoctor-version "1.5.4", + :attribute-missing "skip", + :attribute-undefined "drop-line", + :author "Zeus", + :author-email "zeus@thunderdome.olympus", + :authorcount 1, + :authorinitials "Z", + :authors "Zeus", + :backend "html5", + :backend-html5 "", + :backend-html5-doctype-article "", + :basebackend "html", + :basebackend-html "", + :basebackend-html-doctype-article "", + :caution-caption "Caution", + ; :date-build "2016-09-16", + ; :date-modified "2016-09-16" + :description "Some posts are close to your heart..." + ; :docdate "2016-09-16", + ; :docdatetime "2016-09-16 08:31:58 CEST", + :docdir "", + ; :doctime "08:31:58 CEST", + :doctitle "In my own image: Perun" + :doctype "article", + :doctype-article "", + :draft nil, ;; from frontmatter + :email "zeus@thunderdome.olympus" + :embedded "", + :example-caption "Example", + :figure-caption "Figure", + :filetype "html", + :filetype-html "", + :firstname "Zeus" + :generator "perun", + :htmlsyntax "html", + :iconfont-remote "", + :iconsdir "./icons", + :imagesdir "." + :important-caption "Important", + :last-update-label "Last updated", + :linkcss "", + ; :localdate "2016-09-16", + ; :localdatetime "2016-09-16 08:31:58 CEST", + ; :localtime "08:31:58 CEST", + :manname-title "NAME", + :max-include-depth 64, + :name "In my own image: Perun" ;; frontmatter overwritten by doc + :note-caption "Note", + :notitle "", + :original true, ;; from frontmatter + :outfilesuffix ".html", + :prewrap "", + :revdate "02-08-907" + :safe-mode-level 20, + :safe-mode-name "secure", + :safe-mode-secure "", + :showtitle "", + :skip-front-matter "" + :sectids "", + :stylesdir ".", + :stylesheet "", + :table-caption "Table", + :tip-caption "Tip", + :toc "" + :toc-placement "auto", + :toc-title "Table of Contents", + :untitled-label "Untitled", + :user-home ".", + :version-label "Version", + :warning-caption "Warning", + :webfonts ""}) + +(def diagram-sample " += The way Perun does + +[plantuml, lightning-direction] +.... +Branch --|> Velves +.... +") + +(def expected-diagram-html "

The way Perun does

+
+
+\"lightning +
+
") + +(def expected-extraction + {:meta + {:draft nil + :name "in my own image" + :original true}, + :asciidoc + "= In my own image: Perun\n:author: Zeus\n:email: zeus@thunderdome.olympus\n:revdate: 02-08-907\n:toc:\n:description: Some posts are close to your heart...\n\nI Zeus would like to describe how the god Perun relates to my image.\n\n[quote, Perun, Having struck Veles]\n\"Well, there is your place, remain there!\"\n\n.No power more godlike then the Clojure power of Perun\n[source, clojure]\n----\n(deftask build\n \"Build blog.\"\n []\n (comp (asciidoctor)\n (render :renderer renderer)))\n----\n"}) + +(def n-opts (normalize-options @#'io.perun/+asciidoctor-defaults+)) +; deref the private defintion var to circument the private-ness + +(def container (new-adoc-container n-opts)) + +(deftest test-extract-meta + (let [extraction (extract-meta sample-adoc)] + (is (= expected-extraction extraction)))) + +(deftest test-asciidoc-to-html + "Test the `asciidoc-to-html` function on its actual conversion." + (let [rendered (asciidoc-to-html container (:asciidoc (extract-meta sample-adoc)) n-opts)] + (is (= expected-html rendered)))) + +(deftest test-parse-file-metadata + "Test the metadata extraction by `parse-file-metadata`." + (let [extraction (extract-meta sample-adoc) + adoc-content (:asciidoc extraction) + frontmatter (:meta extraction) + metadata (parse-file-metadata container adoc-content frontmatter n-opts)] + (is (s/subset? (into #{} expected-meta) (into #{} metadata))))) + +(deftest convert-with-asciidoctor-diagram + "Test the handling by the `asciidoctor-diagram` library for built-in images" + (let [rendered (asciidoc-to-html container (:asciidoc (extract-meta diagram-sample)) n-opts)] + (is (= expected-diagram-html rendered))))