Support basic #+INCLUDE (src/example/export block only)

including org files is more complex - e.g. footnotes need to be namespaced to
their source file. org does this by prefixing each included files footnotes
with a number - but even that is not enough as it doesn't guarantee
uniqueness.

As I don't have a usecase for it, I'll avoid the additional complexity for
now.

Makefile

@@ -22,6 +22,9 @@ case=example
 render:
 	go run main.go org/testdata/$(case).org html | html2text
 
+.PHONY: generate
+generate: generate-gh-pages generate-html-fixtures
+
 .PHONY: generate-gh-pages
 generate-gh-pages: build
 	./etc/generate-gh-pages

README.org

@@ -4,13 +4,9 @@ A basic org-mode parser in go
 - have a org-mode AST to play around with building an org-mode language server
 - hopefully add reasonable org-mode support to hugo - sadly [[https://github.com/chaseadamsio/goorgeous][goorgeous]] is broken & abandoned
 * next
-- handle #+RESULTS: raw and stuff
+- lists with checkboxes
+- property drawers
 - support more keywords: https://orgmode.org/manual/In_002dbuffer-settings.html
-  - #+LINK
-  - #+INCLUDE
-*** TODO [[https://github.com/chaseadamsio/goorgeous/issues/10][#10]]: Support noexport
-*** TODO [[https://github.com/chaseadamsio/goorgeous/issues/31][#31]]: Support #+INCLUDE
-- see https://orgmode.org/manual/Include-files.html
 * resources
 - syntax
   - https://orgmode.org/worg/dev/org-syntax.html

main.go

@@ -21,20 +21,22 @@ func main() {
 		log.Println("USAGE: org FILE OUTPUT_FORMAT")
 		log.Fatal("supported output formats: org, html, html-chroma")
 	}
-	bs, err := ioutil.ReadFile(os.Args[1])
+	path := os.Args[1]
+	bs, err := ioutil.ReadFile(path)
 	if err != nil {
 		log.Fatal(err)
 	}
-	r, out, err := bytes.NewReader(bs), "", nil
+	out, err := "", nil
+	d := org.NewDocument().SetPath(path).Parse(bytes.NewReader(bs))
 	switch strings.ToLower(os.Args[2]) {
 	case "org":
-		out, err = org.NewDocument().Parse(r).Write(org.NewOrgWriter())
+		out, err = d.Write(org.NewOrgWriter())
 	case "html":
-		out, err = org.NewDocument().Parse(r).Write(org.NewHTMLWriter())
+		out, err = d.Write(org.NewHTMLWriter())
 	case "html-chroma":
 		writer := org.NewHTMLWriter()
 		writer.HighlightCodeBlock = highlightCodeBlock
-		out, err = org.NewDocument().Parse(r).Write(writer)
+		out, err = d.Write(writer)
 	default:
 		log.Fatal("Unsupported output format")
 	}

org/document.go

@@ -111,6 +111,11 @@ func (d *Document) Parse(input io.Reader) *Document {
 	return d
 }
 
+func (d *Document) SetPath(path string) *Document {
+	d.Path = path
+	return d
+}
+
 func (d *Document) FrontMatter(input io.Reader, f func(string, string) interface{}) (_ map[string]interface{}, err error) {
 	defer func() {
 		d.tokens = nil

org/html.go

@@ -11,7 +11,6 @@ import (
 
 type HTMLWriter struct {
 	stringBuilder
-	document           *Document
 	HighlightCodeBlock func(source, lang string) string
 }
 
@@ -65,6 +64,8 @@ func (w *HTMLWriter) writeNodes(ns ...Node) {
 		switch n := n.(type) {
 		case Keyword:
 			w.writeKeyword(n)
+		case Include:
+			w.writeInclude(n)
 		case Comment:
 			continue
 		case NodeWithMeta:
@@ -144,6 +145,10 @@ func (w *HTMLWriter) writeKeyword(k Keyword) {
 	}
 }
 
+func (w *HTMLWriter) writeInclude(i Include) {
+	w.writeNodes(i.Resolve())
+}
+
 func (w *HTMLWriter) writeFootnoteDefinition(f FootnoteDefinition) {
 	n := f.Name
 	w.WriteString(`<div class="footnote-definition">` + "\n")

org/html_test.go

@@ -8,7 +8,7 @@ import (
 func TestHTMLWriter(t *testing.T) {
 	for _, path := range orgTestFiles() {
 		reader, writer := strings.NewReader(fileString(path)), NewHTMLWriter()
-		actual, err := NewDocument().Parse(reader).Write(writer)
+		actual, err := NewDocument().SetPath(path).Parse(reader).Write(writer)
 		if err != nil {
 			t.Errorf("%s\n got error: %s", path, err)
 			continue

org/keyword.go

@@ -2,10 +2,15 @@ package org
 
 import (
 	"encoding/csv"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
 	"regexp"
 	"strings"
 )
 
+type Comment struct{ Content string }
+
 type Keyword struct {
 	Key   string
 	Value string
@@ -21,11 +26,16 @@ type Metadata struct {
 	HTMLAttributes [][]string
 }
 
-type Comment struct{ Content string }
+type Include struct {
+	Keyword
+	Resolve func() Node
+}
 
 var keywordRegexp = regexp.MustCompile(`^(\s*)#\+([^:]+):(\s+(.*)|(\s*)$)`)
 var commentRegexp = regexp.MustCompile(`^(\s*)#(.*)`)
 
+var includeFileRegexp = regexp.MustCompile(`(?i)^"([^"]+)" (src|example|export) (\w+)$`)
+
 func lexKeywordOrComment(line string) (token, bool) {
 	if m := keywordRegexp.FindStringSubmatch(line); m != nil {
 		return token{"keyword", len(m[1]), m[2], m}, true
@@ -41,18 +51,23 @@ func (d *Document) parseComment(i int, stop stopFn) (int, Node) {
 
 func (d *Document) parseKeyword(i int, stop stopFn) (int, Node) {
 	k := parseKeyword(d.tokens[i])
-	if k.Key == "CAPTION" || k.Key == "ATTR_HTML" {
+	switch k.Key {
+	case "INCLUDE":
+		return d.newInclude(k)
+	case "CAPTION", "ATTR_HTML":
 		consumed, node := d.parseAffiliated(i, stop)
 		if consumed != 0 {
 			return consumed, node
 		}
+		fallthrough
+	default:
+		if _, ok := d.BufferSettings[k.Key]; ok {
+			d.BufferSettings[k.Key] = strings.Join([]string{d.BufferSettings[k.Key], k.Value}, "\n")
+		} else {
+			d.BufferSettings[k.Key] = k.Value
+		}
+		return 1, k
 	}
-	if _, ok := d.BufferSettings[k.Key]; ok {
-		d.BufferSettings[k.Key] = strings.Join([]string{d.BufferSettings[k.Key], k.Value}, "\n")
-	} else {
-		d.BufferSettings[k.Key] = k.Value
-	}
-	return 1, k
 }
 
 func (d *Document) parseAffiliated(i int, stop stopFn) (int, Node) {
@@ -89,3 +104,21 @@ func parseKeyword(t token) Keyword {
 	k = strings.ToUpper(k)
 	return Keyword{k, v}
 }
+
+func (d *Document) newInclude(k Keyword) (int, Node) {
+	resolve := func() Node { panic(fmt.Sprintf("bad include: '#+INCLUDE: %s'", k.Value)) }
+	if m := includeFileRegexp.FindStringSubmatch(k.Value); m != nil {
+		path, kind, lang := m[1], m[2], m[3]
+		if !filepath.IsAbs(path) {
+			path = filepath.Join(filepath.Dir(d.Path), path)
+		}
+		resolve = func() Node {
+			bs, err := ioutil.ReadFile(path)
+			if err != nil {
+				panic(fmt.Sprintf("bad include '#+INCLUDE: %s': %s", k.Value, err))
+			}
+			return Block{strings.ToUpper(kind), []string{lang}, []Node{Text{string(bs)}}}
+		}
+	}
+	return 1, Include{k, resolve}
+}

org/org.go

@@ -54,6 +54,8 @@ func (w *OrgWriter) writeNodes(ns ...Node) {
 			w.writeComment(n)
 		case Keyword:
 			w.writeKeyword(n)
+		case Include:
+			w.writeKeyword(n.Keyword)
 		case NodeWithMeta:
 			w.writeNodeWithMeta(n)
 		case Headline:

org/org_test.go

@@ -14,7 +14,7 @@ func TestOrgWriter(t *testing.T) {
 	for _, path := range orgTestFiles() {
 		expected := fileString(path)
 		reader, writer := strings.NewReader(expected), NewOrgWriter()
-		actual, err := NewDocument().Parse(reader).Write(writer)
+		actual, err := NewDocument().SetPath(path).Parse(reader).Write(writer)
 		if err != nil {
 			t.Errorf("%s\n got error: %s", path, err)
 			continue

org/testdata/headlines.html

@@ -2,6 +2,3 @@
 <h1>Headline with todo status &amp; priority</h1>
 <h1>Headline with TODO status</h1>
 <h1>Headline with tags &amp; priority</h1>
-<p>
-this one is cheating a little as tags are ALWAYS printed right aligned to a given column number…
-</p>

org/testdata/headlines.org

@@ -2,4 +2,4 @@
 * TODO [#B] Headline with todo status & priority
 * DONE Headline with TODO status
 * [#A] Headline with tags & priority                                :foo:bar:
-this one is cheating a little as tags are ALWAYS printed right aligned to a given column number...
+

org/testdata/misc.html

@@ -15,6 +15,74 @@ or even a <strong>totally</strong> <em>custom</em> kind of block
 crazy ain&#39;t it?
 </p>
 </div>
+<h3><a href="https://github.com/chaseadamsio/goorgeous/issues/31">#31</a>: Support #+INCLUDE</h3>
+<p>
+Note that only src/example/export block inclusion is supported for now.
+There&#39;s quite a lot more to include (see the <a href="https://orgmode.org/manual/Include-files.html">org manual for include files</a>) but I
+don&#39;t have a use case for this yet and stuff like namespacing footnotes of included files
+adds quite a bit of complexity.
+</p>
+<p>
+for now files can be included as:
+</p>
+<ul>
+<li>
+<p>
+src block
+</p>
+<div class="highlight">
+<pre>
+* Simple Headline
+* TODO [#B] Headline with todo status &amp; priority
+* DONE Headline with TODO status
+* [#A] Headline with tags &amp; priority                                :foo:bar:
+
+
+</pre>
+</div>
+</li>
+<li>
+<p>
+export block
+</p>
+<p>
+Paragraphs are the default element.
+</p>
+<p>
+Empty lines and other elements end paragraphs - but paragraphs
+can
+obviously
+span
+multiple
+lines.
+</p>
+<p>
+Paragraphs can contain inline markup like <em>emphasis</em> <strong>strong</strong>  and links <a href="https://www.example.com">example.com</a> and stuff.
+</p>
+
+</li>
+<li>
+<p>
+example block
+</p>
+<pre class="example">
+language: go
+script:
+  - make test
+  - make generate-gh-pages
+deploy:
+  provider: pages
+  github-token: $GITHUB_TOKEN  # From travis-ci.org repository settings
+  local-dir: gh-pages
+  target-branch: gh-pages
+  skip-cleanup: true
+  verbose: true
+  on:
+    branch: master
+
+</pre>
+</li>
+</ul>
 <h3><a href="https://github.com/chaseadamsio/goorgeous/issues/33">#33</a>: Wrong output when mixing html with org-mode</h3>
 <div class="outline-2" id="meta" style="color: green;">
 <table>

org/testdata/misc.org

@@ -12,6 +12,19 @@ verse
 or even a *totally* /custom/ kind of block
 crazy ain't it?
 #+END_CUSTOM
+*** DONE [[https://github.com/chaseadamsio/goorgeous/issues/31][#31]]: Support #+INCLUDE
+Note that only src/example/export block inclusion is supported for now.
+There's quite a lot more to include (see the [[https://orgmode.org/manual/Include-files.html][org manual for include files]]) but I
+don't have a use case for this yet and stuff like namespacing footnotes of included files
+adds quite a bit of complexity.
+
+for now files can be included as:
+- src block
+  #+INCLUDE: "./headlines.org" src org
+- export block
+  #+INCLUDE: "./paragraphs.html" export html
+- example block
+  #+INCLUDE: "../../.travis.yml" example yaml
 *** DONE [[https://github.com/chaseadamsio/goorgeous/issues/33][#33]]: Wrong output when mixing html with org-mode
 #+HTML: <div class="outline-2" id="meta" style="color: green;">
 | *foo* | foo |