Add emoji support

This uses the Emoji map from https://github.com/kyokomi/emoji -- but with a custom replacement implementation.

The built-in are fine for most use cases, but in Hugo we do care about pure speed.

The benchmarks below are skewed in Hugo's direction as the source and result is a byte slice,
Kyokomi's implementation works best with strings.

Curious: The easy-to-use `strings.Replacer` is also plenty fast.

```
BenchmarkEmojiKyokomiFprint-4  	   20000	     86038 ns/op	   33960 B/op	     117 allocs/op
BenchmarkEmojiKyokomiSprint-4  	   20000	     83252 ns/op	   38232 B/op	     122 allocs/op
BenchmarkEmojiStringsReplacer-4	  100000	     21092 ns/op	   17248 B/op	      25 allocs/op
BenchmarkHugoEmoji-4           	  500000	      5728 ns/op	     624 B/op	      13 allocs/op
```

Fixes #1891

Author: bep

commands/hugo.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2016 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -301,6 +301,7 @@ func LoadDefaultSettings() {
 	viper.SetDefault("SectionPagesMenu", "")
 	viper.SetDefault("DisablePathToLower", false)
 	viper.SetDefault("HasCJKLanguage", false)
+	viper.SetDefault("EnableEmoji", false)
 }
 
 // InitializeConfig initializes a config file with sensible default configuration flags.

docs/content/overview/configuration.md

@@ -99,6 +99,9 @@ Following is a list of Hugo-defined variables that you can configure and their c
     disableRobotsTXT:           false
     # edit new content with this editor, if provided
     editor:                     ""
+    # Enable Emoji emoticons support for page content.
+    # See www.emoji-cheat-sheet.com
+    enableEmoji:				false
     footnoteAnchorPrefix:       ""
     footnoteReturnLinkContents: ""
     # google analytics tracking id

docs/content/templates/functions.md

@@ -413,6 +413,14 @@ These are formatted with the layout string.
 e.g. `{{ dateFormat "Monday, Jan 2, 2006" "2015-01-21" }}` → "Wednesday, Jan 21, 2015"
 
 
+### emojify
+
+Runs the string through the Emoji emoticons processor. The result will be declared as "safe" so Go templates will not filter it.
+
+See the [Emoji cheat sheet](http://www.emoji-cheat-sheet.com/) for available emoticons.
+
+e.g. `{{ "I :heart: Hugo" | emojify }}`
+
 ### highlight
 Takes a string of code and a language, uses Pygments to return the syntax highlighted code in HTML.
 Used in the [highlight shortcode](/extras/highlighting/).

helpers/emoji.go

@@ -0,0 +1,94 @@
+// Copyright 2016 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package helpers
+
+import (
+	"bytes"
+	"github.com/kyokomi/emoji"
+	"sync"
+)
+
+var (
+	emojiInit sync.Once
+
+	emojis = make(map[string][]byte)
+
+	emojiDelim     = []byte(":")
+	emojiWordDelim = []byte(" ")
+	emojiMaxSize   int
+)
+
+// Emojify "emojifies" the input source.
+// Note that the input byte slice will be modified if needed.
+// See http://www.emoji-cheat-sheet.com/
+func Emojify(source []byte) []byte {
+
+	emojiInit.Do(initEmoji)
+
+	start := 0
+	k := bytes.Index(source[start:], emojiDelim)
+
+	for k != -1 {
+
+		j := start + k
+
+		upper := j + emojiMaxSize
+
+		if upper > len(source) {
+			upper = len(source)
+		}
+
+		endEmoji := bytes.Index(source[j+1:upper], emojiDelim)
+
+		if endEmoji < 0 {
+			break
+		}
+
+		nextWordDelim := bytes.Index(source[j:upper], emojiWordDelim)
+
+		if endEmoji == 0 || (nextWordDelim != -1 && nextWordDelim < endEmoji) {
+			start += endEmoji + 1
+		} else {
+			endKey := endEmoji + j + 2
+			emojiKey := source[j:endKey]
+
+			if emoji, ok := emojis[string(emojiKey)]; ok {
+				source = append(source[:j], append(emoji, source[endKey:]...)...)
+			}
+
+			start += endEmoji
+		}
+
+		if start >= len(source) {
+			break
+		}
+
+		k = bytes.Index(source[start:], emojiDelim)
+	}
+
+	return source
+
+}
+
+func initEmoji() {
+	emojiMap := emoji.CodeMap()
+
+	for k, v := range emojiMap {
+		emojis[k] = []byte(v + emoji.ReplacePadding)
+
+		if len(k) > emojiMaxSize {
+			emojiMaxSize = len(k)
+		}
+	}
+
+}

helpers/emoji_test.go

@@ -0,0 +1,128 @@
+// Copyright 2016 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package helpers
+
+import (
+	"github.com/kyokomi/emoji"
+	"github.com/spf13/hugo/bufferpool"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestEmojiCustom(t *testing.T) {
+	for i, this := range []struct {
+		input  string
+		expect []byte
+	}{
+		{"A :smile: a day", []byte(emoji.Sprint("A :smile: a day"))},
+		{"A few :smile:s a day", []byte(emoji.Sprint("A few :smile:s a day"))},
+		{"A :smile: and a :beer: makes the day for sure.", []byte(emoji.Sprint("A :smile: and a :beer: makes the day for sure."))},
+		{"A :smile: and: a :beer:", []byte(emoji.Sprint("A :smile: and: a :beer:"))},
+		{"A :diamond_shape_with_a_dot_inside: and then some.", []byte(emoji.Sprint("A :diamond_shape_with_a_dot_inside: and then some."))},
+		{":smile:", []byte(emoji.Sprint(":smile:"))},
+		{":smi", []byte(":smi")},
+		{"A :smile:", []byte(emoji.Sprint("A :smile:"))},
+		{":beer:!", []byte(emoji.Sprint(":beer:!"))},
+		{"::smile:", []byte(emoji.Sprint("::smile:"))},
+		{":beer::", []byte(emoji.Sprint(":beer::"))},
+		{" :beer: :", []byte(emoji.Sprint(" :beer: :"))},
+		{":beer: and :smile: and another :beer:!", []byte(emoji.Sprint(":beer: and :smile: and another :beer:!"))},
+		{" :beer: : ", []byte(emoji.Sprint(" :beer: : "))},
+		{"No smilies for you!", []byte("No smilies for you!")},
+		{" The motto: no smiles! ", []byte(" The motto: no smiles! ")},
+		{":hugo_is_the_best_static_gen:", []byte(":hugo_is_the_best_static_gen:")},
+		{"은행 :smile: 은행", []byte(emoji.Sprint("은행 :smile: 은행"))},
+	} {
+		result := Emojify([]byte(this.input))
+
+		if !reflect.DeepEqual(result, this.expect) {
+			t.Errorf("[%d] got '%q' but expected %q", i, result, this.expect)
+		}
+
+	}
+}
+
+// The Emoji benchmarks below are heavily skewed in Hugo's direction:
+//
+// Hugo have a byte slice, wants a byte slice and doesn't mind if the original is modified.
+
+func BenchmarkEmojiKyokomiFprint(b *testing.B) {
+
+	f := func(in []byte) []byte {
+		buff := bufferpool.GetBuffer()
+		defer bufferpool.PutBuffer(buff)
+		emoji.Fprint(buff, string(in))
+
+		bc := make([]byte, buff.Len(), buff.Len())
+		copy(bc, buff.Bytes())
+		return bc
+	}
+
+	doBenchmarkEmoji(b, f)
+}
+
+func BenchmarkEmojiKyokomiSprint(b *testing.B) {
+
+	f := func(in []byte) []byte {
+		return []byte(emoji.Sprint(string(in)))
+	}
+
+	doBenchmarkEmoji(b, f)
+}
+
+func BenchmarkHugoEmoji(b *testing.B) {
+	doBenchmarkEmoji(b, Emojify)
+}
+
+func doBenchmarkEmoji(b *testing.B, f func(in []byte) []byte) {
+
+	type input struct {
+		in     []byte
+		expect []byte
+	}
+
+	data := []struct {
+		input  string
+		expect string
+	}{
+		{"A :smile: a day", emoji.Sprint("A :smile: a day")},
+		{"A :smile: and a :beer: day keeps the doctor away", emoji.Sprint("A :smile: and a :beer: day keeps the doctor away")},
+		{"A :smile: a day and 10 " + strings.Repeat(":beer: ", 10), emoji.Sprint("A :smile: a day and 10 " + strings.Repeat(":beer: ", 10))},
+		{"No smiles today.", "No smiles today."},
+		{"No smiles for you or " + strings.Repeat("you ", 1000), "No smiles for you or " + strings.Repeat("you ", 1000)},
+	}
+
+	var in []input = make([]input, b.N*len(data))
+	var cnt = 0
+	for i := 0; i < b.N; i++ {
+		for _, this := range data {
+			in[cnt] = input{[]byte(this.input), []byte(this.expect)}
+			cnt++
+		}
+	}
+
+	b.ResetTimer()
+	cnt = 0
+	for i := 0; i < b.N; i++ {
+		for j := range data {
+			currIn := in[cnt]
+			cnt++
+			result := f(currIn.in)
+			if len(result) != len(currIn.expect) {
+				b.Fatalf("[%d] emoji std, got \n%q but expected \n%q", j, result, currIn.expect)
+			}
+		}
+
+	}
+}

hugolib/handler_page.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2016 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ import (
 	"github.com/spf13/hugo/source"
 	"github.com/spf13/hugo/tpl"
 	jww "github.com/spf13/jwalterweatherman"
+	"github.com/spf13/viper"
 )
 
 func init() {
@@ -114,6 +115,10 @@ func commonConvert(p *Page, t tpl.Template) HandledResult {
 
 	var err error
 
+	if viper.GetBool("EnableEmoji") {
+		p.rawContent = helpers.Emojify(p.rawContent)
+	}
+
 	renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
 
 	if len(p.contentShortCodes) > 0 {

���tpl/template_funcs.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2016 The Hugo Authors. All rights reserved.
 //
 // Portions Copyright The Go Authors.
 
@@ -1156,6 +1156,19 @@ func jsonify(v interface{}) (template.HTML, error) {
 		return "", err
 	}
 	return template.HTML(b), nil
+
+}
+
+// emojify "emojifies" the given string.
+//
+// See http://www.emoji-cheat-sheet.com/
+func emojify(in interface{}) (template.HTML, error) {
+	str, err := cast.ToStringE(in)
+
+	if err != nil {
+		return "", err
+	}
+	return template.HTML(helpers.Emojify([]byte(str))), nil
 }
 
 func refPage(page interface{}, ref, methodName string) template.HTML {
@@ -1715,6 +1728,7 @@ func init() {
 		"dict":         dictionary,
 		"div":          func(a, b interface{}) (interface{}, error) { return doArithmetic(a, b, '/') },
 		"echoParam":    returnWhenSet,
+		"emojify":      emojify,
 		"eq":           eq,
 		"first":        first,
 		"ge":           ge,