Feature: GetJson and GetCsv in short codes or other layout files.

commands/hugo.go

@@ -55,8 +55,8 @@ Complete documentation is available at http://gohugo.io`,
 var hugoCmdV *cobra.Command
 
 //Flags that are to be added to commands.
-var BuildWatch, Draft, Future, UglyUrls, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, NoTimes bool
-var Source, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string
+var BuildWatch, IgnoreCache, Draft, Future, UglyUrls, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, NoTimes bool
+var Source, CacheDir, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string
 
 //Execute adds all child commands to the root command HugoCmd and sets flags appropriately.
 func Execute() {
@@ -83,6 +83,8 @@ func init() {
 	HugoCmd.PersistentFlags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files")
 	HugoCmd.PersistentFlags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file")
 	HugoCmd.PersistentFlags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from")
+	HugoCmd.PersistentFlags().StringVarP(&CacheDir, "cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
+	HugoCmd.PersistentFlags().BoolVarP(&IgnoreCache, "ignoreCache", "", false, "Ignores the cache directory for reading but still writes to it")
 	HugoCmd.PersistentFlags().StringVarP(&Destination, "destination", "d", "", "filesystem path to write files to")
 	HugoCmd.PersistentFlags().StringVarP(&Theme, "theme", "t", "", "theme to use (located in /themes/THEMENAME/)")
 	HugoCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
@@ -126,6 +128,7 @@ func InitializeConfig() {
 	viper.SetDefault("BuildFuture", false)
 	viper.SetDefault("UglyUrls", false)
 	viper.SetDefault("Verbose", false)
+	viper.SetDefault("IgnoreCache", false)
 	viper.SetDefault("CanonifyUrls", false)
 	viper.SetDefault("Taxonomies", map[string]string{"tag": "tags", "category": "categories"})
 	viper.SetDefault("Permalinks", make(hugolib.PermalinkOverrides, 0))
@@ -203,6 +206,24 @@ func InitializeConfig() {
 		viper.Set("WorkingDir", dir)
 	}
 
+	if hugoCmdV.PersistentFlags().Lookup("ignoreCache").Changed {
+		viper.Set("IgnoreCache", IgnoreCache)
+	}
+
+	if CacheDir != "" {
+		if helpers.FilePathSeparator != CacheDir[len(CacheDir)-1:] {
+			CacheDir = CacheDir + helpers.FilePathSeparator
+		}
+		isDir, err := helpers.DirExists(CacheDir, hugofs.SourceFs)
+		utils.CheckErr(err)
+		if isDir == false {
+			mkdir(CacheDir)
+		}
+		viper.Set("CacheDir", CacheDir)
+	} else {
+		viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.SourceFs))
+	}
+
 	if VerboseLog || Logging || (viper.IsSet("LogFile") && viper.GetString("LogFile") != "") {
 		if viper.IsSet("LogFile") && viper.GetString("LogFile") != "" {
 			jww.SetLogFile(viper.GetString("LogFile"))

docs/content/extras/datafiles.md

@@ -5,7 +5,7 @@ date: 2015-01-22
 menu:
   main:
     parent: extras
-next: /extras/highlighting
+next: /extras/dynamiccontent
 prev: /extras/scratch
 title: Data Files
 weight: 90

docs/content/extras/dynamiccontent.md

@@ -0,0 +1,142 @@
+---
+aliases:
+- /doc/dynamiccontent/
+date: 2015-02-14
+menu:
+  main:
+    parent: extras
+next: /extras/highlighting
+prev: /extras/datafiles
+title: Dynamic Content
+weight: 91
+---
+
+Dynamic content with a static site generator? Yes it is possible!
+
+In addition to the [data files](/extras/datafiles/) feature, we have also
+implemented the feature "Dynamic Content", which lets you load
+any [JSON](http://www.json.org/) or
+[CSV](http://en.wikipedia.org/wiki/Comma-separated_values) file
+from nearly any resource.
+
+"Dynamic Content" currently consists of two functions, `getJson`
+and `getCsv`, which are available in **all template files**.
+
+## Implementation details
+
+### Calling the functions with an URL
+
+In any HTML template or Markdown document call the functions like:
+
+
+	{{ $dataJ := getJson "url" }}
+	{{ $dataC := getCsv "separator" "url" }}
+
+
+or if you use a prefix or postfix for the URL the functions
+accept [variadic arguments](http://en.wikipedia.org/wiki/Variadic_function):
+
+	{{ $dataJ := getJson "url prefix" "arg1" "arg2" "arg n" }}
+	{{ $dataC := getCsv  "separator" "url prefix" "arg1" "arg2" "arg n" }}
+
+The separator for `getCsv` must be put on the first position and can be
+only one character long.
+
+All passed arguments will be joined to the final URL, example:
+
+	{{ $urlPre := "https://api.github.com" }}
+	{{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }}
+
+will resolve internally to:
+
+	{{ $gistJ := getJson "https://api.github.com/users/GITHUB_USERNAME/gists" }}
+
+Eventually you can range over the array. This example will output the
+first 5 Github gists for a user:
+
+	<ul>
+		{{ $urlPre := "https://api.github.com" }}
+		{{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }}
+		{{range first 5 $gistJ }}
+			{{ if .public }}
+				<li><a href="{{ .html_url }}" target="_blank">{{.description}}</a></li>
+			{{ end }}
+		{{end}}
+	</ul>
+
+
+### Example for CSV files
+
+For `getCsv` the one character long separator must be placed on the
+first position followed by the URL.
+
+	<table>
+	  <thead>
+	    <tr>
+		<th>Name</th>
+		<th>Position</th>
+		<th>Salary</th>
+	    </tr>
+	  </thead>
+	  <tbody>
+	  {{ $url := "http://a-big-corp.com/finance/employee-salaries.csv" }}
+	  {{ $sep := "," }}
+	  {{ range $i, $r := getCsv $sep $url }}
+	    <tr>
+	      <td>{{ index $r 0 }}</td>
+	      <td>{{ index $r 1 }}</td>
+	      <td>{{ index $r 2 }}</td>
+	    </tr>
+	  {{ end }}
+	  </tbody>
+	</table>
+
+The expression `{{index $r number}}` must be used to output the nth-column from
+the current row.
+
+### Caching of URLs
+
+Each downloaded URL will be cached in the default folder `$TMPDIR/hugo_cache/`.
+The variable `$TMPDIR` will be resolved to your system dependent
+temporary directory.
+
+With the command line flag `--cacheDir` you can specify any folder on
+your system as a caching directory.
+
+If you don't like caching at all, you can fully disable to read from the
+cache with the command line flag `--ignoreCache`. However Hugo will always
+write, on each build of the site, to the cache folder (silent backup).
+
+### Authentication when using REST URLs
+
+Currently you can only use those authentication methods that can
+be put into an URL. [OAuth](http://en.wikipedia.org/wiki/OAuth) or
+other authentication methods are not implemented.
+
+### Loading local files
+
+To load local files with the two functions `getJson` and `getCsv` the
+source files must reside within Hugos working directory. The file
+extension does not matter but the content.
+
+It applies the same output logic as in the topic: *Calling the functions with an URL*.
+
+## Live reload
+
+There is no chance to trigger a [LiveReload](/extras/livereload/) when
+the content of an URL changes. However when a local JSON/CSV file changes
+then a live reload will be triggered of course. Symlinks not supported.
+
+**URLs and Live reload**: If you change any local file and the live reload
+got triggered Hugo will either read the URL content from the cache or, if
+you have disabled the cache, Hugo will re-download the content.
+This can create huge traffic and you may also reach API limits quickly.
+
+As downloading of content takes a while, Hugo stops with processing
+your markdown files until the content has been downloaded.
+
+## Examples
+
+- Photo gallery JSON powered: [https://github.com/pcdummy/hugo-lightslider-example](https://github.com/pcdummy/hugo-lightslider-example)
+- Github Starred Repositories [in a posts](https://github.com/SchumacherFM/blog-cs/blob/master/content%2Fposts%2Fgithub-starred.md) with the related [short code](https://github.com/SchumacherFM/blog-cs/blob/master/layouts%2Fshortcodes%2FghStarred.html).
+- more?

helpers/path.go

@@ -437,3 +437,35 @@ func WriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) {
 	_, err = io.Copy(file, r)
 	return
 }
+
+// GetTempDir returns the OS default temp directory with trailing slash
+// if subPath is not empty then it will be created recursively
+func GetTempDir(subPath string, fs afero.Fs) string {
+	dir := os.TempDir()
+	if FilePathSeparator != dir[len(dir)-1:] {
+		dir = dir + FilePathSeparator
+	}
+	if subPath != "" {
+		// preserve windows backslash :-(
+		if FilePathSeparator == "\\" {
+			subPath = strings.Replace(subPath, "\\", "____", -1)
+		}
+		dir = dir + MakePath(subPath)
+		if FilePathSeparator == "\\" {
+			dir = strings.Replace(dir, "____", "\\", -1)
+		}
+
+		if exists, _ := Exists(dir, fs); exists {
+			return dir
+		}
+
+		err := fs.MkdirAll(dir, 0777) // rwx, rw, r
+		if err != nil {
+			panic(err)
+		}
+		if FilePathSeparator != dir[len(dir)-1:] {
+			dir = dir + FilePathSeparator
+		}
+	}
+	return dir
+}

helpers/path_test.go

@@ -647,3 +647,31 @@ func TestWriteToDisk(t *testing.T) {
 		reader.Seek(0, 0)
 	}
 }
+
+func TestGetTempDir(t *testing.T) {
+	dir := os.TempDir()
+	if FilePathSeparator != dir[len(dir)-1:] {
+		dir = dir + FilePathSeparator
+	}
+	testDir := "hugoTestFolder" + FilePathSeparator
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"", dir},
+		{testDir + "  Foo bar  ", dir + testDir + "--Foo-bar" + FilePathSeparator},
+		{testDir + "Foo.Bar/foo_Bar-Foo", dir + testDir + "Foo.Bar/foo_Bar-Foo" + FilePathSeparator},
+		{testDir + "fOO,bar:foo%bAR", dir + testDir + "fOObarfoobAR" + FilePathSeparator},
+		{testDir + "FOo/BaR.html", dir + testDir + "FOo/BaR.html" + FilePathSeparator},
+		{testDir + "трям/трям", dir + testDir + "трям/трям" + FilePathSeparator},
+		{testDir + "은행", dir + testDir + "은행" + FilePathSeparator},
+		{testDir + "Банковский кассир", dir + testDir + "Банковский-кассир" + FilePathSeparator},
+	}
+
+	for _, test := range tests {
+		output := GetTempDir(test.input, new(afero.MemMapFs))
+		if output != test.expected {
+			t.Errorf("Expected %#v, got %#v\n", test.expected, output)
+		}
+	}
+}

tpl/template.go

@@ -1300,6 +1300,8 @@ func init() {
 		"replace":     Replace,
 		"trim":        Trim,
 		"dateFormat":  DateFormat,
+		"getJson":     GetJson,
+		"getCsv":      GetCsv,
 	}
 
 }