@@ -24,8 +24,8 @@ import (
2424 "github.com/gohugoio/hugo/common/herrors"
2525 "golang.org/x/text/language"
2626
27- "github.com/gohugoio/go-i18n/v2/i18n"
2827 "github.com/gohugoio/hugo/helpers"
28+ "github.com/nicksnyder/go-i18n/v2/i18n"
2929 toml "github.com/pelletier/go-toml/v2"
3030
3131 "github.com/gohugoio/hugo/deps"
@@ -50,12 +50,7 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error {
5050 if err != nil {
5151 defaultLangTag = language .English
5252 }
53- bundle := i18n .NewBundle (defaultLangTag )
54-
55- bundle .RegisterUnmarshalFunc ("toml" , toml .Unmarshal )
56- bundle .RegisterUnmarshalFunc ("yaml" , metadecoders .UnmarshalYaml )
57- bundle .RegisterUnmarshalFunc ("yml" , metadecoders .UnmarshalYaml )
58- bundle .RegisterUnmarshalFunc ("json" , json .Unmarshal )
53+ builder := newBundleBuilder (defaultLangTag )
5954
6055 w := hugofs .NewWalkway (
6156 hugofs.WalkwayConfig {
@@ -66,53 +61,138 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error {
6661 if info .IsDir () {
6762 return nil
6863 }
69- return addTranslationFile (bundle , source .NewFileInfo (info ))
64+ return builder . addTranslationFile (source .NewFileInfo (info ))
7065 },
7166 })
7267
7368 if err := w .Walk (); err != nil {
7469 return err
7570 }
7671
77- tp .t = NewTranslator (bundle , dst .Conf , dst .Log )
72+ bundle , err := builder .Build ()
73+ if err != nil {
74+ return err
75+ }
76+
77+ tp .t = newTranslator (bundle , dst .Conf , dst .Log )
7878
7979 dst .Translate = tp .t .Func (dst .Conf .Language ().Lang )
8080
8181 return nil
8282}
8383
84- const artificialLangTagPrefix = "art-x-"
84+ func newBundleBuilder (defaultLangTag language.Tag ) * bundleBuilder {
85+ b := i18n .NewBundle (defaultLangTag )
8586
86- func addTranslationFile (bundle * i18n.Bundle , r * source.File ) error {
87- f , err := r .FileInfo ().Meta ().Open ()
88- if err != nil {
89- return fmt .Errorf ("failed to open translations file %q:: %w" , r .LogicalName (), err )
87+ b .RegisterUnmarshalFunc ("toml" , toml .Unmarshal )
88+ b .RegisterUnmarshalFunc ("yaml" , metadecoders .UnmarshalYaml )
89+ b .RegisterUnmarshalFunc ("yml" , metadecoders .UnmarshalYaml )
90+ b .RegisterUnmarshalFunc ("json" , json .Unmarshal )
91+
92+ bb := & bundle {
93+ b : b ,
94+ definedLangs : make (map [language.Tag ]bool ),
95+ undefinedLangs : make (map [language.Tag ]string ),
9096 }
9197
92- b := helpers .ReaderToBytes (f )
93- f .Close ()
98+ return & bundleBuilder {b : bb }
99+ }
100+
101+ type bundleBuilder struct {
102+ b * bundle
103+
104+ // The Go i18n library we use does not support artificial language tags.
105+ // Store them away and add them later using available real language tags.
106+ undefinedLangs []* source.File
107+ }
108+
109+ type bundle struct {
110+ b * i18n.Bundle
111+
112+ // Bundled languages.
113+ definedLangs map [language.Tag ]bool
114+
115+ // Maps an arbitrary but real language tag to a Hugo artificial language key.
116+ undefinedLangs map [language.Tag ]string
117+ }
118+
119+ var errUndefinedLang = fmt .Errorf ("undefined language" )
120+
121+ func (b * bundleBuilder ) Build () (* bundle , error ) {
122+ const retries = 10
123+ for range retries {
124+ if len (b .undefinedLangs ) == 0 {
125+ break
126+ }
127+ var undefinedLangs []* source.File
128+ for _ , r := range b .undefinedLangs {
129+ name := r .LogicalName ()
130+ lang := paths .Filename (name )
131+ var tag language.Tag
132+ // Find an unused language tag.
133+ for _ , t := range languageTags {
134+ if ! b .b .definedLangs [t ] {
135+ tag = t
136+ break
137+ }
138+ }
139+ if tag == language .Und {
140+ return nil , fmt .Errorf ("failed to resolve language for file %q" , r .LogicalName ())
141+ }
142+ ext := paths .Ext (name )
143+ name = tag .String () + ext
144+ if err := b .doAddTranslationFile (r , tag , name ); err != nil {
145+ if err == errUndefinedLang {
146+ undefinedLangs = append (undefinedLangs , r )
147+ continue
148+ }
149+ return nil , err
150+ }
151+ b .b .undefinedLangs [tag ] = lang
152+ }
153+ b .undefinedLangs = undefinedLangs
154+ }
94155
156+ if len (b .undefinedLangs ) != 0 {
157+ return nil , fmt .Errorf ("failed to resolve languages for some translation files" )
158+ }
159+
160+ return b .b , nil
161+ }
162+
163+ func (bb * bundleBuilder ) addTranslationFile (r * source.File ) error {
95164 name := r .LogicalName ()
96165 lang := paths .Filename (name )
97166 tag := language .Make (lang )
98167 if tag == language .Und {
99- try := artificialLangTagPrefix + lang
100- _ , err = language .Parse (try )
101- if err != nil {
102- return fmt .Errorf ("%q: %s" , try , err )
103- }
104- name = artificialLangTagPrefix + name
168+ bb .undefinedLangs = append (bb .undefinedLangs , r )
169+ return nil
170+ }
171+ err := bb .doAddTranslationFile (r , tag , name )
172+
173+ if err == errUndefinedLang {
174+ bb .undefinedLangs = append (bb .undefinedLangs , r )
175+ return nil
105176 }
106177
107- _ , err = bundle .ParseMessageFileBytes (b , name )
178+ return err
179+ }
180+
181+ // Note that name must include the file extension.
182+ func (bb * bundleBuilder ) doAddTranslationFile (r * source.File , tag language.Tag , name string ) error {
183+ f , err := r .FileInfo ().Meta ().Open ()
184+ if err != nil {
185+ return fmt .Errorf ("failed to open translations file %q:: %w" , r .LogicalName (), err )
186+ }
187+
188+ b := helpers .ReaderToBytes (f )
189+ f .Close ()
190+
191+ _ , err = bb .b .b .ParseMessageFileBytes (b , name )
108192 if err != nil {
109193 if strings .Contains (err .Error (), "no plural rule" ) {
110194 // https://github.com/gohugoio/hugo/issues/7798
111- name = artificialLangTagPrefix + name
112- _ , err = bundle .ParseMessageFileBytes (b , name )
113- if err == nil {
114- return nil
115- }
195+ return errUndefinedLang
116196 }
117197 var guidance string
118198 if strings .Contains (err .Error (), "mixed with unreserved keys" ) {
@@ -121,6 +201,8 @@ func addTranslationFile(bundle *i18n.Bundle, r *source.File) error {
121201 return errWithFileContext (fmt .Errorf ("failed to load translations: %w%s" , err , guidance ), r )
122202 }
123203
204+ bb .b .definedLangs [tag ] = true
205+
124206 return nil
125207}
126208
@@ -141,3 +223,86 @@ func errWithFileContext(inerr error, r *source.File) error {
141223
142224 return herrors .NewFileErrorFromName (inerr , realFilename ).UpdateContent (f , nil )
143225}
226+
227+ // A list of languages in no particular order.
228+ var languageTags = []language.Tag {
229+ language .Georgian ,
230+ language .Urdu ,
231+ language .Vietnamese ,
232+ language .Catalan ,
233+ language .Swedish ,
234+ language .Filipino ,
235+ language .Icelandic ,
236+ language .Punjabi ,
237+ language .Persian ,
238+ language .EuropeanPortuguese ,
239+ language .BritishEnglish ,
240+ language .Kannada ,
241+ language .Ukrainian ,
242+ language .EuropeanSpanish ,
243+ language .Arabic ,
244+ language .Kazakh ,
245+ language .Hebrew ,
246+ language .Danish ,
247+ language .Serbian ,
248+ language .SimplifiedChinese ,
249+ language .Lithuanian ,
250+ language .Gujarati ,
251+ language .Italian ,
252+ language .Russian ,
253+ language .Macedonian ,
254+ language .Burmese ,
255+ language .Portuguese ,
256+ language .Bengali ,
257+ language .Swahili ,
258+ language .Tamil ,
259+ language .Zulu ,
260+ language .Croatian ,
261+ language .Dutch ,
262+ language .Khmer ,
263+ language .LatinAmericanSpanish ,
264+ language .Japanese ,
265+ language .AmericanEnglish ,
266+ language .Azerbaijani ,
267+ language .Turkish ,
268+ language .Norwegian ,
269+ language .TraditionalChinese ,
270+ language .Hungarian ,
271+ language .Finnish ,
272+ language .Estonian ,
273+ language .Lao ,
274+ language .Marathi ,
275+ language .Greek ,
276+ language .Korean ,
277+ language .Uzbek ,
278+ language .Latvian ,
279+ language .Nepali ,
280+ language .Albanian ,
281+ language .SerbianLatin ,
282+ language .BrazilianPortuguese ,
283+ language .Romanian ,
284+ language .Chinese ,
285+ language .Amharic ,
286+ language .English ,
287+ language .French ,
288+ language .CanadianFrench ,
289+ language .Indonesian ,
290+ language .Malayalam ,
291+ language .Slovak ,
292+ language .Slovenian ,
293+ language .Telugu ,
294+ language .Thai ,
295+ language .Sinhala ,
296+ language .Armenian ,
297+ language .Czech ,
298+ language .German ,
299+ language .Polish ,
300+ language .Spanish ,
301+ language .Malay ,
302+ language .Mongolian ,
303+ language .Afrikaans ,
304+ language .ModernStandardArabic ,
305+ language .Bulgarian ,
306+ language .Hindi ,
307+ language .Kirghiz ,
308+ }
0 commit comments