path: root/vendor/golang.org/x/text/message/pipeline/pipeline.go

// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package pipeline provides tools for creating translation pipelines.
//
// NOTE: UNDER DEVELOPMENT. API MAY CHANGE.
package pipeline

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/build"
	"go/parser"
	"io/ioutil"
	"log"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"
	"unicode"

	"golang.org/x/text/language"
	"golang.org/x/text/runes"
	"golang.org/x/tools/go/loader"
)

const (
	extractFile  = "extracted.gotext.json"
	outFile      = "out.gotext.json"
	gotextSuffix = "gotext.json"
)

// Config contains configuration for the translation pipeline.
type Config struct {
	// Supported indicates the languages for which data should be generated.
	// The default is to support all locales for which there are matching
	// translation files.
	Supported []language.Tag

	// --- Extraction

	SourceLanguage language.Tag

	Packages []string

	// --- File structure

	// Dir is the root dir for all operations.
	Dir string

	// TranslationsPattern is a regular expression to match incoming translation
	// files. These files may appear in any directory rooted at Dir.
	// language for the translation files is determined as follows:
	//   1. From the Language field in the file.
	//   2. If not present, from a valid language tag in the filename, separated
	//      by dots (e.g. "en-US.json" or "incoming.pt_PT.xmb").
	//   3. If not present, from a the closest subdirectory in which the file
	//      is contained that parses as a valid language tag.
	TranslationsPattern string

	// OutPattern defines the location for translation files for a certain
	// language. The default is "{{.Dir}}/{{.Language}}/out.{{.Ext}}"
	OutPattern string

	// Format defines the file format for generated translation files.
	// The default is XMB. Alternatives are GetText, XLIFF, L20n, GoText.
	Format string

	Ext string

	// TODO:
	// Actions are additional actions to be performed after the initial extract
	// and merge.
	// Actions []struct {
	// 	Name    string
	// 	Options map[string]string
	// }

	// --- Generation

	// GenFile may be in a different package. It is not defined, it will
	// be written to stdout.
	GenFile string

	// GenPackage is the package or relative path into which to generate the
	// file. If not specified it is relative to the current directory.
	GenPackage string

	// DeclareVar defines a variable to which to assing the generated Catalog.
	DeclareVar string

	// SetDefault determines whether to assign the generated Catalog to
	// message.DefaultCatalog. The default for this is true if DeclareVar is
	// not defined, false otherwise.
	SetDefault bool

	// TODO:
	// - Printf-style configuration
	// - Template-style configuration
	// - Extraction options
	// - Rewrite options
	// - Generation options
}

// Operations:
// - extract:       get the strings
// - disambiguate:  find messages with the same key, but possible different meaning.
// - create out:    create a list of messages that need translations
// - load trans:    load the list of current translations
// - merge:         assign list of translations as done
// - (action)expand:    analyze features and create example sentences for each version.
// - (action)googletrans:   pre-populate messages with automatic translations.
// - (action)export:    send out messages somewhere non-standard
// - (action)import:    load messages from somewhere non-standard
// - vet program:   don't pass "foo" + var + "bar" strings. Not using funcs for translated strings.
// - vet trans:     coverage: all translations/ all features.
// - generate:      generate Go code

// State holds all accumulated information on translations during processing.
type State struct {
	Config Config

	Package string
	program *loader.Program

	Extracted Messages `json:"messages"`

	// Messages includes all messages for which there need to be translations.
	// Duplicates may be eliminated. Generation will be done from these messages
	// (usually after merging).
	Messages []Messages

	// Translations are incoming translations for the application messages.
	Translations []Messages
}

func (s *State) dir() string {
	if d := s.Config.Dir; d != "" {
		return d
	}
	return "./locales"
}

func outPattern(s *State) (string, error) {
	c := s.Config
	pat := c.OutPattern
	if pat == "" {
		pat = "{{.Dir}}/{{.Language}}/out.{{.Ext}}"
	}

	ext := c.Ext
	if ext == "" {
		ext = c.Format
	}
	if ext == "" {
		ext = gotextSuffix
	}
	t, err := template.New("").Parse(pat)
	if err != nil {
		return "", wrap(err, "error parsing template")
	}
	buf := bytes.Buffer{}
	err = t.Execute(&buf, map[string]string{
		"Dir":      s.dir(),
		"Language": "%s",
		"Ext":      ext,
	})
	return filepath.FromSlash(buf.String()), wrap(err, "incorrect OutPattern")
}

var transRE = regexp.MustCompile(`.*\.` + gotextSuffix)

// Import loads existing translation files.
func (s *State) Import() error {
	outPattern, err := outPattern(s)
	if err != nil {
		return err
	}
	re := transRE
	if pat := s.Config.TranslationsPattern; pat != "" {
		if re, err = regexp.Compile(pat); err != nil {
			return wrapf(err, "error parsing regexp %q", s.Config.TranslationsPattern)
		}
	}
	x := importer{s, outPattern, re}
	return x.walkImport(s.dir(), s.Config.SourceLanguage)
}

type importer struct {
	state      *State
	outPattern string
	transFile  *regexp.Regexp
}

func (i *importer) walkImport(path string, tag language.Tag) error {
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil
	}
	for _, f := range files {
		name := f.Name()
		tag := tag
		if f.IsDir() {
			if t, err := language.Parse(name); err == nil {
				tag = t
			}
			// We ignore errors
			if err := i.walkImport(filepath.Join(path, name), tag); err != nil {
				return err
			}
			continue
		}
		for _, l := range strings.Split(name, ".") {
			if t, err := language.Parse(l); err == nil {
				tag = t
			}
		}
		file := filepath.Join(path, name)
		// TODO: Should we skip files that match output files?
		if fmt.Sprintf(i.outPattern, tag) == file {
			continue
		}
		// TODO: handle different file formats.
		if !i.transFile.MatchString(name) {
			continue
		}
		b, err := ioutil.ReadFile(file)
		if err != nil {
			return wrap(err, "read file failed")
		}
		var translations Messages
		if err := json.Unmarshal(b, &translations); err != nil {
			return wrap(err, "parsing translation file failed")
		}
		i.state.Translations = append(i.state.Translations, translations)
	}
	return nil
}

// Merge merges the extracted messages with the existing translations.
func (s *State) Merge() error {
	panic("unimplemented")
	return nil

}

// Export writes out the messages to translation out files.
func (s *State) Export() error {
	panic("unimplemented")
	return nil
}

var (
	ws    = runes.In(unicode.White_Space).Contains
	notWS = runes.NotIn(unicode.White_Space).Contains
)

func trimWS(s string) (trimmed, leadWS, trailWS string) {
	trimmed = strings.TrimRightFunc(s, ws)
	trailWS = s[len(trimmed):]
	if i := strings.IndexFunc(trimmed, notWS); i > 0 {
		leadWS = trimmed[:i]
		trimmed = trimmed[i:]
	}
	return trimmed, leadWS, trailWS
}

// NOTE: The command line tool already prefixes with "gotext:".
var (
	wrap = func(err error, msg string) error {
		if err == nil {
			return nil
		}
		return fmt.Errorf("%s: %v", msg, err)
	}
	wrapf = func(err error, msg string, args ...interface{}) error {
		if err == nil {
			return nil
		}
		return wrap(err, fmt.Sprintf(msg, args...))
	}
	errorf = fmt.Errorf
)

func warnf(format string, args ...interface{}) {
	// TODO: don't log.
	log.Printf(format, args...)
}

func loadPackages(conf *loader.Config, args []string) (*loader.Program, error) {
	if len(args) == 0 {
		args = []string{"."}
	}

	conf.Build = &build.Default
	conf.ParserMode = parser.ParseComments

	// Use the initial packages from the command line.
	args, err := conf.FromArgs(args, false)
	if err != nil {
		return nil, wrap(err, "loading packages failed")
	}

	// Load, parse and type-check the whole program.
	return conf.Load()
}