dati

data and template interface
git clone git://git.gearsix.net/dati
Log | Files | Refs | Atom | README | LICENSE

commit 8d5fb64edeea48893381ebe4a85058fb28849cc2
parent d61c1f8a3acd8e66fe7e8d2e81d86cc6367a9087
Author: gearsix <gearsix@tuta.io>
Date:   Tue, 13 Apr 2021 11:18:08 +0100

major rework to data.go; made api more in-line with std golib

- removed type Data = the user will want to sue their own data type to
load data into
- LoadData now writes output to an interface{} (pointer) passed in
params
- LoadDataFile now takes an interface{} (pointer) to write output to
as well
- removed LoadDataFiles() = user responsibility
- removed GenerateSuperData() = superfluous
- removed MergeData() = superfluous
- updated data_test.go to reflect changes

Diffstat:
Mdata.go | 231++++++++-----------------------------------------------------------------------
Mdata_test.go | 155++++++++++++++++---------------------------------------------------------------
2 files changed, 52 insertions(+), 334 deletions(-)

diff --git a/data.go b/data.go @@ -26,232 +26,45 @@ import ( "io/ioutil" "os" "path/filepath" - "sort" - "strings" - "time" ) -// Data is the data type used to represent parsed Data (in any format). -type Data map[string]interface{} - -func getDataType(path string) string { - return strings.TrimPrefix(filepath.Ext(path), ".") -} - -func loadGlobPaths(paths ...string) ([]string, error) { - var err error - var glob []string - for p, path := range paths { - if strings.Contains(path, "*") { - if glob, err = filepath.Glob(path); err == nil { - paths = append(paths, glob...) - if len(glob) > 0 { - paths = append(paths[:p], paths[p+1:]...) - } - } - } - } - return paths, err -} - -// LoadData reads all data from `in` and loads it in the format set in `lang`. -func LoadData(lang string, in io.Reader) (d Data, e error) { - var inbuf []byte - if inbuf, e = ioutil.ReadAll(in); e != nil { - return make(Data), e +// LoadData attempts to load all data from `in` as the data language `lang` +// and writes the result in the pointer `outp`. +func LoadData(lang string, in io.Reader, outp interface{}) error { + inbuf, e := ioutil.ReadAll(in) + if e != nil { + return e } else if len(inbuf) == 0 { - return make(Data), nil + return nil } if lang == "json" { - e = json.Unmarshal(inbuf, &d) + e = json.Unmarshal(inbuf, outp) } else if lang == "yaml" { - e = yaml.Unmarshal(inbuf, &d) + e = yaml.Unmarshal(inbuf, outp) } else if lang == "toml" { - e = toml.Unmarshal(inbuf, &d) + e = toml.Unmarshal(inbuf, outp) } else { e = fmt.Errorf("'%s' is not a supported data language", lang) } - return + return e } // LoadDataFile loads all the data from the file found at `path` into the the -// format of that files file extension (e.g. "x.json" will be loaded as a json). -func LoadDataFile(path string) (d Data, e error) { - var f *os.File - if f, e = os.Open(path); e == nil { - d, e = LoadData(getDataType(path), f) - } - f.Close() - if e != nil { - e = fmt.Errorf("failed to load data '%s':", e.Error()) - } - return -} - -// LoadDataFiles loads all files in `paths` recursively and sorted them in -// `order`. -func LoadDataFiles(order string, paths ...string) (data []Data, err error) { - var stat os.FileInfo - - paths, err = loadGlobPaths(paths...) - - loaded := make(map[string]Data) - - for i := 0; i < len(paths) && err == nil; i++ { - path := paths[i] - if stat, err = os.Stat(path); err == nil { - if stat.IsDir() { - err = filepath.Walk(path, - func(p string, fi os.FileInfo, e error) error { - if e == nil && !fi.IsDir() { - loaded[p], e = LoadDataFile(p) - } - return e - }) - } else { - loaded[path], err = LoadDataFile(path) - } - } - } - - if err == nil { - data, err = sortFileData(loaded, order) - } - - return data, err -} - -func sortFileData(data map[string]Data, order string) ([]Data, error) { - var err error - sorted := make([]Data, 0, len(data)) - - if strings.HasPrefix(order, "filename") { - if order == "filename-desc" { - sorted = sortFileDataFilename("desc", data) - } else if order == "filename-asc" { - sorted = sortFileDataFilename("asc", data) - } else { - sorted = sortFileDataFilename("asc", data) - } - } else if strings.HasPrefix(order, "modified") { - if order == "modified-desc" { - sorted, err = sortFileDataModified("desc", data) - } else if order == "modified-asc" { - sorted, err = sortFileDataModified("asc", data) - } else { - sorted, err = sortFileDataModified("asc", data) - } - } else { - for _, d := range data { - sorted = append(sorted, d) - } - } - - return sorted, err -} - -func sortFileDataFilename(direction string, data map[string]Data) []Data { - sorted := make([]Data, 0, len(data)) - - fnames := make([]string, 0, len(data)) - for fpath := range data { - fnames = append(fnames, filepath.Base(fpath)) - } - sort.Strings(fnames) - - if direction == "desc" { - for i := len(fnames) - 1; i >= 0; i-- { - for fpath, d := range data { - if fnames[i] == filepath.Base(fpath) { - sorted = append(sorted, d) - } - } - } - } else { - for _, fname := range fnames { - for fpath, d := range data { - if fname == filepath.Base(fpath) { - sorted = append(sorted, d) - } - } +// format of that files extension (e.g. "x.json" will be loaded as a json). +// The result is written to the value pointed at by `outp`. +func LoadDataFile(path string, outp interface{}) error { + f, e := os.Open(path) + defer f.Close() + + if e == nil { + lang := filepath.Ext(path)[1:] // don't include '.' + if e = LoadData(lang, f, outp); e != nil { + e = fmt.Errorf("failed to load data '%s':", e.Error()) } } - return sorted + return e } -func sortFileDataModified(direction string, data map[string]Data) ([]Data, error) { - sorted := make([]Data, 0, len(data)) - - stats := make(map[string]os.FileInfo) - for fpath := range data { - if stat, err := os.Stat(fpath); err != nil { - return nil, err - } else { - stats[fpath] = stat - } - } - - modtimes := make([]time.Time, 0, len(data)) - for _, stat := range stats { - modtimes = append(modtimes, stat.ModTime()) - } - - if direction == "desc" { - sort.Slice(modtimes, func(i, j int) bool { - return modtimes[i].After(modtimes[j]) - }) - } else { - sort.Slice(modtimes, func(i, j int) bool { - return modtimes[i].Before(modtimes[j]) - }) - } - - for _, t := range modtimes { - for fpath, stat := range stats { - if stat.ModTime() == t { - sorted = append(sorted, data[fpath]) - delete(stats, fpath) - break - } - } - } - - return sorted, nil -} - -// GenerateSuperData simply addeds a key named `datakey` to `global` and assigns `data` to it -func GenerateSuperData(datakey string, global Data, data []Data) (super Data, err error) { - super = global - - if len(datakey) == 0 { - datakey = "data" - } - - if super[datakey] != nil { - err = fmt.Errorf("datakey '%s' already exists", datakey) - } else { - super[datakey] = data - } - - return -} - -// MergeData combines all keys in `data` into a single Data object. If there's -// a conflict (duplicate key), the first found value is kept and the conflicting -// values are ignored. -func MergeData(data ...Data) (merged Data, conflicts []string) { - merged = make(Data) - for _, d := range data { - for key, val := range d { - if merged[key] == nil { - merged[key] = val - } else { - conflicts = append(conflicts, key) - } - } - } - return -} diff --git a/data_test.go b/data_test.go @@ -24,7 +24,6 @@ import ( "os" "strings" "testing" - "time" ) var good = map[string]string{ @@ -51,15 +50,12 @@ func writeTestFile(t *testing.T, path string, Data string) { return } -func validateData(t *testing.T, d Data, e error, lang string) { +func validateData(t *testing.T, d interface{}, e error, lang string) { var b []byte if e != nil { t.Error(e) } - if len(d) == 0 { - t.Error("no data loaded") - } switch lang { case "json": @@ -79,148 +75,57 @@ func validateData(t *testing.T, d Data, e error, lang string) { } func TestLoadData(t *testing.T) { - var d Data + var d interface{} var e error for lang, data := range good { - d, e = LoadData(lang, strings.NewReader(data)) + e = LoadData(lang, strings.NewReader(data), &d) validateData(t, d, e, lang) - - if d, e = LoadData(lang, strings.NewReader(badData)); e == nil || len(d) > 0 { - t.Errorf("bad %s passed", lang) - } - - if d, e = LoadData(lang, strings.NewReader("")); e != nil { - t.Errorf("empty file failed for json: %s, %s", d, e) - } } - if d, e = LoadData("invalid", strings.NewReader("shouldn't pass")); e == nil || len(d) > 0 { - t.Errorf("invalid data language passed: %s, %s", d, e) + if e = LoadData("json", strings.NewReader(badData), &d); e == nil { + t.Errorf("bad data passed") } - - return -} - -func validateFileData(t *testing.T, e error, d []Data, dlen int, orderedLangs ...string) { - if e != nil { - t.Error(e) + if e = LoadData("toml", strings.NewReader(""), &d); e != nil { + t.Errorf("empty data failed %s, %s", d, e) } - - if dlen != len(orderedLangs) { - t.Errorf("invalid orderedLangs length (%d should be %d)", len(orderedLangs), dlen) - } - - if len(d) != dlen { - t.Errorf("invalid data length (%d should be %d)", len(d), dlen) + if e = LoadData("void", strings.NewReader("shouldn't pass"), &d); e == nil { + t.Errorf("invalid data language passed: %s, %s", d, e) } - for i, lang := range orderedLangs { - validateData(t, d[i], nil, lang) - } + return } -func TestLoadDataFiles(t *testing.T) { +func TestLoadDataFile(t *testing.T) { + var d interface{} var e error - var p []string - var d []Data + var p string tdir := t.TempDir() - p = append(p, tdir+"/1.yaml") - writeTestFile(t, p[len(p)-1], good["yaml"]) - time.Sleep(100 * time.Millisecond) - p = append(p, tdir+"/good.json") - writeTestFile(t, p[len(p)-1], good["json"]) - time.Sleep(100 * time.Millisecond) - p = append(p, tdir+"/good.toml") - writeTestFile(t, p[len(p)-1], good["toml"]) - - d, e = LoadDataFiles("filename", tdir) - validateFileData(t, e, d, len(p), "yaml", "json", "toml") - - d, e = LoadDataFiles("filename-desc", tdir+"/*") - validateFileData(t, e, d, len(p), "toml", "json", "yaml") - - d, e = LoadDataFiles("modified", p...) - validateFileData(t, e, d, len(p), "yaml", "json", "toml") - - d, e = LoadDataFiles("modified-desc", p...) - validateFileData(t, e, d, len(p), "toml", "json", "yaml") - - p = append(p, tdir+"/bad.json") - writeTestFile(t, p[len(p)-1], badData) - if _, e = LoadDataFiles("modified-desc", p...); e == nil { - t.Error("bad.json passed") + for lang, data := range good { + p = tdir + "/good." + lang + writeTestFile(t, p, data) + e = LoadDataFile(p, &d) + validateData(t, d, e, lang) } -} -func TestGenerateSuperData(t *testing.T) { - var data Data - var e error - var gd Data - var d []Data - var sd Data - - if data, e = LoadData("json", strings.NewReader(good["json"])); e == nil { - gd = data - } else { - t.Skip("setup failure:", e) - } - if data, e = LoadData("json", strings.NewReader(good["json"])); e == nil { - d = append(d, data) - } else { - t.Skip("setup failure:", e) - } - if data, e = LoadData("yaml", strings.NewReader(good["yaml"])); e == nil { - d = append(d, data) - } else { - t.Skip("setup failure:", e) + p = tdir + "/bad.json" + writeTestFile(t, p, badData) + e = LoadDataFile(p, &d) + if e == nil { + t.Errorf("bad data passed") } - sd, e = GenerateSuperData("testdata", gd, d) + p = tdir + "/empty.json" + writeTestFile(t, p, "") + e = LoadDataFile(p, &d) if e != nil { - t.Error(e) + t.Errorf("empty file failed: %s", e) } - if sd["testdata"] == nil { - t.Log(sd) - t.Error("datakey is empty") - } - if v, ok := sd["testdata"].([]Data); ok == false { - t.Log(sd) - t.Error("unable to infer datakey 'testdata'") - } else if len(v) < len(data) { - t.Log(sd) - t.Error("datakey is missing data") - } -} -func TestMergeData(t *testing.T) { - var e error - var d []Data - var m Data - var c []string - - if m, e = LoadData("json", strings.NewReader(good["json"])); e == nil { - d = append(d, m) - } else { - t.Skip("setup failure:", e) - } - if m, e = LoadData("json", strings.NewReader(good["json"])); e == nil { - d = append(d, m) - } else { - t.Skip("setup failure:", e) - } - if m, e = LoadData("yaml", strings.NewReader(good["yaml"])); e == nil { - d = append(d, m) - } else { - t.Skip("setup failure:", e) + if e = LoadDataFile("non-existing-file.toml", &d); e == nil { + t.Errorf("non-existing file passed: %s, %s", d, e) } - m, c = MergeData(d...) - if m["eg"] == nil { - t.Error("missing global keys") - } - if len(c) == 0 { - t.Errorf("conflicting keys were not reported") - } + return }