From f8f6f1da873e525425cef175a5d145517b566aff Mon Sep 17 00:00:00 2001 From: hartzell Date: Mon, 2 Dec 2019 16:20:18 -0800 Subject: [PATCH 1/3] First cut at generating Spack resource statements --- main.go | 16 +++++++- spack/resource.go | 87 ++++++++++++++++++++++++++++++++++++++++++ spack/resource_test.go | 83 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 spack/resource.go create mode 100644 spack/resource_test.go diff --git a/main.go b/main.go index cde02e6..6de6fd5 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "path" "text/template" + "github.com/dmgk/modules2tuple/spack" "github.com/dmgk/modules2tuple/tuple" ) @@ -28,7 +29,16 @@ func main() { parser := tuple.NewParser(flagPackagePrefix, flagOffline) tuples, errors := parser.Load(args[0]) if len(tuples) != 0 { - fmt.Println(tuples) + if !flagSpack { + fmt.Println(tuples) + } else { + resources, err := spack.Resources(flagAppVersion, tuples) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + fmt.Println(resources) + } } if errors != nil { fmt.Println(errors) @@ -53,7 +63,9 @@ this commit ID translation can be disabled with -offline flag. var ( flagOffline = false + flagSpack = false flagPackagePrefix = "vendor" + flagAppVersion = "" flagVersion = false ) @@ -63,7 +75,9 @@ func init() { basename := path.Base(os.Args[0]) flag.BoolVar(&flagOffline, "offline", flagOffline, "disable network access") + flag.BoolVar(&flagSpack, "spack", flagSpack, "print Spack resource stanzas") flag.StringVar(&flagPackagePrefix, "prefix", "vendor", "package prefix") + flag.StringVar(&flagAppVersion, "app_version", "", "application version (for spack)") flag.BoolVar(&flagVersion, "v", flagVersion, "show version") flag.Usage = func() { diff --git a/spack/resource.go b/spack/resource.go new file mode 100644 index 0000000..898c03c --- /dev/null +++ b/spack/resource.go @@ -0,0 +1,87 @@ +package spack + +import ( + "bytes" + "fmt" + "reflect" + "regexp" + "text/template" + + "github.com/hartzell/modules2tuple/tuple" +) + +type resource struct { + AppVersion string + Name string + RepoURL string + Fetcher string + CommitIdType string + CommitId string + Placement string +} + +func Resources(appVersion string, tt tuple.Tuples) (string, error) { + buf := bytes.Buffer{} + for _, t := range tt { + r, err := Resource(appVersion, t) + if err != nil { + return "", err + } + buf.WriteString(r) + } + return buf.String(), nil +} + +func Resource(appVersion string, t *tuple.Tuple) (string, error) { + var repoSite string + var repoURL string + var fetcher = "git" + + st := reflect.TypeOf(t.Source) + switch st.String() { + case "tuple.GH": + repoSite = "https://github.com" // tuple.GH.Site() returns ""... + repoURL = fmt.Sprintf("%s/%s/%s", repoSite, t.Account, t.Project) + case "tuple.GL": + repoSite = t.Source.Site() + repoURL = fmt.Sprintf("%s/%s/%s", repoSite, t.Account, t.Project) + default: + return "", fmt.Errorf("Unknown site type: %s", st.String()) + } + + commitIdType := "tag" + commitId := t.Tag + matched, err := regexp.MatchString("[0-9a-f]{12}", t.Tag) + if err != nil { + return "", err + } + if matched { + commitIdType = "commit" + // commitId = commitId[:7] + } + + var buf bytes.Buffer + r := resource{ + AppVersion: appVersion, + Name: t.Package, + RepoURL: repoURL, + Fetcher: fetcher, + CommitIdType: commitIdType, + CommitId: commitId, + Placement: fmt.Sprintf("%s/%s", t.Prefix, t.Package), + } + resource_template := ` + resource(name="{{.Name}}", + {{.Fetcher}}="{{.RepoURL}}", + {{.CommitIdType}}="{{.CommitId}}", + destination=".",{{if .AppVersion}} + when="@{{.AppVersion}}",{{end}} + placement="{{.Placement}}")` + templ := template.Must(template.New("resource").Parse(resource_template)) + err = templ.Execute(&buf, r) + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/spack/resource_test.go b/spack/resource_test.go new file mode 100644 index 0000000..98ebff2 --- /dev/null +++ b/spack/resource_test.go @@ -0,0 +1,83 @@ +package spack + +import ( + "testing" + + "github.com/hartzell/modules2tuple/tuple" +) + +// ExpectedError should only occur if regexp match fails or template +// execution fails. I haven't figured out how to test those cases. +type testCase struct { + AppVersion string + Tuple *tuple.Tuple + Expected string + ExpectedError string +} + +// TestResource tests that an input tuple generates the expected +// resource string. +func TestResource(t *testing.T) { + testCases := []testCase{ + // This should use "commit" + {AppVersion: "1.2.3", + Tuple: tParse("github.com/pkg/errors v1.2.3-20150716171945-2caba252f4dc"), + Expected: ` + resource(name="github.com/pkg/errors", + git="https://github.com/pkg/errors", + commit="2caba252f4dc", + destination=".", + when="@1.2.3", + placement="vendor/github.com/pkg/errors")`, + ExpectedError: "", + }, + // This should use "tag" + {AppVersion: "1.2.3", + Tuple: tParse("github.com/rogpeppe/go-internal v1.3.0"), + Expected: ` + resource(name="github.com/rogpeppe/go-internal", + git="https://github.com/rogpeppe/go-internal", + tag="v1.3.0", + destination=".", + when="@1.2.3", + placement="vendor/github.com/rogpeppe/go-internal")`, + ExpectedError: "", + }, + // empty AppVersion => no when + {AppVersion: "", + Tuple: tParse("github.com/rogpeppe/go-internal v1.3.0"), + Expected: ` + resource(name="github.com/rogpeppe/go-internal", + git="https://github.com/rogpeppe/go-internal", + tag="v1.3.0", + destination=".", + placement="vendor/github.com/rogpeppe/go-internal")`, + ExpectedError: "", + }, + } + for _, c := range testCases { + r, err := Resource(c.AppVersion, c.Tuple) + if err != nil { + if err.Error() != c.ExpectedError { + t.Error(err) + } + } + if c.ExpectedError != "" { + t.Errorf("Did not get expected error: %s", c.ExpectedError) + } + if r != c.Expected { + t.Errorf("Got: %s, expected %s", r, c.Expected) + } + } +} + +// tParse is a helper function that parses a tuple string and returns +// the resulting *tuple.Tuple. It panics if there's a problem parsing +// the string, since that's not what we're testing here. +func tParse(t string) *tuple.Tuple { + tuple, err := tuple.Parse(t, "vendor") + if err != nil { + panic(err) + } + return tuple +} From aa8dc46dc233630d80b2c43df3c81b61587e3ef5 Mon Sep 17 00:00:00 2001 From: hartzell Date: Tue, 3 Dec 2019 11:41:38 -0800 Subject: [PATCH 2/3] Fix pkg references to my fork, add gitlab test I made a hard fork early, thought I'd cleaned it all up, but missed these. Add a test that resolves to a private GitLab repo. --- spack/resource.go | 2 +- spack/resource_test.go | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/spack/resource.go b/spack/resource.go index 898c03c..630104e 100644 --- a/spack/resource.go +++ b/spack/resource.go @@ -7,7 +7,7 @@ import ( "regexp" "text/template" - "github.com/hartzell/modules2tuple/tuple" + "github.com/dmgk/modules2tuple/tuple" ) type resource struct { diff --git a/spack/resource_test.go b/spack/resource_test.go index 98ebff2..a1aabd8 100644 --- a/spack/resource_test.go +++ b/spack/resource_test.go @@ -3,7 +3,7 @@ package spack import ( "testing" - "github.com/hartzell/modules2tuple/tuple" + "github.com/dmgk/modules2tuple/tuple" ) // ExpectedError should only occur if regexp match fails or template @@ -54,6 +54,17 @@ func TestResource(t *testing.T) { placement="vendor/github.com/rogpeppe/go-internal")`, ExpectedError: "", }, + // A nonstandard, GitLab repo + {AppVersion: "", + Tuple: tParse("howett.net/plist v0.0.0-20181124034731-591f970eefbb"), + Expected: ` + resource(name="howett.net/plist", + git="https://gitlab.howett.net/go/plist", + commit="591f970eefbb", + destination=".", + placement="vendor/howett.net/plist")`, + ExpectedError: "", + }, } for _, c := range testCases { r, err := Resource(c.AppVersion, c.Tuple) From c86c7f923b1a9095daa7f756bc8a7f014a9c6dbd Mon Sep 17 00:00:00 2001 From: hartzell Date: Wed, 4 Dec 2019 15:41:34 -0800 Subject: [PATCH 3/3] Produce JSON instead of resource() statements I've reworked the Spack prototype to inhale a JSON array of objects. This updated the spack output to be, wait for it..., a JSON array of objects.... With tests. --- main.go | 9 +++- spack/resource.go | 105 +++++++++++++++++++------------------- spack/resource_test.go | 111 +++++++++++++++++++++++++---------------- 3 files changed, 128 insertions(+), 97 deletions(-) diff --git a/main.go b/main.go index 6de6fd5..41e2e27 100644 --- a/main.go +++ b/main.go @@ -32,12 +32,17 @@ func main() { if !flagSpack { fmt.Println(tuples) } else { - resources, err := spack.Resources(flagAppVersion, tuples) + resources, err := spack.ResourcesFromTuples(flagAppVersion, tuples) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } - fmt.Println(resources) + json, err := resources.ToJson() + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + fmt.Println(string(json)) } } if errors != nil { diff --git a/spack/resource.go b/spack/resource.go index 630104e..6c49201 100644 --- a/spack/resource.go +++ b/spack/resource.go @@ -1,87 +1,88 @@ package spack import ( - "bytes" + "encoding/json" "fmt" "reflect" "regexp" - "text/template" "github.com/dmgk/modules2tuple/tuple" ) -type resource struct { - AppVersion string - Name string - RepoURL string - Fetcher string - CommitIdType string - CommitId string - Placement string +type Resource struct { + Name string `json:"name,omitempty"` + Git string `json:"git,omitempty"` + Tag string `json:"tag,omitempty"` + Commit string `json:"commit,omitempty"` + Placement string `json:"placement,omitempty"` + When string `json:"when,omitempty"` + Destination string `json:"destination,omitempty"` } -func Resources(appVersion string, tt tuple.Tuples) (string, error) { - buf := bytes.Buffer{} +type Resources []*Resource + +func (r Resources) ToJson() ([]byte, error) { + b, err := json.Marshal(r) + if err != nil { + return nil, err + } + return b, nil +} + +func (r Resource) ToJson() ([]byte, error) { + b, err := json.Marshal(r) + if err != nil { + return nil, err + } + return b, nil +} + +func ResourcesFromTuples(appVersion string, tt tuple.Tuples) (Resources, error) { + resources := make(Resources, 0, len(tt)) + for _, t := range tt { - r, err := Resource(appVersion, t) + r, err := resourceFromTuple(appVersion, t) if err != nil { - return "", err + return nil, err } - buf.WriteString(r) + resources = append(resources, r) } - return buf.String(), nil + + return resources, nil } -func Resource(appVersion string, t *tuple.Tuple) (string, error) { - var repoSite string - var repoURL string - var fetcher = "git" +func resourceFromTuple(appVersion string, t *tuple.Tuple) (*Resource, error) { + r := Resource{ + Name: t.Package, + Destination: ".", + Placement: fmt.Sprintf("%s/%s", t.Prefix, t.Package), + } st := reflect.TypeOf(t.Source) switch st.String() { case "tuple.GH": - repoSite = "https://github.com" // tuple.GH.Site() returns ""... - repoURL = fmt.Sprintf("%s/%s/%s", repoSite, t.Account, t.Project) + repoSite := "https://github.com" // tuple.GH.Site() returns ""... + r.Git = fmt.Sprintf("%s/%s/%s", repoSite, t.Account, t.Project) case "tuple.GL": - repoSite = t.Source.Site() - repoURL = fmt.Sprintf("%s/%s/%s", repoSite, t.Account, t.Project) + repoSite := t.Source.Site() + r.Git = fmt.Sprintf("%s/%s/%s", repoSite, t.Account, t.Project) default: - return "", fmt.Errorf("Unknown site type: %s", st.String()) + return nil, fmt.Errorf("Unknown site type: %s", st.String()) } - commitIdType := "tag" - commitId := t.Tag matched, err := regexp.MatchString("[0-9a-f]{12}", t.Tag) if err != nil { - return "", err + return nil, err } if matched { - commitIdType = "commit" - // commitId = commitId[:7] + r.Commit = t.Tag + } else { + r.Tag = t.Tag } - var buf bytes.Buffer - r := resource{ - AppVersion: appVersion, - Name: t.Package, - RepoURL: repoURL, - Fetcher: fetcher, - CommitIdType: commitIdType, - CommitId: commitId, - Placement: fmt.Sprintf("%s/%s", t.Prefix, t.Package), - } - resource_template := ` - resource(name="{{.Name}}", - {{.Fetcher}}="{{.RepoURL}}", - {{.CommitIdType}}="{{.CommitId}}", - destination=".",{{if .AppVersion}} - when="@{{.AppVersion}}",{{end}} - placement="{{.Placement}}")` - templ := template.Must(template.New("resource").Parse(resource_template)) - err = templ.Execute(&buf, r) - if err != nil { - return "", err + if appVersion != "" { + r.When = "@" + appVersion } - return buf.String(), nil + return &r, nil } diff --git a/spack/resource_test.go b/spack/resource_test.go index a1aabd8..17d500c 100644 --- a/spack/resource_test.go +++ b/spack/resource_test.go @@ -1,18 +1,17 @@ package spack import ( + "encoding/json" + "reflect" "testing" "github.com/dmgk/modules2tuple/tuple" ) -// ExpectedError should only occur if regexp match fails or template -// execution fails. I haven't figured out how to test those cases. type testCase struct { - AppVersion string - Tuple *tuple.Tuple - Expected string - ExpectedError string + AppVersion string + Tuple *tuple.Tuple + Expected string } // TestResource tests that an input tuple generates the expected @@ -21,62 +20,71 @@ func TestResource(t *testing.T) { testCases := []testCase{ // This should use "commit" {AppVersion: "1.2.3", - Tuple: tParse("github.com/pkg/errors v1.2.3-20150716171945-2caba252f4dc"), + Tuple: tParse("github.com/rivo/uniseg v0.0.0-20190313204849-f699dde9c340"), Expected: ` - resource(name="github.com/pkg/errors", - git="https://github.com/pkg/errors", - commit="2caba252f4dc", - destination=".", - when="@1.2.3", - placement="vendor/github.com/pkg/errors")`, - ExpectedError: "", + { + "name": "github.com/rivo/uniseg", + "git": "https://github.com/rivo/uniseg", + "commit": "f699dde9c340", + "placement": "vendor/github.com/rivo/uniseg", + "when": "@1.2.3", + "destination": "." + }`, }, - // This should use "tag" + // This should use "tag" and resolve to github {AppVersion: "1.2.3", - Tuple: tParse("github.com/rogpeppe/go-internal v1.3.0"), + Tuple: tParse("golang.org/x/text v0.3.2"), Expected: ` - resource(name="github.com/rogpeppe/go-internal", - git="https://github.com/rogpeppe/go-internal", - tag="v1.3.0", - destination=".", - when="@1.2.3", - placement="vendor/github.com/rogpeppe/go-internal")`, - ExpectedError: "", + { + "name": "golang.org/x/text", + "git": "https://github.com/golang/text", + "tag": "v0.3.2", + "placement": "vendor/golang.org/x/text", + "when": "@1.2.3", + "destination": "." + }`, }, // empty AppVersion => no when {AppVersion: "", - Tuple: tParse("github.com/rogpeppe/go-internal v1.3.0"), + Tuple: tParse("golang.org/x/text v0.3.2"), Expected: ` - resource(name="github.com/rogpeppe/go-internal", - git="https://github.com/rogpeppe/go-internal", - tag="v1.3.0", - destination=".", - placement="vendor/github.com/rogpeppe/go-internal")`, - ExpectedError: "", + { + "name": "golang.org/x/text", + "git": "https://github.com/golang/text", + "tag": "v0.3.2", + "placement": "vendor/golang.org/x/text", + "destination": "." + }`, }, // A nonstandard, GitLab repo {AppVersion: "", Tuple: tParse("howett.net/plist v0.0.0-20181124034731-591f970eefbb"), Expected: ` - resource(name="howett.net/plist", - git="https://gitlab.howett.net/go/plist", - commit="591f970eefbb", - destination=".", - placement="vendor/howett.net/plist")`, - ExpectedError: "", + { + "name": "howett.net/plist", + "git": "https://gitlab.howett.net/go/plist", + "commit": "591f970eefbb", + "placement": "vendor/howett.net/plist", + "destination": "." + }`, }, } + for _, c := range testCases { - r, err := Resource(c.AppVersion, c.Tuple) + expected := []byte(c.Expected) + r, err := resourceFromTuple(c.AppVersion, c.Tuple) + if err != nil { + t.Error(err) + } + got, err := r.ToJson() if err != nil { - if err.Error() != c.ExpectedError { - t.Error(err) - } + t.Error(err) } - if c.ExpectedError != "" { - t.Errorf("Did not get expected error: %s", c.ExpectedError) + equal, err := jsonCompare(expected, got) + if err != nil { + t.Error(err) } - if r != c.Expected { + if !equal { t.Errorf("Got: %s, expected %s", r, c.Expected) } } @@ -92,3 +100,20 @@ func tParse(t string) *tuple.Tuple { } return tuple } + +// After https://gist.github.com/turtlemonvh/e4f7404e28387fadb8ad275a99596f67 +func jsonCompare(j1, j2 []byte) (bool, error) { + var o1 interface{} + var o2 interface{} + + err := json.Unmarshal([]byte(j1), &o1) + if err != nil { + return false, err + } + err = json.Unmarshal([]byte(j2), &o2) + if err != nil { + return false, err + } + + return reflect.DeepEqual(o1, o2), nil +}