Implement `QueryJSON` and introduce new way for consuming Terraform's structured logging (#539)
Co-authored-by: Radek Simko <radeksimko@users.noreply.github.com>
Co-authored-by: Sarah French <15078782+SarahFrench@users.noreply.github.com>
diff --git a/go.mod b/go.mod
index 19545b4..cd72c26 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@
github.com/google/go-cmp v0.7.0
github.com/hashicorp/go-version v1.7.0
github.com/hashicorp/hc-install v0.9.2
- github.com/hashicorp/terraform-json v0.26.0
+ github.com/hashicorp/terraform-json v0.27.1
github.com/zclconf/go-cty v1.16.4
github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b
)
diff --git a/go.sum b/go.sum
index 9b15158..64bde7f 100644
--- a/go.sum
+++ b/go.sum
@@ -51,8 +51,8 @@
github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/hc-install v0.9.2 h1:v80EtNX4fCVHqzL9Lg/2xkp62bbvQMnvPQ0G+OmtO24=
github.com/hashicorp/hc-install v0.9.2/go.mod h1:XUqBQNnuT4RsxoxiM9ZaUk0NX8hi2h+Lb6/c0OZnC/I=
-github.com/hashicorp/terraform-json v0.26.0 h1:+BnJavhRH+oyNWPnfzrfQwVWCZBFMvjdiH2Vi38Udz4=
-github.com/hashicorp/terraform-json v0.26.0/go.mod h1:eyWCeC3nrZamyrKLFnrvwpc3LQPIJsx8hWHQ/nu2/v4=
+github.com/hashicorp/terraform-json v0.27.1 h1:zWhEracxJW6lcjt/JvximOYyc12pS/gaKSy/wzzE7nY=
+github.com/hashicorp/terraform-json v0.27.1/go.mod h1:GzPLJ1PLdUG5xL6xn1OXWIjteQRT2CNT9o/6A9mi9hE=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
diff --git a/tfexec/cmd.go b/tfexec/cmd.go
index cd5a7e2..47e0e68 100644
--- a/tfexec/cmd.go
+++ b/tfexec/cmd.go
@@ -12,12 +12,14 @@
"fmt"
"io"
"io/ioutil"
+ "iter"
"os"
"os/exec"
"runtime"
"strings"
"github.com/hashicorp/terraform-exec/internal/version"
+ tfjson "github.com/hashicorp/terraform-json"
)
const (
@@ -216,6 +218,72 @@
return dec.Decode(v)
}
+func (tf *Terraform) runTerraformCmdJSONLog(ctx context.Context, cmd *exec.Cmd) iter.Seq[NextMessage] {
+ pr, pw := io.Pipe()
+ tf.SetStdout(pw)
+
+ emitter := newLogMsgEmitter(pr)
+
+ go func() {
+ err := tf.runTerraformCmd(ctx, cmd)
+ emitter.done <- errors.Join(err, pw.Close())
+ }()
+
+ return func(yield func(msg NextMessage) bool) {
+ for {
+ nextMsg := emitter.NextMessage()
+ ok := yield(nextMsg)
+ if !ok || nextMsg.Msg == nil {
+ return
+ }
+ }
+ }
+}
+
+func newLogMsgEmitter(stdoutReader io.ReadCloser) *logMsgEmitter {
+ return &logMsgEmitter{
+ scanner: bufio.NewScanner(stdoutReader),
+ stdoutReader: stdoutReader,
+ done: make(chan error, 1),
+ }
+}
+
+type logMsgEmitter struct {
+ scanner *bufio.Scanner
+ stdoutReader io.Closer
+ done chan error
+}
+
+type NextMessage struct {
+ Msg tfjson.LogMsg
+ Err error
+}
+
+// NextMessage returns next decoded message, if any, along with any errors.
+// Stdout reader is closed when the last message is received.
+//
+// Error returned can be related to decoding of the message, the Terraform command
+// or closing of stdout reader.
+//
+// Any error coming from Terraform (such as wrong configuration syntax) is
+// represented as LogMsg of Level [tfjson.Error].
+func (e *logMsgEmitter) NextMessage() NextMessage {
+ if e.scanner.Scan() {
+ msg, err := tfjson.UnmarshalLogMessage(e.scanner.Bytes())
+ return NextMessage{
+ Msg: msg,
+ Err: err,
+ }
+ }
+
+ err := <-e.done
+ err = errors.Join(err, e.scanner.Err(), e.stdoutReader.Close())
+ return NextMessage{
+ Msg: nil,
+ Err: err,
+ }
+}
+
// mergeUserAgent does some minor deduplication to ensure we aren't
// just using the same append string over and over.
func mergeUserAgent(uas ...string) string {
diff --git a/tfexec/internal/e2etest/query_test.go b/tfexec/internal/e2etest/query_test.go
new file mode 100644
index 0000000..0ca5cfd
--- /dev/null
+++ b/tfexec/internal/e2etest/query_test.go
@@ -0,0 +1,82 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package e2etest
+
+import (
+ "context"
+ "regexp"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/hashicorp/go-version"
+ "github.com/hashicorp/terraform-exec/tfexec"
+ "github.com/hashicorp/terraform-exec/tfexec/internal/testutil"
+ tfjson "github.com/hashicorp/terraform-json"
+)
+
+func TestQueryJSON_TF112(t *testing.T) {
+ versions := []string{testutil.Latest_v1_12}
+
+ runTestWithVersions(t, versions, "query", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
+ err := tf.Init(context.Background())
+ if err != nil {
+ t.Fatalf("error running Init in test directory: %s", err)
+ }
+
+ re := regexp.MustCompile("terraform query -json was added in 1.14.0")
+
+ _, err = tf.QueryJSON(context.Background())
+ if err != nil && !re.MatchString(err.Error()) {
+ t.Fatalf("error running Query: %s", err)
+ }
+ })
+}
+
+func TestQueryJSON_TF114(t *testing.T) {
+ versions := []string{testutil.Latest_Alpha_v1_14}
+
+ runTestWithVersions(t, versions, "query", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
+ err := tf.Init(context.Background())
+ if err != nil {
+ t.Fatalf("error running Init in test directory: %s", err)
+ }
+
+ iter, err := tf.QueryJSON(context.Background())
+ if err != nil {
+ t.Fatalf("error running Query: %s", err)
+ }
+
+ results := 0
+ listingStarted := 0
+ var completeData tfjson.ListCompleteData
+ for nextMsg := range iter {
+ if nextMsg.Err != nil {
+ t.Fatalf("error getting next message: %s", err)
+ }
+ switch m := nextMsg.Msg.(type) {
+ case tfjson.ListStartMessage:
+ listingStarted++
+ case tfjson.ListResourceFoundMessage:
+ results++
+ case tfjson.ListCompleteMessage:
+ completeData = m.ListComplete
+ }
+ }
+
+ if listingStarted != 1 {
+ t.Fatalf("expected exactly 1 list start message, got %d", listingStarted)
+ }
+ if results != 5 {
+ t.Fatalf("expected 5 query results, got %d", results)
+ }
+ expectedData := tfjson.ListCompleteData{
+ Address: "list.concept_pet.pets",
+ ResourceType: "concept_pet",
+ Total: 5,
+ }
+ if diff := cmp.Diff(expectedData, completeData); diff != "" {
+ t.Fatalf("unexpected complete message data: %s", diff)
+ }
+ })
+}
diff --git a/tfexec/internal/e2etest/testdata/query/main.tf b/tfexec/internal/e2etest/testdata/query/main.tf
new file mode 100644
index 0000000..a6cb2c2
--- /dev/null
+++ b/tfexec/internal/e2etest/testdata/query/main.tf
@@ -0,0 +1,8 @@
+terraform {
+ required_providers {
+ concept = {
+ source = "dbanck/concept"
+ version = "0.1.0"
+ }
+ }
+}
diff --git a/tfexec/internal/e2etest/testdata/query/main.tfquery.hcl b/tfexec/internal/e2etest/testdata/query/main.tfquery.hcl
new file mode 100644
index 0000000..99a5b2a
--- /dev/null
+++ b/tfexec/internal/e2etest/testdata/query/main.tfquery.hcl
@@ -0,0 +1,3 @@
+list "concept_pet" "pets" {
+ provider = concept
+}
diff --git a/tfexec/internal/testutil/tfcache.go b/tfexec/internal/testutil/tfcache.go
index 6c4d09d..fa7d260 100644
--- a/tfexec/internal/testutil/tfcache.go
+++ b/tfexec/internal/testutil/tfcache.go
@@ -36,6 +36,7 @@
Latest_Alpha_v1_10 = "1.10.0-alpha20240926"
Latest_v1_11 = "1.11.4"
Latest_v1_12 = "1.12.2"
+ Latest_Alpha_v1_14 = "1.14.0-alpha20250903"
)
const appendUserAgent = "tfexec-testutil"
diff --git a/tfexec/options.go b/tfexec/options.go
index 339bf39..0129bd5 100644
--- a/tfexec/options.go
+++ b/tfexec/options.go
@@ -184,6 +184,14 @@
return &FromModuleOption{source}
}
+type GenerateConfigOutOption struct {
+ path string
+}
+
+func GenerateConfigOut(path string) *GenerateConfigOutOption {
+ return &GenerateConfigOutOption{path}
+}
+
type GetOption struct {
get bool
}
diff --git a/tfexec/query.go b/tfexec/query.go
new file mode 100644
index 0000000..04b515f
--- /dev/null
+++ b/tfexec/query.go
@@ -0,0 +1,127 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package tfexec
+
+import (
+ "context"
+ "fmt"
+ "iter"
+ "os/exec"
+)
+
+type queryConfig struct {
+ dir string
+ generateConfig string
+ reattachInfo ReattachInfo
+ vars []string
+ varFiles []string
+}
+
+var defaultQueryOptions = queryConfig{}
+
+// QueryOption represents options used in the Query method.
+type QueryOption interface {
+ configureQuery(*queryConfig)
+}
+
+func (opt *DirOption) configureQuery(conf *queryConfig) {
+ conf.dir = opt.path
+}
+
+func (opt *GenerateConfigOutOption) configureQuery(conf *queryConfig) {
+ conf.generateConfig = opt.path
+}
+
+func (opt *ReattachOption) configureQuery(conf *queryConfig) {
+ conf.reattachInfo = opt.info
+}
+
+func (opt *VarFileOption) configureQuery(conf *queryConfig) {
+ conf.varFiles = append(conf.varFiles, opt.path)
+}
+
+func (opt *VarOption) configureQuery(conf *queryConfig) {
+ conf.vars = append(conf.vars, opt.assignment)
+}
+
+// QueryJSON executes `terraform query` with the specified options as well as the
+// `-json` flag and waits for it to complete.
+//
+// Using the `-json` flag will result in
+// [machine-readable](https://developer.hashicorp.com/terraform/internals/machine-readable-ui)
+// JSON being written to the supplied `io.Writer`.
+//
+// The returned error is nil if `terraform query` has been executed and exits
+// with 0.
+//
+// QueryJSON is likely to be removed in a future major version in favour of
+// query returning JSON by default.
+func (tf *Terraform) QueryJSON(ctx context.Context, opts ...QueryOption) (iter.Seq[NextMessage], error) {
+ err := tf.compatible(ctx, tf1_14_0, nil)
+ if err != nil {
+ return nil, fmt.Errorf("terraform query -json was added in 1.14.0: %w", err)
+ }
+
+ queryCmd, err := tf.queryJSONCmd(ctx, opts...)
+ if err != nil {
+ return nil, err
+ }
+
+ return tf.runTerraformCmdJSONLog(ctx, queryCmd), nil
+}
+
+func (tf *Terraform) queryJSONCmd(ctx context.Context, opts ...QueryOption) (*exec.Cmd, error) {
+ c := defaultQueryOptions
+
+ for _, o := range opts {
+ o.configureQuery(&c)
+ }
+
+ args, err := tf.buildQueryArgs(ctx, c)
+ if err != nil {
+ return nil, err
+ }
+
+ args = append(args, "-json")
+
+ return tf.buildQueryCmd(ctx, c, args)
+}
+
+func (tf *Terraform) buildQueryArgs(ctx context.Context, c queryConfig) ([]string, error) {
+ args := []string{"query", "-no-color"}
+
+ if c.generateConfig != "" {
+ args = append(args, "-generate-config-out="+c.generateConfig)
+ }
+
+ for _, vf := range c.varFiles {
+ args = append(args, "-var-file="+vf)
+ }
+
+ if c.vars != nil {
+ for _, v := range c.vars {
+ args = append(args, "-var", v)
+ }
+ }
+
+ return args, nil
+}
+
+func (tf *Terraform) buildQueryCmd(ctx context.Context, c queryConfig, args []string) (*exec.Cmd, error) {
+ // optional positional argument
+ if c.dir != "" {
+ args = append(args, c.dir)
+ }
+
+ mergeEnv := map[string]string{}
+ if c.reattachInfo != nil {
+ reattachStr, err := c.reattachInfo.marshalString()
+ if err != nil {
+ return nil, err
+ }
+ mergeEnv[reattachEnvVar] = reattachStr
+ }
+
+ return tf.buildTerraformCmd(ctx, mergeEnv, args...), nil
+}
diff --git a/tfexec/query_test.go b/tfexec/query_test.go
new file mode 100644
index 0000000..9291667
--- /dev/null
+++ b/tfexec/query_test.go
@@ -0,0 +1,59 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package tfexec
+
+import (
+ "context"
+ "testing"
+
+ "github.com/hashicorp/terraform-exec/tfexec/internal/testutil"
+)
+
+func TestQueryJSONCmd(t *testing.T) {
+ td := t.TempDir()
+
+ tf, err := NewTerraform(td, tfVersion(t, testutil.Latest_Alpha_v1_14))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // empty env, to avoid environ mismatch in testing
+ tf.SetEnv(map[string]string{})
+
+ t.Run("defaults", func(t *testing.T) {
+ queryCmd, err := tf.queryJSONCmd(context.Background())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ assertCmd(t, []string{
+ "query",
+ "-no-color",
+ "-json",
+ }, nil, queryCmd)
+ })
+
+ t.Run("override all", func(t *testing.T) {
+ queryCmd, err := tf.queryJSONCmd(context.Background(),
+ GenerateConfigOut("generated.tf"),
+ Var("android=paranoid"),
+ Var("brain_size=planet"),
+ VarFile("trillian"),
+ Dir("earth"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ assertCmd(t, []string{
+ "query",
+ "-no-color",
+ "-generate-config-out=generated.tf",
+ "-var-file=trillian",
+ "-var", "android=paranoid",
+ "-var", "brain_size=planet",
+ "-json",
+ "earth",
+ }, nil, queryCmd)
+ })
+}
diff --git a/tfexec/version.go b/tfexec/version.go
index 87addd1..40b9301 100644
--- a/tfexec/version.go
+++ b/tfexec/version.go
@@ -34,6 +34,8 @@
tf1_4_0 = version.Must(version.NewVersion("1.4.0"))
tf1_6_0 = version.Must(version.NewVersion("1.6.0"))
tf1_9_0 = version.Must(version.NewVersion("1.9.0"))
+ tf1_13_0 = version.Must(version.NewVersion("1.13.0"))
+ tf1_14_0 = version.Must(version.NewVersion("1.14.0"))
)
// Version returns structured output from the terraform version command including both the Terraform CLI version
