Skip to content

Commit

Permalink
(TFECO-7366) Stacks: parse terraform-sources.json to support remote…
Browse files Browse the repository at this point in the history
… component sources (#1836)

* support parsing terraform-sources.json files created by tfstacks cli to support completions for remote modules in stacks projects
* log errors
* add tests
* add changie entry
* fix: ensure test also succeeds on windows using backslashes in paths
* simplify terraform sources parsing code
* parse registry modules from terraform-sources.json as well
* Bump terraform-schema to `1392f74`
  • Loading branch information
ansgarm authored Nov 28, 2024
1 parent ce3cac1 commit 0a91911
Show file tree
Hide file tree
Showing 17 changed files with 610 additions and 35 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/ENHANCEMENTS-20241028-090453.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: ENHANCEMENTS
body: 'Stacks: parse terraform-sources.json to support remote component sources'
time: 2024-10-28T09:04:53.004252+01:00
custom:
Issue: "1836"
Repository: terraform-ls
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-memdb v1.3.4
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-slug v0.16.0
github.com/hashicorp/go-uuid v1.0.3
github.com/hashicorp/go-version v1.7.0
github.com/hashicorp/hc-install v0.9.0
Expand All @@ -18,7 +19,7 @@ require (
github.com/hashicorp/terraform-exec v0.21.0
github.com/hashicorp/terraform-json v0.23.0
github.com/hashicorp/terraform-registry-address v0.2.3
github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e
github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe
github.com/mcuadros/go-defaults v1.2.0
github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
github.com/mitchellh/cli v1.1.5
Expand All @@ -37,6 +38,7 @@ require (

require (
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/apparentlymart/go-versions v1.0.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand Down
11 changes: 9 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/apparentlymart/go-versions v1.0.1 h1:ECIpSn0adcYNsBfSRwdDdz9fWlL+S/6EUd9+irwkBgU=
github.com/apparentlymart/go-versions v1.0.1/go.mod h1:YF5j7IQtrOAOnsGkniupEA5bfCjzd7i14yu0shZavyM=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
Expand Down Expand Up @@ -130,6 +132,7 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
Expand Down Expand Up @@ -210,6 +213,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/go-slug v0.16.0 h1:S/ko9fms1gf6305ktJNUKGxFmscZ+yWvAtsas0SYUyA=
github.com/hashicorp/go-slug v0.16.0/go.mod h1:THWVTAXwJEinbsp4/bBRcmbaO5EYNLTqxbG4tZ3gCYQ=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
Expand All @@ -233,8 +238,8 @@ github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2
github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c=
github.com/hashicorp/terraform-registry-address v0.2.3 h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI=
github.com/hashicorp/terraform-registry-address v0.2.3/go.mod h1:lFHA76T8jfQteVfT7caREqguFrW3c4MFSPhZB7HHgUM=
github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e h1:yozV7l40vatIUPCYTeTqpPvOKsPDOd00kg8Tetf4VeQ=
github.com/hashicorp/terraform-schema v0.0.0-20241115125457-9097f9e5a82e/go.mod h1:hwYMiQp/tVcJtYfbNSxEEK+ilauXwwtZgpLXmeUBVGg=
github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe h1:2pVtzihaLjn6PTIyKom+X491QlLupxGoLRf5Ik8zpYM=
github.com/hashicorp/terraform-schema v0.0.0-20241128095320-1392f740c4fe/go.mod h1:3vDqHlpaMuTeBXSC4LWDM/m2QdEe9DmC90IgyuhdgZw=
github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ=
github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc=
github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=
Expand Down Expand Up @@ -279,6 +284,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
Expand Down
42 changes: 42 additions & 0 deletions internal/features/rootmodules/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package rootmodules

import (
"context"
"path/filepath"

"github.com/hashicorp/terraform-ls/internal/document"
"github.com/hashicorp/terraform-ls/internal/features/rootmodules/ast"
Expand Down Expand Up @@ -85,6 +86,18 @@ func (f *RootModulesFeature) didOpen(ctx context.Context, dir document.DirHandle
}
ids = append(ids, modManifestId)

terraformSourcesId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
Dir: dir,
Func: func(ctx context.Context) error {
return jobs.ParseTerraformSources(ctx, f.fs, f.Store, dir.Path())
},
Type: op.OpTypeParseTerraformSources.String(),
})
if err != nil {
return ids, err
}
ids = append(ids, terraformSourcesId)

pSchemaVerId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
Dir: dir,
Func: func(ctx context.Context) error {
Expand Down Expand Up @@ -168,6 +181,9 @@ func (f *RootModulesFeature) manifestChange(ctx context.Context, dir document.Di
if changeType == protocol.Deleted {
// Manifest is deleted, so we clear the manifest from the store
f.Store.UpdateModManifest(path, nil, nil)
// We also delete the Terraform Sources (if they exist), since delete changes can also happen if the
// entire .terraform directory is deleted and there should only be either a manifest or terraform sources anyway
f.Store.UpdateTerraformSources(path, nil, nil)
return ids, nil
}

Expand All @@ -186,6 +202,21 @@ func (f *RootModulesFeature) manifestChange(ctx context.Context, dir document.Di
}
ids = append(ids, modManifestId)

terraformSourcesId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
Dir: dir,
Func: func(ctx context.Context) error {
return jobs.ParseTerraformSources(ctx, f.fs, f.Store, dir.Path())
},
Type: op.OpTypeParseTerraformSources.String(),
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
return f.indexTerraformSourcesDirs(ctx, dir)
},
})
if err != nil {
return ids, err
}
ids = append(ids, terraformSourcesId)

return ids, nil
}

Expand All @@ -204,3 +235,14 @@ func (f *RootModulesFeature) indexInstalledModuleCalls(ctx context.Context, dir

return jobIds, nil
}

func (f *RootModulesFeature) indexTerraformSourcesDirs(ctx context.Context, dir document.DirHandle) (job.IDs, error) {
jobIds := make(job.IDs, 0)

for _, subDir := range f.Store.TerraformSourcesDirectories(dir.Path()) {
dh := document.DirHandleFromPath(filepath.Join(dir.Path(), subDir))
f.stateStore.WalkerPaths.EnqueueDir(ctx, dh)
}

return jobIds, nil
}
56 changes: 56 additions & 0 deletions internal/features/rootmodules/jobs/terraform_sources.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package jobs

import (
"context"
"fmt"

"github.com/hashicorp/terraform-ls/internal/document"
"github.com/hashicorp/terraform-ls/internal/features/rootmodules/state"
"github.com/hashicorp/terraform-ls/internal/job"
"github.com/hashicorp/terraform-ls/internal/terraform/datadir"
op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
)

// ParseTerraformSources parses the NEW* "module manifest" which
// contains records of installed modules, e.g. where they're
// installed on the filesystem.
// This is useful for processing any modules which are not local
// nor hosted in the Registry (which would be handled by
// [GetModuleDataFromRegistry]).
// NEW* as there is a new terraform-sources.json file format which currently only exists for stacks.
func ParseTerraformSources(ctx context.Context, fs ReadOnlyFS, rootStore *state.RootStore, modPath string) error {
mod, err := rootStore.RootRecordByPath(modPath)
if err != nil {
return err
}

// Avoid parsing if it is already in progress or already known
if mod.TerraformSourcesState != op.OpStateUnknown && !job.IgnoreState(ctx) {
return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
}

err = rootStore.SetTerraformSourcesState(modPath, op.OpStateLoading)
if err != nil {
return err
}

tfs, err := datadir.ParseTerraformSourcesFromFile(modPath)
if err != nil {
err := fmt.Errorf("failed to parse terraform sources: %w", err)
sErr := rootStore.UpdateTerraformSources(modPath, nil, err)
if sErr != nil {
return sErr
}
return err
}

sErr := rootStore.UpdateTerraformSources(modPath, tfs, err)

if sErr != nil {
return sErr
}
return err
}
Loading

0 comments on commit 0a91911

Please sign in to comment.