Skip to content

Commit 13190e9

Browse files
nikpivkinfwereade
nikpivkin
and
fwereade
authored
fix(terraform): eval submodules (aquasecurity#6411)
Co-authored-by: William Reade <[email protected]>
1 parent 6bca7c3 commit 13190e9

File tree

4 files changed

+223
-41
lines changed

4 files changed

+223
-41
lines changed

‎pkg/iac/scanners/terraform/parser/evaluator.go

+87-32
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/hashicorp/hcl/v2"
1010
"github.com/hashicorp/hcl/v2/ext/typeexpr"
11+
"github.com/samber/lo"
1112
"github.com/zclconf/go-cty/cty"
1213
"github.com/zclconf/go-cty/cty/convert"
1314
"golang.org/x/exp/slices"
@@ -102,6 +103,7 @@ func (e *evaluator) evaluateStep() {
102103

103104
e.ctx.Set(e.getValuesByBlockType("data"), "data")
104105
e.ctx.Set(e.getValuesByBlockType("output"), "output")
106+
e.ctx.Set(e.getValuesByBlockType("module"), "module")
105107
}
106108

107109
// exportOutputs is used to export module outputs to the parent module
@@ -126,48 +128,100 @@ func (e *evaluator) EvaluateAll(ctx context.Context) (terraform.Modules, map[str
126128
fsMap := make(map[string]fs.FS)
127129
fsMap[fsKey] = e.filesystem
128130

129-
var lastContext hcl.EvalContext
130131
e.debug.Log("Starting module evaluation...")
131-
for i := 0; i < maxContextIterations; i++ {
132+
e.evaluateSteps()
132133

133-
e.evaluateStep()
134+
// expand out resources and modules via count, for-each and dynamic
135+
// (not a typo, we do this twice so every order is processed)
136+
e.blocks = e.expandBlocks(e.blocks)
137+
e.blocks = e.expandBlocks(e.blocks)
134138

135-
// if ctx matches the last evaluation, we can bail, nothing left to resolve
136-
if i > 0 && reflect.DeepEqual(lastContext.Variables, e.ctx.Inner().Variables) {
137-
break
138-
}
139+
e.debug.Log("Starting submodule evaluation...")
140+
submodules := e.loadSubmodules(ctx)
139141

140-
if len(e.ctx.Inner().Variables) != len(lastContext.Variables) {
141-
lastContext.Variables = make(map[string]cty.Value, len(e.ctx.Inner().Variables))
142+
for i := 0; i < maxContextIterations; i++ {
143+
changed := false
144+
for _, sm := range submodules {
145+
changed = changed || e.evaluateSubmodule(ctx, sm)
142146
}
143-
for k, v := range e.ctx.Inner().Variables {
144-
lastContext.Variables[k] = v
147+
if !changed {
148+
e.debug.Log("All submodules are evaluated at i=%d", i)
149+
break
145150
}
146151
}
147152

148-
// expand out resources and modules via count, for-each and dynamic
149-
// (not a typo, we do this twice so every order is processed)
150-
e.blocks = e.expandBlocks(e.blocks)
151-
e.blocks = e.expandBlocks(e.blocks)
153+
e.debug.Log("Starting post-submodule evaluation...")
154+
e.evaluateSteps()
152155

153-
e.debug.Log("Starting submodule evaluation...")
154156
var modules terraform.Modules
157+
for _, sm := range submodules {
158+
modules = append(modules, sm.modules...)
159+
fsMap = lo.Assign(fsMap, sm.fsMap)
160+
}
161+
162+
e.debug.Log("Finished processing %d submodule(s).", len(modules))
163+
164+
e.debug.Log("Module evaluation complete.")
165+
rootModule := terraform.NewModule(e.projectRootPath, e.modulePath, e.blocks, e.ignores)
166+
return append(terraform.Modules{rootModule}, modules...), fsMap
167+
}
168+
169+
type submodule struct {
170+
definition *ModuleDefinition
171+
eval *evaluator
172+
modules terraform.Modules
173+
lastState map[string]cty.Value
174+
fsMap map[string]fs.FS
175+
}
176+
177+
func (e *evaluator) loadSubmodules(ctx context.Context) []*submodule {
178+
var submodules []*submodule
179+
155180
for _, definition := range e.loadModules(ctx) {
156-
submodules, outputs, err := definition.Parser.EvaluateAll(ctx)
157-
if err != nil {
158-
e.debug.Log("Failed to evaluate submodule '%s': %s.", definition.Name, err)
181+
eval, err := definition.Parser.Load(ctx)
182+
if errors.Is(err, ErrNoFiles) {
183+
continue
184+
} else if err != nil {
185+
e.debug.Log("Failed to load submodule '%s': %s.", definition.Name, err)
159186
continue
160187
}
161-
// export module outputs
162-
e.ctx.Set(outputs, "module", definition.Name)
163-
modules = append(modules, submodules...)
164-
for key, val := range definition.Parser.GetFilesystemMap() {
165-
fsMap[key] = val
188+
189+
submodules = append(submodules, &submodule{
190+
definition: definition,
191+
eval: eval,
192+
fsMap: make(map[string]fs.FS),
193+
})
194+
}
195+
196+
return submodules
197+
}
198+
199+
func (e *evaluator) evaluateSubmodule(ctx context.Context, sm *submodule) bool {
200+
inputVars := sm.definition.inputVars()
201+
if len(sm.modules) > 0 {
202+
if reflect.DeepEqual(inputVars, sm.lastState) {
203+
e.debug.Log("Submodule %s inputs unchanged", sm.definition.Name)
204+
return false
166205
}
167206
}
168-
e.debug.Log("Finished processing %d submodule(s).", len(modules))
169207

170-
e.debug.Log("Starting post-submodule evaluation...")
208+
e.debug.Log("Evaluating submodule %s", sm.definition.Name)
209+
sm.eval.inputVars = inputVars
210+
sm.modules, sm.fsMap = sm.eval.EvaluateAll(ctx)
211+
outputs := sm.eval.exportOutputs()
212+
213+
// lastState needs to be captured after applying outputs – so that they
214+
// don't get treated as changes – but before running post-submodule
215+
// evaluation, so that changes from that can trigger re-evaluations of
216+
// the submodule if/when they feed back into inputs.
217+
e.ctx.Set(outputs, "module", sm.definition.Name)
218+
sm.lastState = sm.definition.inputVars()
219+
e.evaluateSteps()
220+
return true
221+
}
222+
223+
func (e *evaluator) evaluateSteps() {
224+
var lastContext hcl.EvalContext
171225
for i := 0; i < maxContextIterations; i++ {
172226

173227
e.evaluateStep()
@@ -176,18 +230,13 @@ func (e *evaluator) EvaluateAll(ctx context.Context) (terraform.Modules, map[str
176230
if i > 0 && reflect.DeepEqual(lastContext.Variables, e.ctx.Inner().Variables) {
177231
break
178232
}
179-
180233
if len(e.ctx.Inner().Variables) != len(lastContext.Variables) {
181234
lastContext.Variables = make(map[string]cty.Value, len(e.ctx.Inner().Variables))
182235
}
183236
for k, v := range e.ctx.Inner().Variables {
184237
lastContext.Variables[k] = v
185238
}
186239
}
187-
188-
e.debug.Log("Module evaluation complete.")
189-
rootModule := terraform.NewModule(e.projectRootPath, e.modulePath, e.blocks, e.ignores)
190-
return append(terraform.Modules{rootModule}, modules...), fsMap
191240
}
192241

193242
func (e *evaluator) expandBlocks(blocks terraform.Blocks) terraform.Blocks {
@@ -217,7 +266,9 @@ func (e *evaluator) expandDynamicBlock(b *terraform.Block) {
217266
b.InjectBlock(content, blockName)
218267
}
219268
}
220-
sub.MarkExpanded()
269+
if len(expanded) > 0 {
270+
sub.MarkExpanded()
271+
}
221272
}
222273
}
223274

@@ -246,6 +297,10 @@ func (e *evaluator) expandBlockForEaches(blocks terraform.Blocks, isDynamic bool
246297
clones := make(map[string]cty.Value)
247298
_ = forEachAttr.Each(func(key cty.Value, val cty.Value) {
248299

300+
if val.IsNull() {
301+
return
302+
}
303+
249304
// instances are identified by a map key (or set member) from the value provided to for_each
250305
idx, err := convert.Convert(key, cty.String)
251306
if err != nil {

‎pkg/iac/scanners/terraform/parser/load_module.go

+8
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@ type ModuleDefinition struct {
2222
External bool
2323
}
2424

25+
func (d *ModuleDefinition) inputVars() map[string]cty.Value {
26+
inputs := d.Definition.Values().AsValueMap()
27+
if inputs == nil {
28+
return make(map[string]cty.Value)
29+
}
30+
return inputs
31+
}
32+
2533
// loadModules reads all module blocks and loads them
2634
func (e *evaluator) loadModules(ctx context.Context) []*ModuleDefinition {
2735
var moduleDefinitions []*ModuleDefinition

‎pkg/iac/scanners/terraform/parser/parser.go

+19-9
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package parser
22

33
import (
44
"context"
5+
"errors"
56
"io"
67
"io/fs"
78
"os"
@@ -224,18 +225,19 @@ func (p *Parser) ParseFS(ctx context.Context, dir string) error {
224225
return nil
225226
}
226227

227-
func (p *Parser) EvaluateAll(ctx context.Context) (terraform.Modules, cty.Value, error) {
228+
var ErrNoFiles = errors.New("no files found")
228229

230+
func (p *Parser) Load(ctx context.Context) (*evaluator, error) {
229231
p.debug.Log("Evaluating module...")
230232

231233
if len(p.files) == 0 {
232234
p.debug.Log("No files found, nothing to do.")
233-
return nil, cty.NilVal, nil
235+
return nil, ErrNoFiles
234236
}
235237

236238
blocks, ignores, err := p.readBlocks(p.files)
237239
if err != nil {
238-
return nil, cty.NilVal, err
240+
return nil, err
239241
}
240242
p.debug.Log("Read %d block(s) and %d ignore(s) for module '%s' (%d file[s])...", len(blocks), len(ignores), p.moduleName, len(p.files))
241243

@@ -246,7 +248,7 @@ func (p *Parser) EvaluateAll(ctx context.Context) (terraform.Modules, cty.Value,
246248
} else {
247249
inputVars, err = loadTFVars(p.configsFS, p.tfvarsPaths)
248250
if err != nil {
249-
return nil, cty.NilVal, err
251+
return nil, err
250252
}
251253
p.debug.Log("Added %d variables from tfvars.", len(inputVars))
252254
}
@@ -260,10 +262,10 @@ func (p *Parser) EvaluateAll(ctx context.Context) (terraform.Modules, cty.Value,
260262

261263
workingDir, err := os.Getwd()
262264
if err != nil {
263-
return nil, cty.NilVal, err
265+
return nil, err
264266
}
265267
p.debug.Log("Working directory for module evaluation is '%s'", workingDir)
266-
evaluator := newEvaluator(
268+
return newEvaluator(
267269
p.moduleFS,
268270
p,
269271
p.projectRoot,
@@ -278,11 +280,19 @@ func (p *Parser) EvaluateAll(ctx context.Context) (terraform.Modules, cty.Value,
278280
p.debug.Extend("evaluator"),
279281
p.allowDownloads,
280282
p.skipCachedModules,
281-
)
282-
modules, fsMap := evaluator.EvaluateAll(ctx)
283+
), nil
284+
}
285+
286+
func (p *Parser) EvaluateAll(ctx context.Context) (terraform.Modules, cty.Value, error) {
287+
288+
e, err := p.Load(ctx)
289+
if errors.Is(err, ErrNoFiles) {
290+
return nil, cty.NilVal, nil
291+
}
292+
modules, fsMap := e.EvaluateAll(ctx)
283293
p.debug.Log("Finished parsing module '%s'.", p.moduleName)
284294
p.fsMap = fsMap
285-
return modules, evaluator.exportOutputs(), nil
295+
return modules, e.exportOutputs(), nil
286296
}
287297

288298
func (p *Parser) GetFilesystemMap() map[string]fs.FS {

‎pkg/iac/scanners/terraform/parser/parser_test.go

+109
Original file line numberDiff line numberDiff line change
@@ -1522,3 +1522,112 @@ func compareSets(a []int, b []int) bool {
15221522

15231523
return true
15241524
}
1525+
1526+
func TestModuleRefersToOutputOfAnotherModule(t *testing.T) {
1527+
files := map[string]string{
1528+
"main.tf": `
1529+
module "module2" {
1530+
source = "./modules/foo"
1531+
}
1532+
1533+
module "module1" {
1534+
source = "./modules/bar"
1535+
test_var = module.module2.test_out
1536+
}
1537+
`,
1538+
"modules/foo/main.tf": `
1539+
output "test_out" {
1540+
value = "test_value"
1541+
}
1542+
`,
1543+
"modules/bar/main.tf": `
1544+
variable "test_var" {}
1545+
1546+
resource "test_resource" "this" {
1547+
dynamic "dynamic_block" {
1548+
for_each = [var.test_var]
1549+
content {
1550+
some_attr = dynamic_block.value
1551+
}
1552+
}
1553+
}
1554+
`,
1555+
}
1556+
1557+
modules := parse(t, files)
1558+
require.Len(t, modules, 3)
1559+
1560+
resources := modules.GetResourcesByType("test_resource")
1561+
require.Len(t, resources, 1)
1562+
1563+
attr, _ := resources[0].GetNestedAttribute("dynamic_block.some_attr")
1564+
require.NotNil(t, attr)
1565+
1566+
assert.Equal(t, "test_value", attr.GetRawValue())
1567+
}
1568+
1569+
func TestCyclicModules(t *testing.T) {
1570+
files := map[string]string{
1571+
"main.tf": `
1572+
module "module2" {
1573+
source = "./modules/foo"
1574+
test_var = passthru.handover.from_1
1575+
}
1576+
1577+
// Demonstrates need for evaluateSteps between submodule evaluations.
1578+
resource "passthru" "handover" {
1579+
from_1 = module.module1.test_out
1580+
from_2 = module.module2.test_out
1581+
}
1582+
1583+
module "module1" {
1584+
source = "./modules/bar"
1585+
test_var = passthru.handover.from_2
1586+
}
1587+
`,
1588+
"modules/foo/main.tf": `
1589+
variable "test_var" {}
1590+
1591+
resource "test_resource" "this" {
1592+
dynamic "dynamic_block" {
1593+
for_each = [var.test_var]
1594+
content {
1595+
some_attr = dynamic_block.value
1596+
}
1597+
}
1598+
}
1599+
1600+
output "test_out" {
1601+
value = "test_value"
1602+
}
1603+
`,
1604+
"modules/bar/main.tf": `
1605+
variable "test_var" {}
1606+
1607+
resource "test_resource" "this" {
1608+
dynamic "dynamic_block" {
1609+
for_each = [var.test_var]
1610+
content {
1611+
some_attr = dynamic_block.value
1612+
}
1613+
}
1614+
}
1615+
1616+
output "test_out" {
1617+
value = test_resource.this.dynamic_block.some_attr
1618+
}
1619+
`,
1620+
}
1621+
1622+
modules := parse(t, files)
1623+
require.Len(t, modules, 3)
1624+
1625+
resources := modules.GetResourcesByType("test_resource")
1626+
require.Len(t, resources, 2)
1627+
1628+
for _, res := range resources {
1629+
attr, _ := res.GetNestedAttribute("dynamic_block.some_attr")
1630+
require.NotNil(t, attr, res.FullName())
1631+
assert.Equal(t, "test_value", attr.GetRawValue())
1632+
}
1633+
}

0 commit comments

Comments
 (0)