Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit cb66ba8

Browse files
chivalryqchivalryq
authoredAug 12, 2022
Fix: legacy backend state reading and GC (#338)
1 parent 8620dbb commit cb66ba8

17 files changed

+633
-279
lines changed
 

‎.github/workflows/e2e-test.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ jobs:
6161
make configuration
6262
env:
6363
TERRAFORM_BACKEND_NAMESPACE: terraform
64+
- name: dump controller logs
65+
if: ${{ always() }}
66+
run: kubectl logs deploy/terraform-controller -n terraform
6467

6568
- name: Upload coverage report
6669
uses: codecov/codecov-action@v2

‎Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,8 @@ custom: custom-credentials custom-provider
267267

268268

269269
configuration:
270-
go test -coverprofile=e2e-coverage1.xml -v ./e2e/... -count=1
270+
go test -coverprofile=e2e-coverage1.xml -v $(go list ./e2e/...|grep -v controllernamespace) -count=1
271+
go test -v ./e2e/controllernamespace/...
271272

272273
e2e-setup: install-chart alibaba
273274

‎controllers/configuration/backend/backend.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ var backendInitFuncMap = map[string]backendInitFunc{
5656
}
5757

5858
// ParseConfigurationBackend parses backend Conf from the v1beta2.Configuration
59-
func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient client.Client, credentials map[string]string) (Backend, error) {
59+
func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient client.Client, credentials map[string]string, controllerNSSpecified bool) (Backend, error) {
6060
backend := configuration.Spec.Backend
6161

6262
var (
@@ -68,7 +68,7 @@ func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient c
6868
switch {
6969
case backend == nil || (backend.Inline == "" && backend.BackendType == ""):
7070
// use the default k8s backend
71-
return handleDefaultBackend(configuration, k8sClient)
71+
return handleDefaultBackend(configuration, k8sClient, controllerNSSpecified)
7272

7373
case backend.Inline != "" && backend.BackendType != "":
7474
return nil, errors.New("it's not allowed to set `spec.backend.inline` and `spec.backend.backendType` at the same time")
@@ -79,6 +79,8 @@ func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient c
7979

8080
case backend.BackendType != "":
8181
// In this case, use the explicit custom backend
82+
// we don't change backend secret suffix to UID of configuration here.
83+
// If backend specified, it's user's responsibility to set the right secret suffix, to avoid conflict.
8284
backendType, backendConf, err = handleExplicitBackend(backend)
8385
}
8486
if err != nil {
@@ -92,7 +94,7 @@ func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient c
9294
return initFunc(k8sClient, backendConf, credentials)
9395
}
9496

95-
func handleDefaultBackend(configuration *v1beta2.Configuration, k8sClient client.Client) (Backend, error) {
97+
func handleDefaultBackend(configuration *v1beta2.Configuration, k8sClient client.Client, controllerNSSpecified bool) (Backend, error) {
9698
if configuration.Spec.Backend != nil {
9799
if configuration.Spec.Backend.SecretSuffix == "" {
98100
configuration.Spec.Backend.SecretSuffix = configuration.Name
@@ -104,7 +106,7 @@ func handleDefaultBackend(configuration *v1beta2.Configuration, k8sClient client
104106
InClusterConfig: true,
105107
}
106108
}
107-
return newDefaultK8SBackend(configuration.Spec.Backend.SecretSuffix, k8sClient, configuration.Namespace), nil
109+
return newDefaultK8SBackend(configuration, k8sClient, controllerNSSpecified), nil
108110
}
109111

110112
func handleInlineBackendHCL(hclCode string) (string, interface{}, error) {

‎controllers/configuration/backend/backend_test.go

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ import (
1313

1414
func TestParseConfigurationBackend(t *testing.T) {
1515
type args struct {
16-
configuration *v1beta2.Configuration
17-
credentials map[string]string
16+
configuration *v1beta2.Configuration
17+
credentials map[string]string
18+
controllerNSSpecified bool
1819
}
1920
type want struct {
2021
backend Backend
@@ -299,17 +300,46 @@ terraform {
299300
errMsg: "it's not allowed to set `spec.backend.inline` and `spec.backend.backendType` at the same time",
300301
},
301302
},
303+
{
304+
name: "backend is nil, specify controller namespace, generate backend with legacy secret suffix",
305+
args: args{
306+
configuration: &v1beta2.Configuration{
307+
ObjectMeta: metav1.ObjectMeta{Name: "name", Namespace: "ns", UID: "xxxx-xxxx"},
308+
Spec: v1beta2.ConfigurationSpec{
309+
Backend: nil,
310+
},
311+
},
312+
controllerNSSpecified: true,
313+
},
314+
want: want{
315+
backend: &K8SBackend{
316+
LegacySecretSuffix: "name",
317+
SecretNS: "ns",
318+
SecretSuffix: "xxxx-xxxx",
319+
Client: k8sClient,
320+
HCLCode: `
321+
terraform {
322+
backend "kubernetes" {
323+
secret_suffix = "xxxx-xxxx"
324+
in_cluster_config = true
325+
namespace = "ns"
326+
}
327+
}
328+
`,
329+
},
330+
},
331+
},
302332
}
303333

304334
for _, tc := range testcases {
305335
t.Run(tc.name, func(t *testing.T) {
306-
got, err := ParseConfigurationBackend(tc.args.configuration, k8sClient, tc.args.credentials)
336+
got, err := ParseConfigurationBackend(tc.args.configuration, k8sClient, tc.args.credentials, tc.args.controllerNSSpecified)
307337
if tc.want.errMsg != "" && !strings.Contains(err.Error(), tc.want.errMsg) {
308338
t.Errorf("ValidConfigurationObject() error = %v, wantErr %v", err, tc.want.errMsg)
309339
return
310340
}
311341
if !reflect.DeepEqual(tc.want.backend, got) {
312-
t.Errorf("got %#v, want %#v", got, tc.want.backend)
342+
t.Errorf("\ngot %#v,\nwant %#v", got, tc.want.backend)
313343
}
314344
})
315345
}

‎controllers/configuration/backend/kubernetes.go

Lines changed: 73 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ import (
2121
"fmt"
2222
"os"
2323

24-
"github.com/oam-dev/terraform-controller/api/v1beta2"
25-
"github.com/oam-dev/terraform-controller/controllers/util"
2624
"github.com/pkg/errors"
2725
v1 "k8s.io/api/core/v1"
2826
"k8s.io/klog/v2"
2927
"sigs.k8s.io/controller-runtime/pkg/client"
28+
29+
"github.com/oam-dev/terraform-controller/api/v1beta2"
30+
"github.com/oam-dev/terraform-controller/controllers/util"
31+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3032
)
3133

3234
const (
@@ -47,19 +49,31 @@ type K8SBackend struct {
4749
SecretSuffix string
4850
// SecretNS is the namespace of the Terraform backend secret
4951
SecretNS string
52+
// LegacySecretSuffix is the same as SecretSuffix, but only used when `--controller-namespace` is specified
53+
LegacySecretSuffix string
5054
}
5155

52-
func newDefaultK8SBackend(suffix string, client client.Client, namespace string) *K8SBackend {
56+
func newDefaultK8SBackend(configuration *v1beta2.Configuration, client client.Client, controllerNSSpecified bool) *K8SBackend {
5357
ns := os.Getenv("TERRAFORM_BACKEND_NAMESPACE")
5458
if ns == "" {
55-
ns = namespace
59+
ns = configuration.GetNamespace()
60+
}
61+
62+
var (
63+
suffix = configuration.Spec.Backend.SecretSuffix
64+
legacySuffix string
65+
)
66+
if controllerNSSpecified {
67+
legacySuffix = suffix
68+
suffix = string(configuration.GetUID())
5669
}
5770
hcl := renderK8SBackendHCL(suffix, ns)
5871
return &K8SBackend{
59-
Client: client,
60-
HCLCode: hcl,
61-
SecretSuffix: suffix,
62-
SecretNS: ns,
72+
Client: client,
73+
HCLCode: hcl,
74+
SecretSuffix: suffix,
75+
SecretNS: ns,
76+
LegacySecretSuffix: legacySuffix,
6377
}
6478
}
6579

@@ -95,15 +109,23 @@ terraform {
95109
return fmt.Sprintf(fmtStr, suffix, ns)
96110
}
97111

98-
func (k K8SBackend) secretName() string {
112+
func (k *K8SBackend) secretName() string {
99113
return fmt.Sprintf(TFBackendSecret, terraformWorkspace, k.SecretSuffix)
100114
}
101115

116+
func (k *K8SBackend) legacySecretName() string {
117+
return fmt.Sprintf(TFBackendSecret, terraformWorkspace, k.LegacySecretSuffix)
118+
}
119+
102120
// GetTFStateJSON gets Terraform state json from the Terraform kubernetes backend
103121
func (k *K8SBackend) GetTFStateJSON(ctx context.Context) ([]byte, error) {
104122
var s = v1.Secret{}
105-
if err := k.Client.Get(ctx, client.ObjectKey{Name: k.secretName(), Namespace: k.SecretNS}, &s); err != nil {
106-
return nil, errors.Wrap(err, "terraform state file backend secret is not generated")
123+
// Try to get legacy secret first, if it doesn't exist, try to get new secret
124+
err := k.Client.Get(ctx, client.ObjectKey{Name: k.legacySecretName(), Namespace: k.SecretNS}, &s)
125+
if err != nil {
126+
if err = k.Client.Get(ctx, client.ObjectKey{Name: k.secretName(), Namespace: k.SecretNS}, &s); err != nil {
127+
return nil, errors.Wrap(err, "terraform state file backend secret is not generated")
128+
}
107129
}
108130
tfStateData, ok := s.Data[TerraformStateNameInSecret]
109131
if !ok {
@@ -119,8 +141,15 @@ func (k *K8SBackend) GetTFStateJSON(ctx context.Context) ([]byte, error) {
119141

120142
// CleanUp will delete the Terraform kubernetes backend secret when deleting the configuration object
121143
func (k *K8SBackend) CleanUp(ctx context.Context) error {
122-
klog.InfoS("Deleting the secret which stores Kubernetes backend", "Name", k.secretName())
144+
klog.InfoS("Deleting the legacy secret which stores Kubernetes backend", "Name", k.legacySecretName())
123145
var kubernetesBackendSecret v1.Secret
146+
if err := k.Client.Get(ctx, client.ObjectKey{Name: k.legacySecretName(), Namespace: k.SecretNS}, &kubernetesBackendSecret); err == nil {
147+
if err := k.Client.Delete(ctx, &kubernetesBackendSecret); err != nil {
148+
return err
149+
}
150+
}
151+
152+
klog.InfoS("Deleting the secret which stores Kubernetes backend", "Name", k.secretName())
124153
if err := k.Client.Get(ctx, client.ObjectKey{Name: k.secretName(), Namespace: k.SecretNS}, &kubernetesBackendSecret); err == nil {
125154
if err := k.Client.Delete(ctx, &kubernetesBackendSecret); err != nil {
126155
return err
@@ -131,8 +160,40 @@ func (k *K8SBackend) CleanUp(ctx context.Context) error {
131160

132161
// HCL returns the backend hcl code string
133162
func (k *K8SBackend) HCL() string {
163+
if k.LegacySecretSuffix != "" {
164+
err := k.migrateLegacySecret()
165+
if err != nil {
166+
klog.ErrorS(err, "Failed to migrate legacy secret")
167+
}
168+
}
169+
134170
if k.HCLCode == "" {
135171
k.HCLCode = renderK8SBackendHCL(k.SecretSuffix, k.SecretNS)
136172
}
137173
return k.HCLCode
138174
}
175+
176+
// migrateLegacySecret will migrate the legacy secret to the new secret if the legacy secret exists
177+
// This is needed when the --controller-namespace is specified and restart the controller
178+
func (k *K8SBackend) migrateLegacySecret() error {
179+
ctx := context.TODO()
180+
s := v1.Secret{}
181+
if err := k.Client.Get(ctx, client.ObjectKey{Name: k.legacySecretName(), Namespace: k.SecretNS}, &s); err == nil {
182+
klog.InfoS("Migrating legacy secret to new secret", "LegacyName", k.legacySecretName(), "NewName", k.secretName(), "Namespace", k.SecretNS)
183+
newSecret := v1.Secret{
184+
ObjectMeta: metav1.ObjectMeta{
185+
Name: k.secretName(),
186+
Namespace: k.SecretNS,
187+
},
188+
Data: s.Data,
189+
}
190+
err = k.Client.Create(ctx, &newSecret)
191+
if err != nil {
192+
return errors.Wrapf(err, "Fail to create new secret, Name: %s, Namespace: %s", k.secretName(), k.SecretNS)
193+
} else if err = k.Client.Delete(ctx, &s); err != nil {
194+
// Only delete the legacy secret if the new secret is successfully created
195+
return errors.Wrapf(err, "Fail to delete legacy secret, Name: %s, Namespace: %s", k.legacySecretName(), k.SecretNS)
196+
}
197+
}
198+
return nil
199+
}

‎controllers/configuration/backend/kubernetes_test.go

Lines changed: 78 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package backend
33
import (
44
"context"
55
"encoding/base64"
6+
"gotest.tools/assert"
67
"reflect"
78
"testing"
89

@@ -82,17 +83,19 @@ terraform {
8283
}
8384

8485
func TestK8SBackend_GetTFStateJSON(t *testing.T) {
86+
const UID = "xxxx-xxxx"
8587
type fields struct {
86-
Client client.Client
87-
HCLCode string
88-
SecretSuffix string
89-
SecretNS string
88+
Client client.Client
89+
HCLCode string
90+
SecretSuffix string
91+
SecretNS string
92+
LegacySecretSuffix string
9093
}
9194
type args struct {
9295
ctx context.Context
9396
}
9497
tfStateData, _ := base64.StdEncoding.DecodeString("H4sIAAAAAAAA/4SQzarbMBCF934KoXUdPKNf+1VKCWNp5AocO8hyaSl592KlcBd3cZfnHPHpY/52QshfXI68b3IS+tuVK5dCaS+P+8ci4TbcULb94JJplZPAFte8MS18PQrKBO8Q+xk59SHa1AMA9M4YmoN3FGJ8M/azPs96yElcCkLIsG+V8sblnqOc3uXlRuvZ0GxSSuiCRUYbw2gGHRFGPxitEgJYQDQ0a68I2ChNo1cAZJ2bR20UtW8bsv55NuJRS94W2erXe5X5QQs3A/FZ4fhJaOwUgZTVMRjto1HGpSGSQuuD955hdDDPcR6NY1ZpQJ/YwagTRAvBpsi8LXn7Pa1U+ahfWHX/zWThYz9L4Otg3390r+5fAAAA//8hmcuNuQEAAA==")
95-
secret := &v1.Secret{
98+
baseSecret := &v1.Secret{
9699
ObjectMeta: metav1.ObjectMeta{
97100
Name: "tfstate-default-a",
98101
Namespace: "default",
@@ -102,7 +105,8 @@ func TestK8SBackend_GetTFStateJSON(t *testing.T) {
102105
TerraformStateNameInSecret: tfStateData,
103106
},
104107
}
105-
k8sClient := fake.NewClientBuilder().WithObjects(secret).Build()
108+
109+
k8sClient := fake.NewClientBuilder().WithObjects(baseSecret).Build()
106110
k8sClient2 := fake.NewClientBuilder().Build()
107111
tests := []struct {
108112
name string
@@ -120,24 +124,7 @@ func TestK8SBackend_GetTFStateJSON(t *testing.T) {
120124
SecretNS: "default",
121125
},
122126
args: args{ctx: context.Background()},
123-
want: []byte(`{
124-
"version": 4,
125-
"terraform_version": "1.0.2",
126-
"serial": 2,
127-
"lineage": "c35c8722-b2ef-cd6f-1111-755abc87acdd",
128-
"outputs": {
129-
"container_id":{
130-
"value": "e5fff27c62e26dc9504d21980543f21161225ab483a1e534a98311a677b9453a",
131-
"type": "string"
132-
},
133-
"image_id": {
134-
"value": "sha256:d1a364dc548d5357f0da3268c888e1971bbdb957ee3f028fe7194f1d61c6fdeenginx:latest",
135-
"type": "string"
136-
}
137-
},
138-
"resources": []
139-
}
140-
`),
127+
want: tfStateJson,
141128
},
142129
{
143130
name: "secret doesn't exist",
@@ -151,14 +138,26 @@ func TestK8SBackend_GetTFStateJSON(t *testing.T) {
151138
want: nil,
152139
wantErr: true,
153140
},
141+
{
142+
name: "got a legacy secret",
143+
fields: fields{
144+
Client: k8sClient,
145+
HCLCode: "",
146+
LegacySecretSuffix: "a",
147+
SecretNS: "default",
148+
SecretSuffix: UID,
149+
},
150+
want: tfStateJson,
151+
},
154152
}
155153
for _, tt := range tests {
156154
t.Run(tt.name, func(t *testing.T) {
157155
k := &K8SBackend{
158-
Client: tt.fields.Client,
159-
HCLCode: tt.fields.HCLCode,
160-
SecretSuffix: tt.fields.SecretSuffix,
161-
SecretNS: tt.fields.SecretNS,
156+
Client: tt.fields.Client,
157+
HCLCode: tt.fields.HCLCode,
158+
SecretSuffix: tt.fields.SecretSuffix,
159+
SecretNS: tt.fields.SecretNS,
160+
LegacySecretSuffix: tt.fields.LegacySecretSuffix,
162161
}
163162
got, err := k.GetTFStateJSON(tt.args.ctx)
164163
if (err != nil) != tt.wantErr {
@@ -221,3 +220,54 @@ func TestK8SBackend_CleanUp(t *testing.T) {
221220
})
222221
}
223222
}
223+
224+
func TestMigrateLegacySecret(t *testing.T) {
225+
secretNS := "default"
226+
secret := v1.Secret{
227+
ObjectMeta: metav1.ObjectMeta{
228+
Name: "tfstate-default-a",
229+
Namespace: secretNS,
230+
},
231+
Type: v1.SecretTypeOpaque,
232+
}
233+
k8sClient := fake.NewClientBuilder().WithObjects(&secret).Build()
234+
fakeUID := "xxxx-xxxx"
235+
k := &K8SBackend{
236+
Client: k8sClient,
237+
HCLCode: "",
238+
SecretSuffix: fakeUID,
239+
SecretNS: secretNS,
240+
LegacySecretSuffix: "a",
241+
}
242+
err := k.migrateLegacySecret()
243+
assert.NilError(t, err)
244+
NoLegacySecK8sClient := fake.NewClientBuilder().Build()
245+
k = &K8SBackend{
246+
Client: NoLegacySecK8sClient,
247+
HCLCode: "",
248+
SecretSuffix: fakeUID,
249+
SecretNS: secretNS,
250+
LegacySecretSuffix: "a",
251+
}
252+
err = k.migrateLegacySecret()
253+
assert.NilError(t, err)
254+
}
255+
256+
var tfStateJson = []byte(`{
257+
"version": 4,
258+
"terraform_version": "1.0.2",
259+
"serial": 2,
260+
"lineage": "c35c8722-b2ef-cd6f-1111-755abc87acdd",
261+
"outputs": {
262+
"container_id":{
263+
"value": "e5fff27c62e26dc9504d21980543f21161225ab483a1e534a98311a677b9453a",
264+
"type": "string"
265+
},
266+
"image_id": {
267+
"value": "sha256:d1a364dc548d5357f0da3268c888e1971bbdb957ee3f028fe7194f1d61c6fdeenginx:latest",
268+
"type": "string"
269+
}
270+
},
271+
"resources": []
272+
}
273+
`)

‎controllers/configuration/configuration.go

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"strconv"
77
"strings"
88

9-
"github.com/oam-dev/terraform-controller/controllers/configuration/backend"
109
"github.com/pkg/errors"
1110
kerrors "k8s.io/apimachinery/pkg/api/errors"
1211
apitypes "k8s.io/apimachinery/pkg/types"
@@ -50,25 +49,6 @@ func ValidConfigurationObject(configuration *v1beta2.Configuration) (types.Confi
5049
return "", nil
5150
}
5251

53-
// RenderConfiguration will compose the Terraform configuration with hcl/json and backend
54-
func RenderConfiguration(configuration *v1beta2.Configuration, k8sClient client.Client, configurationType types.ConfigurationType, credentials map[string]string) (string, backend.Backend, error) {
55-
backendInterface, err := backend.ParseConfigurationBackend(configuration, k8sClient, credentials)
56-
if err != nil {
57-
return "", nil, errors.Wrap(err, "failed to prepare Terraform backend configuration")
58-
}
59-
60-
switch configurationType {
61-
case types.ConfigurationHCL:
62-
completedConfiguration := configuration.Spec.HCL
63-
completedConfiguration += "\n" + backendInterface.HCL()
64-
return completedConfiguration, backendInterface, nil
65-
case types.ConfigurationRemote:
66-
return backendInterface.HCL(), backendInterface, nil
67-
default:
68-
return "", nil, errors.New("Unsupported Configuration Type")
69-
}
70-
}
71-
7252
// SetRegion will set the region for Configuration
7353
func SetRegion(ctx context.Context, k8sClient client.Client, namespace, name string, providerObj *v1beta1.Provider) (string, error) {
7454
configuration, err := Get(ctx, k8sClient, apitypes.NamespacedName{Namespace: namespace, Name: name})

‎controllers/configuration/configuration_test.go

Lines changed: 0 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@ package configuration
22

33
import (
44
"context"
5-
"reflect"
65
"strings"
76
"testing"
87

9-
"github.com/oam-dev/terraform-controller/controllers/configuration/backend"
108
"github.com/stretchr/testify/assert"
119
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1210
"k8s.io/apimachinery/pkg/runtime"
@@ -103,134 +101,6 @@ func TestValidConfigurationObject(t *testing.T) {
103101

104102
}
105103

106-
func TestRenderConfiguration(t *testing.T) {
107-
type args struct {
108-
configuration *v1beta2.Configuration
109-
configurationType types.ConfigurationType
110-
credentials map[string]string
111-
}
112-
type want struct {
113-
cfg string
114-
backendInterface backend.Backend
115-
errMsg string
116-
}
117-
118-
k8sClient := fake.NewClientBuilder().Build()
119-
120-
testcases := []struct {
121-
name string
122-
args args
123-
want want
124-
}{
125-
{
126-
name: "backend is not nil, configuration is hcl",
127-
args: args{
128-
configuration: &v1beta2.Configuration{
129-
ObjectMeta: metav1.ObjectMeta{
130-
Namespace: "n1",
131-
},
132-
Spec: v1beta2.ConfigurationSpec{
133-
Backend: &v1beta2.Backend{},
134-
HCL: "image_id=123",
135-
},
136-
},
137-
configurationType: types.ConfigurationHCL,
138-
},
139-
want: want{
140-
cfg: `image_id=123
141-
142-
terraform {
143-
backend "kubernetes" {
144-
secret_suffix = ""
145-
in_cluster_config = true
146-
namespace = "n1"
147-
}
148-
}
149-
`,
150-
backendInterface: &backend.K8SBackend{
151-
Client: k8sClient,
152-
HCLCode: `
153-
terraform {
154-
backend "kubernetes" {
155-
secret_suffix = ""
156-
in_cluster_config = true
157-
namespace = "n1"
158-
}
159-
}
160-
`,
161-
SecretSuffix: "",
162-
SecretNS: "n1",
163-
},
164-
},
165-
},
166-
{
167-
name: "backend is nil, configuration is remote",
168-
args: args{
169-
configuration: &v1beta2.Configuration{
170-
ObjectMeta: metav1.ObjectMeta{
171-
Namespace: "n2",
172-
},
173-
Spec: v1beta2.ConfigurationSpec{
174-
Remote: "https://github.com/a/b.git",
175-
},
176-
},
177-
configurationType: types.ConfigurationRemote,
178-
},
179-
want: want{
180-
cfg: `
181-
terraform {
182-
backend "kubernetes" {
183-
secret_suffix = ""
184-
in_cluster_config = true
185-
namespace = "n2"
186-
}
187-
}
188-
`,
189-
backendInterface: &backend.K8SBackend{
190-
Client: k8sClient,
191-
HCLCode: `
192-
terraform {
193-
backend "kubernetes" {
194-
secret_suffix = ""
195-
in_cluster_config = true
196-
namespace = "n2"
197-
}
198-
}
199-
`,
200-
SecretSuffix: "",
201-
SecretNS: "n2",
202-
},
203-
},
204-
},
205-
{
206-
name: "backend is nil, configuration is not supported",
207-
args: args{
208-
configuration: &v1beta2.Configuration{
209-
Spec: v1beta2.ConfigurationSpec{},
210-
},
211-
},
212-
want: want{
213-
errMsg: "Unsupported Configuration Type",
214-
},
215-
},
216-
}
217-
218-
for _, tc := range testcases {
219-
t.Run(tc.name, func(t *testing.T) {
220-
got, backendConf, err := RenderConfiguration(tc.args.configuration, k8sClient, tc.args.configurationType, tc.args.credentials)
221-
if tc.want.errMsg != "" && !strings.Contains(err.Error(), tc.want.errMsg) {
222-
t.Errorf("ValidConfigurationObject() error = %v, wantErr %v", err, tc.want.errMsg)
223-
return
224-
}
225-
assert.Equal(t, tc.want.cfg, got)
226-
227-
if !reflect.DeepEqual(tc.want.backendInterface, backendConf) {
228-
t.Errorf("backendInterface is not equal.\n got %#v\n, want %#v", backendConf, tc.want.backendInterface)
229-
}
230-
})
231-
}
232-
}
233-
234104
func TestReplaceTerraformSource(t *testing.T) {
235105
testcases := []struct {
236106
remote string

‎controllers/configuration_controller.go

Lines changed: 83 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
113113
return ctrl.Result{}, client.IgnoreNotFound(err)
114114
}
115115

116-
meta := initTFConfigurationMeta(req, configuration)
116+
meta := initTFConfigurationMeta(req, configuration, r.Client)
117117
if r.ControllerNamespace != "" {
118118
uid := string(configuration.GetUID())
119119
// @step: since we are using a single namespace to run these, we must ensure the names
@@ -124,11 +124,7 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
124124
meta.ConfigurationCMName = fmt.Sprintf(TFInputConfigMapName, uid)
125125
meta.VariableSecretName = fmt.Sprintf(TFVariableSecret, uid)
126126
meta.ControllerNamespace = r.ControllerNamespace
127-
128-
configuration.Spec.Backend = &v1beta2.Backend{
129-
InClusterConfig: true,
130-
SecretSuffix: uid,
131-
}
127+
meta.ControllerNSSpecified = true
132128
}
133129

134130
// add finalizer
@@ -147,15 +143,6 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
147143
return ctrl.Result{}, err
148144
}
149145

150-
var tfExecutionJob = &batchv1.Job{}
151-
if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: meta.ControllerNamespace}, tfExecutionJob); err == nil {
152-
if !meta.EnvChanged && tfExecutionJob.Status.Succeeded == int32(1) {
153-
if err := meta.updateApplyStatus(ctx, r.Client, types.Available, types.MessageCloudResourceDeployed); err != nil {
154-
return ctrl.Result{}, err
155-
}
156-
}
157-
}
158-
159146
if isDeleting {
160147
// terraform destroy
161148
klog.InfoS("performing Configuration Destroy", "Namespace", req.Namespace, "Name", req.Name, "JobName", meta.DestroyJobName)
@@ -188,6 +175,14 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
188175
return ctrl.Result{}, nil
189176
}
190177

178+
var tfExecutionJob = &batchv1.Job{}
179+
if err := meta.getApplyJob(ctx, r.Client, tfExecutionJob); err == nil {
180+
if !meta.EnvChanged && tfExecutionJob.Status.Succeeded == int32(1) {
181+
err = meta.updateApplyStatus(ctx, r.Client, types.Available, types.MessageCloudResourceDeployed)
182+
return ctrl.Result{}, err
183+
}
184+
}
185+
191186
// Terraform apply (create or update)
192187
klog.InfoS("performing Terraform Apply (cloud resource create/update)", "Namespace", req.Namespace, "Name", req.Name)
193188
if err := r.terraformApply(ctx, configuration, meta); err != nil {
@@ -220,6 +215,17 @@ type LegacySubResources struct {
220215
VariableSecretName string
221216
}
222217

218+
type ResourceQuota struct {
219+
ResourcesLimitsCPU string
220+
ResourcesLimitsCPUQuantity resource.Quantity
221+
ResourcesLimitsMemory string
222+
ResourcesLimitsMemoryQuantity resource.Quantity
223+
ResourcesRequestsCPU string
224+
ResourcesRequestsCPUQuantity resource.Quantity
225+
ResourcesRequestsMemory string
226+
ResourcesRequestsMemoryQuantity resource.Quantity
227+
}
228+
223229
// TFConfigurationMeta is all the metadata of a Configuration
224230
type TFConfigurationMeta struct {
225231
Name string
@@ -251,20 +257,16 @@ type TFConfigurationMeta struct {
251257
BusyboxImage string
252258
GitImage string
253259

254-
// Resources series Variables are for Setting Compute Resources required by this container
255-
ResourcesLimitsCPU string
256-
ResourcesLimitsCPUQuantity resource.Quantity
257-
ResourcesLimitsMemory string
258-
ResourcesLimitsMemoryQuantity resource.Quantity
259-
ResourcesRequestsCPU string
260-
ResourcesRequestsCPUQuantity resource.Quantity
261-
ResourcesRequestsMemory string
262-
ResourcesRequestsMemoryQuantity resource.Quantity
260+
// ResourceQuota series Variables are for Setting Compute Resources required by this container
261+
ResourceQuota ResourceQuota
262+
263+
LegacySubResources LegacySubResources
264+
ControllerNSSpecified bool
263265

264-
LegacySubResources LegacySubResources
266+
K8sClient client.Client
265267
}
266268

267-
func initTFConfigurationMeta(req ctrl.Request, configuration v1beta2.Configuration) *TFConfigurationMeta {
269+
func initTFConfigurationMeta(req ctrl.Request, configuration v1beta2.Configuration, k8sClient client.Client) *TFConfigurationMeta {
268270
var meta = &TFConfigurationMeta{
269271
ControllerNamespace: req.Namespace,
270272
Namespace: req.Namespace,
@@ -273,6 +275,7 @@ func initTFConfigurationMeta(req ctrl.Request, configuration v1beta2.Configurati
273275
VariableSecretName: fmt.Sprintf(TFVariableSecret, req.Name),
274276
ApplyJobName: req.Name + "-" + string(TerraformApply),
275277
DestroyJobName: req.Name + "-" + string(TerraformDestroy),
278+
K8sClient: k8sClient,
276279
}
277280

278281
jobNodeSelectorStr := os.Getenv("JOB_NODE_SELECTOR")
@@ -368,7 +371,7 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, configur
368371
}
369372
}
370373
if err := meta.updateTerraformJobIfNeeded(ctx, k8sClient, destroyJob); err != nil {
371-
klog.ErrorS(err, types.ErrUpdateTerraformApplyJob, "Name", meta.ApplyJobName)
374+
klog.ErrorS(err, types.ErrUpdateTerraformApplyJob, "Name", meta.DestroyJobName)
372375
return errors.Wrap(err, types.ErrUpdateTerraformApplyJob)
373376
}
374377
}
@@ -439,45 +442,45 @@ func (r *ConfigurationReconciler) cleanUpSubResources(ctx context.Context, confi
439442

440443
func (r *ConfigurationReconciler) preCheckResourcesSetting(meta *TFConfigurationMeta) error {
441444

442-
meta.ResourcesLimitsCPU = os.Getenv("RESOURCES_LIMITS_CPU")
443-
if meta.ResourcesLimitsCPU != "" {
444-
limitsCPU, err := resource.ParseQuantity(meta.ResourcesLimitsCPU)
445+
meta.ResourceQuota.ResourcesLimitsCPU = os.Getenv("RESOURCES_LIMITS_CPU")
446+
if meta.ResourceQuota.ResourcesLimitsCPU != "" {
447+
limitsCPU, err := resource.ParseQuantity(meta.ResourceQuota.ResourcesLimitsCPU)
445448
if err != nil {
446449
errMsg := "failed to parse env variable RESOURCES_LIMITS_CPU into resource.Quantity"
447450
klog.ErrorS(err, errMsg)
448451
return errors.Wrap(err, errMsg)
449452
}
450-
meta.ResourcesLimitsCPUQuantity = limitsCPU
453+
meta.ResourceQuota.ResourcesLimitsCPUQuantity = limitsCPU
451454
}
452-
meta.ResourcesLimitsMemory = os.Getenv("RESOURCES_LIMITS_MEMORY")
453-
if meta.ResourcesLimitsMemory != "" {
454-
limitsMemory, err := resource.ParseQuantity(meta.ResourcesLimitsMemory)
455+
meta.ResourceQuota.ResourcesLimitsMemory = os.Getenv("RESOURCES_LIMITS_MEMORY")
456+
if meta.ResourceQuota.ResourcesLimitsMemory != "" {
457+
limitsMemory, err := resource.ParseQuantity(meta.ResourceQuota.ResourcesLimitsMemory)
455458
if err != nil {
456459
errMsg := "failed to parse env variable RESOURCES_LIMITS_MEMORY into resource.Quantity"
457460
klog.ErrorS(err, errMsg)
458461
return errors.Wrap(err, errMsg)
459462
}
460-
meta.ResourcesLimitsMemoryQuantity = limitsMemory
463+
meta.ResourceQuota.ResourcesLimitsMemoryQuantity = limitsMemory
461464
}
462-
meta.ResourcesRequestsCPU = os.Getenv("RESOURCES_REQUESTS_CPU")
463-
if meta.ResourcesRequestsCPU != "" {
464-
requestsCPU, err := resource.ParseQuantity(meta.ResourcesRequestsCPU)
465+
meta.ResourceQuota.ResourcesRequestsCPU = os.Getenv("RESOURCES_REQUESTS_CPU")
466+
if meta.ResourceQuota.ResourcesRequestsCPU != "" {
467+
requestsCPU, err := resource.ParseQuantity(meta.ResourceQuota.ResourcesRequestsCPU)
465468
if err != nil {
466469
errMsg := "failed to parse env variable RESOURCES_REQUESTS_CPU into resource.Quantity"
467470
klog.ErrorS(err, errMsg)
468471
return errors.Wrap(err, errMsg)
469472
}
470-
meta.ResourcesRequestsCPUQuantity = requestsCPU
473+
meta.ResourceQuota.ResourcesRequestsCPUQuantity = requestsCPU
471474
}
472-
meta.ResourcesRequestsMemory = os.Getenv("RESOURCES_REQUESTS_MEMORY")
473-
if meta.ResourcesRequestsMemory != "" {
474-
requestsMemory, err := resource.ParseQuantity(meta.ResourcesRequestsMemory)
475+
meta.ResourceQuota.ResourcesRequestsMemory = os.Getenv("RESOURCES_REQUESTS_MEMORY")
476+
if meta.ResourceQuota.ResourcesRequestsMemory != "" {
477+
requestsMemory, err := resource.ParseQuantity(meta.ResourceQuota.ResourcesRequestsMemory)
475478
if err != nil {
476479
errMsg := "failed to parse env variable RESOURCES_REQUESTS_MEMORY into resource.Quantity"
477480
klog.ErrorS(err, errMsg)
478481
return errors.Wrap(err, errMsg)
479482
}
480-
meta.ResourcesRequestsMemoryQuantity = requestsMemory
483+
meta.ResourceQuota.ResourcesRequestsMemoryQuantity = requestsMemory
481484
}
482485
return nil
483486
}
@@ -533,7 +536,7 @@ func (r *ConfigurationReconciler) preCheck(ctx context.Context, configuration *v
533536
}
534537

535538
// Render configuration with backend
536-
completeConfiguration, backendConf, err := tfcfg.RenderConfiguration(configuration, r.Client, configurationType, meta.Credentials)
539+
completeConfiguration, backendConf, err := meta.RenderConfiguration(configuration, configurationType)
537540
if err != nil {
538541
return err
539542
}
@@ -610,6 +613,7 @@ func (meta *TFConfigurationMeta) updateApplyStatus(ctx context.Context, k8sClien
610613
if state == types.Available {
611614
outputs, err := meta.getTFOutputs(ctx, k8sClient, configuration)
612615
if err != nil {
616+
klog.InfoS("Failed to get outputs", "error", err)
613617
configuration.Status.Apply = v1beta2.ConfigurationApplyStatus{
614618
State: types.GeneratingOutputs,
615619
Message: types.ErrGenerateOutputs + ": " + err.Error(),
@@ -650,7 +654,7 @@ func (meta *TFConfigurationMeta) assembleAndTriggerJob(ctx context.Context, k8sC
650654
return k8sClient.Create(ctx, job)
651655
}
652656

653-
// updateTerraformJob will set deletion finalizer to the Terraform job if its envs are changed, which will result in
657+
// updateTerraformJobIfNeeded will set deletion finalizer to the Terraform job if its envs are changed, which will result in
654658
// deleting the job. Finally, a new Terraform job will be generated
655659
func (meta *TFConfigurationMeta) updateTerraformJobIfNeeded(ctx context.Context, k8sClient client.Client, job batchv1.Job) error {
656660
// if either one changes, delete the job
@@ -768,25 +772,25 @@ func (meta *TFConfigurationMeta) assembleTerraformJob(executionType TerraformExe
768772
Env: meta.Envs,
769773
}
770774

771-
if meta.ResourcesLimitsCPU != "" || meta.ResourcesLimitsMemory != "" ||
772-
meta.ResourcesRequestsCPU != "" || meta.ResourcesRequestsMemory != "" {
775+
if meta.ResourceQuota.ResourcesLimitsCPU != "" || meta.ResourceQuota.ResourcesLimitsMemory != "" ||
776+
meta.ResourceQuota.ResourcesRequestsCPU != "" || meta.ResourceQuota.ResourcesRequestsMemory != "" {
773777
resourceRequirements := v1.ResourceRequirements{}
774-
if meta.ResourcesLimitsCPU != "" || meta.ResourcesLimitsMemory != "" {
778+
if meta.ResourceQuota.ResourcesLimitsCPU != "" || meta.ResourceQuota.ResourcesLimitsMemory != "" {
775779
resourceRequirements.Limits = map[v1.ResourceName]resource.Quantity{}
776-
if meta.ResourcesLimitsCPU != "" {
777-
resourceRequirements.Limits["cpu"] = meta.ResourcesLimitsCPUQuantity
780+
if meta.ResourceQuota.ResourcesLimitsCPU != "" {
781+
resourceRequirements.Limits["cpu"] = meta.ResourceQuota.ResourcesLimitsCPUQuantity
778782
}
779-
if meta.ResourcesLimitsMemory != "" {
780-
resourceRequirements.Limits["memory"] = meta.ResourcesLimitsMemoryQuantity
783+
if meta.ResourceQuota.ResourcesLimitsMemory != "" {
784+
resourceRequirements.Limits["memory"] = meta.ResourceQuota.ResourcesLimitsMemoryQuantity
781785
}
782786
}
783-
if meta.ResourcesRequestsCPU != "" || meta.ResourcesLimitsMemory != "" {
787+
if meta.ResourceQuota.ResourcesRequestsCPU != "" || meta.ResourceQuota.ResourcesLimitsMemory != "" {
784788
resourceRequirements.Requests = map[v1.ResourceName]resource.Quantity{}
785-
if meta.ResourcesRequestsCPU != "" {
786-
resourceRequirements.Requests["cpu"] = meta.ResourcesRequestsCPUQuantity
789+
if meta.ResourceQuota.ResourcesRequestsCPU != "" {
790+
resourceRequirements.Requests["cpu"] = meta.ResourceQuota.ResourcesRequestsCPUQuantity
787791
}
788-
if meta.ResourcesRequestsMemory != "" {
789-
resourceRequirements.Requests["memory"] = meta.ResourcesRequestsMemoryQuantity
792+
if meta.ResourceQuota.ResourcesRequestsMemory != "" {
793+
resourceRequirements.Requests["memory"] = meta.ResourceQuota.ResourcesRequestsMemoryQuantity
790794
}
791795
}
792796
container.Resources = resourceRequirements
@@ -960,6 +964,7 @@ func (meta *TFConfigurationMeta) getTFOutputs(ctx context.Context, k8sClient cli
960964
gotSecret.Namespace, name,
961965
ownerNamespace, ownerName,
962966
)
967+
klog.ErrorS(err, "fail to update backend secret")
963968
return nil, errors.New(errMsg)
964969
}
965970
gotSecret.Data = data
@@ -1232,8 +1237,29 @@ func (meta *TFConfigurationMeta) KeepLegacySubResourceMetas() {
12321237

12331238
func (meta *TFConfigurationMeta) getApplyJob(ctx context.Context, k8sClient client.Client, job *batchv1.Job) error {
12341239
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.LegacySubResources.ApplyJobName, Namespace: meta.LegacySubResources.Namespace}, job); err == nil {
1240+
klog.InfoS("Found legacy apply job", "Configuration", fmt.Sprintf("%s/%s", meta.Name, meta.Namespace),
1241+
"Job", fmt.Sprintf("%s/%s", meta.LegacySubResources.Namespace, meta.LegacySubResources.ApplyJobName))
12351242
return nil
12361243
}
12371244
err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: meta.ControllerNamespace}, job)
12381245
return err
12391246
}
1247+
1248+
// RenderConfiguration will compose the Terraform configuration with hcl/json and backend
1249+
func (meta *TFConfigurationMeta) RenderConfiguration(configuration *v1beta2.Configuration, configurationType types.ConfigurationType) (string, backend.Backend, error) {
1250+
backendInterface, err := backend.ParseConfigurationBackend(configuration, meta.K8sClient, meta.Credentials, meta.ControllerNSSpecified)
1251+
if err != nil {
1252+
return "", nil, errors.Wrap(err, "failed to prepare Terraform backend configuration")
1253+
}
1254+
1255+
switch configurationType {
1256+
case types.ConfigurationHCL:
1257+
completedConfiguration := configuration.Spec.HCL
1258+
completedConfiguration += "\n" + backendInterface.HCL()
1259+
return completedConfiguration, backendInterface, nil
1260+
case types.ConfigurationRemote:
1261+
return backendInterface.HCL(), backendInterface, nil
1262+
default:
1263+
return "", nil, errors.New("Unsupported Configuration Type")
1264+
}
1265+
}

‎controllers/configuration_controller_test.go

Lines changed: 193 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func TestInitTFConfigurationMeta(t *testing.T) {
114114

115115
for _, tc := range testcases {
116116
t.Run(tc.name, func(t *testing.T) {
117-
meta := initTFConfigurationMeta(req, tc.configuration)
117+
meta := initTFConfigurationMeta(req, tc.configuration, nil)
118118
if !reflect.DeepEqual(meta.Name, tc.want.Name) {
119119
t.Errorf("initTFConfigurationMeta = %v, want %v", meta, tc.want)
120120
}
@@ -176,7 +176,7 @@ func TestInitTFConfigurationMetaWithDeleteResource(t *testing.T) {
176176
}
177177
for _, tc := range testcases {
178178
t.Run(tc.name, func(t *testing.T) {
179-
meta := initTFConfigurationMeta(req, tc.configuration)
179+
meta := initTFConfigurationMeta(req, tc.configuration, nil)
180180
if !reflect.DeepEqual(meta.DeleteResource, tc.meta.DeleteResource) {
181181
t.Errorf("initTFConfigurationMeta = %v, want %v", meta, tc.meta)
182182
}
@@ -197,7 +197,7 @@ func TestInitTFConfigurationMetaWithJobNodeSelector(t *testing.T) {
197197
},
198198
Spec: v1beta2.ConfigurationSpec{},
199199
}
200-
meta := initTFConfigurationMeta(req, configuration)
200+
meta := initTFConfigurationMeta(req, configuration, nil)
201201
assert.Equal(t, meta.JobNodeSelector, map[string]string{"ssd": "true"})
202202
}
203203

@@ -1437,14 +1437,16 @@ func TestAssembleTerraformJobWithResourcesSetting(t *testing.T) {
14371437
TerraformImage: "f",
14381438
RemoteGit: "g",
14391439

1440-
ResourcesLimitsCPU: "10m",
1441-
ResourcesLimitsCPUQuantity: quantityLimitsCPU,
1442-
ResourcesLimitsMemory: "10Mi",
1443-
ResourcesLimitsMemoryQuantity: quantityLimitsMemory,
1444-
ResourcesRequestsCPU: "100m",
1445-
ResourcesRequestsCPUQuantity: quantityRequestsCPU,
1446-
ResourcesRequestsMemory: "5Gi",
1447-
ResourcesRequestsMemoryQuantity: quantityRequestsMemory,
1440+
ResourceQuota: ResourceQuota{
1441+
ResourcesLimitsCPU: "10m",
1442+
ResourcesLimitsCPUQuantity: quantityLimitsCPU,
1443+
ResourcesLimitsMemory: "10Mi",
1444+
ResourcesLimitsMemoryQuantity: quantityLimitsMemory,
1445+
ResourcesRequestsCPU: "100m",
1446+
ResourcesRequestsCPUQuantity: quantityRequestsCPU,
1447+
ResourcesRequestsMemory: "5Gi",
1448+
ResourcesRequestsMemoryQuantity: quantityRequestsMemory,
1449+
},
14481450
}
14491451

14501452
job := meta.assembleTerraformJob(TerraformApply)
@@ -2174,3 +2176,183 @@ func TestGetApplyJob(t *testing.T) {
21742176
})
21752177
}
21762178
}
2179+
2180+
func TestRenderConfiguration(t *testing.T) {
2181+
type args struct {
2182+
configuration *v1beta2.Configuration
2183+
configurationType types.ConfigurationType
2184+
credentials map[string]string
2185+
controllerNSSpecified bool
2186+
}
2187+
type want struct {
2188+
cfg string
2189+
backendInterface backend.Backend
2190+
errMsg string
2191+
}
2192+
2193+
k8sClient := fake.NewClientBuilder().Build()
2194+
baseMeta := TFConfigurationMeta{
2195+
K8sClient: k8sClient,
2196+
}
2197+
2198+
testcases := []struct {
2199+
name string
2200+
args args
2201+
want want
2202+
}{
2203+
{
2204+
name: "backend is not nil, configuration is hcl",
2205+
args: args{
2206+
configuration: &v1beta2.Configuration{
2207+
ObjectMeta: metav1.ObjectMeta{
2208+
Namespace: "n1",
2209+
},
2210+
Spec: v1beta2.ConfigurationSpec{
2211+
Backend: &v1beta2.Backend{},
2212+
HCL: "image_id=123",
2213+
},
2214+
},
2215+
configurationType: types.ConfigurationHCL,
2216+
},
2217+
want: want{
2218+
cfg: `image_id=123
2219+
2220+
terraform {
2221+
backend "kubernetes" {
2222+
secret_suffix = ""
2223+
in_cluster_config = true
2224+
namespace = "n1"
2225+
}
2226+
}
2227+
`,
2228+
backendInterface: &backend.K8SBackend{
2229+
Client: k8sClient,
2230+
HCLCode: `
2231+
terraform {
2232+
backend "kubernetes" {
2233+
secret_suffix = ""
2234+
in_cluster_config = true
2235+
namespace = "n1"
2236+
}
2237+
}
2238+
`,
2239+
SecretSuffix: "",
2240+
SecretNS: "n1",
2241+
},
2242+
},
2243+
},
2244+
{
2245+
name: "backend is nil, configuration is remote",
2246+
args: args{
2247+
configuration: &v1beta2.Configuration{
2248+
ObjectMeta: metav1.ObjectMeta{
2249+
Namespace: "n2",
2250+
},
2251+
Spec: v1beta2.ConfigurationSpec{
2252+
Remote: "https://github.com/a/b.git",
2253+
},
2254+
},
2255+
configurationType: types.ConfigurationRemote,
2256+
},
2257+
want: want{
2258+
cfg: `
2259+
terraform {
2260+
backend "kubernetes" {
2261+
secret_suffix = ""
2262+
in_cluster_config = true
2263+
namespace = "n2"
2264+
}
2265+
}
2266+
`,
2267+
backendInterface: &backend.K8SBackend{
2268+
Client: k8sClient,
2269+
HCLCode: `
2270+
terraform {
2271+
backend "kubernetes" {
2272+
secret_suffix = ""
2273+
in_cluster_config = true
2274+
namespace = "n2"
2275+
}
2276+
}
2277+
`,
2278+
SecretSuffix: "",
2279+
SecretNS: "n2",
2280+
},
2281+
},
2282+
},
2283+
{
2284+
name: "backend is nil, configuration is not supported",
2285+
args: args{
2286+
configuration: &v1beta2.Configuration{
2287+
Spec: v1beta2.ConfigurationSpec{},
2288+
},
2289+
},
2290+
want: want{
2291+
errMsg: "Unsupported Configuration Type",
2292+
},
2293+
},
2294+
{
2295+
name: "controller-namespace specified, backend should have legacy secret suffix",
2296+
args: args{
2297+
configuration: &v1beta2.Configuration{
2298+
Spec: v1beta2.ConfigurationSpec{
2299+
Backend: nil,
2300+
},
2301+
ObjectMeta: metav1.ObjectMeta{
2302+
UID: "xxxx-xxxx",
2303+
Namespace: "n2",
2304+
Name: "name",
2305+
},
2306+
},
2307+
controllerNSSpecified: true,
2308+
configurationType: types.ConfigurationRemote,
2309+
},
2310+
want: want{
2311+
cfg: `
2312+
terraform {
2313+
backend "kubernetes" {
2314+
secret_suffix = "xxxx-xxxx"
2315+
in_cluster_config = true
2316+
namespace = "n2"
2317+
}
2318+
}
2319+
`,
2320+
backendInterface: &backend.K8SBackend{
2321+
Client: k8sClient,
2322+
HCLCode: `
2323+
terraform {
2324+
backend "kubernetes" {
2325+
secret_suffix = "xxxx-xxxx"
2326+
in_cluster_config = true
2327+
namespace = "n2"
2328+
}
2329+
}
2330+
`,
2331+
SecretSuffix: "xxxx-xxxx",
2332+
SecretNS: "n2",
2333+
LegacySecretSuffix: "name",
2334+
}},
2335+
},
2336+
}
2337+
2338+
for _, tc := range testcases {
2339+
t.Run(tc.name, func(t *testing.T) {
2340+
meta := baseMeta
2341+
meta.ControllerNSSpecified = tc.args.controllerNSSpecified
2342+
got, backendConf, err := meta.RenderConfiguration(tc.args.configuration, tc.args.configurationType)
2343+
if tc.want.errMsg != "" && !strings.Contains(err.Error(), tc.want.errMsg) {
2344+
t.Errorf("ValidConfigurationObject() error = %v, wantErr %v", err, tc.want.errMsg)
2345+
return
2346+
}
2347+
if tc.want.errMsg == "" && err != nil {
2348+
t.Errorf("ValidConfigurationObject() error = %v, wantErr nil", err)
2349+
return
2350+
}
2351+
assert.Equal(t, tc.want.cfg, got)
2352+
2353+
if !reflect.DeepEqual(tc.want.backendInterface, backendConf) {
2354+
t.Errorf("backendInterface is not equal.\n got %#v\n, want %#v", backendConf, tc.want.backendInterface)
2355+
}
2356+
})
2357+
}
2358+
}

‎e2e/controllernamespace/controllernamespace_test.go

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
package controllernamespace
2+
3+
import (
4+
"context"
5+
types2 "github.com/oam-dev/terraform-controller/api/types"
6+
"strings"
7+
"time"
8+
9+
crossplane "github.com/oam-dev/terraform-controller/api/types/crossplane-runtime"
10+
. "github.com/onsi/ginkgo"
11+
. "github.com/onsi/gomega"
12+
"github.com/pkg/errors"
13+
appv1 "k8s.io/api/apps/v1"
14+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/types"
16+
pkgClient "sigs.k8s.io/controller-runtime/pkg/client"
17+
"sigs.k8s.io/controller-runtime/pkg/client/config"
18+
19+
"github.com/oam-dev/terraform-controller/api/v1beta2"
20+
)
21+
22+
var _ = Describe("Restart with controller-namespace", func() {
23+
const (
24+
defaultNamespace = "default"
25+
controllerNamespace = "terraform"
26+
chartNamespace = "terraform"
27+
)
28+
var (
29+
controllerDeployMeta = types.NamespacedName{Name: "terraform-controller", Namespace: chartNamespace}
30+
)
31+
ctx := context.Background()
32+
33+
// create k8s rest config
34+
restConf, err := config.GetConfig()
35+
Expect(err).NotTo(HaveOccurred())
36+
k8sClient, err := pkgClient.New(restConf, pkgClient.Options{})
37+
s := k8sClient.Scheme()
38+
_ = v1beta2.AddToScheme(s)
39+
Expect(err).NotTo(HaveOccurred())
40+
41+
configuration := &v1beta2.Configuration{
42+
ObjectMeta: v1.ObjectMeta{
43+
Name: "e2e-for-ctrl-ns",
44+
Namespace: defaultNamespace,
45+
},
46+
Spec: v1beta2.ConfigurationSpec{
47+
HCL: `
48+
resource "random_id" "server" {
49+
byte_length = 8
50+
}
51+
52+
output "random_id" {
53+
value = random_id.server.hex
54+
}`,
55+
InlineCredentials: true,
56+
WriteConnectionSecretToReference: &crossplane.SecretReference{
57+
Name: "some-conn",
58+
Namespace: defaultNamespace,
59+
},
60+
},
61+
}
62+
AfterEach(func() {
63+
_ = k8sClient.Delete(ctx, configuration)
64+
})
65+
It("Restart with controller namespace", func() {
66+
By("apply configuration without --controller-namespace", func() {
67+
err = k8sClient.Create(ctx, configuration)
68+
Expect(err).NotTo(HaveOccurred())
69+
var cfg = &v1beta2.Configuration{}
70+
Eventually(func() error {
71+
err = k8sClient.Get(ctx, types.NamespacedName{Name: configuration.Name, Namespace: configuration.Namespace}, cfg)
72+
if err != nil {
73+
return err
74+
}
75+
if cfg.Status.Apply.State != types2.Available {
76+
return errors.Errorf("configuration is not available, status now: %s", cfg.Status.Apply.State)
77+
}
78+
return nil
79+
}, time.Second*60, time.Second*5).Should(Succeed())
80+
})
81+
By("restart controller with --controller-namespace", func() {
82+
ctrlDeploy := appv1.Deployment{}
83+
err = k8sClient.Get(ctx, controllerDeployMeta, &ctrlDeploy)
84+
Expect(err).NotTo(HaveOccurred())
85+
ctrlDeploy.Spec.Template.Spec.Containers[0].Args = append(ctrlDeploy.Spec.Template.Spec.Containers[0].Args, "--controller-namespace="+controllerNamespace)
86+
err := k8sClient.Update(ctx, &ctrlDeploy)
87+
Expect(err).NotTo(HaveOccurred())
88+
89+
Eventually(func() error {
90+
err := k8sClient.Get(ctx, controllerDeployMeta, &ctrlDeploy)
91+
if err != nil {
92+
return err
93+
}
94+
if ctrlDeploy.Status.UnavailableReplicas == 1 {
95+
return errors.New("controller is not updated")
96+
}
97+
return nil
98+
}, time.Second*60, time.Second*5).Should(Succeed())
99+
100+
})
101+
By("configuration should be still available", func() {
102+
// wait about half minute to check configuration's state isn't changed
103+
for i := 0; i < 30; i++ {
104+
err := k8sClient.Get(ctx, types.NamespacedName{
105+
Name: configuration.Name, Namespace: configuration.Namespace,
106+
}, configuration)
107+
Expect(err).NotTo(HaveOccurred())
108+
time.Sleep(time.Second)
109+
}
110+
})
111+
By("restore controller", func() {
112+
ctrlDeploy := appv1.Deployment{}
113+
err = k8sClient.Get(ctx, controllerDeployMeta, &ctrlDeploy)
114+
Expect(err).NotTo(HaveOccurred())
115+
cmds := make([]string, 0)
116+
for _, cmd := range ctrlDeploy.Spec.Template.Spec.Containers[0].Args {
117+
if !strings.HasPrefix(cmd, "--controller-namespace") {
118+
cmds = append(cmds, cmd)
119+
}
120+
}
121+
ctrlDeploy.Spec.Template.Spec.Containers[0].Args = cmds
122+
err := k8sClient.Update(ctx, &ctrlDeploy)
123+
Expect(err).NotTo(HaveOccurred())
124+
})
125+
})
126+
})

‎e2e/controllernamespace/e2e_suite_test.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package controllernamespace_test
2+
3+
import (
4+
"testing"
5+
6+
. "github.com/onsi/ginkgo"
7+
. "github.com/onsi/gomega"
8+
)
9+
10+
func TestE2e(t *testing.T) {
11+
RegisterFailHandler(Fail)
12+
defer GinkgoRecover()
13+
RunSpecs(t, "E2e Suite")
14+
}

‎e2e/configuration_test.go renamed to ‎e2e/normal/configuration_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
package e2e
1+
package normal
22

33
import (
4+
"context"
45
"fmt"
56
"os"
67
"os/exec"
@@ -9,7 +10,6 @@ import (
910
"testing"
1011
"time"
1112

12-
"golang.org/x/net/context"
1313
"gotest.tools/assert"
1414
kerrors "k8s.io/apimachinery/pkg/api/errors"
1515
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -29,6 +29,8 @@ var (
2929
"examples/alibaba/oss/configuration_hcl_bucket.yaml",
3030
}
3131
testConfigurationsForceDelete = "examples/random/configuration_force_delete.yaml"
32+
33+
chartNamespace = "terraform"
3234
)
3335

3436
type ConfigurationAttr struct {

‎e2e/regression.go renamed to ‎e2e/normal/regression.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package e2e
1+
package normal
22

33
import (
44
"fmt"

‎gitee/gitee_configuration_test.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
package gitee
22

33
import (
4+
"github.com/oam-dev/terraform-controller/e2e/normal"
45
"testing"
5-
6-
"github.com/oam-dev/terraform-controller/e2e"
76
)
87

98
var (
@@ -16,5 +15,5 @@ var (
1615
func TestGiteeConfigurationRegression(t *testing.T) {
1716
var retryTimes = 240
1817

19-
e2e.Regression(t, giteeConfigurationsRegression, retryTimes)
18+
normal.Regression(t, giteeConfigurationsRegression, retryTimes)
2019
}

‎go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,9 @@ require (
1212
github.com/hashicorp/hcl/v2 v2.12.0
1313
github.com/jinzhu/copier v0.3.5
1414
github.com/onsi/ginkgo v1.16.4
15-
github.com/onsi/gomega v1.14.0
15+
github.com/onsi/gomega v1.19.0
1616
github.com/pkg/errors v0.9.1
1717
github.com/stretchr/testify v1.7.0
18-
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
1918
gopkg.in/yaml.v2 v2.4.0
2019
gotest.tools v2.2.0+incompatible
2120
k8s.io/api v0.21.3
@@ -60,8 +59,9 @@ require (
6059
github.com/prometheus/procfs v0.6.0 // indirect
6160
github.com/spf13/pflag v1.0.5 // indirect
6261
github.com/zclconf/go-cty v1.10.0 // indirect
62+
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
6363
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
64-
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
64+
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect
6565
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
6666
golang.org/x/text v0.3.7 // indirect
6767
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect

‎go.sum

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI
199199
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
200200
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
201201
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
202+
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
202203
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
203204
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
204205
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -246,6 +247,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
246247
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
247248
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
248249
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
250+
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
249251
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
250252
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
251253
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -337,12 +339,16 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
337339
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
338340
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
339341
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
342+
github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc=
343+
github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
340344
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
341345
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
342346
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
343347
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
344-
github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI=
345348
github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
349+
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
350+
github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
351+
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
346352
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
347353
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
348354
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -531,8 +537,9 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
531537
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
532538
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
533539
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
534-
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
535540
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
541+
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
542+
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
536543
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
537544
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
538545
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -594,8 +601,9 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w
594601
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
595602
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
596603
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
597-
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
598604
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
605+
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
606+
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
599607
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
600608
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
601609
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

0 commit comments

Comments
 (0)
Please sign in to comment.