This is an automated email from the ASF dual-hosted git repository. astefanutti pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 8e97ba7a2e7cdd2ec808198c52dc5b8a7e288121 Author: Pasquale Congiusti <[email protected]> AuthorDate: Fri Jan 29 15:25:38 2021 +0100 feat(trait): leverage ConfigMap binary data for resources * Any binary data will be using BinaryData ConfiMap instead of Data. We let the cluster to encode/decode the resource * Any text resource will be still using the Data (plain) ConfigMap * The `compression` feature can be run both on binary and text resources, providing a base64 encoded file. * Added unit test to check all the possible scenarios Close #1946, close #1881 --- pkg/apis/camel/v1/common_types.go | 2 + pkg/cmd/run.go | 46 +++++++--- pkg/cmd/run_test.go | 44 +++++++++ pkg/cmd/util_content.go | 37 +++++--- pkg/cmd/util_content_test.go | 99 ++++++++++++++++++++ pkg/cmd/util_dependencies.go | 2 +- pkg/trait/trait_test.go | 185 ++++++++++++++++++++++++++------------ pkg/trait/trait_types.go | 15 +++- 8 files changed, 344 insertions(+), 86 deletions(-) diff --git a/pkg/apis/camel/v1/common_types.go b/pkg/apis/camel/v1/common_types.go index b940b2f..2d50208 100644 --- a/pkg/apis/camel/v1/common_types.go +++ b/pkg/apis/camel/v1/common_types.go @@ -207,8 +207,10 @@ const ( type DataSpec struct { Name string `json:"name,omitempty"` Content string `json:"content,omitempty"` + RawContent []byte `json:"rawContent,omitempty"` ContentRef string `json:"contentRef,omitempty"` ContentKey string `json:"contentKey,omitempty"` + ContentType string `json:"contentType,omitempty"` Compression bool `json:"compression,omitempty"` } diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 695f01f..8ac482e 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -495,23 +495,20 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string, } for _, resource := range o.Resources { - data, compressed, err := loadContent(resource, o.Compression, o.CompressBinary) + rawData, contentType, err := loadRawContent(resource) if err != nil { return nil, err } - integration.Spec.AddResources(v1.ResourceSpec{ - DataSpec: v1.DataSpec{ - Name: path.Base(resource), - Content: data, - Compression: compressed, - }, - Type: v1.ResourceTypeData, - }) + resourceSpec, err := binaryOrTextResource(path.Base(resource), rawData, contentType, o.Compression) + if err != nil { + return nil, err + } + integration.Spec.AddResources(resourceSpec) } for _, resource := range o.OpenAPIs { - data, compressed, err := loadContent(resource, o.Compression, o.CompressBinary) + data, _, compressed, err := loadTextContent(resource, o.Compression) if err != nil { return nil, err } @@ -623,6 +620,35 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string, return &integration, nil } +func binaryOrTextResource(fileName string, data []byte, contentType string, base64Compression bool) (v1.ResourceSpec, error) { + resourceSpec := v1.ResourceSpec{ + DataSpec: v1.DataSpec{ + Name: fileName, + ContentKey: fileName, + ContentType: contentType, + Compression: false, + }, + Type: v1.ResourceTypeData, + } + + if !base64Compression && isBinary(contentType) { + resourceSpec.RawContent = data + return resourceSpec, nil + } + // either is a text resource or base64 compression is enabled + if base64Compression { + content, err := compressToString(data) + if err != nil { + return resourceSpec, err + } + resourceSpec.Content = content + resourceSpec.Compression = true + } else { + resourceSpec.Content = string(data) + } + return resourceSpec, nil +} + func (o *runCmdOptions) GetIntegrationName(sources []string) string { name := "" if o.IntegrationName != "" { diff --git a/pkg/cmd/run_test.go b/pkg/cmd/run_test.go index ce9527f..2dda764 100644 --- a/pkg/cmd/run_test.go +++ b/pkg/cmd/run_test.go @@ -477,3 +477,47 @@ func TestRunWithSavedValues(t *testing.T) { assert.Nil(t, err) assert.Len(t, runCmdOptions.Sources, 2) }*/ + +func TestRunBinaryResource(t *testing.T) { + binaryResourceSpec, err := binaryOrTextResource("file.ext", []byte{1, 2, 3, 4}, "application/octet-stream", false) + assert.Nil(t, err) + assert.Equal(t, "", binaryResourceSpec.Content) + assert.NotNil(t, binaryResourceSpec.RawContent) + assert.Equal(t, "file.ext", binaryResourceSpec.Name) + assert.Equal(t, "application/octet-stream", binaryResourceSpec.ContentType) + assert.False(t, binaryResourceSpec.Compression) +} + +func TestRunBinaryCompressedResource(t *testing.T) { + data := []byte{1, 2, 3, 4} + base64Compressed, _ := compressToString(data) + binaryResourceSpec, err := binaryOrTextResource("file.ext", data, "application/octet-stream", true) + assert.Nil(t, err) + assert.Equal(t, base64Compressed, binaryResourceSpec.Content) + assert.Nil(t, binaryResourceSpec.RawContent) + assert.Equal(t, "file.ext", binaryResourceSpec.Name) + assert.Equal(t, "application/octet-stream", binaryResourceSpec.ContentType) + assert.True(t, binaryResourceSpec.Compression) +} + +func TestRunTextResource(t *testing.T) { + textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello world"), "text/plain", false) + assert.Nil(t, err) + assert.Equal(t, "hello world", textResourceSpec.Content) + assert.Nil(t, textResourceSpec.RawContent) + assert.Equal(t, "file.ext", textResourceSpec.Name) + assert.Equal(t, "text/plain", textResourceSpec.ContentType) + assert.False(t, textResourceSpec.Compression) +} + +func TestRunTextCompressedResource(t *testing.T) { + data := []byte("hello horld") + base64Compressed, _ := compressToString(data) + textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello horld"), "text/plain", true) + assert.Nil(t, err) + assert.Equal(t, base64Compressed, textResourceSpec.Content) + assert.Nil(t, textResourceSpec.RawContent) + assert.Equal(t, "file.ext", textResourceSpec.Name) + assert.Equal(t, "text/plain", textResourceSpec.ContentType) + assert.True(t, textResourceSpec.Compression) +} diff --git a/pkg/cmd/util_content.go b/pkg/cmd/util_content.go index c210870..7950e44 100644 --- a/pkg/cmd/util_content.go +++ b/pkg/cmd/util_content.go @@ -26,7 +26,7 @@ import ( "strings" ) -func loadContent(source string, compress bool, compressBinary bool) (string, bool, error) { +func loadRawContent(source string) ([]byte, string, error) { var content []byte var err error @@ -35,7 +35,7 @@ func loadContent(source string, compress bool, compressBinary bool) (string, boo } else { u, err := url.Parse(source) if err != nil { - return "", false, err + return nil, "", err } switch u.Scheme { @@ -46,27 +46,36 @@ func loadContent(source string, compress bool, compressBinary bool) (string, boo case "https": content, err = loadContentHTTP(u) default: - return "", false, fmt.Errorf("unsupported scheme %s", u.Scheme) + return nil, "", fmt.Errorf("unsupported scheme %s", u.Scheme) } } if err != nil { - return "", false, err + return nil, "", err } - doCompress := compress - if !doCompress && compressBinary { - contentType := http.DetectContentType(content) - if strings.HasPrefix(contentType, "application/octet-stream") { - doCompress = true - } + + contentType := http.DetectContentType(content) + return content, contentType, nil +} + +func isBinary(contentType string) bool { + // According the http.DetectContentType method + // also json and other "text" application mime types would be reported as text + return !strings.HasPrefix(contentType, "text") +} + +func loadTextContent(source string, base64Compression bool) (string, string, bool, error) { + content, contentType, err := loadRawContent(source) + if err != nil { + return "", "", false, err } - if doCompress { - answer, err := compressToString(content) - return answer, true, err + if base64Compression { + base64Compressed, err := compressToString(content) + return base64Compressed, contentType, true, err } - return string(content), false, nil + return string(content), contentType, false, nil } func loadContentHTTP(u *url.URL) ([]byte, error) { diff --git a/pkg/cmd/util_content_test.go b/pkg/cmd/util_content_test.go new file mode 100644 index 0000000..10d6610 --- /dev/null +++ b/pkg/cmd/util_content_test.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRawContentFileMissing(t *testing.T) { + _, _, err := loadRawContent("dsadas") + assert.NotNil(t, err) +} + +func TestRawBinaryContentType(t *testing.T) { + var tmpFile *os.File + var err error + if tmpFile, err = ioutil.TempFile("", "camel-k-*.json"); err != nil { + t.Error(err) + } + assert.Nil(t, tmpFile.Close()) + assert.Nil(t, ioutil.WriteFile(tmpFile.Name(), []byte{1, 2, 3, 4, 5, 6}, 0644)) + + data, contentType, err := loadRawContent(tmpFile.Name()) + assert.Nil(t, err) + assert.Equal(t, []byte{1, 2, 3, 4, 5, 6}, data) + assert.True(t, isBinary(contentType)) +} + +func TestRawApplicationContentType(t *testing.T) { + var tmpFile *os.File + var err error + if tmpFile, err = ioutil.TempFile("", "camel-k-*.json"); err != nil { + t.Error(err) + } + assert.Nil(t, tmpFile.Close()) + assert.Nil(t, ioutil.WriteFile(tmpFile.Name(), []byte(`{"hello":"world"}`), 0644)) + + data, contentType, err := loadRawContent(tmpFile.Name()) + assert.Nil(t, err) + assert.Equal(t, `{"hello":"world"}`, string(data)) + assert.False(t, isBinary(contentType)) +} + +func TestTextContentType(t *testing.T) { + var tmpFile *os.File + var err error + if tmpFile, err = ioutil.TempFile("", "camel-k-*.json"); err != nil { + t.Error(err) + } + assert.Nil(t, tmpFile.Close()) + assert.Nil(t, ioutil.WriteFile(tmpFile.Name(), []byte(`{"hello":"world"}`), 0644)) + + data, contentType, compressed, err := loadTextContent(tmpFile.Name(), false) + assert.Nil(t, err) + assert.Equal(t, `{"hello":"world"}`, string(data)) + assert.False(t, isBinary(contentType)) + assert.False(t, compressed) +} + +func TestTextCompressed(t *testing.T) { + var tmpFile *os.File + var err error + if tmpFile, err = ioutil.TempFile("", "camel-k-*.json"); err != nil { + t.Error(err) + } + assert.Nil(t, tmpFile.Close()) + assert.Nil(t, ioutil.WriteFile(tmpFile.Name(), []byte(`{"hello":"world"}`), 0644)) + + data, contentType, compressed, err := loadTextContent(tmpFile.Name(), true) + assert.Nil(t, err) + assert.NotEqual(t, `{"hello":"world"}`, string(data)) + assert.False(t, isBinary(contentType)) + assert.True(t, compressed) +} + +func TestIsBinary(t *testing.T) { + assert.True(t, isBinary("image/jpeg")) + assert.True(t, isBinary("application/zip")) + assert.False(t, isBinary("text/plain")) +} diff --git a/pkg/cmd/util_dependencies.go b/pkg/cmd/util_dependencies.go index 94cc757..f987aa7 100644 --- a/pkg/cmd/util_dependencies.go +++ b/pkg/cmd/util_dependencies.go @@ -81,7 +81,7 @@ func getTopLevelDependencies(catalog *camel.RuntimeCatalog, args []string) ([]st // Invoke the dependency inspector code for each source file for _, source := range args { - data, _, err := loadContent(source, false, false) + data, _, _, err := loadTextContent(source, false) if err != nil { return []string{}, err } diff --git a/pkg/trait/trait_test.go b/pkg/trait/trait_test.go index 6336fd7..2b4f2a0 100644 --- a/pkg/trait/trait_test.go +++ b/pkg/trait/trait_test.go @@ -207,65 +207,7 @@ func TestTraitHierarchyDecode(t *testing.T) { assert.Equal(t, 15, *kns.Target) } -func TestConfigureVolumesAndMountsSources(t *testing.T) { - env := Environment{ - Resources: kubernetes.NewCollection(), - Integration: &v1.Integration{ - ObjectMeta: metav1.ObjectMeta{ - Name: TestDeploymentName, - Namespace: "ns", - }, - Spec: v1.IntegrationSpec{ - Sources: []v1.SourceSpec{ - { - DataSpec: v1.DataSpec{ - Name: "source1.java", - ContentRef: "my-cm1", - ContentKey: "source1.java", - }, - Type: "data", - }, - { - DataSpec: v1.DataSpec{ - Name: "source2.java", - ContentRef: "my-cm2", - }, - Type: "data", - }, - }, - }, - }, - } - - vols := make([]corev1.Volume, 0) - mnts := make([]corev1.VolumeMount, 0) - - env.Resources.AddAll(env.ComputeConfigMaps()) - env.ConfigureVolumesAndMounts(&vols, &mnts) - - assert.Len(t, vols, 2) - assert.Len(t, mnts, 2) - - v := findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm1" }) - assert.NotNil(t, v) - assert.NotNil(t, v.VolumeSource.ConfigMap) - assert.Len(t, v.VolumeSource.ConfigMap.Items, 1) - assert.Equal(t, "source1.java", v.VolumeSource.ConfigMap.Items[0].Key) - - m := findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name }) - assert.NotNil(t, m) - - v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm2" }) - assert.NotNil(t, v) - assert.NotNil(t, v.VolumeSource.ConfigMap) - assert.Len(t, v.VolumeSource.ConfigMap.Items, 1) - assert.Equal(t, "content", v.VolumeSource.ConfigMap.Items[0].Key) - - m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name }) - assert.NotNil(t, m) -} - -func TestConfigureVolumesAndMountsResourcesAndProperties(t *testing.T) { +func TestConfigureVolumesAndMountsTextResourcesAndProperties(t *testing.T) { env := Environment{ Resources: kubernetes.NewCollection(), Integration: &v1.Integration{ @@ -425,6 +367,131 @@ func TestConfigureVolumesAndMountsResourcesAndProperties(t *testing.T) { assert.Equal(t, "/foo/bar", m.MountPath) } +func TestConfigureVolumesAndMountsSources(t *testing.T) { + env := Environment{ + Resources: kubernetes.NewCollection(), + Integration: &v1.Integration{ + ObjectMeta: metav1.ObjectMeta{ + Name: TestDeploymentName, + Namespace: "ns", + }, + Spec: v1.IntegrationSpec{ + Sources: []v1.SourceSpec{ + { + DataSpec: v1.DataSpec{ + Name: "source1.java", + ContentRef: "my-cm1", + ContentKey: "source1.java", + }, + Type: "data", + }, + { + DataSpec: v1.DataSpec{ + Name: "source2.java", + ContentRef: "my-cm2", + }, + Type: "data", + }, + }, + }, + }, + } + + vols := make([]corev1.Volume, 0) + mnts := make([]corev1.VolumeMount, 0) + + env.Resources.AddAll(env.ComputeConfigMaps()) + env.ConfigureVolumesAndMounts(&vols, &mnts) + + assert.Len(t, vols, 2) + assert.Len(t, mnts, 2) + + v := findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm1" }) + assert.NotNil(t, v) + assert.NotNil(t, v.VolumeSource.ConfigMap) + assert.Len(t, v.VolumeSource.ConfigMap.Items, 1) + assert.Equal(t, "source1.java", v.VolumeSource.ConfigMap.Items[0].Key) + + m := findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name }) + assert.NotNil(t, m) + + v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm2" }) + assert.NotNil(t, v) + assert.NotNil(t, v.VolumeSource.ConfigMap) + assert.Len(t, v.VolumeSource.ConfigMap.Items, 1) + assert.Equal(t, "content", v.VolumeSource.ConfigMap.Items[0].Key) + + m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name }) + assert.NotNil(t, m) +} + +func TestConfigureVolumesAndMountsBinaryAndTextResources(t *testing.T) { + env := Environment{ + Resources: kubernetes.NewCollection(), + Integration: &v1.Integration{ + ObjectMeta: metav1.ObjectMeta{ + Name: TestDeploymentName, + Namespace: "ns", + }, + Spec: v1.IntegrationSpec{ + Resources: []v1.ResourceSpec{ + { + DataSpec: v1.DataSpec{ + Name: "res1.bin", + RawContent: []byte{1, 2, 3, 4}, + ContentRef: "my-cm1", + ContentKey: "my-binary", + ContentType: "application/octet-stream", + }, + Type: "data", + }, + { + DataSpec: v1.DataSpec{ + Name: "res2.txt", + ContentRef: "my-cm2", + Content: "hello", + ContentKey: "my-text", + ContentType: "text/plain", + }, + Type: "data", + }, + }, + }, + }, + } + + vols := make([]corev1.Volume, 0) + mnts := make([]corev1.VolumeMount, 0) + + env.Resources.AddAll(env.ComputeConfigMaps()) + env.ConfigureVolumesAndMounts(&vols, &mnts) + + assert.Len(t, vols, 2) + assert.Len(t, mnts, 2) + + v := findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm1" }) + assert.NotNil(t, v) + assert.NotNil(t, v.VolumeSource.ConfigMap) + assert.Len(t, v.VolumeSource.ConfigMap.Items, 1) + assert.Equal(t, "my-binary", v.VolumeSource.ConfigMap.Items[0].Key) + assert.Equal(t, "res1.bin", v.VolumeSource.ConfigMap.Items[0].Path) + + m := findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name }) + assert.NotNil(t, m) + assert.Equal(t, "/etc/camel/resources/i-resource-000", m.MountPath) + + v = findVolume(vols, func(v corev1.Volume) bool { return v.ConfigMap.Name == "my-cm2" }) + assert.NotNil(t, v) + assert.NotNil(t, v.VolumeSource.ConfigMap) + assert.Len(t, v.VolumeSource.ConfigMap.Items, 1) + assert.Equal(t, "my-text", v.VolumeSource.ConfigMap.Items[0].Key) + assert.Equal(t, "res2.txt", v.VolumeSource.ConfigMap.Items[0].Path) + + m = findVVolumeMount(mnts, func(m corev1.VolumeMount) bool { return m.Name == v.Name }) + assert.NotNil(t, m) + assert.Equal(t, "/etc/camel/resources/i-resource-001", m.MountPath) +} + func TestOnlySomeTraitsInfluenceBuild(t *testing.T) { c := NewTraitTestCatalog() buildTraits := []string{"builder", "quarkus"} diff --git a/pkg/trait/trait_types.go b/pkg/trait/trait_types.go index e896b3b..10a99c0 100644 --- a/pkg/trait/trait_types.go +++ b/pkg/trait/trait_types.go @@ -464,9 +464,20 @@ func (e *Environment) ComputeConfigMaps() []runtime.Object { "camel.apache.org/resource.compression": strconv.FormatBool(r.Compression), }, }, - Data: map[string]string{ + } + + if r.ContentType != "" { + cm.Annotations["camel.apache.org/resource.content-type"] = r.ContentType + } + + if r.RawContent != nil { + cm.BinaryData = map[string][]byte{ + cmKey: []byte(r.RawContent), + } + } else { + cm.Data = map[string]string{ cmKey: r.Content, - }, + } } maps = append(maps, &cm)
