This is an automated email from the ASF dual-hosted git repository. astefanutti pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 7d1db84ff176021dbc524adb96cfcf9e03b871a4 Author: Antonin Stefanutti <[email protected]> AuthorDate: Thu Mar 18 16:02:58 2021 +0100 feat: Coordinate task working directories automatically --- pkg/builder/image.go | 4 ++- pkg/controller/build/schedule_pod.go | 43 ++++++++++++++--------------- pkg/controller/build/schedule_routine.go | 46 ++++++++++++++++++++++++++------ pkg/trait/builder.go | 23 +++------------- 4 files changed, 67 insertions(+), 49 deletions(-) diff --git a/pkg/builder/image.go b/pkg/builder/image.go index a94aa9c..9ec8bf1 100644 --- a/pkg/builder/image.go +++ b/pkg/builder/image.go @@ -31,6 +31,8 @@ import ( "github.com/apache/camel-k/pkg/util/controller" ) +const ContextDir = "context" + type artifactsSelector func(ctx *builderContext) error func standardImageContext(ctx *builderContext) error { @@ -84,7 +86,7 @@ func imageContext(ctx *builderContext, selector artifactsSelector) error { return err } - contextDir := path.Join(ctx.Path, "context") + contextDir := path.Join(ctx.Path, ContextDir) err = os.MkdirAll(contextDir, 0777) if err != nil { diff --git a/pkg/controller/build/schedule_pod.go b/pkg/controller/build/schedule_pod.go index 1e2f3ac..8876222 100644 --- a/pkg/controller/build/schedule_pod.go +++ b/pkg/controller/build/schedule_pod.go @@ -41,7 +41,10 @@ import ( "github.com/apache/camel-k/pkg/util/kubernetes" ) -const builderVolume = "camel-k-builder" +const ( + builderDir = "/builder" + builderVolume = "camel-k-builder" +) type schedulePodAction struct { baseAction @@ -246,7 +249,15 @@ func (action *schedulePodAction) newBuildPod(ctx context.Context, build *v1.Buil } func (action *schedulePodAction) addBuilderTaskToPod(build *v1.Build, task *v1.BuilderTask, pod *corev1.Pod) { - pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{ + // Add the EmptyDir volume used to share the build state across tasks + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: builderVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + + container := corev1.Container{ Name: task.Name, Image: action.operatorImage, ImagePullPolicy: corev1.PullIfNotPresent, @@ -260,20 +271,10 @@ func (action *schedulePodAction) addBuilderTaskToPod(build *v1.Build, task *v1.B "--task-name", task.Name, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: builderVolume, - MountPath: builderDir, - }, - }, - }) + WorkingDir: path.Join(builderDir, build.Name), + } - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: builderVolume, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }) + action.addContainerToPod(build, container, pod) } func (action *schedulePodAction) addBuildahTaskToPod(ctx context.Context, build *v1.Build, task *v1.BuildahTask, pod *corev1.Pod) error { @@ -357,13 +358,13 @@ func (action *schedulePodAction) addBuildahTaskToPod(ctx context.Context, build Command: []string{"/bin/sh", "-c"}, Args: []string{strings.Join(args, " && ")}, Env: env, - WorkingDir: path.Join(builderDir, e.IntegrationKit.Name, "context"), + WorkingDir: path.Join(builderDir, build.Name, builder.ContextDir), VolumeMounts: volumeMounts, } pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) - action.addContainerToPod(container, pod) + action.addContainerToPod(build, container, pod) return nil } @@ -376,7 +377,7 @@ func (action *schedulePodAction) addKanikoTaskToPod(ctx context.Context, build * args := []string{ "--dockerfile=Dockerfile", - "--context=" + path.Join(builderDir, e.IntegrationKit.Name, "context"), + "--context=" + path.Join(builderDir, build.Name, builder.ContextDir), "--destination=" + task.Image, "--cache=" + strconv.FormatBool(cache), "--cache-dir=" + builder.KanikoCacheDir, @@ -473,16 +474,16 @@ func (action *schedulePodAction) addKanikoTaskToPod(ctx context.Context, build * pod.Spec.Affinity = affinity pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) - action.addContainerToPod(container, pod) + action.addContainerToPod(build, container, pod) return nil } -func (action *schedulePodAction) addContainerToPod(container corev1.Container, pod *corev1.Pod) { +func (action *schedulePodAction) addContainerToPod(build *v1.Build, container corev1.Container, pod *corev1.Pod) { if action.hasBuilderVolume(pod) { container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ Name: builderVolume, - MountPath: builderDir, + MountPath: path.Join(builderDir, build.Name), }) } diff --git a/pkg/controller/build/schedule_routine.go b/pkg/controller/build/schedule_routine.go index e2cfc63..1f31852 100644 --- a/pkg/controller/build/schedule_routine.go +++ b/pkg/controller/build/schedule_routine.go @@ -19,6 +19,9 @@ package build import ( "context" + "io/ioutil" + "os" + "path" "sync" "time" @@ -114,28 +117,55 @@ func (action *scheduleRoutineAction) runBuild(ctx context.Context, build *v1.Bui return } - // FIXME: Clean-up build directory + buildDir := "" + defer os.RemoveAll(buildDir) + for i, task := range build.Spec.Tasks { - status := action.builder.Build(build).Task(task).Do(ctx) + // Coordinate the build and context directories across the sequence of tasks + if t := task.Builder; t != nil { + if t.BuildDir == "" { + tmpDir, err := ioutil.TempDir(os.TempDir(), build.Name+"-") + if err != nil { + status.Failed(err) + break + } + t.BuildDir = tmpDir + } + buildDir = t.BuildDir + } else if t := task.Spectrum; t != nil && t.ContextDir == "" { + t.ContextDir = path.Join(buildDir, builder.ContextDir) + } else if t := task.S2i; t != nil && t.ContextDir == "" { + t.ContextDir = path.Join(buildDir, builder.ContextDir) + } + + // Execute the task + status = action.builder.Build(build).Task(task).Do(ctx) lastTask := i == len(build.Spec.Tasks)-1 taskFailed := status.Phase == v1.BuildPhaseFailed || status.Phase == v1.BuildPhaseError if lastTask && !taskFailed { status.Phase = v1.BuildPhaseSucceeded } - if lastTask || taskFailed { - duration := metav1.Now().Sub(build.Status.StartedAt.Time) - status.Duration = duration.String() - // Account for the Build metrics - observeBuildResult(build, status.Phase, duration) + if lastTask || taskFailed { + // Spare a redundant update + break } + // Update the Build status err := action.updateBuildStatus(ctx, build, status) - if err != nil || taskFailed { + if err != nil { + status.Failed(err) break } } + + duration := metav1.Now().Sub(build.Status.StartedAt.Time) + status.Duration = duration.String() + // Account for the Build metrics + observeBuildResult(build, status.Phase, duration) + + _ = action.updateBuildStatus(ctx, build, status) } func (action *scheduleRoutineAction) updateBuildStatus(ctx context.Context, build *v1.Build, status v1.BuildStatus) error { diff --git a/pkg/trait/builder.go b/pkg/trait/builder.go index ede0e8e..07d7401 100644 --- a/pkg/trait/builder.go +++ b/pkg/trait/builder.go @@ -18,9 +18,6 @@ limitations under the License. package trait import ( - "io/ioutil" - "os" - "path" "sort" v1 "github.com/apache/camel-k/pkg/apis/camel/v1" @@ -64,16 +61,6 @@ func (t *builderTrait) Configure(e *Environment) (bool, error) { func (t *builderTrait) Apply(e *Environment) error { builderTask := t.builderTask(e) - switch e.Platform.Status.Build.PublishStrategy { - case v1.IntegrationPlatformBuildPublishStrategyBuildah, v1.IntegrationPlatformBuildPublishStrategyKaniko: - builderTask.BuildDir = path.Join(builderDir, e.IntegrationKit.Name) - default: - tmpDir, err := ioutil.TempDir(os.TempDir(), e.IntegrationKit.Name+"-") - if err != nil { - return err - } - builderTask.BuildDir = tmpDir - } e.BuildTasks = append(e.BuildTasks, v1.Task{Builder: builderTask}) switch e.Platform.Status.Build.PublishStrategy { @@ -83,10 +70,9 @@ func (t *builderTrait) Apply(e *Environment) error { Name: "spectrum", }, PublishTask: v1.PublishTask{ - ContextDir: path.Join(builderTask.BuildDir, "context"), - BaseImage: e.Platform.Status.Build.BaseImage, - Image: getImageName(e), - Registry: e.Platform.Status.Build.Registry, + BaseImage: e.Platform.Status.Build.BaseImage, + Image: getImageName(e), + Registry: e.Platform.Status.Build.Registry, }, }}) @@ -95,8 +81,7 @@ func (t *builderTrait) Apply(e *Environment) error { BaseTask: v1.BaseTask{ Name: "s2i", }, - ContextDir: path.Join(builderTask.BuildDir, "context"), - Tag: e.IntegrationKit.ResourceVersion, + Tag: e.IntegrationKit.ResourceVersion, }}) case v1.IntegrationPlatformBuildPublishStrategyBuildah:
