This is an automated email from the ASF dual-hosted git repository. wenjin272 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-agents.git
commit eca17418481aae6fb7547bc81c4d01f80fd253ec Author: WenjinXie <[email protected]> AuthorDate: Wed May 6 17:54:29 2026 +0800 [api][runtime][java] Support agent skills in Java. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]> --- .../apache/flink/agents/api/annotation/Skills.java | 39 ++++ .../agents/api/chat/model/BaseChatModelSetup.java | 52 +++++ .../org/apache/flink/agents/api/skills/Skills.java | 86 ++++++++ .../chat/model/BaseChatModelSetupSkillsTest.java | 179 ++++++++++++++++ .../agents/api/skills/SkillsResourceTest.java | 53 +++++ .../integration/test/SkillsIntegrationAgent.java | 119 +++++++++++ .../integration/test/SkillsIntegrationTest.java | 226 +++++++++++++++++++++ .../test/resources/skills/joke-generator/SKILL.md | 18 ++ .../skills/joke-generator/scripts/gen_joke.py | 20 ++ .../test/resources/skills/math-calculator/SKILL.md | 57 ++++++ .../org/apache/flink/agents/plan/AgentPlan.java | 62 ++++++ .../flink/agents/plan/actions/ChatModelAction.java | 47 ++++- .../agents/plan/AgentPlanDeclareSkillsTest.java | 131 ++++++++++++ .../runtime/java/java_resource_wrapper.py | 14 +- runtime/pom.xml | 6 + .../runtime/python/utils/JavaResourceAdapter.java | 15 ++ .../runtime/resource/ResourceContextImpl.java | 46 ++++- .../flink/agents/runtime/skill/AgentSkill.java | 147 ++++++++++++++ .../flink/agents/runtime/skill/LoadSkillTool.java | 146 +++++++++++++ .../flink/agents/runtime/skill/SkillManager.java | 148 ++++++++++++++ .../flink/agents/runtime/skill/SkillParser.java | 121 +++++++++++ .../agents/runtime/skill/SkillPromptProvider.java | 50 +++++ .../agents/runtime/skill/SkillRepository.java | 44 ++++ .../repository/FileSystemSkillRepository.java | 184 +++++++++++++++++ .../skill/FileSystemSkillRepositoryTest.java | 105 ++++++++++ .../agents/runtime/skill/LoadSkillToolTest.java | 117 +++++++++++ .../agents/runtime/skill/SkillManagerTest.java | 92 +++++++++ .../agents/runtime/skill/SkillParserTest.java | 122 +++++++++++ .../src/test/resources/skill_discovery_prompt.txt | 24 +++ runtime/src/test/resources/skills/github/SKILL.md | 47 +++++ .../test/resources/skills/nano-banana-pro/SKILL.md | 130 ++++++++++++ .../resources/skills/nano-banana-pro/_meta.json | 6 + .../nano-banana-pro/scripts/generate_image.py | 165 +++++++++++++++ 33 files changed, 2809 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/org/apache/flink/agents/api/annotation/Skills.java b/api/src/main/java/org/apache/flink/agents/api/annotation/Skills.java new file mode 100644 index 00000000..072d360c --- /dev/null +++ b/api/src/main/java/org/apache/flink/agents/api/annotation/Skills.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.api.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks a static method that returns an {@link org.apache.flink.agents.api.skills.Skills} resource + * describing where to load agent skills from. + * + * <p>Mirrors the Python {@code @skills} decorator. Multiple {@code @Skills} methods on the same + * agent are merged at plan-build time. + * + * <p>Note: this annotation shares its simple name with {@link + * org.apache.flink.agents.api.skills.Skills} (different package). When importing both, one of them + * must be referenced by its fully-qualified name. + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Skills {} diff --git a/api/src/main/java/org/apache/flink/agents/api/chat/model/BaseChatModelSetup.java b/api/src/main/java/org/apache/flink/agents/api/chat/model/BaseChatModelSetup.java index b0f73d80..af7ed10b 100644 --- a/api/src/main/java/org/apache/flink/agents/api/chat/model/BaseChatModelSetup.java +++ b/api/src/main/java/org/apache/flink/agents/api/chat/model/BaseChatModelSetup.java @@ -25,6 +25,7 @@ import org.apache.flink.agents.api.resource.Resource; import org.apache.flink.agents.api.resource.ResourceContext; import org.apache.flink.agents.api.resource.ResourceDescriptor; import org.apache.flink.agents.api.resource.ResourceType; +import org.apache.flink.agents.api.skills.Skills; import org.apache.flink.agents.api.tools.Tool; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.util.Preconditions; @@ -42,6 +43,10 @@ public abstract class BaseChatModelSetup extends Resource { protected String model; protected Object prompt; protected List<String> toolNames; + @Nullable protected List<String> skills; + @Nullable protected String skillDiscoveryPrompt; + protected List<String> allowedCommands; + protected List<String> allowedScriptDirs; @Nullable protected BaseChatModelConnection connection; protected final List<Tool> tools = new ArrayList<>(); @@ -52,6 +57,15 @@ public abstract class BaseChatModelSetup extends Resource { this.model = descriptor.getArgument("model"); this.prompt = descriptor.getArgument("prompt"); this.toolNames = descriptor.getArgument("tools"); + this.skills = descriptor.getArgument("skills"); + List<String> declaredCommands = descriptor.getArgument("allowed_commands"); + this.allowedCommands = + declaredCommands == null ? new ArrayList<>() : new ArrayList<>(declaredCommands); + List<String> declaredScriptDirs = descriptor.getArgument("allowed_script_dirs"); + this.allowedScriptDirs = + declaredScriptDirs == null + ? new ArrayList<>() + : new ArrayList<>(declaredScriptDirs); } /** @@ -71,6 +85,19 @@ public abstract class BaseChatModelSetup extends Resource { this.prompt = this.resourceContext.getResource((String) this.prompt, ResourceType.PROMPT); } + if (this.skills != null) { + this.skillDiscoveryPrompt = + this.resourceContext.generateAvailableSkillsPrompt(this.skills); + List<String> mutable = + this.toolNames == null ? new ArrayList<>() : new ArrayList<>(this.toolNames); + if (!mutable.contains(Skills.LOAD_SKILL_TOOL)) { + mutable.add(Skills.LOAD_SKILL_TOOL); + } + if (!mutable.contains(Skills.BASH_TOOL)) { + mutable.add(Skills.BASH_TOOL); + } + this.toolNames = mutable; + } if (this.toolNames != null) { for (String name : this.toolNames) { this.tools.add((Tool) this.resourceContext.getResource(name, ResourceType.TOOL)); @@ -115,6 +142,13 @@ public abstract class BaseChatModelSetup extends Resource { messages = promptMessages; } + if (this.skillDiscoveryPrompt != null && !this.skillDiscoveryPrompt.isEmpty()) { + int idx = ChatMessage.findFirstSystemMessage(messages); + List<ChatMessage> mutated = new ArrayList<>(messages); + mutated.add(idx + 1, new ChatMessage(MessageRole.SYSTEM, this.skillDiscoveryPrompt)); + messages = mutated; + } + Map<String, Object> params = this.getParameters(); params.putAll(parameters); return connection.chat(messages, tools, params); @@ -144,4 +178,22 @@ public abstract class BaseChatModelSetup extends Resource { public List<String> getToolNames() { return toolNames; } + + @Nullable + public List<String> getSkills() { + return skills; + } + + @Nullable + public String getSkillDiscoveryPrompt() { + return skillDiscoveryPrompt; + } + + public List<String> getAllowedCommands() { + return allowedCommands; + } + + public List<String> getAllowedScriptDirs() { + return allowedScriptDirs; + } } diff --git a/api/src/main/java/org/apache/flink/agents/api/skills/Skills.java b/api/src/main/java/org/apache/flink/agents/api/skills/Skills.java new file mode 100644 index 00000000..c4a8de7b --- /dev/null +++ b/api/src/main/java/org/apache/flink/agents/api/skills/Skills.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.api.skills; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.flink.agents.api.resource.ResourceType; +import org.apache.flink.agents.api.resource.SerializableResource; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Configuration resource describing where to load agent skills from. + * + * <p>Mirrors the Python {@code flink_agents.api.skills.Skills}. Use {@link + * #fromLocalDir(String...)} to construct. + * + * <p>Multiple {@code @Skills} declarations on the same agent are merged at plan-build time. + */ +@JsonIgnoreProperties( + ignoreUnknown = true, + value = {"metricGroup", "resourceType"}) +public class Skills extends SerializableResource { + + /** Reserved resource name under which AgentPlan registers the merged Skills config. */ + public static final String SKILLS_CONFIG = "_skills_config"; + + /** Reserved name of the built-in skill loader tool. */ + public static final String LOAD_SKILL_TOOL = "load_skill"; + + /** Reserved name of the built-in bash tool used to execute skill scripts. */ + public static final String BASH_TOOL = "bash"; + + private List<String> paths; + + /** Required by Jackson. */ + public Skills() { + this.paths = Collections.emptyList(); + } + + @JsonCreator + public Skills(@JsonProperty("paths") List<String> paths) { + this.paths = paths == null ? Collections.emptyList() : List.copyOf(paths); + } + + /** + * Create a {@link Skills} resource from one or more local filesystem directories. + * + * <p>Each path points to a directory whose immediate subdirectories each contain a {@code + * SKILL.md} file. + */ + public static Skills fromLocalDir(String... paths) { + return new Skills(Arrays.asList(paths)); + } + + @JsonProperty("paths") + public List<String> getPaths() { + return paths; + } + + @JsonIgnore + @Override + public ResourceType getResourceType() { + return ResourceType.SKILLS; + } +} diff --git a/api/src/test/java/org/apache/flink/agents/api/chat/model/BaseChatModelSetupSkillsTest.java b/api/src/test/java/org/apache/flink/agents/api/chat/model/BaseChatModelSetupSkillsTest.java new file mode 100644 index 00000000..a31d9971 --- /dev/null +++ b/api/src/test/java/org/apache/flink/agents/api/chat/model/BaseChatModelSetupSkillsTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.api.chat.model; + +import org.apache.flink.agents.api.chat.messages.ChatMessage; +import org.apache.flink.agents.api.chat.messages.MessageRole; +import org.apache.flink.agents.api.resource.Resource; +import org.apache.flink.agents.api.resource.ResourceContext; +import org.apache.flink.agents.api.resource.ResourceDescriptor; +import org.apache.flink.agents.api.resource.ResourceType; +import org.apache.flink.agents.api.skills.Skills; +import org.apache.flink.agents.api.tools.Tool; +import org.apache.flink.agents.api.tools.ToolMetadata; +import org.apache.flink.agents.api.tools.ToolParameters; +import org.apache.flink.agents.api.tools.ToolResponse; +import org.apache.flink.agents.api.tools.ToolType; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BaseChatModelSetupSkillsTest { + + /** Stub chat model setup that exposes a configurable parameters map. */ + private static class StubChatSetup extends BaseChatModelSetup { + public StubChatSetup(ResourceDescriptor descriptor, ResourceContext resourceContext) { + super(descriptor, resourceContext); + } + + @Override + public Map<String, Object> getParameters() { + return new HashMap<>(); + } + } + + private static class StubConnection extends BaseChatModelConnection { + List<ChatMessage> capturedMessages; + List<Tool> capturedTools; + + StubConnection(ResourceDescriptor d, ResourceContext c) { + super(d, c); + } + + @Override + public ChatMessage chat( + List<ChatMessage> messages, List<Tool> tools, Map<String, Object> arguments) { + this.capturedMessages = new ArrayList<>(messages); + this.capturedTools = new ArrayList<>(tools); + return new ChatMessage(MessageRole.ASSISTANT, "ok"); + } + } + + private static class StubTool extends Tool { + public StubTool(String name) { + super(new ToolMetadata(name, "stub", "{}")); + } + + @Override + public ToolType getToolType() { + return ToolType.FUNCTION; + } + + @Override + public ToolResponse call(ToolParameters parameters) { + return ToolResponse.success(""); + } + } + + @Test + void openInjectsSkillToolsAndDiscoveryPrompt() throws Exception { + Map<String, Resource> store = new HashMap<>(); + StubConnection connection = new StubConnection(new ResourceDescriptor("X", Map.of()), null); + Tool loadSkillTool = new StubTool(Skills.LOAD_SKILL_TOOL); + Tool bashTool = new StubTool(Skills.BASH_TOOL); + store.put("conn", connection); + store.put(Skills.LOAD_SKILL_TOOL, loadSkillTool); + store.put(Skills.BASH_TOOL, bashTool); + ResourceContext ctx = + new ResourceContext() { + @Override + public Resource getResource(String name, ResourceType type) { + return store.get(name); + } + + @Override + public String generateAvailableSkillsPrompt(List<String> skillNames) { + return "<available_skills>\n<skill>\n<name>" + + skillNames.get(0) + + "</name>\n</skill>\n</available_skills>"; + } + + @Override + public List<String> getSkillDirs(List<String> skillNames) { + return List.of(); + } + }; + + Map<String, Object> args = new HashMap<>(); + args.put("connection", "conn"); + args.put("skills", Arrays.asList("github")); + ResourceDescriptor descriptor = new ResourceDescriptor("X", args); + + StubChatSetup setup = new StubChatSetup(descriptor, ctx); + setup.open(); + + assertNotNull(setup.getSkillDiscoveryPrompt()); + assertTrue(setup.getSkillDiscoveryPrompt().contains("<available_skills>")); + assertTrue(setup.getToolNames().contains(Skills.LOAD_SKILL_TOOL)); + assertTrue(setup.getToolNames().contains(Skills.BASH_TOOL)); + } + + @Test + void chatInjectsSkillPromptAfterFirstSystemMessage() throws Exception { + Map<String, Resource> store = new HashMap<>(); + StubConnection connection = new StubConnection(new ResourceDescriptor("X", Map.of()), null); + store.put("conn", connection); + store.put(Skills.LOAD_SKILL_TOOL, new StubTool(Skills.LOAD_SKILL_TOOL)); + store.put(Skills.BASH_TOOL, new StubTool(Skills.BASH_TOOL)); + ResourceContext ctx = + new ResourceContext() { + @Override + public Resource getResource(String name, ResourceType type) { + return store.get(name); + } + + @Override + public String generateAvailableSkillsPrompt(List<String> skillNames) { + return "<available_skills>marker</available_skills>"; + } + + @Override + public List<String> getSkillDirs(List<String> skillNames) { + return List.of(); + } + }; + Map<String, Object> args = new HashMap<>(); + args.put("connection", "conn"); + args.put("skills", Arrays.asList("github")); + StubChatSetup setup = new StubChatSetup(new ResourceDescriptor("X", args), ctx); + setup.open(); + + List<ChatMessage> input = + Arrays.asList( + new ChatMessage(MessageRole.SYSTEM, "you are an agent"), + new ChatMessage(MessageRole.USER, "hi")); + setup.chat(input); + + // Expected: SYSTEM, SYSTEM(skill_prompt), USER + assertEquals(3, connection.capturedMessages.size()); + assertEquals(MessageRole.SYSTEM, connection.capturedMessages.get(0).getRole()); + assertEquals("you are an agent", connection.capturedMessages.get(0).getContent()); + assertEquals(MessageRole.SYSTEM, connection.capturedMessages.get(1).getRole()); + assertTrue(connection.capturedMessages.get(1).getContent().contains("<available_skills>")); + assertEquals(MessageRole.USER, connection.capturedMessages.get(2).getRole()); + } +} diff --git a/api/src/test/java/org/apache/flink/agents/api/skills/SkillsResourceTest.java b/api/src/test/java/org/apache/flink/agents/api/skills/SkillsResourceTest.java new file mode 100644 index 00000000..864eae42 --- /dev/null +++ b/api/src/test/java/org/apache/flink/agents/api/skills/SkillsResourceTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.api.skills; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.agents.api.resource.ResourceType; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class SkillsResourceTest { + + @Test + void fromLocalDirCarriesPaths() { + Skills skills = Skills.fromLocalDir("/tmp/a", "/tmp/b"); + assertEquals(List.of("/tmp/a", "/tmp/b"), skills.getPaths()); + assertEquals(ResourceType.SKILLS, skills.getResourceType()); + } + + @Test + void roundTripsThroughJackson() throws Exception { + Skills original = Skills.fromLocalDir("/tmp/skill1", "/tmp/skill2"); + ObjectMapper mapper = new ObjectMapper(); + String json = mapper.writeValueAsString(original); + Skills restored = mapper.readValue(json, Skills.class); + assertEquals(original.getPaths(), restored.getPaths()); + } + + @Test + void reservedNamesMatchPython() { + assertEquals("_skills_config", Skills.SKILLS_CONFIG); + assertEquals("load_skill", Skills.LOAD_SKILL_TOOL); + assertEquals("bash", Skills.BASH_TOOL); + } +} diff --git a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/SkillsIntegrationAgent.java b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/SkillsIntegrationAgent.java new file mode 100644 index 00000000..b3533824 --- /dev/null +++ b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/SkillsIntegrationAgent.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.integration.test; + +import org.apache.flink.agents.api.InputEvent; +import org.apache.flink.agents.api.OutputEvent; +import org.apache.flink.agents.api.agents.Agent; +import org.apache.flink.agents.api.annotation.Action; +import org.apache.flink.agents.api.annotation.ChatModelConnection; +import org.apache.flink.agents.api.annotation.ChatModelSetup; +import org.apache.flink.agents.api.annotation.Prompt; +import org.apache.flink.agents.api.chat.messages.ChatMessage; +import org.apache.flink.agents.api.chat.messages.MessageRole; +import org.apache.flink.agents.api.context.RunnerContext; +import org.apache.flink.agents.api.event.ChatRequestEvent; +import org.apache.flink.agents.api.event.ChatResponseEvent; +import org.apache.flink.agents.api.resource.ResourceDescriptor; +import org.apache.flink.agents.api.resource.ResourceName; +import org.apache.flink.agents.api.skills.Skills; + +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Agent that exercises the agent-skills feature end-to-end. Mirrors the Python {@code + * agent_skills_test.SkillTestAgent}: declares two skills (math-calculator, joke-generator) and a + * system prompt that instructs the model to load the skill before answering. + */ +public class SkillsIntegrationAgent extends Agent { + + /** Same model name as the Python {@code agent_skills_test} (a dashscope-hosted Qwen). */ + public static final String MODEL = "qwen3.6-plus"; + + /** + * Same default endpoint as the Python test β overridden by the {@code ACTION_BASE_URL} CI env + * var. + */ + public static final String DEFAULT_BASE_URL = "https://coding.dashscope.aliyuncs.com/v1";; + + @ChatModelConnection + public static ResourceDescriptor openaiConnection() { + String apiKey = System.getenv("ACTION_API_KEY"); + String baseUrl = System.getenv().getOrDefault("ACTION_BASE_URL", DEFAULT_BASE_URL); + return ResourceDescriptor.Builder.newBuilder( + ResourceName.ChatModel.OPENAI_COMPLETIONS_CONNECTION) + .addInitialArgument("api_key", apiKey) + .addInitialArgument("api_base_url", baseUrl) + .build(); + } + + @ChatModelSetup + public static ResourceDescriptor openaiSetup() { + return ResourceDescriptor.Builder.newBuilder( + ResourceName.ChatModel.OPENAI_COMPLETIONS_SETUP) + .addInitialArgument("connection", "openaiConnection") + .addInitialArgument("model", MODEL) + .addInitialArgument("skills", List.of("math-calculator", "joke-generator")) + .addInitialArgument("allowed_commands", List.of("echo", "bc", "python", "python3")) + .addInitialArgument("prompt", "systemPrompt") + .build(); + } + + /** Resolve the {@code skills/} test resource directory to an absolute filesystem path. */ + @org.apache.flink.agents.api.annotation.Skills + public static Skills mySkills() { + URL url = + Objects.requireNonNull( + SkillsIntegrationAgent.class.getClassLoader().getResource("skills"), + "skills/ test resources are missing"); + Path path = Paths.get(url.getPath()); + return Skills.fromLocalDir(path.toString()); + } + + @Prompt + public static org.apache.flink.agents.api.prompt.Prompt systemPrompt() { + return org.apache.flink.agents.api.prompt.Prompt.fromMessages( + Collections.singletonList( + new ChatMessage( + MessageRole.SYSTEM, + "You are a helpful assistant. Use the math-calculator skill when " + + "asked to evaluate an expression, and the joke-generator " + + "skill when asked for a joke. You must load the skill " + + "first and strictly follow the instructions of the skill."))); + } + + @Action(listenEventTypes = {InputEvent.EVENT_TYPE}) + public static void process(InputEvent event, RunnerContext ctx) throws Exception { + ctx.sendEvent( + new ChatRequestEvent( + "openaiSetup", + Collections.singletonList( + new ChatMessage(MessageRole.USER, (String) event.getInput())))); + } + + @Action(listenEventTypes = {ChatResponseEvent.EVENT_TYPE}) + public static void processChatResponse(ChatResponseEvent event, RunnerContext ctx) { + ctx.sendEvent(new OutputEvent(event.getResponse().getContent())); + } +} diff --git a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/SkillsIntegrationTest.java b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/SkillsIntegrationTest.java new file mode 100644 index 00000000..65360910 --- /dev/null +++ b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/SkillsIntegrationTest.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.integration.test; + +import org.apache.flink.agents.api.AgentsExecutionEnvironment; +import org.apache.flink.agents.api.agents.Agent; +import org.apache.flink.agents.api.agents.ReActAgent; +import org.apache.flink.agents.api.chat.messages.ChatMessage; +import org.apache.flink.agents.api.chat.messages.MessageRole; +import org.apache.flink.agents.api.prompt.Prompt; +import org.apache.flink.agents.api.resource.ResourceDescriptor; +import org.apache.flink.agents.api.resource.ResourceName; +import org.apache.flink.agents.api.resource.ResourceType; +import org.apache.flink.agents.api.skills.Skills; +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +import java.net.URL; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.apache.flink.agents.api.agents.AgentExecutionOptions.ERROR_HANDLING_STRATEGY; +import static org.apache.flink.agents.api.agents.AgentExecutionOptions.MAX_RETRIES; + +/** + * End-to-end tests for agent skills. Mirrors the Python {@code + * python/flink_agents/e2e_tests/e2e_tests_integration/agent_skills_test.py}, including both the + * workflow-style agent and the {@link ReActAgent} variant. + * + * <ul> + * <li>{@link #testWorkflowWithSkills()} β feeds prompts through {@link SkillsIntegrationAgent} + * (workflow agent) and asserts on the math/joke responses. + * <li>{@link #testReActAgentWithSkills()} β uses {@link ReActAgent} with a structured output + * schema; asserts the parsed {@code result} field equals 8 ({@code 2 ^ 3}). + * </ul> + * + * <p>Skipped unless {@code ACTION_API_KEY} (the GitHub Actions-injected env var, mirroring the + * Python test) is set; small local models do not reliably handle the multi-turn skill-loading flow. + * {@code ACTION_BASE_URL} optionally overrides the default dashscope endpoint. + */ +public class SkillsIntegrationTest { + + @Test + public void testWorkflowWithSkills() throws Exception { + Assumptions.assumeTrue( + System.getenv().get("ACTION_API_KEY") != null, + "ACTION_API_KEY is required for the skills end-to-end test."); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + + DataStream<String> inputStream = + env.fromData( + "Please evaluate the expression: (2 ^ 3)", "Tell me a joke about cat."); + + AgentsExecutionEnvironment agentsEnv = + AgentsExecutionEnvironment.getExecutionEnvironment(env); + + DataStream<Object> outputStream = + agentsEnv + .fromDataStream(inputStream, (KeySelector<String, String>) value -> value) + .apply(new SkillsIntegrationAgent()) + .toDataStream(); + + CloseableIterator<Object> results = outputStream.collectAsync(); + agentsEnv.execute(); + + List<String> responses = new ArrayList<>(); + while (results.hasNext()) { + responses.add(String.valueOf(results.next())); + } + + Assertions.assertEquals( + 2, responses.size(), String.format("Expected 2 responses, got: %s", responses)); + + String text = String.join("\n", responses); + Assertions.assertTrue( + text.contains("8"), + String.format("Math response should contain '8'. Full responses: %s", text)); + Assertions.assertTrue( + text.contains("Too many cheetahs"), + String.format( + "Joke response should contain script punchline 'Too many cheetahs'. " + + "Full responses: %s", + text)); + } + + @Test + public void testReActAgentWithSkills() throws Exception { + Assumptions.assumeTrue( + System.getenv().get("ACTION_API_KEY") != null, + "ACTION_API_KEY is required for the skills end-to-end test."); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); + + AgentsExecutionEnvironment agentsEnv = + AgentsExecutionEnvironment.getExecutionEnvironment(env, tableEnv); + + String apiKey = System.getenv("ACTION_API_KEY"); + String baseUrl = + System.getenv() + .getOrDefault("ACTION_BASE_URL", SkillsIntegrationAgent.DEFAULT_BASE_URL); + + // Resolve the bundled skills/ test resource directory (same fixtures as the workflow test). + URL url = + Objects.requireNonNull( + SkillsIntegrationTest.class.getClassLoader().getResource("skills"), + "skills/ test resources are missing"); + String skillsPath = Paths.get(url.getPath()).toString(); + + agentsEnv + .addResource( + "openai", + ResourceType.CHAT_MODEL_CONNECTION, + ResourceDescriptor.Builder.newBuilder( + ResourceName.ChatModel.OPENAI_COMPLETIONS_CONNECTION) + .addInitialArgument("api_key", apiKey) + .addInitialArgument("api_base_url", baseUrl) + .build()) + .addResource("my_skill", ResourceType.SKILLS, Skills.fromLocalDir(skillsPath)); + + agentsEnv.getConfig().set(ERROR_HANDLING_STRATEGY, ReActAgent.ErrorHandlingStrategy.RETRY); + agentsEnv.getConfig().set(MAX_RETRIES, 3); + + ResourceDescriptor chatModelDescriptor = + ResourceDescriptor.Builder.newBuilder( + ResourceName.ChatModel.OPENAI_COMPLETIONS_SETUP) + .addInitialArgument("connection", "openai") + .addInitialArgument("model", SkillsIntegrationAgent.MODEL) + .addInitialArgument("skills", List.of("math-calculator")) + .addInitialArgument("allowed_commands", List.of("echo", "bc")) + .build(); + + Prompt prompt = + Prompt.fromMessages( + List.of( + new ChatMessage( + MessageRole.SYSTEM, + "You are a math calculate assistant. Use the math-calculator " + + "skill when asked to evaluate an expression. You " + + "must load the skill first and strictly follow the " + + "instructions of the skill."), + new ChatMessage( + MessageRole.USER, + "Please evaluate the expression: {a} ^ {b}"))); + + RowTypeInfo outputTypeInfo = + new RowTypeInfo( + new TypeInformation[] {BasicTypeInfo.INT_TYPE_INFO}, + new String[] {"result"}); + + Agent agent = new ReActAgent(chatModelDescriptor, prompt, outputTypeInfo); + + Table inputTable = + tableEnv.fromValues( + DataTypes.ROW( + DataTypes.FIELD("a", DataTypes.INT()), + DataTypes.FIELD("b", DataTypes.INT())), + Row.of(2, 3)); + + Schema outputSchema = + Schema.newBuilder() + .column("f0", DataTypes.ROW(DataTypes.FIELD("result", DataTypes.INT()))) + .build(); + + Table outputTable = + agentsEnv + .fromTable( + inputTable, + (KeySelector<Object, Integer>) + value -> (Integer) ((Row) value).getField("a")) + .apply(agent) + .toTable(outputSchema); + + CloseableIterator<Row> results = + tableEnv.toDataStream(outputTable) + .map((MapFunction<Row, Row>) x -> (Row) x.getField("f0")) + .collectAsync(); + + env.execute(); + + Assertions.assertTrue( + results.hasNext(), + "ReAct agent did not produce any output β the LLM response may not have matched the " + + "output schema; rerun if so."); + Row row = (Row) results.next(); + Object result = row.getField("result"); + Assertions.assertNotNull(result, String.format("Missing result field in row %s", row)); + Assertions.assertEquals( + 8, ((Integer) result).intValue(), String.format("Expected 2 ^ 3 = 8, got %s", row)); + } +} diff --git a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/joke-generator/SKILL.md b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/joke-generator/SKILL.md new file mode 100644 index 00000000..eadf07ab --- /dev/null +++ b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/joke-generator/SKILL.md @@ -0,0 +1,18 @@ +--- +name: joke-generator +description: Tell a joke about cat. +--- + +# Math Calculator Skill + +This skill provides the ability to tell a joke about cat. + +## When to Use + +Use this skill when user want to get a joke about cat. + +## Methods + +```bash +python scripts/gen_joke.py +``` diff --git a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/joke-generator/scripts/gen_joke.py b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/joke-generator/scripts/gen_joke.py new file mode 100755 index 00000000..3c681eba --- /dev/null +++ b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/joke-generator/scripts/gen_joke.py @@ -0,0 +1,20 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +if __name__ == "__main__": + print("Why don't cats play poker in the jungle? Too many cheetahs. π±") diff --git a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/math-calculator/SKILL.md b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/math-calculator/SKILL.md new file mode 100644 index 00000000..b00a1592 --- /dev/null +++ b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/resources/skills/math-calculator/SKILL.md @@ -0,0 +1,57 @@ +--- +name: math-calculator +description: Calculate simple mathematical expressions using shell commands. Use when the user asks to perform arithmetic calculations like addition, subtraction, multiplication, division, or more complex math expressions. +license: Apache-2.0 +compatibility: Requires bash with bc (basic calculator) +--- + +# Math Calculator Skill + +This skill provides the ability to calculate mathematical expressions using shell commands. + +## When to Use + +Use this skill when: +- Performing arithmetic calculations (add, subtract, multiply, divide) +- Evaluating mathematical expressions with parentheses +- Computing percentages or powers +- Any numeric computation requested by the user + +## Methods + +### Using `bc` (Basic Calculator) + +The `bc` command is a powerful calculator that supports: +- Basic arithmetic: `+`, `-`, `*`, `/` +- Power: `^` +- Parentheses for grouping +- Scale for decimal precision + +**Example:** +```bash +echo "2 + 3 * 4" | bc +# Output: 14 + +echo "scale=2; 10 / 3" | bc +# Output: 3.33 + +echo "(2 + 3) * 4" | bc +# Output: 20 +``` + +## Supported Operations + +| Operation | Symbol | Example | +|-----------|--------|---------| +| Addition | `+` | `5 + 3 = 8` | +| Subtraction | `-` | `10 - 4 = 6` | +| Multiplication | `*` | `6 * 7 = 42` | +| Division | `/` | `15 / 3 = 5` | +| Power | `^` (bc) or `**` (Python) | `2 ^ 3 = 8` | +| Modulo | `%` | `17 % 5 = 2` | +| Square Root | `sqrt()` (bc) | `sqrt(16) = 4` | + +## Notes + +- Use `scale=N` in `bc` to set decimal precision (default is 0, integer only) +- For floating-point division, always set `scale` in `bc` diff --git a/plan/src/main/java/org/apache/flink/agents/plan/AgentPlan.java b/plan/src/main/java/org/apache/flink/agents/plan/AgentPlan.java index 0dde929e..3fb15109 100644 --- a/plan/src/main/java/org/apache/flink/agents/plan/AgentPlan.java +++ b/plan/src/main/java/org/apache/flink/agents/plan/AgentPlan.java @@ -28,6 +28,7 @@ import org.apache.flink.agents.api.resource.ResourceDescriptor; import org.apache.flink.agents.api.resource.ResourceType; import org.apache.flink.agents.api.resource.SerializableResource; import org.apache.flink.agents.api.resource.python.PythonResourceWrapper; +import org.apache.flink.agents.api.skills.Skills; import org.apache.flink.agents.api.tools.ToolMetadata; import org.apache.flink.agents.plan.actions.Action; import org.apache.flink.agents.plan.actions.ChatModelAction; @@ -42,6 +43,7 @@ import org.apache.flink.agents.plan.serializer.AgentPlanJsonDeserializer; import org.apache.flink.agents.plan.serializer.AgentPlanJsonSerializer; import org.apache.flink.agents.plan.tools.FunctionTool; import org.apache.flink.agents.plan.tools.ToolMetadataFactory; +import org.apache.flink.agents.plan.tools.bash.BashTool; import org.apache.flink.api.java.tuple.Tuple3; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +58,8 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -337,6 +341,9 @@ public class AgentPlan implements Serializable { private void extractResourceProvidersFromAgent(Agent agent) throws Exception { Class<?> agentClass = agent.getClass(); + // Collect Skills declarations from both @Skills methods and Agent.addResource(SKILLS, ...) + Map<String, Skills> skillsObjects = new LinkedHashMap<>(); + // Scan all fields in the agent class for @Tool and @ChatModel annotations for (Field field : agentClass.getDeclaredFields()) { field.setAccessible(true); // Allow access to private fields @@ -411,6 +418,17 @@ public class AgentPlan implements Serializable { extractResource(ResourceType.EMBEDDING_MODEL_CONNECTION, method); } else if (method.isAnnotationPresent(VectorStore.class)) { extractResource(ResourceType.VECTOR_STORE, method); + } else if (method.isAnnotationPresent( + org.apache.flink.agents.api.annotation.Skills.class) + && Modifier.isStatic(method.getModifiers())) { + Object value = method.invoke(null); + if (!(value instanceof Skills)) { + throw new IllegalStateException( + "@Skills method " + + method.getName() + + " must return org.apache.flink.agents.api.skills.Skills"); + } + skillsObjects.put(method.getName(), (Skills) value); } else if (method.isAnnotationPresent(MCPServer.class)) { // Check the MCPServer annotation version to determine which version to use. MCPServer MCPServerAnnotation = method.getAnnotation(MCPServer.class); @@ -476,8 +494,52 @@ public class AgentPlan implements Serializable { ((org.apache.flink.agents.api.tools.FunctionTool) kv.getValue()) .getMethod()); } + } else if (type == ResourceType.SKILLS) { + for (Map.Entry<String, Object> kv : entry.getValue().entrySet()) { + if (kv.getValue() instanceof Skills) { + skillsObjects.put(kv.getKey(), (Skills) kv.getValue()); + } + } } } + + addSkills(skillsObjects); + } + + /** + * Mirror of Python {@code _add_skills}: register the merged Skills config under {@link + * Skills#SKILLS_CONFIG} plus the built-in {@code load_skill} and {@code bash} tools. + * + * <p>{@link BashTool} lives in this module so we can reference its class directly; {@code + * LoadSkillTool} lives in the runtime module and is referenced by FQN string to avoid a reverse + * dependency. + */ + private void addSkills(Map<String, Skills> skillsObjects) throws Exception { + if (skillsObjects.isEmpty()) { + return; + } + + addResourceProvider( + new JavaResourceProvider( + Skills.LOAD_SKILL_TOOL, + ResourceType.TOOL, + new ResourceDescriptor( + "org.apache.flink.agents.runtime.skill.LoadSkillTool", + new HashMap<>()))); + addResourceProvider( + new JavaResourceProvider( + Skills.BASH_TOOL, + ResourceType.TOOL, + new ResourceDescriptor(BashTool.class.getName(), new HashMap<>()))); + + LinkedHashSet<String> paths = new LinkedHashSet<>(); + for (Skills s : skillsObjects.values()) { + paths.addAll(s.getPaths()); + } + Skills merged = Skills.fromLocalDir(paths.toArray(new String[0])); + addResourceProvider( + JavaSerializableResourceProvider.createResourceProvider( + Skills.SKILLS_CONFIG, ResourceType.SKILLS, merged)); } /** diff --git a/plan/src/main/java/org/apache/flink/agents/plan/actions/ChatModelAction.java b/plan/src/main/java/org/apache/flink/agents/plan/actions/ChatModelAction.java index becec471..997fb28b 100644 --- a/plan/src/main/java/org/apache/flink/agents/plan/actions/ChatModelAction.java +++ b/plan/src/main/java/org/apache/flink/agents/plan/actions/ChatModelAction.java @@ -36,6 +36,7 @@ import org.apache.flink.agents.api.event.ToolRequestEvent; import org.apache.flink.agents.api.event.ToolResponseEvent; import org.apache.flink.agents.api.metrics.FlinkAgentsMetricGroup; import org.apache.flink.agents.api.resource.ResourceType; +import org.apache.flink.agents.api.skills.Skills; import org.apache.flink.agents.api.tools.ToolResponse; import org.apache.flink.agents.plan.JavaFunction; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -182,6 +183,7 @@ public class ChatModelAction { ChatMessage response, UUID initialRequestId, String model, + BaseChatModelSetup chatModel, List<ChatMessage> messages, Object outputSchema, RunnerContext ctx) @@ -192,6 +194,8 @@ public class ChatModelAction { messages, Collections.singletonList(response)); + injectBashToolArgs(response.getToolCalls(), chatModel); + ToolRequestEvent toolRequestEvent = new ToolRequestEvent(model, response.getToolCalls()); saveToolRequestEventContext( @@ -204,6 +208,46 @@ public class ChatModelAction { ctx.sendEvent(toolRequestEvent); } + /** + * Inject framework-controlled args ({@code allowed_commands}, {@code allowed_script_dirs}) into + * bash tool calls so they remain hidden from the LLM. Mirrors Python {@code + * _inject_bash_tool_args}. + */ + @SuppressWarnings("unchecked") + private static void injectBashToolArgs( + List<Map<String, Object>> toolCalls, BaseChatModelSetup chatModel) throws Exception { + if (toolCalls == null || toolCalls.isEmpty()) { + return; + } + List<String> scriptDirs = new ArrayList<>(chatModel.getAllowedScriptDirs()); + List<String> declaredSkills = chatModel.getSkills(); + if (declaredSkills != null + && !declaredSkills.isEmpty() + && chatModel.getResourceContext() != null) { + scriptDirs.addAll(chatModel.getResourceContext().getSkillDirs(declaredSkills)); + } + for (Map<String, Object> call : toolCalls) { + Object function = call.get("function"); + if (!(function instanceof Map)) { + continue; + } + Map<String, Object> functionMap = (Map<String, Object>) function; + if (!Skills.BASH_TOOL.equals(functionMap.get("name"))) { + continue; + } + Object argsObj = functionMap.get("arguments"); + Map<String, Object> args; + if (argsObj instanceof Map) { + args = (Map<String, Object>) argsObj; + } else { + args = new HashMap<>(); + functionMap.put("arguments", args); + } + args.put("allowed_commands", new ArrayList<>(chatModel.getAllowedCommands())); + args.put("allowed_script_dirs", scriptDirs); + } + } + static String cleanLlmResponse(String rawResponse) { String trimmed = rawResponse.trim(); if (trimmed.startsWith("```")) { @@ -348,7 +392,8 @@ public class ChatModelAction { } if (!Objects.requireNonNull(response).getToolCalls().isEmpty()) { - handleToolCalls(response, initialRequestId, model, messages, outputSchema, ctx); + handleToolCalls( + response, initialRequestId, model, chatModel, messages, outputSchema, ctx); } else { Map<String, Long> retryStats = getRetryStats(ctx.getSensoryMemory(), initialRequestId); int totalRetryCount = retryStats.get(TOTAL_RETRY_COUNT).intValue(); diff --git a/plan/src/test/java/org/apache/flink/agents/plan/AgentPlanDeclareSkillsTest.java b/plan/src/test/java/org/apache/flink/agents/plan/AgentPlanDeclareSkillsTest.java new file mode 100644 index 00000000..e7605368 --- /dev/null +++ b/plan/src/test/java/org/apache/flink/agents/plan/AgentPlanDeclareSkillsTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.plan; + +import org.apache.flink.agents.api.agents.Agent; +import org.apache.flink.agents.api.resource.ResourceType; +import org.apache.flink.agents.api.skills.Skills; +import org.apache.flink.agents.plan.resourceprovider.JavaResourceProvider; +import org.apache.flink.agents.plan.resourceprovider.JavaSerializableResourceProvider; +import org.apache.flink.agents.plan.resourceprovider.ResourceProvider; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class AgentPlanDeclareSkillsTest { + + public static class SingleSkillsAgent extends Agent { + @org.apache.flink.agents.api.annotation.Skills + public static Skills mySkills() { + return Skills.fromLocalDir("/tmp/skill-a", "/tmp/skill-b"); + } + } + + public static class MultiSkillsAgent extends Agent { + @org.apache.flink.agents.api.annotation.Skills + public static Skills first() { + return Skills.fromLocalDir("/tmp/skill-a", "/tmp/skill-b"); + } + + @org.apache.flink.agents.api.annotation.Skills + public static Skills second() { + return Skills.fromLocalDir("/tmp/skill-b", "/tmp/skill-c"); + } + } + + public static class NoSkillsAgent extends Agent {} + + @Test + void singleSkillsRegistersConfigAndBuiltInTools() throws Exception { + AgentPlan plan = new AgentPlan(new SingleSkillsAgent()); + Map<ResourceType, Map<String, ResourceProvider>> providers = plan.getResourceProviders(); + + // Skills config under reserved name + assertNotNull(providers.get(ResourceType.SKILLS)); + ResourceProvider configProvider = + providers.get(ResourceType.SKILLS).get(Skills.SKILLS_CONFIG); + assertNotNull(configProvider); + assertTrue(configProvider instanceof JavaSerializableResourceProvider); + + // load_skill + bash tools as JavaResourceProviders pointing at runtime / plan classes + Map<String, ResourceProvider> tools = providers.get(ResourceType.TOOL); + assertNotNull(tools); + assertTrue(tools.get(Skills.LOAD_SKILL_TOOL) instanceof JavaResourceProvider); + assertEquals( + "org.apache.flink.agents.runtime.skill.LoadSkillTool", + ((JavaResourceProvider) tools.get(Skills.LOAD_SKILL_TOOL)) + .getDescriptor() + .getClazz()); + assertEquals( + "org.apache.flink.agents.plan.tools.bash.BashTool", + ((JavaResourceProvider) tools.get(Skills.BASH_TOOL)).getDescriptor().getClazz()); + } + + @Test + void multipleSkillsMethodsMergePathsWithDeduplication() throws Exception { + AgentPlan plan = new AgentPlan(new MultiSkillsAgent()); + ResourceProvider configProvider = + plan.getResourceProviders().get(ResourceType.SKILLS).get(Skills.SKILLS_CONFIG); + Skills merged = + (Skills) + ((JavaSerializableResourceProvider) configProvider) + .provide( + org.apache.flink.agents.api.resource.ResourceContext + .fromGetResource((n, t) -> null)); + // Order is preserved; "/tmp/skill-b" appears once. + assertEquals(3, merged.getPaths().size()); + assertTrue(merged.getPaths().contains("/tmp/skill-a")); + assertTrue(merged.getPaths().contains("/tmp/skill-b")); + assertTrue(merged.getPaths().contains("/tmp/skill-c")); + } + + @Test + void noSkillsLeavesNoConfigProvider() throws Exception { + AgentPlan plan = new AgentPlan(new NoSkillsAgent()); + Map<String, ResourceProvider> skillsMap = + plan.getResourceProviders().getOrDefault(ResourceType.SKILLS, Map.of()); + assertNull(skillsMap.get(Skills.SKILLS_CONFIG)); + Map<String, ResourceProvider> tools = + plan.getResourceProviders().getOrDefault(ResourceType.TOOL, Map.of()); + assertNull(tools.get(Skills.LOAD_SKILL_TOOL)); + assertNull(tools.get(Skills.BASH_TOOL)); + } + + @Test + void programmaticSkillsAddResourceParticipates() throws Exception { + Agent agent = new NoSkillsAgent(); + agent.addResource("more", ResourceType.SKILLS, Skills.fromLocalDir("/tmp/skill-d")); + AgentPlan plan = new AgentPlan(agent); + ResourceProvider configProvider = + plan.getResourceProviders().get(ResourceType.SKILLS).get(Skills.SKILLS_CONFIG); + assertNotNull(configProvider); + Skills merged = + (Skills) + ((JavaSerializableResourceProvider) configProvider) + .provide( + org.apache.flink.agents.api.resource.ResourceContext + .fromGetResource((n, t) -> null)); + assertEquals(java.util.List.of("/tmp/skill-d"), merged.getPaths()); + } +} diff --git a/python/flink_agents/runtime/java/java_resource_wrapper.py b/python/flink_agents/runtime/java/java_resource_wrapper.py index cc929b6f..83305731 100644 --- a/python/flink_agents/runtime/java/java_resource_wrapper.py +++ b/python/flink_agents/runtime/java/java_resource_wrapper.py @@ -93,11 +93,17 @@ class JavaResourceContextWrapper(ResourceContext): @override def generate_available_skills_prompt(self, *skill_names: str) -> str: - """Generate the skill discovery prompt for the given skill names.""" - # TODO: Implement after java supports agent skills. + """Generate the skill discovery prompt for the given skill names. + + Forwards to the Java ``JavaResourceAdapter#generateAvailableSkillsPrompt`` + so that a Python chat model running inside a Java agent can use skills + declared on the Java side. + """ + result = self._j_resource_adapter.generateAvailableSkillsPrompt(list(skill_names)) + return result if result is not None else "" @override def get_skill_dirs(self, *skill_names: str) -> List[str]: """Return absolute directory paths for the given skill names.""" - # TODO: Implement after java supports agent skills. - return [] + result = self._j_resource_adapter.getSkillDirs(list(skill_names)) + return list(result) if result is not None else [] diff --git a/runtime/pom.xml b/runtime/pom.xml index 39679389..168fb2be 100644 --- a/runtime/pom.xml +++ b/runtime/pom.xml @@ -41,6 +41,12 @@ under the License. <version>${project.version}</version> </dependency> + <!-- YAML parsing for SKILL.md frontmatter --> + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-yaml</artifactId> + </dependency> + <!-- flink --> <dependency> <groupId>org.apache.flink</groupId> diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/python/utils/JavaResourceAdapter.java b/runtime/src/main/java/org/apache/flink/agents/runtime/python/utils/JavaResourceAdapter.java index 81336ecb..fbb0365e 100644 --- a/runtime/src/main/java/org/apache/flink/agents/runtime/python/utils/JavaResourceAdapter.java +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/python/utils/JavaResourceAdapter.java @@ -28,6 +28,7 @@ import org.apache.flink.agents.api.vectorstores.VectorStoreQueryMode; import pemja.core.PythonInterpreter; import pemja.core.object.PyObject; +import java.util.List; import java.util.Map; /** Adapter for managing Java resources and facilitating Python-Java interoperability. */ @@ -54,6 +55,20 @@ public class JavaResourceAdapter { return resourceContext.getResource(name, ResourceType.fromValue(typeValue)); } + /** + * Generate the available skills prompt for the given skill names. Used by the Python {@code + * JavaResourceContextWrapper} when a Python chat model running in a Java agent needs the skill + * discovery prompt. + */ + public String generateAvailableSkillsPrompt(List<String> skillNames) throws Exception { + return resourceContext.generateAvailableSkillsPrompt(skillNames); + } + + /** Return absolute directory paths for the given skill names. */ + public List<String> getSkillDirs(List<String> skillNames) throws Exception { + return resourceContext.getSkillDirs(skillNames); + } + /** * Convert a Python chat message to a Java chat message. This method is intended for use by the * Python interpreter. diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/resource/ResourceContextImpl.java b/runtime/src/main/java/org/apache/flink/agents/runtime/resource/ResourceContextImpl.java index 11c1cb63..b7dc0eed 100644 --- a/runtime/src/main/java/org/apache/flink/agents/runtime/resource/ResourceContextImpl.java +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/resource/ResourceContextImpl.java @@ -21,6 +21,10 @@ package org.apache.flink.agents.runtime.resource; import org.apache.flink.agents.api.resource.Resource; import org.apache.flink.agents.api.resource.ResourceContext; import org.apache.flink.agents.api.resource.ResourceType; +import org.apache.flink.agents.api.skills.Skills; +import org.apache.flink.agents.runtime.skill.SkillManager; + +import javax.annotation.Nullable; import java.util.Collections; import java.util.List; @@ -30,13 +34,17 @@ import java.util.function.BiFunction; * Default {@link ResourceContext} implementation that delegates resource lookup to a {@link * BiFunction} (typically the underlying {@code ResourceCache::getResource}). * - * <p>Mirrors the Python {@code flink_agents.runtime.resource_context.ResourceContextImpl}. Skill - * methods return safe defaults; callers without skills configured see empty values. + * <p>Mirrors the Python {@code flink_agents.runtime.resource_context.ResourceContextImpl}. The + * skill methods lazily build a {@link SkillManager} from the {@code _skills_config} resource β if + * no such resource is registered they return safe defaults (empty string / empty list). */ public class ResourceContextImpl implements ResourceContext { private final BiFunction<String, ResourceType, Resource> getResource; + @Nullable private volatile SkillManager skillManager; + @Nullable private volatile Skills cachedSkillsConfig; + public ResourceContextImpl(BiFunction<String, ResourceType, Resource> getResource) { this.getResource = getResource; } @@ -55,11 +63,41 @@ public class ResourceContextImpl implements ResourceContext { @Override public String generateAvailableSkillsPrompt(List<String> skillNames) throws Exception { - return ""; + SkillManager manager = ensureSkillManager(); + return manager == null ? "" : manager.generateDiscoveryPrompt(skillNames); } @Override public List<String> getSkillDirs(List<String> skillNames) throws Exception { - return Collections.emptyList(); + SkillManager manager = ensureSkillManager(); + return manager == null ? Collections.emptyList() : manager.getSkillDirs(skillNames); + } + + /** + * Returns the cached {@link SkillManager} for this context, or {@code null} if not configured. + */ + @Nullable + public synchronized SkillManager getSkillManager() throws Exception { + return ensureSkillManager(); + } + + @Nullable + private synchronized SkillManager ensureSkillManager() throws Exception { + Skills config; + try { + Resource r = getResource(Skills.SKILLS_CONFIG, ResourceType.SKILLS); + if (!(r instanceof Skills)) { + return null; + } + config = (Skills) r; + } catch (Exception e) { + // No skills config registered β that's fine, return null. + return null; + } + if (config != cachedSkillsConfig) { + cachedSkillsConfig = config; + skillManager = new SkillManager(config); + } + return skillManager; } } diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/AgentSkill.java b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/AgentSkill.java new file mode 100644 index 00000000..7b32ade7 --- /dev/null +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/AgentSkill.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import org.apache.flink.util.Preconditions; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Runtime representation of one parsed {@code SKILL.md}. + * + * <p>Mirrors the Python {@code flink_agents.runtime.skill.agent_skill.AgentSkill}. Resources are + * lazily loaded on first access. + */ +public final class AgentSkill { + + private final String name; + private final String description; + private final String content; + @Nullable private final String license; + @Nullable private final String compatibility; + @Nullable private final Map<String, String> metadata; + @Nullable private volatile Map<String, String> resources; + @Nullable private Supplier<Map<String, String>> resourceLoader; + private volatile boolean activated; + + public AgentSkill( + String name, + String description, + String content, + @Nullable String license, + @Nullable String compatibility, + @Nullable Map<String, String> metadata) { + this(name, description, content, license, compatibility, metadata, null); + } + + public AgentSkill( + String name, + String description, + String content, + @Nullable String license, + @Nullable String compatibility, + @Nullable Map<String, String> metadata, + @Nullable Map<String, String> resources) { + Preconditions.checkArgument( + name != null && !name.isEmpty() && name.length() <= 64, + "Skill name must be 1..64 characters: %s", + name); + Preconditions.checkArgument( + description != null && !description.isEmpty() && description.length() <= 1024, + "Skill description must be 1..1024 characters"); + Preconditions.checkArgument( + content != null && !content.isEmpty(), "Skill content must not be empty"); + Preconditions.checkArgument( + compatibility == null || compatibility.length() <= 500, + "Skill compatibility must be at most 500 characters"); + this.name = name; + this.description = description; + this.content = content; + this.license = license; + this.compatibility = compatibility; + this.metadata = metadata; + this.resources = resources; + this.activated = resources != null; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getContent() { + return content; + } + + @Nullable + public String getLicense() { + return license; + } + + @Nullable + public String getCompatibility() { + return compatibility; + } + + @Nullable + public Map<String, String> getMetadata() { + return metadata; + } + + /** Set a lazy resource loader. Must be called before the first {@link #getResource(String)}. */ + public void setResourceLoader(Supplier<Map<String, String>> loader) { + this.resourceLoader = loader; + } + + /** + * Return the content of the named resource (relative path from the skill root) or {@code null} + * if no such resource is registered. + */ + @Nullable + public String getResource(String relativePath) { + activate(); + return resources == null ? null : resources.get(relativePath); + } + + /** Return all registered resource relative paths (sorted, may be empty). */ + public List<String> getResourcePaths() { + activate(); + if (resources == null) { + return List.of(); + } + List<String> keys = new ArrayList<>(resources.keySet()); + keys.sort(String::compareTo); + return keys; + } + + private synchronized void activate() { + if (!activated && resourceLoader != null) { + resources = resourceLoader.get(); + activated = true; + } + } +} diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/LoadSkillTool.java b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/LoadSkillTool.java new file mode 100644 index 00000000..6e08bd76 --- /dev/null +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/LoadSkillTool.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import org.apache.flink.agents.api.resource.ResourceContext; +import org.apache.flink.agents.api.resource.ResourceDescriptor; +import org.apache.flink.agents.api.tools.Tool; +import org.apache.flink.agents.api.tools.ToolMetadata; +import org.apache.flink.agents.api.tools.ToolParameters; +import org.apache.flink.agents.api.tools.ToolResponse; +import org.apache.flink.agents.api.tools.ToolType; +import org.apache.flink.agents.runtime.resource.ResourceContextImpl; + +import java.nio.file.Path; +import java.util.List; + +/** + * Built-in tool that returns the body of a SKILL.md (default) or a specific bundled resource. + * + * <p>Mirrors the Python {@code flink_agents.runtime.skill.skill_tools.LoadSkillTool}. Auto-loaded + * by {@code AgentPlan.addSkills} when an agent declares any {@code @Skills} method. + */ +public class LoadSkillTool extends Tool { + + private static final String DESCRIPTION = + "Load a skill's content or a specific resource. Use this to access skill instructions and resources."; + + private static final String INPUT_SCHEMA = + "{\"type\":\"object\"," + + "\"properties\":{" + + "\"name\":{\"type\":\"string\"," + + "\"description\":\"The name of the skill to load (e.g., 'pdf-processing').\"}," + + "\"path\":{\"type\":\"string\"," + + "\"description\":\"Optional path to a specific resource within the skill. If not provided, returns the full SKILL.md content.\"," + + "\"default\":\"SKILL.md\"}}," + + "\"required\":[\"name\"]}"; + + public LoadSkillTool(ResourceDescriptor descriptor, ResourceContext resourceContext) { + super(new ToolMetadata("load_skill", DESCRIPTION, INPUT_SCHEMA)); + this.resourceContext = resourceContext; + } + + @Override + public ToolType getToolType() { + return ToolType.FUNCTION; + } + + @Override + public ToolResponse call(ToolParameters parameters) { + String name = parameters.getParameter("name", String.class); + String path = + parameters.hasParameter("path") + ? parameters.getParameter("path", String.class) + : "SKILL.md"; + + SkillManager manager; + try { + manager = resolveSkillManager(); + } catch (Exception e) { + return ToolResponse.success( + "Skill manager not available. No skills have been registered."); + } + if (manager == null) { + return ToolResponse.success( + "Skill manager not available. No skills have been registered."); + } + + AgentSkill skill; + try { + skill = manager.getSkill(name); + } catch (IllegalArgumentException e) { + List<String> available = manager.getAllSkillNames(); + String availableStr = + available.isEmpty() ? "No skills available." : String.join(", ", available); + return ToolResponse.success( + "Skill '" + name + "' not found. Available skills: " + availableStr); + } + + if (path == null || "SKILL.md".equals(path)) { + Path skillDir = manager.getSkillDir(name); + if (skillDir != null) { + StringBuilder files = new StringBuilder(); + for (String rel : skill.getResourcePaths()) { + files.append("<file>") + .append(skillDir.resolve(rel)) + .append("</file>") + .append('\n'); + } + String filesSection = files.length() == 0 ? "" : files.toString().stripTrailing(); + return ToolResponse.success( + "<skill_content name=\"" + + name + + "\">\n" + + "# Skill: " + + name + + "\n\n" + + skill.getContent().strip() + + "\n\n" + + "Base directory for this skill: " + + skillDir + + "\n" + + "Relative paths in this skill are relative to this base directory.\n" + + "<skill_files>\n" + + filesSection + + "\n</skill_files>\n" + + "</skill_content>"); + } + return ToolResponse.success(skill.getContent()); + } + + String content = skill.getResource(path); + if (content == null) { + return ToolResponse.success( + "Resource '" + + path + + "' not found in skill '" + + name + + "', Available resources: " + + skill.getResourcePaths()); + } + return ToolResponse.success(content); + } + + private SkillManager resolveSkillManager() throws Exception { + if (resourceContext instanceof ResourceContextImpl) { + return ((ResourceContextImpl) resourceContext).getSkillManager(); + } + return null; + } +} diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillManager.java b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillManager.java new file mode 100644 index 00000000..8778bd0e --- /dev/null +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillManager.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import org.apache.flink.agents.api.skills.Skills; +import org.apache.flink.agents.runtime.skill.repository.FileSystemSkillRepository; + +import javax.annotation.Nullable; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Loads and indexes all skills referenced by a {@link Skills} configuration. + * + * <p>Mirrors the Python {@code flink_agents.runtime.skill.skill_manager.SkillManager}. + */ +public class SkillManager { + + private final Skills config; + private final Map<String, AgentSkill> skills = new LinkedHashMap<>(); + private final Map<String, SkillRepository> repos = new HashMap<>(); + + public SkillManager(Skills config) { + this.config = config; + loadFromPaths(); + } + + public int size() { + return skills.size(); + } + + public AgentSkill getSkill(String name) { + AgentSkill skill = skills.get(name); + if (skill == null) { + throw new IllegalArgumentException( + "Skill " + + name + + " not found, available skill names are: " + + getAllSkillNames()); + } + return skill; + } + + public List<String> getAllSkillNames() { + return new ArrayList<>(skills.keySet()); + } + + @Nullable + public String loadSkillResource(String skillName, String resourcePath) { + return getSkill(skillName).getResource(resourcePath); + } + + /** Build the {@code <available_skills>} system prompt for the given skill names. */ + public String generateDiscoveryPrompt(List<String> names) { + if (size() == 0) { + return ""; + } + StringBuilder sb = new StringBuilder(SkillPromptProvider.SKILL_DISCOVERY_PROMPT); + for (String name : names) { + AgentSkill skill = getSkill(name); + sb.append( + String.format( + SkillPromptProvider.AVAILABLE_SKILL_TEMPLATE, + skill.getName(), + skill.getDescription())); + } + sb.append(SkillPromptProvider.AVAILABLE_SKILLS_TAG_END); + return sb.toString(); + } + + /** + * Absolute directory paths for the listed skill names (filesystem-backed only). When called + * with an empty or {@code null} list, returns directories for all filesystem-backed skills. + */ + public List<String> getSkillDirs(List<String> names) { + Iterable<String> selected = (names == null || names.isEmpty()) ? repos.keySet() : names; + List<String> dirs = new ArrayList<>(); + for (String skillName : selected) { + SkillRepository repo = repos.get(skillName); + if (repo instanceof FileSystemSkillRepository) { + Path dir = ((FileSystemSkillRepository) repo).getBaseDir().resolve(skillName); + dirs.add(dir.toString()); + } + } + return dirs; + } + + /** Return absolute directory path for a single skill, if filesystem-backed. */ + @Nullable + public Path getSkillDir(String skillName) { + SkillRepository repo = repos.get(skillName); + if (repo instanceof FileSystemSkillRepository) { + return ((FileSystemSkillRepository) repo).getBaseDir().resolve(skillName); + } + return null; + } + + /** Resolve a skill resource's relative path to an absolute path, or {@code null} if missing. */ + @Nullable + public Path resolveResourcePath(String skillName, String resourcePath) { + SkillRepository repo = repos.get(skillName); + if (repo instanceof FileSystemSkillRepository) { + Path resolved = + ((FileSystemSkillRepository) repo) + .getBaseDir() + .resolve(skillName) + .resolve(resourcePath); + if (Files.isRegularFile(resolved)) { + return resolved; + } + } + return null; + } + + private void loadFromPaths() { + for (String path : config.getPaths()) { + FileSystemSkillRepository repo = new FileSystemSkillRepository(path); + for (AgentSkill skill : repo.getSkills()) { + final String skillName = skill.getName(); + skill.setResourceLoader(() -> repo.getResources(skillName)); + skills.put(skillName, skill); + repos.put(skillName, repo); + } + } + } +} diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillParser.java b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillParser.java new file mode 100644 index 00000000..02dcdfda --- /dev/null +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillParser.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parser that splits a {@code SKILL.md} file into YAML frontmatter and markdown body, then + * constructs an {@link AgentSkill}. + * + * <p>Mirrors the Python {@code flink_agents.runtime.skill.skill_parser}. + */ +public final class SkillParser { + + private static final Pattern FRONTMATTER = + Pattern.compile( + "^---\\s*[\\r\\n]+(.*?)[\\r\\n]*---(?:\\s*[\\r\\n]+)?(.*)", Pattern.DOTALL); + + private static final ObjectMapper YAML = new ObjectMapper(new YAMLFactory()); + + private SkillParser() {} + + /** Result of splitting a markdown file. */ + public static final class ParsedMarkdown { + public final Map<String, Object> metadata; + public final String content; + + public ParsedMarkdown(Map<String, Object> metadata, String content) { + this.metadata = metadata == null ? Collections.emptyMap() : metadata; + this.content = content == null ? "" : content; + } + } + + /** Split the markdown into YAML frontmatter and the remaining body. */ + public static ParsedMarkdown parseMarkdown(String markdown) { + if (markdown == null || markdown.isEmpty()) { + return new ParsedMarkdown(Collections.emptyMap(), ""); + } + Matcher m = FRONTMATTER.matcher(markdown); + if (!m.matches()) { + return new ParsedMarkdown(Collections.emptyMap(), markdown); + } + String yaml = m.group(1).trim(); + String body = m.group(2); + if (yaml.isEmpty()) { + return new ParsedMarkdown(Collections.emptyMap(), body); + } + try { + Map<String, Object> metadata = + YAML.readValue(yaml, new TypeReference<Map<String, Object>>() {}); + return new ParsedMarkdown(metadata, body); + } catch (Exception e) { + throw new IllegalArgumentException( + "Invalid YAML frontmatter syntax: " + e.getMessage(), e); + } + } + + /** Parse a SKILL.md content into an {@link AgentSkill}. */ + public static AgentSkill parseSkill(String skillMdContent) { + ParsedMarkdown parsed = parseMarkdown(skillMdContent); + Map<String, Object> metadata = parsed.metadata; + + Object name = metadata.get("name"); + if (!(name instanceof String) || ((String) name).trim().isEmpty()) { + throw new IllegalArgumentException( + "The SKILL.md must have a YAML frontmatter including 'name' field."); + } + Object description = metadata.get("description"); + if (!(description instanceof String) || ((String) description).trim().isEmpty()) { + throw new IllegalArgumentException( + "The SKILL.md must have a YAML frontmatter including 'description' field."); + } + if (parsed.content == null || parsed.content.isEmpty()) { + throw new IllegalArgumentException( + "The SKILL.md must have a markdown content after YAML frontmatter."); + } + + Object license = metadata.get("license"); + Object compatibility = metadata.get("compatibility"); + Object inner = metadata.get("metadata"); + Map<String, String> innerMetadata = null; + if (inner instanceof Map) { + innerMetadata = new HashMap<>(); + for (Map.Entry<?, ?> e : ((Map<?, ?>) inner).entrySet()) { + innerMetadata.put(String.valueOf(e.getKey()), String.valueOf(e.getValue())); + } + } + + return new AgentSkill( + ((String) name).trim(), + ((String) description).trim(), + parsed.content, + license == null ? null : license.toString(), + compatibility == null ? null : compatibility.toString(), + innerMetadata); + } +} diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillPromptProvider.java b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillPromptProvider.java new file mode 100644 index 00000000..2cf84386 --- /dev/null +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillPromptProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +/** + * System-prompt templates for skill discovery and activation. + * + * <p>Mirrors the Python {@code flink_agents.runtime.skill.skill_prompt_provider} byte-for-byte. + * Descriptions injected into {@link #AVAILABLE_SKILL_TEMPLATE} must not contain {@code %} (no skill + * fixture currently does β the SKILL.md schema bounds descriptions to plain text). + */ +public final class SkillPromptProvider { + + public static final String SKILL_DISCOVERY_PROMPT = + "## Available Skills\n" + + "\n" + + "<usage>\n" + + "Skills provide specialized capabilities and domain knowledge. Use them when they match your current task.\n" + + "\n" + + "Load a skill with `load_skill(name=\"<skill-name>\")` to get its full instructions.\n" + + "Individual resources (scripts, references, assets) can be loaded with a `path` argument.\n" + + "\n" + + "The loaded content includes the skill's base directory and the absolute paths of its resources.\n" + + "</usage>\n" + + "\n" + + "<available_skills>\n"; + + public static final String AVAILABLE_SKILL_TEMPLATE = + "\n<skill>\n<name>%s</name>\n<description>%s</description>\n</skill>\n"; + + public static final String AVAILABLE_SKILLS_TAG_END = "\n</available_skills>\n"; + + private SkillPromptProvider() {} +} diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillRepository.java b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillRepository.java new file mode 100644 index 00000000..9d0e48b8 --- /dev/null +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/SkillRepository.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; + +/** + * Source of skills. Mirrors the Python {@code + * flink_agents.runtime.skill.skill_repository.SkillRepository}. + */ +public interface SkillRepository { + + /** Return the named skill, or {@code null} if not found. */ + @Nullable + AgentSkill getSkill(String name); + + /** Return all skills in the repository (order is implementation-specific). */ + List<AgentSkill> getSkills(); + + /** + * Return the resource map for the named skill β keys are relative paths from the skill root, + * values are the file contents. + */ + Map<String, String> getResources(String name); +} diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/repository/FileSystemSkillRepository.java b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/repository/FileSystemSkillRepository.java new file mode 100644 index 00000000..91e16df4 --- /dev/null +++ b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/repository/FileSystemSkillRepository.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill.repository; + +import org.apache.flink.agents.runtime.skill.AgentSkill; +import org.apache.flink.agents.runtime.skill.SkillParser; +import org.apache.flink.agents.runtime.skill.SkillRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.nio.charset.MalformedInputException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +/** + * Filesystem-backed {@link SkillRepository}. Each immediate subdirectory of the configured base + * directory that contains a {@code SKILL.md} file is treated as a skill. + * + * <p>Mirrors the Python {@code + * flink_agents.runtime.skill.repository.filesystem_repository.FileSystemSkillRepository}. + */ +public class FileSystemSkillRepository implements SkillRepository { + + private static final Logger LOG = LoggerFactory.getLogger(FileSystemSkillRepository.class); + + public static final String SKILL_MD_FILE = "SKILL.md"; + + private final Path baseDir; + + public FileSystemSkillRepository(Path baseDir) { + if (baseDir == null) { + throw new IllegalArgumentException("Base directory cannot be null"); + } + Path resolved = baseDir.toAbsolutePath().normalize(); + if (!Files.exists(resolved)) { + throw new IllegalArgumentException("Base directory does not exist: " + resolved); + } + if (!Files.isDirectory(resolved)) { + throw new IllegalArgumentException("Base directory is not a directory: " + resolved); + } + this.baseDir = resolved; + } + + public FileSystemSkillRepository(String baseDir) { + this(Path.of(baseDir)); + } + + public Path getBaseDir() { + return baseDir; + } + + @Override + @Nullable + public AgentSkill getSkill(String name) { + Path skillDir = baseDir.resolve(name); + Path skillMd = skillDir.resolve(SKILL_MD_FILE); + if (!Files.exists(skillMd)) { + return null; + } + return loadSkill(skillDir); + } + + @Override + public List<AgentSkill> getSkills() { + List<AgentSkill> skills = new ArrayList<>(); + for (String skillName : listSkillNames()) { + AgentSkill skill = getSkill(skillName); + if (skill != null) { + skills.add(skill); + } + } + return skills; + } + + @Override + public Map<String, String> getResources(String name) { + Path skillDir = baseDir.resolve(name); + if (!Files.isDirectory(skillDir)) { + return Collections.emptyMap(); + } + return loadResources(skillDir); + } + + private List<String> listSkillNames() { + List<String> names = new ArrayList<>(); + try (Stream<Path> entries = Files.list(baseDir)) { + entries.forEach( + entry -> { + if (Files.isDirectory(entry) + && Files.exists(entry.resolve(SKILL_MD_FILE))) { + names.add(entry.getFileName().toString()); + } + }); + } catch (IOException e) { + throw new IllegalStateException("Failed to list skills under " + baseDir, e); + } + names.sort(String::compareTo); + return names; + } + + private AgentSkill loadSkill(Path skillDir) { + Path skillMd = skillDir.resolve(SKILL_MD_FILE); + if (!Files.exists(skillMd)) { + return null; + } + try { + String content = Files.readString(skillMd, StandardCharsets.UTF_8); + AgentSkill skill = SkillParser.parseSkill(content); + if (!skill.getName().equals(skillDir.getFileName().toString())) { + LOG.warn( + "The skill name {} is different from the base directory {}.", + skill.getName(), + skillDir.getFileName()); + } + return skill; + } catch (Exception e) { + throw new IllegalArgumentException("Failed to load skill from " + skillDir, e); + } + } + + private Map<String, String> loadResources(Path skillDir) { + Map<String, String> resources = new HashMap<>(); + try (Stream<Path> walk = Files.walk(skillDir)) { + walk.filter(Files::isRegularFile) + .forEach( + file -> { + if (file.getFileName().toString().equals(SKILL_MD_FILE)) { + return; + } + String rel = skillDir.relativize(file).toString(); + try { + resources.put( + rel, Files.readString(file, StandardCharsets.UTF_8)); + } catch (MalformedInputException mie) { + try { + byte[] bytes = Files.readAllBytes(file); + resources.put( + rel, + "base64: " + + Base64.getEncoder() + .encodeToString(bytes)); + } catch (IOException e) { + LOG.warn( + "Failed to read resource file {} as binary.", + file, + e); + } + } catch (IOException e) { + LOG.warn("Failed to read resource file {}.", file, e); + } + }); + } catch (IOException e) { + throw new IllegalStateException("Failed to walk skill dir " + skillDir, e); + } + return resources; + } +} diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/skill/FileSystemSkillRepositoryTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/FileSystemSkillRepositoryTest.java new file mode 100644 index 00000000..b4c3a778 --- /dev/null +++ b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/FileSystemSkillRepositoryTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import org.apache.flink.agents.runtime.skill.repository.FileSystemSkillRepository; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FileSystemSkillRepositoryTest { + + private static Path resourcesRoot() { + return Path.of("src/test/resources/skills").toAbsolutePath(); + } + + @Test + void getSkillsReturnsSortedSkillNames() { + FileSystemSkillRepository repo = new FileSystemSkillRepository(resourcesRoot()); + List<AgentSkill> skills = repo.getSkills(); + assertEquals(2, skills.size()); + assertEquals("github", skills.get(0).getName()); + assertEquals("nano-banana-pro", skills.get(1).getName()); + } + + @Test + void getSkillReturnsNullForUnknown() { + FileSystemSkillRepository repo = new FileSystemSkillRepository(resourcesRoot()); + assertNull(repo.getSkill("does-not-exist")); + } + + @Test + void getResourcesReadsBundledFiles() { + FileSystemSkillRepository repo = new FileSystemSkillRepository(resourcesRoot()); + Map<String, String> resources = repo.getResources("nano-banana-pro"); + assertNotNull(resources); + assertTrue( + resources.containsKey("scripts/generate_image.py"), + "expected scripts/generate_image.py to be loaded as a resource"); + assertTrue(resources.containsKey("_meta.json")); + } + + @Test + void resourceLoaderIsLazy() { + FileSystemSkillRepository repo = new FileSystemSkillRepository(resourcesRoot()); + AgentSkill skill = repo.getSkill("nano-banana-pro"); + assertNotNull(skill); + // resources are not loaded until requested through the loader hook. + skill.setResourceLoader(() -> repo.getResources("nano-banana-pro")); + assertEquals(2, skill.getResourcePaths().size()); + } + + @Test + void missingBaseDirRaises(@TempDir Path tempDir) { + Path missing = tempDir.resolve("missing"); + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> new FileSystemSkillRepository(missing)); + assertTrue(ex.getMessage().contains("does not exist")); + } + + @Test + void binaryResourceFallsBackToBase64(@TempDir Path tempDir) throws IOException { + Path skillDir = Files.createDirectory(tempDir.resolve("binary-skill")); + Files.writeString( + skillDir.resolve("SKILL.md"), + "---\nname: binary-skill\ndescription: holds a binary resource\n---\n# Body\n", + StandardCharsets.UTF_8); + // Bytes that are not valid UTF-8 (start of a 4-byte sequence with no continuation bytes). + byte[] bad = new byte[] {(byte) 0xF8, (byte) 0x88, (byte) 0x80, (byte) 0x80}; + Files.write(skillDir.resolve("blob.bin"), bad); + + FileSystemSkillRepository repo = new FileSystemSkillRepository(tempDir); + Map<String, String> resources = repo.getResources("binary-skill"); + assertTrue(resources.get("blob.bin").startsWith("base64: ")); + } +} diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/skill/LoadSkillToolTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/LoadSkillToolTest.java new file mode 100644 index 00000000..f048920d --- /dev/null +++ b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/LoadSkillToolTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import org.apache.flink.agents.api.resource.Resource; +import org.apache.flink.agents.api.resource.ResourceDescriptor; +import org.apache.flink.agents.api.resource.ResourceType; +import org.apache.flink.agents.api.skills.Skills; +import org.apache.flink.agents.api.tools.ToolParameters; +import org.apache.flink.agents.api.tools.ToolResponse; +import org.apache.flink.agents.runtime.resource.ResourceContextImpl; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LoadSkillToolTest { + + private static ResourceContextImpl contextWithSkills() { + Skills skills = + Skills.fromLocalDir( + Path.of("src/test/resources/skills").toAbsolutePath().toString()); + Map<String, Resource> store = new HashMap<>(); + store.put(Skills.SKILLS_CONFIG, skills); + return new ResourceContextImpl( + (name, type) -> { + if (type == ResourceType.SKILLS) { + return store.get(name); + } + return null; + }); + } + + private static LoadSkillTool tool(ResourceContextImpl ctx) { + return new LoadSkillTool( + new ResourceDescriptor(LoadSkillTool.class.getName(), Map.of()), ctx); + } + + private static ToolParameters args(String name, String path) { + Map<String, Object> m = new HashMap<>(); + m.put("name", name); + if (path != null) { + m.put("path", path); + } + return new ToolParameters(m); + } + + @Test + void unknownSkillReturnsAvailableList() { + LoadSkillTool t = tool(contextWithSkills()); + ToolResponse resp = t.call(args("does-not-exist", null)); + String out = (String) resp.getResult(); + assertTrue(out.contains("not found")); + assertTrue(out.contains("github")); + assertTrue(out.contains("nano-banana-pro")); + } + + @Test + void defaultPathReturnsSkillContentEnvelope() { + LoadSkillTool t = tool(contextWithSkills()); + ToolResponse resp = t.call(args("github", null)); + String out = (String) resp.getResult(); + assertTrue(out.startsWith("<skill_content name=\"github\">")); + assertTrue(out.contains("# Skill: github")); + assertTrue(out.contains("Base directory for this skill: ")); + assertTrue(out.contains("</skill_content>")); + } + + @Test + void resourcePathReturnsRawContent() { + LoadSkillTool t = tool(contextWithSkills()); + ToolResponse resp = t.call(args("nano-banana-pro", "scripts/generate_image.py")); + String out = (String) resp.getResult(); + // The script file should be returned verbatim (not wrapped in <skill_content>). + assertTrue(!out.startsWith("<skill_content")); + assertTrue(out.length() > 0); + } + + @Test + void missingResourceReportsAvailable() { + LoadSkillTool t = tool(contextWithSkills()); + ToolResponse resp = t.call(args("nano-banana-pro", "no-such.txt")); + String out = (String) resp.getResult(); + assertTrue(out.contains("Resource 'no-such.txt' not found")); + assertTrue(out.contains("Available resources")); + } + + @Test + void noSkillsRegisteredReturnsFriendlyMessage() { + // Empty resource context β no _skills_config registered. + ResourceContextImpl ctx = new ResourceContextImpl((name, type) -> null); + LoadSkillTool t = tool(ctx); + ToolResponse resp = t.call(args("anything", null)); + assertEquals( + "Skill manager not available. No skills have been registered.", resp.getResult()); + } +} diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/skill/SkillManagerTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/SkillManagerTest.java new file mode 100644 index 00000000..54e8178e --- /dev/null +++ b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/SkillManagerTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import org.apache.flink.agents.api.skills.Skills; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SkillManagerTest { + + private static Skills configFromResources() { + return Skills.fromLocalDir( + Path.of("src/test/resources/skills").toAbsolutePath().toString()); + } + + @Test + void sizeAndAllSkillNames() { + SkillManager manager = new SkillManager(configFromResources()); + assertEquals(2, manager.size()); + assertEquals(List.of("github", "nano-banana-pro"), manager.getAllSkillNames()); + } + + @Test + void getSkillThrowsWithAvailableNames() { + SkillManager manager = new SkillManager(configFromResources()); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> manager.getSkill("missing")); + assertTrue(ex.getMessage().contains("github")); + assertTrue(ex.getMessage().contains("nano-banana-pro")); + } + + @Test + void generateDiscoveryPromptMatchesGoldenFile() throws IOException { + SkillManager manager = new SkillManager(configFromResources()); + String prompt = manager.generateDiscoveryPrompt(List.of("github", "nano-banana-pro")); + String expected = + Files.readString( + Path.of("src/test/resources/skill_discovery_prompt.txt"), + StandardCharsets.UTF_8); + assertEquals(expected, prompt); + } + + @Test + void getSkillDirsEmptyArgumentReturnsAllFsBacked() { + SkillManager manager = new SkillManager(configFromResources()); + List<String> dirs = manager.getSkillDirs(List.of()); + assertEquals(2, dirs.size()); + assertTrue(dirs.get(0).endsWith("github") || dirs.get(0).endsWith("nano-banana-pro")); + } + + @Test + void getSkillDirsReturnsNamedSkillsInOrder() { + SkillManager manager = new SkillManager(configFromResources()); + List<String> dirs = manager.getSkillDirs(List.of("github")); + assertEquals(1, dirs.size()); + assertTrue(dirs.get(0).endsWith("github")); + } + + @Test + void resolveResourcePathLocatesBundledFile() { + SkillManager manager = new SkillManager(configFromResources()); + Path resolved = manager.resolveResourcePath("nano-banana-pro", "scripts/generate_image.py"); + assertNotNull(resolved); + assertTrue(Files.isRegularFile(resolved)); + } +} diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/skill/SkillParserTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/SkillParserTest.java new file mode 100644 index 00000000..270e1374 --- /dev/null +++ b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/SkillParserTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.skill; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SkillParserTest { + + @Test + void parseMarkdownSplitsFrontmatterFromBody() { + String md = + "---\n" + + "name: my-skill\n" + + "description: Does X\n" + + "---\n" + + "# Body\n" + + "Some markdown.\n"; + SkillParser.ParsedMarkdown parsed = SkillParser.parseMarkdown(md); + assertEquals("my-skill", parsed.metadata.get("name")); + assertEquals("Does X", parsed.metadata.get("description")); + assertTrue(parsed.content.startsWith("# Body")); + } + + @Test + void parseMarkdownNoFrontmatterReturnsRawContent() { + String md = "# Just a body\nNo frontmatter."; + SkillParser.ParsedMarkdown parsed = SkillParser.parseMarkdown(md); + assertTrue(parsed.metadata.isEmpty()); + assertEquals(md, parsed.content); + } + + @Test + void parseMarkdownHandlesCrlfLineEndings() { + String md = "---\r\nname: x\r\ndescription: y\r\n---\r\nBody\r\n"; + SkillParser.ParsedMarkdown parsed = SkillParser.parseMarkdown(md); + assertEquals("x", parsed.metadata.get("name")); + assertTrue(parsed.content.contains("Body")); + } + + @Test + void parseSkillRequiresName() { + String md = "---\ndescription: y\n---\nBody\n"; + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> SkillParser.parseSkill(md)); + assertTrue(ex.getMessage().contains("name")); + } + + @Test + void parseSkillRequiresDescription() { + String md = "---\nname: x\n---\nBody\n"; + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> SkillParser.parseSkill(md)); + assertTrue(ex.getMessage().contains("description")); + } + + @Test + void parseSkillRequiresBody() { + String md = "---\nname: x\ndescription: y\n---\n"; + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> SkillParser.parseSkill(md)); + assertTrue(ex.getMessage().contains("markdown content")); + } + + @Test + void parseSkillSurfacesYamlSyntaxError() { + String md = "---\nname: x\n bad-indent: : :\n---\nBody\n"; + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> SkillParser.parseSkill(md)); + assertTrue(ex.getMessage().startsWith("Invalid YAML frontmatter syntax")); + } + + @Test + void parseSkillTrimsNameAndDescription() { + String md = + "---\nname: \" trimmed-name \"\ndescription: \" trimmed desc \"\n---\nBody\n"; + AgentSkill skill = SkillParser.parseSkill(md); + assertEquals("trimmed-name", skill.getName()); + assertEquals("trimmed desc", skill.getDescription()); + } + + @Test + void parseSkillCarriesOptionalFields() { + String md = + "---\n" + + "name: test\n" + + "description: Does X\n" + + "license: Apache-2.0\n" + + "compatibility: macOS, Linux\n" + + "metadata:\n" + + " author: alice\n" + + " version: \"1.2\"\n" + + "---\n" + + "# Body\n"; + AgentSkill skill = SkillParser.parseSkill(md); + assertEquals("Apache-2.0", skill.getLicense()); + assertEquals("macOS, Linux", skill.getCompatibility()); + assertNotNull(skill.getMetadata()); + assertEquals("alice", skill.getMetadata().get("author")); + assertEquals("1.2", skill.getMetadata().get("version")); + } +} diff --git a/runtime/src/test/resources/skill_discovery_prompt.txt b/runtime/src/test/resources/skill_discovery_prompt.txt new file mode 100644 index 00000000..11d2e6ac --- /dev/null +++ b/runtime/src/test/resources/skill_discovery_prompt.txt @@ -0,0 +1,24 @@ +## Available Skills + +<usage> +Skills provide specialized capabilities and domain knowledge. Use them when they match your current task. + +Load a skill with `load_skill(name="<skill-name>")` to get its full instructions. +Individual resources (scripts, references, assets) can be loaded with a `path` argument. + +The loaded content includes the skill's base directory and the absolute paths of its resources. +</usage> + +<available_skills> + +<skill> +<name>github</name> +<description>Interact with GitHub using the `gh` CLI. Use `gh issue`, `gh pr`, `gh run`, and `gh api` for issues, PRs, CI runs, and advanced queries.</description> +</skill> + +<skill> +<name>nano-banana-pro</name> +<description>Generate/edit images with Nano Banana Pro (Gemini 3 Pro Image). Use for image create/modify requests incl. edits. Supports text-to-image + image-to-image; 1K/2K/4K; use --input-image.</description> +</skill> + +</available_skills> diff --git a/runtime/src/test/resources/skills/github/SKILL.md b/runtime/src/test/resources/skills/github/SKILL.md new file mode 100644 index 00000000..2c9356ce --- /dev/null +++ b/runtime/src/test/resources/skills/github/SKILL.md @@ -0,0 +1,47 @@ +--- +name: github +description: "Interact with GitHub using the `gh` CLI. Use `gh issue`, `gh pr`, `gh run`, and `gh api` for issues, PRs, CI runs, and advanced queries." +--- + +# GitHub Skill + +Use the `gh` CLI to interact with GitHub. Always specify `--repo owner/repo` when not in a git directory, or use URLs directly. + +## Pull Requests + +Check CI status on a PR: +```bash +gh pr checks 55 --repo owner/repo +``` + +List recent workflow runs: +```bash +gh run list --repo owner/repo --limit 10 +``` + +View a run and see which steps failed: +```bash +gh run view <run-id> --repo owner/repo +``` + +View logs for failed steps only: +```bash +gh run view <run-id> --repo owner/repo --log-failed +``` + +## API for Advanced Queries + +The `gh api` command is useful for accessing data not available through other subcommands. + +Get PR with specific fields: +```bash +gh api repos/owner/repo/pulls/55 --jq '.title, .state, .user.login' +``` + +## JSON Output + +Most commands support `--json` for structured output. You can use `--jq` to filter: + +```bash +gh issue list --repo owner/repo --json number,title --jq '.[] | "\(.number): \(.title)"' +``` \ No newline at end of file diff --git a/runtime/src/test/resources/skills/nano-banana-pro/SKILL.md b/runtime/src/test/resources/skills/nano-banana-pro/SKILL.md new file mode 100644 index 00000000..711ee3ff --- /dev/null +++ b/runtime/src/test/resources/skills/nano-banana-pro/SKILL.md @@ -0,0 +1,130 @@ +--- +name: nano-banana-pro +description: Generate/edit images with Nano Banana Pro (Gemini 3 Pro Image). Use for image create/modify requests incl. edits. Supports text-to-image + image-to-image; 1K/2K/4K; use --input-image. +--- + +# Nano Banana Pro Image Generation & Editing + +Generate new images or edit existing ones using Google's Nano Banana Pro API (Gemini 3 Pro Image). + +## Usage + +Run the script using absolute path (do NOT cd to skill directory first): + +**Generate new image:** +```bash +uv run ~/.codex/skills/nano-banana-pro/scripts/generate_image.py --prompt "your image description" --filename "output-name.png" [--resolution 1K|2K|4K] [--api-key KEY] +``` + +**Edit existing image:** +```bash +uv run ~/.codex/skills/nano-banana-pro/scripts/generate_image.py --prompt "editing instructions" --filename "output-name.png" --input-image "path/to/input.png" [--resolution 1K|2K|4K] [--api-key KEY] +``` + +**Important:** Always run from the user's current working directory so images are saved where the user is working, not in the skill directory. + +## Default Workflow (draft β iterate β final) + +Goal: fast iteration without burning time on 4K until the prompt is correct. + +- Draft (1K): quick feedback loop + - `uv run ~/.codex/skills/nano-banana-pro/scripts/generate_image.py --prompt "<draft prompt>" --filename "yyyy-mm-dd-hh-mm-ss-draft.png" --resolution 1K` +- Iterate: adjust prompt in small diffs; keep filename new per run + - If editing: keep the same `--input-image` for every iteration until youβre happy. +- Final (4K): only when prompt is locked + - `uv run ~/.codex/skills/nano-banana-pro/scripts/generate_image.py --prompt "<final prompt>" --filename "yyyy-mm-dd-hh-mm-ss-final.png" --resolution 4K` + +## Resolution Options + +The Gemini 3 Pro Image API supports three resolutions (uppercase K required): + +- **1K** (default) - ~1024px resolution +- **2K** - ~2048px resolution +- **4K** - ~4096px resolution + +Map user requests to API parameters: +- No mention of resolution β `1K` +- "low resolution", "1080", "1080p", "1K" β `1K` +- "2K", "2048", "normal", "medium resolution" β `2K` +- "high resolution", "high-res", "hi-res", "4K", "ultra" β `4K` + +## API Key + +The script checks for API key in this order: +1. `--api-key` argument (use if user provided key in chat) +2. `GEMINI_API_KEY` environment variable + +If neither is available, the script exits with an error message. + +## Preflight + Common Failures (fast fixes) + +- Preflight: + - `command -v uv` (must exist) + - `test -n \"$GEMINI_API_KEY\"` (or pass `--api-key`) + - If editing: `test -f \"path/to/input.png\"` + +- Common failures: + - `Error: No API key provided.` β set `GEMINI_API_KEY` or pass `--api-key` + - `Error loading input image:` β wrong path / unreadable file; verify `--input-image` points to a real image + - βquota/permission/403β style API errors β wrong key, no access, or quota exceeded; try a different key/account + +## Filename Generation + +Generate filenames with the pattern: `yyyy-mm-dd-hh-mm-ss-name.png` + +**Format:** `{timestamp}-{descriptive-name}.png` +- Timestamp: Current date/time in format `yyyy-mm-dd-hh-mm-ss` (24-hour format) +- Name: Descriptive lowercase text with hyphens +- Keep the descriptive part concise (1-5 words typically) +- Use context from user's prompt or conversation +- If unclear, use random identifier (e.g., `x9k2`, `a7b3`) + +Examples: +- Prompt "A serene Japanese garden" β `2025-11-23-14-23-05-japanese-garden.png` +- Prompt "sunset over mountains" β `2025-11-23-15-30-12-sunset-mountains.png` +- Prompt "create an image of a robot" β `2025-11-23-16-45-33-robot.png` +- Unclear context β `2025-11-23-17-12-48-x9k2.png` + +## Image Editing + +When the user wants to modify an existing image: +1. Check if they provide an image path or reference an image in the current directory +2. Use `--input-image` parameter with the path to the image +3. The prompt should contain editing instructions (e.g., "make the sky more dramatic", "remove the person", "change to cartoon style") +4. Common editing tasks: add/remove elements, change style, adjust colors, blur background, etc. + +## Prompt Handling + +**For generation:** Pass user's image description as-is to `--prompt`. Only rework if clearly insufficient. + +**For editing:** Pass editing instructions in `--prompt` (e.g., "add a rainbow in the sky", "make it look like a watercolor painting") + +Preserve user's creative intent in both cases. + +## Prompt Templates (high hit-rate) + +Use templates when the user is vague or when edits must be precise. + +- Generation template: + - βCreate an image of: <subject>. Style: <style>. Composition: <camera/shot>. Lighting: <lighting>. Background: <background>. Color palette: <palette>. Avoid: <list>.β + +- Editing template (preserve everything else): + - βChange ONLY: <single change>. Keep identical: subject, composition/crop, pose, lighting, color palette, background, text, and overall style. Do not add new objects. If text exists, keep it unchanged.β + +## Output + +- Saves PNG to current directory (or specified path if filename includes directory) +- Script outputs the full path to the generated image +- **Do not read the image back** - just inform the user of the saved path + +## Examples + +**Generate new image:** +```bash +uv run ~/.codex/skills/nano-banana-pro/scripts/generate_image.py --prompt "A serene Japanese garden with cherry blossoms" --filename "2025-11-23-14-23-05-japanese-garden.png" --resolution 4K +``` + +**Edit existing image:** +```bash +uv run ~/.codex/skills/nano-banana-pro/scripts/generate_image.py --prompt "make the sky more dramatic with storm clouds" --filename "2025-11-23-14-25-30-dramatic-sky.png" --input-image "original-photo.jpg" --resolution 2K +``` diff --git a/runtime/src/test/resources/skills/nano-banana-pro/_meta.json b/runtime/src/test/resources/skills/nano-banana-pro/_meta.json new file mode 100644 index 00000000..9d100c1d --- /dev/null +++ b/runtime/src/test/resources/skills/nano-banana-pro/_meta.json @@ -0,0 +1,6 @@ +{ + "ownerId": "kn70pywhg0fyz996kpa8xj89s57yhv26", + "slug": "nano-banana-pro", + "version": "1.0.1", + "publishedAt": 1767651987917 +} \ No newline at end of file diff --git a/runtime/src/test/resources/skills/nano-banana-pro/scripts/generate_image.py b/runtime/src/test/resources/skills/nano-banana-pro/scripts/generate_image.py new file mode 100644 index 00000000..84176c3e --- /dev/null +++ b/runtime/src/test/resources/skills/nano-banana-pro/scripts/generate_image.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "google-genai>=1.0.0", +# "pillow>=10.0.0", +# ] +# /// +"""Generate images using Google's Nano Banana Pro (Gemini 3 Pro Image) API. + +Usage: + uv run generate_image.py --prompt "your image description" --filename "output.png" [--resolution 1K|2K|4K] [--api-key KEY] +""" + +import argparse +import os +import sys +from pathlib import Path + + +def get_api_key(provided_key: str | None) -> str | None: + """Get API key from argument first, then environment.""" + if provided_key: + return provided_key + return os.environ.get("GEMINI_API_KEY") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate images using Nano Banana Pro (Gemini 3 Pro Image)" + ) + parser.add_argument( + "--prompt", "-p", required=True, help="Image description/prompt" + ) + parser.add_argument( + "--filename", + "-f", + required=True, + help="Output filename (e.g., sunset-mountains.png)", + ) + parser.add_argument( + "--input-image", "-i", help="Optional input image path for editing/modification" + ) + parser.add_argument( + "--resolution", + "-r", + choices=["1K", "2K", "4K"], + default="1K", + help="Output resolution: 1K (default), 2K, or 4K", + ) + parser.add_argument( + "--api-key", "-k", help="Gemini API key (overrides GEMINI_API_KEY env var)" + ) + + args = parser.parse_args() + + # Get API key + api_key = get_api_key(args.api_key) + if not api_key: + print("Error: No API key provided.", file=sys.stderr) + print("Please either:", file=sys.stderr) + print(" 1. Provide --api-key argument", file=sys.stderr) + print(" 2. Set GEMINI_API_KEY environment variable", file=sys.stderr) + sys.exit(1) + + # Import here after checking API key to avoid slow import on error + from google import genai + from google.genai import types + from PIL import Image as PILImage + + # Initialise client + client = genai.Client(api_key=api_key) + + # Set up output path + output_path = Path(args.filename) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Load input image if provided + input_image = None + output_resolution = args.resolution + if args.input_image: + try: + input_image = PILImage.open(args.input_image) + print(f"Loaded input image: {args.input_image}") + + # Auto-detect resolution if not explicitly set by user + if args.resolution == "1K": # Default value + # Map input image size to resolution + width, height = input_image.size + max_dim = max(width, height) + if max_dim >= 3000: + output_resolution = "4K" + elif max_dim >= 1500: + output_resolution = "2K" + else: + output_resolution = "1K" + print( + f"Auto-detected resolution: {output_resolution} (from input {width}x{height})" + ) + except Exception as e: + print(f"Error loading input image: {e}", file=sys.stderr) + sys.exit(1) + + # Build contents (image first if editing, prompt only if generating) + if input_image: + contents = [input_image, args.prompt] + print(f"Editing image with resolution {output_resolution}...") + else: + contents = args.prompt + print(f"Generating image with resolution {output_resolution}...") + + try: + response = client.models.generate_content( + model="gemini-3-pro-image-preview", + contents=contents, + config=types.GenerateContentConfig( + response_modalities=["TEXT", "IMAGE"], + image_config=types.ImageConfig(image_size=output_resolution), + ), + ) + + # Process response and convert to PNG + image_saved = False + for part in response.parts: + if part.text is not None: + print(f"Model response: {part.text}") + elif part.inline_data is not None: + # Convert inline data to PIL Image and save as PNG + from io import BytesIO + + # inline_data.data is already bytes, not base64 + image_data = part.inline_data.data + if isinstance(image_data, str): + # If it's a string, it might be base64 + import base64 + + image_data = base64.b64decode(image_data) + + image = PILImage.open(BytesIO(image_data)) + + # Ensure RGB mode for PNG (convert RGBA to RGB with white background if needed) + if image.mode == "RGBA": + rgb_image = PILImage.new("RGB", image.size, (255, 255, 255)) + rgb_image.paste(image, mask=image.split()[3]) + rgb_image.save(str(output_path), "PNG") + elif image.mode == "RGB": + image.save(str(output_path), "PNG") + else: + image.convert("RGB").save(str(output_path), "PNG") + image_saved = True + + if image_saved: + full_path = output_path.resolve() + print(f"\nImage saved: {full_path}") + else: + print("Error: No image was generated in the response.", file=sys.stderr) + sys.exit(1) + + except Exception as e: + print(f"Error generating image: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main()
