This is an automated email from the ASF dual-hosted git repository. DImuthuUpe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
commit 983beccd15dba842fa601258e6d85be23039f4f2 Author: DImuthuUpe <[email protected]> AuthorDate: Mon May 18 02:11:10 2026 -0400 Creating a slurm account when an compute allocation was created --- cmd/server/main.go | 4 +- connectors/SLURM/Association-Mapper/README.md | 30 +++++++++++++ .../internal/operations/accounts.go | 8 ++-- .../internal/operations/accounts_test.go | 8 ++-- .../internal/operations/associations.go | 6 +-- .../internal/operations/associations_test.go | 7 ++-- .../internal/operations/client.go | 20 +++++---- .../internal/operations/client_test.go | 4 +- .../internal/subscribers/account.go | 49 +++++++++++++++++++++- .../internal/subscribers/subscriber.go | 5 ++- .../SLURM/Association-Mapper/pkg/smapper/loader.go | 22 ++++++++-- docs/API-Docs.md | 4 +- internal/connectors/loader.go | 5 ++- 13 files changed, 134 insertions(+), 38 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index ea0ebd697..e2cc07006 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -72,12 +72,12 @@ func run() error { // Create a new event bus instance to async messaging between service and connectors eventBus := events.New() + svc := service.New(database, eventBus) - if err := connectors.LoadConnectors(eventBus); err != nil { + if err := connectors.LoadConnectors(eventBus, svc); err != nil { return err } - svc := service.New(database, eventBus) handler := server.LoggingMiddleware(server.New(svc)) httpServer := &http.Server{ diff --git a/connectors/SLURM/Association-Mapper/README.md b/connectors/SLURM/Association-Mapper/README.md index 9d9bfc6f0..2dff57ba3 100644 --- a/connectors/SLURM/Association-Mapper/README.md +++ b/connectors/SLURM/Association-Mapper/README.md @@ -19,6 +19,36 @@ This package is part of the root `github.com/apache/airavata-custos` module. Import path: `github.com/apache/airavata-custos/connectors/SLURM/Association-Mapper/internal/operations`. +## Configuration + +The connector reads its `slurmrestd` connection details from the following environment variables: + +| Variable | Description | +|---------------|-----------------------------------------------------------------------------| +| `SLURM_API` | Base URL of the `slurmrestd` endpoint, e.g. `https://slurm.example.org:6820` | +| `SLURM_USER` | SLURM user name to authenticate as (sent in the `X-SLURM-USER-NAME` header) | +| `SLURM_TOKEN` | JWT token for that user (sent in the `X-SLURM-USER-TOKEN` header) | +| `SLURM_API_VERSION` | API version of `slurmrestd`. You can check it from `slurmrestd -d list` | + +Example: + +```bash +export SLURM_API='https://slurm.example.org:6820' +export SLURM_USER='slurm' +export SLURM_TOKEN="$(scontrol token lifespan=3600 | awk -F= '{print $2}')" +export SLURM_API_VERSION='40' +``` + +All three are required for any live `slurmrestd` interaction. Tests do not need them. + +You can check the functionality of the SLURM REST API through +```bash +curl -sS \ + -H "X-SLURM-USER-NAME: $SLURM_USER" \ + -H "X-SLURM-USER-TOKEN: $SLURM_TOKEN" \ + "$SLURM_API/slurm/v0.0.$SLURM_API_VERSION/ping" | jq +``` + ## Test ```bash diff --git a/connectors/SLURM/Association-Mapper/internal/operations/accounts.go b/connectors/SLURM/Association-Mapper/internal/operations/accounts.go index ea7d87708..18cc0e456 100644 --- a/connectors/SLURM/Association-Mapper/internal/operations/accounts.go +++ b/connectors/SLURM/Association-Mapper/internal/operations/accounts.go @@ -9,7 +9,7 @@ type accountsResponse struct { func (c *Client) ListAccounts() ([]Account, error) { var out accountsResponse - if _, err := c.do("GET", "/slurmdb/v0.0.41/accounts", nil, &out); err != nil { + if _, err := c.do("GET", "/slurmdb/v0.0."+c.apiVersion+"/accounts", nil, &out); err != nil { return nil, err } return out.Accounts, nil @@ -17,7 +17,7 @@ func (c *Client) ListAccounts() ([]Account, error) { func (c *Client) GetAccount(name string) (*Account, error) { var out accountsResponse - if _, err := c.do("GET", "/slurmdb/v0.0.41/account/"+name, nil, &out); err != nil { + if _, err := c.do("GET", "/slurmdb/v0.0."+c.apiVersion+"/account/"+name, nil, &out); err != nil { return nil, err } if len(out.Accounts) == 0 { @@ -43,11 +43,11 @@ func (c *Client) CreateAccount(a Account, cluster string) error { "organization": a.Organization, }, } - _, err := c.do("POST", "/slurmdb/v0.0.41/accounts_association/", body, nil) + _, err := c.do("POST", "/slurmdb/v0.0."+c.apiVersion+"/accounts_association/", body, nil) return err } func (c *Client) DeleteAccount(name string) error { - _, err := c.do("DELETE", "/slurmdb/v0.0.41/account/"+name, nil, nil) + _, err := c.do("DELETE", "/slurmdb/v0.0."+c.apiVersion+"/account/"+name, nil, nil) return err } diff --git a/connectors/SLURM/Association-Mapper/internal/operations/accounts_test.go b/connectors/SLURM/Association-Mapper/internal/operations/accounts_test.go index a427116e5..aaeae99df 100644 --- a/connectors/SLURM/Association-Mapper/internal/operations/accounts_test.go +++ b/connectors/SLURM/Association-Mapper/internal/operations/accounts_test.go @@ -18,7 +18,7 @@ func TestListAccounts(t *testing.T) { })) defer srv.Close() - c := New(srv.URL, "root", "t") + c := New(srv.URL, "root", "t", "41") accts, err := c.ListAccounts() if err != nil { t.Fatal(err) @@ -58,7 +58,7 @@ func TestCreateAccount(t *testing.T) { })) defer srv.Close() - c := New(srv.URL, "root", "t") + c := New(srv.URL, "root", "t", "41") if err := c.CreateAccount(Account{Name: "eng", Description: "engineering"}, "artisan"); err != nil { t.Fatal(err) } @@ -73,7 +73,7 @@ func TestDeleteAccount(t *testing.T) { })) defer srv.Close() - c := New(srv.URL, "root", "t") + c := New(srv.URL, "root", "t", "41") if err := c.DeleteAccount("eng"); err != nil { t.Fatal(err) } @@ -88,7 +88,7 @@ func TestGetAccount(t *testing.T) { })) defer srv.Close() - c := New(srv.URL, "root", "t") + c := New(srv.URL, "root", "t", "41") a, err := c.GetAccount("eng") if err != nil { t.Fatal(err) diff --git a/connectors/SLURM/Association-Mapper/internal/operations/associations.go b/connectors/SLURM/Association-Mapper/internal/operations/associations.go index ab34cd931..fc1607480 100644 --- a/connectors/SLURM/Association-Mapper/internal/operations/associations.go +++ b/connectors/SLURM/Association-Mapper/internal/operations/associations.go @@ -32,7 +32,7 @@ type associationsResponse struct { } func (c *Client) ListAssociations(f AssocFilter) ([]Association, error) { - path := "/slurmdb/v0.0.41/associations" + path := "/slurmdb/v0.0." + c.apiVersion + "/associations" if q := f.query(); q != "" { path += "?" + q } @@ -48,12 +48,12 @@ func (c *Client) ListAssociations(f AssocFilter) ([]Association, error) { // triple exists, it's updated; otherwise created. func (c *Client) UpsertAssociation(a Association) error { body := map[string]any{"associations": []Association{a}} - _, err := c.do("POST", "/slurmdb/v0.0.41/associations", body, nil) + _, err := c.do("POST", "/slurmdb/v0.0."+c.apiVersion+"/associations", body, nil) return err } func (c *Client) DeleteAssociation(f AssocFilter) error { - path := "/slurmdb/v0.0.41/association" + path := "/slurmdb/v0.0." + c.apiVersion + "/association" if q := f.query(); q != "" { path += "?" + q } diff --git a/connectors/SLURM/Association-Mapper/internal/operations/associations_test.go b/connectors/SLURM/Association-Mapper/internal/operations/associations_test.go index 9082927b0..d16da5186 100644 --- a/connectors/SLURM/Association-Mapper/internal/operations/associations_test.go +++ b/connectors/SLURM/Association-Mapper/internal/operations/associations_test.go @@ -10,7 +10,6 @@ import ( "testing" ) - func TestListAssociationsByAccount(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/slurmdb/v0.0.41/associations" { @@ -22,7 +21,7 @@ func TestListAssociationsByAccount(t *testing.T) { _, _ = w.Write([]byte(`{"associations":[{"account":"eng","cluster":"artisan","user":"alice","id_association":5}]}`)) })) defer srv.Close() - c := New(srv.URL, "root", "t") + c := New(srv.URL, "root", "t", "41") assocs, err := c.ListAssociations(AssocFilter{Account: "eng"}) if err != nil { t.Fatal(err) @@ -49,7 +48,7 @@ func TestCreateAssociation(t *testing.T) { _, _ = w.Write([]byte(`{}`)) })) defer srv.Close() - c := New(srv.URL, "root", "t") + c := New(srv.URL, "root", "t", "41") err := c.UpsertAssociation(Association{Account: "eng", Cluster: "artisan", User: "alice"}) if err != nil { t.Fatal(err) @@ -67,7 +66,7 @@ func TestDeleteAssociation(t *testing.T) { _, _ = w.Write([]byte(`{}`)) })) defer srv.Close() - c := New(srv.URL, "root", "t") + c := New(srv.URL, "root", "t", "41") if err := c.DeleteAssociation(AssocFilter{Account: "eng", User: "alice"}); err != nil { t.Fatal(err) } diff --git a/connectors/SLURM/Association-Mapper/internal/operations/client.go b/connectors/SLURM/Association-Mapper/internal/operations/client.go index 62bd25b99..a60ab883a 100644 --- a/connectors/SLURM/Association-Mapper/internal/operations/client.go +++ b/connectors/SLURM/Association-Mapper/internal/operations/client.go @@ -12,18 +12,20 @@ import ( ) type Client struct { - baseURL string - user string - token string - http *http.Client + baseURL string + user string + token string + apiVersion string + http *http.Client } -func New(baseURL, user, token string) *Client { +func New(baseURL, user, token, apiVersion string) *Client { return &Client{ - baseURL: strings.TrimRight(baseURL, "/"), - user: user, - token: token, - http: &http.Client{Timeout: 30 * time.Second}, + baseURL: strings.TrimRight(baseURL, "/"), + user: user, + token: token, + apiVersion: apiVersion, + http: &http.Client{Timeout: 30 * time.Second}, } } diff --git a/connectors/SLURM/Association-Mapper/internal/operations/client_test.go b/connectors/SLURM/Association-Mapper/internal/operations/client_test.go index 3d07b6456..c73d3ecb5 100644 --- a/connectors/SLURM/Association-Mapper/internal/operations/client_test.go +++ b/connectors/SLURM/Association-Mapper/internal/operations/client_test.go @@ -18,7 +18,7 @@ func TestDoSetsHeaders(t *testing.T) { })) defer srv.Close() - c := New(srv.URL, "root", "tok123") + c := New(srv.URL, "root", "tok123", "41") if _, err := c.do("GET", "/slurm/v0.0.41/ping", nil, nil); err != nil { t.Fatal(err) } @@ -36,7 +36,7 @@ func TestDoUnwrapsErrors(t *testing.T) { _, _ = w.Write([]byte(`{"errors":[{"description":"nope","error_number":42,"error":"Bad","source":"test"}]}`)) })) defer srv.Close() - c := New(srv.URL, "root", "tok") + c := New(srv.URL, "root", "tok", "41") _, err := c.do("GET", "/x", nil, nil) if err == nil { t.Fatal("expected error") diff --git a/connectors/SLURM/Association-Mapper/internal/subscribers/account.go b/connectors/SLURM/Association-Mapper/internal/subscribers/account.go index 9c2f29c6f..a4a41f99c 100644 --- a/connectors/SLURM/Association-Mapper/internal/subscribers/account.go +++ b/connectors/SLURM/Association-Mapper/internal/subscribers/account.go @@ -1,10 +1,55 @@ package subscribers -import "github.com/apache/airavata-custos/pkg/models" -import "log/slog" +import ( + "log/slog" + + "context" + "time" + + client "github.com/apache/airavata-custos/connectors/SLURM/Association-Mapper/internal/operations" + "github.com/apache/airavata-custos/pkg/models" +) func (a *AssociationSubscriber) SubscribeToComputeAccountCreation(computeAccount models.ComputeAllocation) { slog.Info("Received compute account creation event", "account", computeAccount) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cluster, err := a.coreService.GetComputeCluster(ctx, computeAccount.ComputeClusterID) + if err != nil { + slog.Error("Failed to get compute cluster for account creation", "error", err) + return + } + + project, err := a.coreService.GetProject(ctx, computeAccount.ProjectID) + if err != nil { + slog.Error("Failed to get project for account creation", "error", err) + return + } + + pi, err := a.coreService.GetUser(ctx, project.ProjectPIID) + if err != nil { + slog.Error("Failed to get project PI for account creation", "error", err) + return + } + + organization, err := a.coreService.GetOrganization(ctx, pi.OrganizationID) + if err != nil { + slog.Error("Failed to get organization for account creation", "error", err) + return + } + + slurmAccount := client.Account{ + Name: computeAccount.Name, + Description: computeAccount.Name, + Organization: organization.Name, + } + + err = a.slurmClient.CreateAccount(slurmAccount, cluster.Name) // TODO: where to get cluster name from? + if err != nil { + slog.Error("Failed to create SLURM account", "error", err) + } } func (a *AssociationSubscriber) SubscribeToComputeAccountDeletion(computeAccount models.ComputeAllocation) { diff --git a/connectors/SLURM/Association-Mapper/internal/subscribers/subscriber.go b/connectors/SLURM/Association-Mapper/internal/subscribers/subscriber.go index 129bfcff9..30b03f9af 100644 --- a/connectors/SLURM/Association-Mapper/internal/subscribers/subscriber.go +++ b/connectors/SLURM/Association-Mapper/internal/subscribers/subscriber.go @@ -2,16 +2,19 @@ package subscribers import client "github.com/apache/airavata-custos/connectors/SLURM/Association-Mapper/internal/operations" import "github.com/apache/airavata-custos/pkg/events" +import "github.com/apache/airavata-custos/pkg/service" type AssociationSubscriber struct { slurmClient *client.Client eventBus *events.Bus + coreService *service.Service } -func NewAssociationSubscriber(slurmClient *client.Client, eventBus *events.Bus) *AssociationSubscriber { +func NewAssociationSubscriber(slurmClient *client.Client, eventBus *events.Bus, coreService *service.Service) *AssociationSubscriber { return &AssociationSubscriber{ slurmClient: slurmClient, eventBus: eventBus, + coreService: coreService, } } diff --git a/connectors/SLURM/Association-Mapper/pkg/smapper/loader.go b/connectors/SLURM/Association-Mapper/pkg/smapper/loader.go index 9c172b4f8..8f5dc39be 100644 --- a/connectors/SLURM/Association-Mapper/pkg/smapper/loader.go +++ b/connectors/SLURM/Association-Mapper/pkg/smapper/loader.go @@ -3,9 +3,25 @@ package smapper import "github.com/apache/airavata-custos/pkg/events" import "github.com/apache/airavata-custos/connectors/SLURM/Association-Mapper/internal/subscribers" import client "github.com/apache/airavata-custos/connectors/SLURM/Association-Mapper/internal/operations" +import "os" +import "log/slog" +import "github.com/apache/airavata-custos/pkg/service" -func LoadConnector(eventBus *events.Bus) error { - slurmClient := client.New("localhost:8080", "", "") // Replace with actual SLURM client initialization. - subscribers.NewAssociationSubscriber(slurmClient, eventBus).RegisterSubscribers() +func LoadConnector(eventBus *events.Bus, coreService *service.Service) error { + + // Read url, username, and password from environment variables + apiUrl := os.Getenv("SLURM_API") + user := os.Getenv("SLURM_USER") + token := os.Getenv("SLURM_TOKEN") + apiVersion := os.Getenv("SLURM_API_VERSION") + if apiUrl == "" || user == "" || token == "" || apiVersion == "" { + slog.Info("SLURM API credentials not fully provided, skipping SLURM Association Mapper connector") + // print valualues of the env vars for debugging + slog.Info("SLURM API credentials", "apiUrl", apiUrl, "user", user, "token", token, "apiVersion", apiVersion) + return nil // skip loading if any of the required env vars are missing + } + + slurmClient := client.New(apiUrl, user, token, apiVersion) + subscribers.NewAssociationSubscriber(slurmClient, eventBus, coreService).RegisterSubscribers() return nil } diff --git a/docs/API-Docs.md b/docs/API-Docs.md index 2a308db05..6931d0304 100644 --- a/docs/API-Docs.md +++ b/docs/API-Docs.md @@ -1052,7 +1052,7 @@ BASE=http://localhost:8080 ORG_ID=$(curl -s -X POST $BASE/organizations \ -H 'Content-Type: application/json' \ - -d '{"name":"University of Example","originated_id":"ACCESS-ORG-001"}' \ + -d '{"name":"Georgia Institute of Technology","originated_id":"ACCESS-ORG-001"}' \ | jq -r .id) USER_ID=$(curl -s -X POST $BASE/users \ @@ -1067,7 +1067,7 @@ PROJ_ID=$(curl -s -X POST $BASE/projects \ CLUSTER_ID=$(curl -s -X POST $BASE/compute-clusters \ -H 'Content-Type: application/json' \ - -d '{"name":"Delta"}' | jq -r .id) + -d '{"name":"nexus-dev"}' | jq -r .id) ALLOC_ID=$(curl -s -X POST $BASE/compute-allocations \ -H 'Content-Type: application/json' \ diff --git a/internal/connectors/loader.go b/internal/connectors/loader.go index 1b603abd1..c1e129661 100644 --- a/internal/connectors/loader.go +++ b/internal/connectors/loader.go @@ -3,13 +3,14 @@ package connectors import "github.com/apache/airavata-custos/connectors/SLURM/Association-Mapper/pkg/smapper" import "github.com/apache/airavata-custos/pkg/events" import "log/slog" +import "github.com/apache/airavata-custos/pkg/service" -func LoadConnectors(eventBus *events.Bus) error { +func LoadConnectors(eventBus *events.Bus, coreService *service.Service) error { slog.Info("loading connectors") slog.Info("loading SLURM Association Mapper connector") - err := smapper.LoadConnector(eventBus) + err := smapper.LoadConnector(eventBus, coreService) if err != nil { slog.Error("failed to load SLURM Association Mapper connector", "error", err) return err
