damccorm commented on code in PR #26026:
URL: https://github.com/apache/beam/pull/26026#discussion_r1183743346
##########
playground/backend/internal/db/datastore/datastore_db.go:
##########
@@ -63,19 +69,42 @@ func (d *Datastore) deleteObsoleteSnippets(ctx
context.Context, snipKey *datasto
Namespace(utils.GetNamespace(ctx)).
FilterField("persistenceKey", "=", persistenceKey)
- // At the moment, datastore emulator doesn't allow != filters,
- // hence this crutches
- //
https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues
- // When it's fixed, post-query filter could be replaced with
- //
- // FilterField("__key__", "!=", snipKey)
+ // At the moment, datastore emulator doesn't allow != filters,
+ // hence this crutches
+ //
https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues
+ // When it's fixed, post-query filter could be replaced with
+ //
+ // FilterField("__key__", "!=", snipKey)
return d.deleteSnippets(ctx, snippetQuery, snipKey)
}
-// PutSnippet puts the snippet entity to datastore
+// PutSnippet puts the snippet entity to datastore using cloud function proxy
func (d *Datastore) PutSnippet(ctx context.Context, snipId string, snip
*entity.Snippet) error {
logger.Debugf("putting snippet %q, persistent key %q...", snipId,
snip.Snippet.PersistenceKey)
+
+ var err error
+ if d.externalFunctions != nil {
+ err = d.externalFunctions.PutSnippet(ctx, snipId, snip)
+ }
+ if err != nil || d.externalFunctions == nil {
+ if err != nil {
+ logger.Errorf("Datastore: PutSnippet(): error during
the PutSnippet() call to the cloud function, "+
+ "accessing the datastore directly, err: %s\n",
err.Error())
+ }
+ if d.externalFunctions == nil {
+ logger.Warnf("Datastore: PutSnippet(): external
functions are not set, " +
+ "accessing the datastore directly")
+ }
+ return d.PutSnippetDirect(ctx, snipId, snip)
+ }
+
+ return nil
+}
+
+// PutSnippetDirect puts the snippet entity to datastore
+func (d *Datastore) PutSnippetDirect(ctx context.Context, snipId string, snip
*entity.Snippet) error {
Review Comment:
Should this be private (`putSnippetDirect`)?
##########
playground/backend/internal/tasks/task.go:
##########
@@ -34,11 +34,11 @@ func New(ctx context.Context) *ScheduledTask {
return &ScheduledTask{ctx: ctx, taskScheduler:
chrono.NewDefaultTaskScheduler()}
}
-func (st *ScheduledTask) StartRemovingExtraSnippets(cron string, dayDiff
int32, db db.Database) error {
+func (st *ScheduledTask) StartRemovingExtraSnippets(cron string,
externalFunction external_functions.ExternalFunctions) error {
task, err := st.taskScheduler.ScheduleWithCron(func(ctx
context.Context) {
logger.Info("ScheduledTask: StartRemovingExtraSnippets() is
running...\n")
startDate := time.Now()
- if err := db.DeleteUnusedSnippets(ctx, dayDiff); err != nil {
+ if err := externalFunction.CleanupSnippets(ctx); err != nil {
Review Comment:
Any reason to call the externalFunction directly here, but use datastore_db
as the mechanism for doing it elsewhere?
##########
playground/backend/internal/db/datastore/datastore_db.go:
##########
@@ -63,19 +69,42 @@ func (d *Datastore) deleteObsoleteSnippets(ctx
context.Context, snipKey *datasto
Namespace(utils.GetNamespace(ctx)).
FilterField("persistenceKey", "=", persistenceKey)
- // At the moment, datastore emulator doesn't allow != filters,
- // hence this crutches
- //
https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues
- // When it's fixed, post-query filter could be replaced with
- //
- // FilterField("__key__", "!=", snipKey)
+ // At the moment, datastore emulator doesn't allow != filters,
+ // hence this crutches
+ //
https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues
+ // When it's fixed, post-query filter could be replaced with
+ //
+ // FilterField("__key__", "!=", snipKey)
return d.deleteSnippets(ctx, snippetQuery, snipKey)
}
-// PutSnippet puts the snippet entity to datastore
+// PutSnippet puts the snippet entity to datastore using cloud function proxy
func (d *Datastore) PutSnippet(ctx context.Context, snipId string, snip
*entity.Snippet) error {
logger.Debugf("putting snippet %q, persistent key %q...", snipId,
snip.Snippet.PersistenceKey)
+
+ var err error
+ if d.externalFunctions != nil {
+ err = d.externalFunctions.PutSnippet(ctx, snipId, snip)
+ }
+ if err != nil || d.externalFunctions == nil {
+ if err != nil {
+ logger.Errorf("Datastore: PutSnippet(): error during
the PutSnippet() call to the cloud function, "+
+ "accessing the datastore directly, err: %s\n",
err.Error())
+ }
+ if d.externalFunctions == nil {
+ logger.Warnf("Datastore: PutSnippet(): external
functions are not set, " +
+ "accessing the datastore directly")
+ }
+ return d.PutSnippetDirect(ctx, snipId, snip)
Review Comment:
Why do we need this fallback? It seems concerning/like it opens us up to the
same issues this PR is trying to avoid. Are there ever cases where the fallback
is expected/valid?
##########
playground/backend/internal/db/datastore/datastore_db.go:
##########
@@ -63,19 +69,42 @@ func (d *Datastore) deleteObsoleteSnippets(ctx
context.Context, snipKey *datasto
Namespace(utils.GetNamespace(ctx)).
FilterField("persistenceKey", "=", persistenceKey)
- // At the moment, datastore emulator doesn't allow != filters,
- // hence this crutches
- //
https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues
- // When it's fixed, post-query filter could be replaced with
- //
- // FilterField("__key__", "!=", snipKey)
+ // At the moment, datastore emulator doesn't allow != filters,
+ // hence this crutches
+ //
https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues
+ // When it's fixed, post-query filter could be replaced with
+ //
+ // FilterField("__key__", "!=", snipKey)
return d.deleteSnippets(ctx, snippetQuery, snipKey)
}
-// PutSnippet puts the snippet entity to datastore
+// PutSnippet puts the snippet entity to datastore using cloud function proxy
func (d *Datastore) PutSnippet(ctx context.Context, snipId string, snip
*entity.Snippet) error {
logger.Debugf("putting snippet %q, persistent key %q...", snipId,
snip.Snippet.PersistenceKey)
+
+ var err error
+ if d.externalFunctions != nil {
+ err = d.externalFunctions.PutSnippet(ctx, snipId, snip)
+ }
+ if err != nil || d.externalFunctions == nil {
+ if err != nil {
+ logger.Errorf("Datastore: PutSnippet(): error during
the PutSnippet() call to the cloud function, "+
+ "accessing the datastore directly, err: %s\n",
err.Error())
+ }
+ if d.externalFunctions == nil {
+ logger.Warnf("Datastore: PutSnippet(): external
functions are not set, " +
+ "accessing the datastore directly")
+ }
+ return d.PutSnippetDirect(ctx, snipId, snip)
Review Comment:
Same question applies below (both this and the private question)
##########
playground/backend/internal/tasks/task.go:
##########
@@ -34,11 +34,11 @@ func New(ctx context.Context) *ScheduledTask {
return &ScheduledTask{ctx: ctx, taskScheduler:
chrono.NewDefaultTaskScheduler()}
}
-func (st *ScheduledTask) StartRemovingExtraSnippets(cron string, dayDiff
int32, db db.Database) error {
+func (st *ScheduledTask) StartRemovingExtraSnippets(cron string,
externalFunction external_functions.ExternalFunctions) error {
task, err := st.taskScheduler.ScheduleWithCron(func(ctx
context.Context) {
logger.Info("ScheduledTask: StartRemovingExtraSnippets() is
running...\n")
startDate := time.Now()
- if err := db.DeleteUnusedSnippets(ctx, dayDiff); err != nil {
+ if err := externalFunction.CleanupSnippets(ctx); err != nil {
Review Comment:
I think that sometimes using datastore_db and sometimes using the
externalFunctions is a little confusing, I'd prefer we standardize on one (my
vote would be to standardize on calling externalFunctions directly since
they're not inherently tied to always invoking the datastore_db.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
