0

I have the code below to update an entity in the Google data store.

I'm now getting an internal error on the model.Collection.put(vhighest)

sync.yaml


service: sync
instance_class: F2
automatic_scaling:
 max_instances: 1
runtime: python312
app_engine_apis: true
entrypoint: gunicorn -b :$PORT sync:app 
#inbound_services:
#- warmup
#libraries:
#- name: jinja2
# version: latest
#- name: ssl
# version: latest
# taskqueue and cron tasks can access admin urls
handlers:
- url: /.*
 script: sync.app
 secure: always
 redirect_http_response_code: 301
env_variables:
 MEMCACHE_USE_CROSS_COMPATIBLE_PROTOCOL: "True"
 NDB_USE_CROSS_COMPATIBLE_PICKLE_PROTOCOL: "True"
 DEFERRED_USE_CROSS_COMPATIBLE_PICKLE_PROTOCOL: "True"
 CURRENT_VERSION_TIMESTAMP: "1677721600"

google.cloud.ndb.tasklets.Return: Key('Collection', 6266129674665984)

Can I ask for some guidance, did something with how i should use these methods?


import flask
import config
import util
app = flask.Flask(__name__)
from google.appengine.api import taskqueue, search, memcache
from apiclient.discovery import build, HttpError
from google.cloud import ndb
from apiclient.http import MediaIoBaseUpload
from datetime import datetime, timedelta
from functools import partial
from io import BytesIO
import os
from os.path import splitext, basename
from model import Config
from model import VideosToCollections
from pytz import timezone
import datetime
import httplib2
import iso8601
import time
from flask import request
from operator import attrgetter
import model
from model import CallBack
import re
import config
import google.appengine.api
client = ndb.Client()
def ndb_wsgi_middleware(wsgi_app):
 def middleware(environ, start_response):
 with client.context():
 return wsgi_app(environ, start_response)
 return middleware
app.wsgi_app = ndb_wsgi_middleware(app.wsgi_app)
 
#
################################################################################
## Flush all caches, rebuild search index, and sync all videos
################################################################################
#
@app.route('/rx/', methods=['GET'])
def rx():
 GAE_APP_ID = os.environ['GOOGLE_CLOUD_PROJECT']
 index = search.Index('general-index')
 while True:
 document_ids = [
 document.doc_id
 for document
 in index.get_range(ids_only=True)]
#
 # If no IDs were returned, we've deleted everything.
 if not document_ids:
 break
#
 # Delete the documents for the given IDs
 index.delete(document_ids)
#
 # flush memcache
 memcache.flush_all()
#
# # get/put all collections so they are reindexed
 collections_dbs_keys, cursor = model.Collection.get_dbs(keys_only=True, limit=-1)
 collections_dbs = ndb.get_multi(collections_dbs_keys)
 for collection in collections_dbs:
 collection.search_update_index()
#
# # sync all videos
 taskqueue.add(url=flask.url_for('sync_video'),
 params={'syncthumb': True},
 method='GET')
#
 return 'ok. flushed everything and started video sync.'
#
#
################################################################################
## Sync video(s) task worker
################################################################################
#
@app.route('/sync/', methods=['GET'])
@app.route('/sync/<yt_video_id>', methods=['POST','GET'])
def sync_video(yt_video_id=None):
 GAE_APP_ID = os.environ['GOOGLE_CLOUD_PROJECT']
 syncthumb = util.param('syncthumb', bool)
 if not syncthumb:
 syncthumb = False
 if yt_video_id:
 util.sync_video_worker(yt_video_id, syncthumb=syncthumb)
 success = 'ok: synced ' + yt_video_id
 return success
 
 index = search.Index('general-index')
 while True:
 document_ids = [
 document.doc_id
 for document
 in index.get_range(ids_only=True)]
 # If no IDs were returned, we've deleted everything.
 if not document_ids:
 break
 # Delete the documents for the given IDs
 index.delete(document_ids)
 # get/put all collections so they are reindexed
 collections_dbs_keys, cursor = model.Collection.get_dbs(keys_only=True, limit=-1)
 collections_dbs = ndb.get_multi(collections_dbs_keys)
 for collection in collections_dbs:
 collection.search_update_index()
 
 video_dbs, video_cursor = model.Video.get_dbs(limit=-1)
 tasks = [taskqueue.Task(
 url='/sync/' + video_db.yt_video_id,
 params={'syncthumb': syncthumb},
 ) for video_db in video_dbs]
 for batches in [tasks[i:i + 5] for i in range(0, len(tasks), 5)]:
 rpc = taskqueue.Queue('sync').add_async(batches)
 rpc.wait()
 success = 'ok: dispatched ' + str(len(tasks)) + ' videos for sync tasks'
 return success
###############################################################################
# Populate Collections
###############################################################################
@app.route('/collectionsync/', methods=['GET'])
#@ndb.transactional
def collectionsync():
 GAE_APP_ID = os.environ['GOOGLE_CLOUD_PROJECT']
 vnew=model.Collection.query(model.Collection.slug=='new').get()
 for p in model.VideosToCollections.query(model.VideosToCollections.collection_key==vnew.key):
 p.key.delete()
 
 #populate newest collection
 tot=0
 ct=0
 for p in model.Video.query().order(-model.Video.launch_date):
 #print(p)
 if(p.launch_date):
 if p.get_launch_date().date() > (datetime.datetime.now(timezone("UTC")).date()):
 continue
 if(p.yt_date_added is not None):
 if(p.yt_date_added > (datetime.datetime.today() - timedelta(days=30))):
 ct+=1
 vc = VideosToCollections()
 vc.video_key = p.key
 vc.collection_key = vnew.key
 vc.order = ct
 vc.launch_date = datetime.datetime.now(timezone("UTC")).date()
 model.VideosToCollections.put(vc)
 if(ct==1):
 vnew.featured_primary=p.key
 model.Collection.put(vnew)
 if(ct==2):
 vnew.featured_secondary=p.key 
 model.Collection.put(vnew) 
 if(ct>=25):
 break
 tot+=ct
 #populate highest rated collection
 ct=0
 vhighest=model.Collection.query(model.Collection.slug=='highest-rated').get()
 for p in model.VideosToCollections.query(model.VideosToCollections.collection_key==vhighest.key):
 p.key.delete()
 for p in model.Video.query().order(-model.Video.approval):
 if(p.launch_date):
 if p.get_launch_date().date() > (datetime.datetime.now(timezone("UTC")).date()):
 continue
 if(p.yt_views > 25000):
 ct+=1
 vc = VideosToCollections()
 vc.video_key = p.key
 vc.collection_key = vhighest.key
 vc.launch_date = datetime.datetime.now(timezone("UTC")).date()
 vc.order = ct
 model.VideosToCollections.put(vc)
 if(ct==1):
 vhighest.featured_primary=p.key
 model.Collection.put(vhighest)
 if(ct==2):
 vhighest.featured_secondary=p.key
 model.Collection.put(vhighest)
 if(ct>=25):
 break
 tot+=ct
 # flush memcache
 #memcache.flush_all() 
 success = 'ok: dispatched ' + str(tot) + ' videos into collections'
 return success

2025年05月16日 15:24:21 sync[20250516t112108] "GET /collectionsync/ HTTP/1.1" 500
2025年05月16日 15:24:21 sync[20250516t112108] [2025年05月16日 15:24:21 +0000] [11] [INFO] Starting gunicorn 22.0.0
2025年05月16日 15:24:21 sync[20250516t112108] [2025年05月16日 15:24:21 +0000] [11] [INFO] Listening at: http://0.0.0.0:8081 (11)
2025年05月16日 15:24:21 sync[20250516t112108] [2025年05月16日 15:24:21 +0000] [11] [INFO] Using worker: sync
2025年05月16日 15:24:21 sync[20250516t112108] [2025年05月16日 15:24:21 +0000] [15] [INFO] Booting worker with pid: 15
2025年05月16日 15:24:25 sync[20250516t112108] [2025年05月16日 15:24:25,568] ERROR in app: Exception on /collectionsync/ [GET]
2025年05月16日 15:24:25 sync[20250516t112108] Traceback (most recent call last): File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/tasklets.py", line 323, in _advance_tasklet yielded = self.generator.send(send_value)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/model.py", line 5518, in put
2025年05月16日 15:24:25 sync[20250516t112108] raise tasklets.Return(self._key)
2025年05月16日 15:24:25 sync[20250516t112108] google.cloud.ndb.tasklets.Return: Key('Collection', 6266129674665984)
2025年05月16日 15:24:25 sync[20250516t112108] During handling of the above exception, another exception occurred:
2025年05月16日 15:24:25 sync[20250516t112108] Traceback (most recent call last): File "/layers/google.python.pip/pip/lib/python3.12/site-packages/flask/app.py", line 2529, in wsgi_app response = self.full_dispatch_request()
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/flask/app.py", line 1825, in full_dispatch_request
2025年05月16日 15:24:25 sync[20250516t112108] rv = self.handle_user_exception(e)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/flask/app.py", line 1823, in full_dispatch_request
2025年05月16日 15:24:25 sync[20250516t112108] rv = self.dispatch_request()
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/flask/app.py", line 1799, in dispatch_request
2025年05月16日 15:24:25 sync[20250516t112108] return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/workspace/sync.py", line 210, in collectionsync
2025年05月16日 15:24:25 sync[20250516t112108] model.Collection.put(vhighest)
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/_options.py", line 102, in wrapper
2025年05月16日 15:24:25 sync[20250516t112108] return wrapped(*pass_args, **kwargs)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/utils.py", line 118, in wrapper
2025年05月16日 15:24:25 sync[20250516t112108] return wrapped(*args, **new_kwargs)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/utils.py", line 150, in positional_wrapper
2025年05月16日 15:24:25 sync[20250516t112108] return wrapped(*args, **kwds)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/model.py", line 5449, in _put
2025年05月16日 15:24:25 sync[20250516t112108] return self._put_async(_options=kwargs["_options"]).result()
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/tasklets.py", line 210, in result
2025年05月16日 15:24:25 sync[20250516t112108] self.check_success()
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/tasklets.py", line 154, in check_success
2025年05月16日 15:24:25 sync[20250516t112108] self.wait()
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/tasklets.py", line 145, in wait
2025年05月16日 15:24:25 sync[20250516t112108] if not _eventloop.run1():
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/_eventloop.py", line 390, in run1
2025年05月16日 15:24:25 sync[20250516t112108] return loop.run1()
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/_eventloop.py", line 326, in run1
2025年05月16日 15:24:25 sync[20250516t112108] delay = self.run0()
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/_eventloop.py", line 286, in run0
2025年05月16日 15:24:25 sync[20250516t112108] if self._run_current() or self.run_idle():
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/_eventloop.py", line 276, in _run_current
2025年05月16日 15:24:25 sync[20250516t112108] callback(*args, **kwargs)
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/tasklets.py", line 337, in _advance_tasklet
2025年05月16日 15:24:25 sync[20250516t112108] self.set_result(_get_return_value(stop))
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/tasklets.py", line 170, in set_result
2025年05月16日 15:24:25 sync[20250516t112108] self._finish()
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/cloud/ndb/tasklets.py", line 199, in _finish
2025年05月16日 15:24:25 sync[20250516t112108] callback(self)
2025年05月16日 15:24:25 sync[20250516t112108] File "/workspace/NdbSearchableBase/SearchableModel.py", line 203, in _post_put_hook
2025年05月16日 15:24:25 sync[20250516t112108] self.search_update_index()
2025年05月16日 15:24:25 sync[20250516t112108] File "/workspace/NdbSearchableBase/SearchableModel.py", line 144, in search_update_index
2025年05月16日 15:24:25 sync[20250516t112108] index.put(document)
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/appengine/datastore/datastore_rpc.py", line 94, in positional_wrapper
2025年05月16日 15:24:25 sync[20250516t112108] return wrapped(*args, **kwds)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/appengine/api/search/search.py", line 3610, in put
2025年05月16日 15:24:25 sync[20250516t112108] return self.put_async(documents, deadline=deadline).get_result()
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/appengine/datastore/datastore_rpc.py", line 94, in positional_wrapper
2025年05月16日 15:24:25 sync[20250516t112108] return wrapped(*args, **kwds)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/appengine/api/search/search.py", line 3667, in put_async
2025年05月16日 15:24:25 sync[20250516t112108] return _PutOperationFuture(self, request, response, deadline, hook)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/appengine/api/search/search.py", line 281, in __init__
2025年05月16日 15:24:25 sync[20250516t112108] super(_PutOperationFuture, self).__init__('IndexDocument', request,
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/appengine/api/search/search.py", line 265, in __init__
2025年05月16日 15:24:25 sync[20250516t112108] self._rpc = apiproxy_stub_map.UserRPC('search', deadline=deadline)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/appengine/api/apiproxy_stub_map.py", line 444, in __init__
2025年05月16日 15:24:25 sync[20250516t112108] self.__rpc = CreateRPC(service, stubmap)
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025年05月16日 15:24:25 sync[20250516t112108] File "/layers/google.python.pip/pip/lib/python3.12/site-packages/google/appengine/api/apiproxy_stub_map.py", line 69, in CreateRPC
2025年05月16日 15:24:25 sync[20250516t112108] assert stub, 'No api proxy found for service "%s"' % service
2025年05月16日 15:24:25 sync[20250516t112108] ^^^^
2025年05月16日 15:24:26 sync[20250516t112108] AssertionError: No api proxy found for service "search"
asked May 16 at 15:26
8
  • Same comment as on the post you deleted - you're using memcache and taskqueue and those are from bundled SDK which you haven't initialized and they only work locally when your app is run with dev_appserver.py Commented May 16 at 15:30
  • Anytime you see, from google.appengine, it means you're using a version of the API/library that is bundled with appengine (Google has done the heavy lifting underneath so that it works in app engine without your having to create a client and specify a project, etc; it will only work within appengine). Instead of using that version, you can use the cloud version which works anywhere (GAE, GCE, Cloud Run, etc) but you have to handle initializing the client and specifying your project, etc Commented May 16 at 15:35
  • Thank you for the second comment, that really helps me digest what I've been reading What is the initialization memcache is expecting? I have this at the top. (I couldn't find a documentation reference explictly mentioning memache). " import google.appengine.api client = ndb.Client() def ndb_wsgi_middleware(wsgi_app): def middleware(environ, start_response): with client.context(): return wsgi_app(environ, start_response) return middleware app.wsgi_app = ndb_wsgi_middleware(app.wsgi_app)" Commented May 16 at 15:38
  • I linked to the bundled SDK GitHub page and it explains the needed steps. Search for Example for a Flask app:. Actually read the entire thing under Using the SDK Commented May 16 at 15:40
  • I've done those steps. (We actually worked through this a year ago for app.yaml) The difference of entrypoint for the sync service has been 'sus' to me. sync.yaml service: sync instance_class: F2 automatic_scaling: max_instances: 1 runtime: python312 app_engine_apis: true entrypoint: gunicorn -b :$PORT sync:app Commented May 16 at 15:44

0

Know someone who can answer? Share a link to this question via email, Twitter, or Facebook.

Your Answer

Draft saved
Draft discarded

Sign up or log in

Sign up using Google
Sign up using Email and Password

Post as a guest

Required, but never shown

Post as a guest

Required, but never shown

By clicking "Post Your Answer", you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.