diff --git a/api/feeds/models.py b/api/feeds/models.py index 9a3b22b..75a473d 100644 --- a/api/feeds/models.py +++ b/api/feeds/models.py @@ -21,7 +21,7 @@ def createFeed(cls, dic): @classmethod def getFeeds(cls): - return FeedModel.all() + return FeedModel.all().run() @classmethod def getFeedById(cls, feedId): @@ -29,32 +29,65 @@ def getFeedById(cls, feedId): @classmethod def getEntries(cls, pagingKey): - return EntryModel.gql('WHERE pagingKey < :1 ORDER BY pagingKey DESC LIMIT 100', float(pagingKey)) + return EntryModel.all().filter('pagingKey <', float(pagingKey)).order('-pagingKey').run(limit=100) @classmethod def getAllEntriesByFeed(cls, feed): - return EntryModel.gql('WHERE ANCESTOR IS :1', feed) + return feed.entrymodel_set.order('-pagingKey').run() @classmethod def getUnreadEntries(cls, pagingKey): - return EntryModel.gql('WHERE pagingKey < :1 AND read = :2 ORDER BY pagingKey DESC LIMIT 100', float(pagingKey), False) + return EntryModel.all().filter('pagingKey <', float(pagingKey)).filter('read = ', False).order('-pagingKey').run(limit=100) @classmethod def getEntriesByFeed(cls, feed, pagingKey): - return EntryModel.gql('WHERE ANCESTOR IS :1 AND pagingKey < :2 ORDER BY pagingKey DESC LIMIT 100', feed, float(pagingKey)) + return feed.entrymodel_set.filter('pagingKey <', float(pagingKey)).order('-pagingKey').run(limit=100) @classmethod def getUnreadEntriesByFeed(cls, feed, pagingKey): - return EntryModel.gql('WHERE ANCESTOR IS :1 AND pagingKey < :2 AND read = :3 ORDER BY pagingKey DESC LIMIT 100', feed, float(pagingKey), False) + return feed.entrymodel_set.filter('pagingKey <', float(pagingKey)).filter('read = ', False).order('-pagingKey').run(limit=100) @classmethod def getEntryById(cls, feedId, entryId): + newEntry = EntryModel.get_by_key_name(entryId) + + oldEntry = cls.getOldStyleEntry(feedId, entryId) + if not oldEntry: + return newEntry + + return cls.mergeEntry(newEntry, oldEntry) + + @classmethod + def getOldStyleEntry(cls, feedId, entryId): feed = cls.getFeedById(feedId) if not feed: return None return EntryModel.get_by_key_name(entryId, parent=feed) + @classmethod + def mergeEntry(cls, newEntry, oldEntry): + if not newEntry: + newEntry = EntryModel(key_name=oldEntry.key().name()) + + newEntry.feed = oldEntry.feed + newEntry.title = oldEntry.title + newEntry.url = oldEntry.url + newEntry.description = oldEntry.description + newEntry.read = oldEntry.read + newEntry.created = oldEntry.created + newEntry.modified = oldEntry.modified + newEntry.pagingKey = oldEntry.pagingKey + + oldEntry.delete() + newEntry.put() + + return newEntry + + @classmethod + def getOldStyleEntries(cls, feed): + return EntryModel.gql('WHERE ANCESTOR IS :1', feed) + @classmethod def setEntryStatus(cls, entry, status): entry.read = status @@ -79,9 +112,9 @@ def updateEntries(cls, feed): for entryDict in parser.entries(): key = entryDict['key'] - entry = EntryModel.get_by_key_name(key, parent=feed) + entry = cls.getEntryById(feed.key().id(), key) if not entry: - entry = EntryModel(parent=feed, key_name=key) + entry = EntryModel(key_name=key) entry.feed = feed feed.total += 1 @@ -114,12 +147,12 @@ def updateAttrFromDict(self, keys, dic): return updateRequired class FeedModel(ModelBase): - title = db.StringProperty(multiline=False) - url = db.StringProperty(multiline=False) - created = db.DateTimeProperty(auto_now_add=True) - modified = db.DateTimeProperty(auto_now=True) - total = db.IntegerProperty(default=0) - unread = db.IntegerProperty(default=0) + title = db.StringProperty(indexed=False, multiline=False) + url = db.StringProperty(indexed=False, multiline=False) + created = db.DateTimeProperty(indexed=False, auto_now_add=True) + modified = db.DateTimeProperty(indexed=False, auto_now=True) + total = db.IntegerProperty(indexed=False, default=0) + unread = db.IntegerProperty(indexed=False, default=0) def fromDict(self, dic): return self.updateAttrFromDict(['title', 'url'], dic) @@ -135,12 +168,12 @@ def toDict(self): class EntryModel(ModelBase): feed = db.ReferenceProperty(FeedModel) - title = db.StringProperty(multiline=False) - url = db.StringProperty(multiline=False) - description = db.TextProperty() + title = db.StringProperty(indexed=False, multiline=False) + url = db.StringProperty(indexed=False, multiline=False) + description = db.TextProperty(indexed=False) read = db.BooleanProperty(default=False) - created = db.DateTimeProperty(auto_now_add=True) - modified = db.DateTimeProperty(auto_now=True) + created = db.DateTimeProperty(indexed=False, auto_now_add=True) + modified = db.DateTimeProperty(indexed=False, auto_now=True) pagingKey = db.FloatProperty() def setPagingKey(self, key): diff --git a/api/feeds/reader.py b/api/feeds/reader.py index 990de78..0b6ac5f 100644 --- a/api/feeds/reader.py +++ b/api/feeds/reader.py @@ -4,6 +4,7 @@ from google.appengine.ext import db from google.appengine.ext import webapp from google.appengine.ext.webapp.util import run_wsgi_app +from google.appengine.api import taskqueue import time @@ -107,12 +108,13 @@ def delete(self, feedId): self.writeNotFoundResponse() return - feeds = models.FeedManager.getAllEntriesByFeed(feed) - if not feeds: - self.writeNotFoundResponse() - return + entries = models.FeedManager.getAllEntriesByFeed(feed) + queue = taskqueue.Queue('low-priority-task') + for entry in entries: + entryUrl = '/api/feeds/' + feedId + '/' + entry.key().name() + task = taskqueue.Task(method='DELETE', url=entryUrl) + queue.add(task) - db.delete(feeds) feed.delete() self.writeNoContentResponse() @@ -139,6 +141,25 @@ def get(self, feedId, action, pagingKey=None): self.writeJsonResponse({'entries': entries}) +class EntryHandler(HandlerBase): + def get(self, feedId, entryId): + entry = models.FeedManager.getEntryById(feedId, entryId) + if not entry: + self.writeNotFoundResponse() + return + + self.writeJsonResponse(entry.toDict()) + + def delete(self, feedId, entryId): + entry = models.FeedManager.getEntryById(feedId, entryId) + if not entry: + self.writeNotFoundResponse() + return + + entry.delete() + + self.writeNoContentResponse() + class EntryReadUnreadHandler(HandlerBase): def post(self, feedId, entryId, action): entry = models.FeedManager.getEntryById(feedId, entryId) @@ -151,17 +172,53 @@ def post(self, feedId, entryId, action): self.writeJsonResponse(entry.toDict()) -application = webapp.WSGIApplication( - [('/api/feeds/?', FeedsHandler), - ('/api/feeds/(all|unread)/?', FeedsEntryHandler), - ('/api/feeds/(all|unread)/([0-9.]+)/?', FeedsEntryHandler), - ('/api/feeds/import', FeedImportHandler), - ('/api/feeds/update', FeedUpdateHandler), - ('/api/feeds/(\d+)/?', FeedHandler), - ('/api/feeds/(\d+)/(all|unread)/?', FeedEntryHandler), - ('/api/feeds/(\d+)/(all|unread)/([0-9.]+)/?', FeedEntryHandler), - ('/api/feeds/(\d+)/(entry-[a-z0-9]+)/(read|unread)', EntryReadUnreadHandler)], - debug=True) +class MigrationStatusHandler(HandlerBase): + def get(self): + feeds = [] + for feed in models.FeedManager.getFeeds(): + feedDict = feed.toDict() + feedDict['oldStyleEntry'] = models.FeedManager.getOldStyleEntries(feed).count() + feeds.append(feedDict) + + self.writeJsonResponse({'feeds': feeds}) + +class MigrationExecuteHandler(HandlerBase): + def get(self, feedId): + feed = models.FeedManager.getFeedById(feedId) + if not feed: + self.writeNoContentResponse() + + queue = taskqueue.Queue('low-priority-task') + for entryKey in models.FeedManager.getOldStyleEntries(feed).run(keys_only=True): + entryUrl = '/api/feeds/v0.2.0/migration/execute/' + feedId + '/' + entryKey.name() + task = taskqueue.Task(url=entryUrl) + queue.add(task) + + self.writeNoContentResponse() + + def post(self, feedId, entryId): + entry = models.FeedManager.getEntryById(feedId, entryId) + if not entry: + self.writeNoContentResponse() + return + + self.writeJsonResponse(entry.toDict()) + +application = webapp.WSGIApplication([ + ('/api/feeds/?', FeedsHandler), + ('/api/feeds/(all|unread)/?', FeedsEntryHandler), + ('/api/feeds/(all|unread)/([0-9.]+)/?', FeedsEntryHandler), + ('/api/feeds/import', FeedImportHandler), + ('/api/feeds/update', FeedUpdateHandler), + ('/api/feeds/(\d+)/?', FeedHandler), + ('/api/feeds/(\d+)/(all|unread)/?', FeedEntryHandler), + ('/api/feeds/(\d+)/(all|unread)/([0-9.]+)/?', FeedEntryHandler), + ('/api/feeds/(\d+)/(entry-[a-z0-9]+)/?', EntryHandler), + ('/api/feeds/(\d+)/(entry-[a-z0-9]+)/(read|unread)', EntryReadUnreadHandler), + ('/api/feeds/v0.2.0/migration/status', MigrationStatusHandler), + ('/api/feeds/v0.2.0/migration/execute/(\d+)/?', MigrationExecuteHandler), + ('/api/feeds/v0.2.0/migration/execute/(\d+)/(entry-[a-z0-9]+)/?', MigrationExecuteHandler) + ], debug=True) def main(): run_wsgi_app(application) diff --git a/index.yaml b/index.yaml index 09d56e7..b22938a 100644 --- a/index.yaml +++ b/index.yaml @@ -12,18 +12,18 @@ indexes: - kind: EntryModel properties: - - name: read + - name: feed - name: pagingKey direction: desc - kind: EntryModel - ancestor: yes properties: + - name: feed + - name: read - name: pagingKey direction: desc - kind: EntryModel - ancestor: yes properties: - name: read - name: pagingKey diff --git a/queue.yaml b/queue.yaml new file mode 100644 index 0000000..f88b917 --- /dev/null +++ b/queue.yaml @@ -0,0 +1,6 @@ +queue: +- name: low-priority-task + rate: 1/m + retry_parameters: + task_retry_limit: 1 +