Skip to content
This repository was archived by the owner on Aug 30, 2021. It is now read-only.

[WIP] Delete taskqueue #14

Open
goru wants to merge 14 commits into
base: develop
Choose a base branch
from
Open

[WIP] Delete taskqueue #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
f8d739c
add handler for getting/deleting single entry
goru Feb 22, 2015
7b9225d
delete all entries using TaskQueue
goru Feb 22, 2015
d459848
remove parent
goru Feb 24, 2015
4798ca6
don't use default queue
goru Feb 24, 2015
9ba1c18
add method for converting old style entry model
goru Feb 25, 2015
fedf152
add migration handlers
goru Feb 25, 2015
94cdaef
change rate to lower
goru Feb 25, 2015
ed4053a
migrate specified feed
goru Feb 25, 2015
44c7996
query keys only
goru Feb 26, 2015
1cdbfb4
use count
goru Feb 27, 2015
b9bf65c
decrease rate of queue
goru Feb 28, 2015
fb57185
several properties changed to unindexed.
goru Feb 28, 2015
2359d83
if both style records is exists, remove old one.
goru Feb 28, 2015
7dfc775
decrease rate of queue
goru Mar 4, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 52 additions & 19 deletions api/feeds/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,40 +21,73 @@ def createFeed(cls, dic):

@classmethod
def getFeeds(cls):
return FeedModel.all()
return FeedModel.all().run()

@classmethod
def getFeedById(cls, feedId):
return FeedModel.get_by_id(long(feedId))

@classmethod
def getEntries(cls, pagingKey):
return EntryModel.gql('WHERE pagingKey < :1 ORDER BY pagingKey DESC LIMIT 100', float(pagingKey))
return EntryModel.all().filter('pagingKey <', float(pagingKey)).order('-pagingKey').run(limit=100)

@classmethod
def getAllEntriesByFeed(cls, feed):
return EntryModel.gql('WHERE ANCESTOR IS :1', feed)
return feed.entrymodel_set.order('-pagingKey').run()

@classmethod
def getUnreadEntries(cls, pagingKey):
return EntryModel.gql('WHERE pagingKey < :1 AND read = :2 ORDER BY pagingKey DESC LIMIT 100', float(pagingKey), False)
return EntryModel.all().filter('pagingKey <', float(pagingKey)).filter('read = ', False).order('-pagingKey').run(limit=100)

@classmethod
def getEntriesByFeed(cls, feed, pagingKey):
return EntryModel.gql('WHERE ANCESTOR IS :1 AND pagingKey < :2 ORDER BY pagingKey DESC LIMIT 100', feed, float(pagingKey))
return feed.entrymodel_set.filter('pagingKey <', float(pagingKey)).order('-pagingKey').run(limit=100)

@classmethod
def getUnreadEntriesByFeed(cls, feed, pagingKey):
return EntryModel.gql('WHERE ANCESTOR IS :1 AND pagingKey < :2 AND read = :3 ORDER BY pagingKey DESC LIMIT 100', feed, float(pagingKey), False)
return feed.entrymodel_set.filter('pagingKey <', float(pagingKey)).filter('read = ', False).order('-pagingKey').run(limit=100)

@classmethod
def getEntryById(cls, feedId, entryId):
newEntry = EntryModel.get_by_key_name(entryId)

oldEntry = cls.getOldStyleEntry(feedId, entryId)
if not oldEntry:
return newEntry

return cls.mergeEntry(newEntry, oldEntry)

@classmethod
def getOldStyleEntry(cls, feedId, entryId):
feed = cls.getFeedById(feedId)
if not feed:
return None

return EntryModel.get_by_key_name(entryId, parent=feed)

@classmethod
def mergeEntry(cls, newEntry, oldEntry):
if not newEntry:
newEntry = EntryModel(key_name=oldEntry.key().name())

newEntry.feed = oldEntry.feed
newEntry.title = oldEntry.title
newEntry.url = oldEntry.url
newEntry.description = oldEntry.description
newEntry.read = oldEntry.read
newEntry.created = oldEntry.created
newEntry.modified = oldEntry.modified
newEntry.pagingKey = oldEntry.pagingKey

oldEntry.delete()
newEntry.put()

return newEntry

@classmethod
def getOldStyleEntries(cls, feed):
return EntryModel.gql('WHERE ANCESTOR IS :1', feed)

@classmethod
def setEntryStatus(cls, entry, status):
entry.read = status
Expand All @@ -79,9 +112,9 @@ def updateEntries(cls, feed):
for entryDict in parser.entries():
key = entryDict['key']

entry = EntryModel.get_by_key_name(key, parent=feed)
entry = cls.getEntryById(feed.key().id(), key)
if not entry:
entry = EntryModel(parent=feed, key_name=key)
entry = EntryModel(key_name=key)
entry.feed = feed

feed.total += 1
Expand Down Expand Up @@ -114,12 +147,12 @@ def updateAttrFromDict(self, keys, dic):
return updateRequired

class FeedModel(ModelBase):
title = db.StringProperty(multiline=False)
url = db.StringProperty(multiline=False)
created = db.DateTimeProperty(auto_now_add=True)
modified = db.DateTimeProperty(auto_now=True)
total = db.IntegerProperty(default=0)
unread = db.IntegerProperty(default=0)
title = db.StringProperty(indexed=False, multiline=False)
url = db.StringProperty(indexed=False, multiline=False)
created = db.DateTimeProperty(indexed=False, auto_now_add=True)
modified = db.DateTimeProperty(indexed=False, auto_now=True)
total = db.IntegerProperty(indexed=False, default=0)
unread = db.IntegerProperty(indexed=False, default=0)

def fromDict(self, dic):
return self.updateAttrFromDict(['title', 'url'], dic)
Expand All @@ -135,12 +168,12 @@ def toDict(self):

class EntryModel(ModelBase):
feed = db.ReferenceProperty(FeedModel)
title = db.StringProperty(multiline=False)
url = db.StringProperty(multiline=False)
description = db.TextProperty()
title = db.StringProperty(indexed=False, multiline=False)
url = db.StringProperty(indexed=False, multiline=False)
description = db.TextProperty(indexed=False)
read = db.BooleanProperty(default=False)
created = db.DateTimeProperty(auto_now_add=True)
modified = db.DateTimeProperty(auto_now=True)
created = db.DateTimeProperty(indexed=False, auto_now_add=True)
modified = db.DateTimeProperty(indexed=False, auto_now=True)
pagingKey = db.FloatProperty()

def setPagingKey(self, key):
Expand Down
89 changes: 73 additions & 16 deletions api/feeds/reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.api import taskqueue

import time

Expand Down Expand Up @@ -107,12 +108,13 @@ def delete(self, feedId):
self.writeNotFoundResponse()
return

feeds = models.FeedManager.getAllEntriesByFeed(feed)
if not feeds:
self.writeNotFoundResponse()
return
entries = models.FeedManager.getAllEntriesByFeed(feed)
queue = taskqueue.Queue('low-priority-task')
for entry in entries:
entryUrl = '/api/feeds/' + feedId + '/' + entry.key().name()
task = taskqueue.Task(method='DELETE', url=entryUrl)
queue.add(task)

db.delete(feeds)
feed.delete()

self.writeNoContentResponse()
Expand All @@ -139,6 +141,25 @@ def get(self, feedId, action, pagingKey=None):

self.writeJsonResponse({'entries': entries})

class EntryHandler(HandlerBase):
def get(self, feedId, entryId):
entry = models.FeedManager.getEntryById(feedId, entryId)
if not entry:
self.writeNotFoundResponse()
return

self.writeJsonResponse(entry.toDict())

def delete(self, feedId, entryId):
entry = models.FeedManager.getEntryById(feedId, entryId)
if not entry:
self.writeNotFoundResponse()
return

entry.delete()

self.writeNoContentResponse()

class EntryReadUnreadHandler(HandlerBase):
def post(self, feedId, entryId, action):
entry = models.FeedManager.getEntryById(feedId, entryId)
Expand All @@ -151,17 +172,53 @@ def post(self, feedId, entryId, action):

self.writeJsonResponse(entry.toDict())

application = webapp.WSGIApplication(
[('/api/feeds/?', FeedsHandler),
('/api/feeds/(all|unread)/?', FeedsEntryHandler),
('/api/feeds/(all|unread)/([0-9.]+)/?', FeedsEntryHandler),
('/api/feeds/import', FeedImportHandler),
('/api/feeds/update', FeedUpdateHandler),
('/api/feeds/(\d+)/?', FeedHandler),
('/api/feeds/(\d+)/(all|unread)/?', FeedEntryHandler),
('/api/feeds/(\d+)/(all|unread)/([0-9.]+)/?', FeedEntryHandler),
('/api/feeds/(\d+)/(entry-[a-z0-9]+)/(read|unread)', EntryReadUnreadHandler)],
debug=True)
class MigrationStatusHandler(HandlerBase):
def get(self):
feeds = []
for feed in models.FeedManager.getFeeds():
feedDict = feed.toDict()
feedDict['oldStyleEntry'] = models.FeedManager.getOldStyleEntries(feed).count()
feeds.append(feedDict)

self.writeJsonResponse({'feeds': feeds})

class MigrationExecuteHandler(HandlerBase):
def get(self, feedId):
feed = models.FeedManager.getFeedById(feedId)
if not feed:
self.writeNoContentResponse()

queue = taskqueue.Queue('low-priority-task')
for entryKey in models.FeedManager.getOldStyleEntries(feed).run(keys_only=True):
entryUrl = '/api/feeds/v0.2.0/migration/execute/' + feedId + '/' + entryKey.name()
task = taskqueue.Task(url=entryUrl)
queue.add(task)

self.writeNoContentResponse()

def post(self, feedId, entryId):
entry = models.FeedManager.getEntryById(feedId, entryId)
if not entry:
self.writeNoContentResponse()
return

self.writeJsonResponse(entry.toDict())

application = webapp.WSGIApplication([
('/api/feeds/?', FeedsHandler),
('/api/feeds/(all|unread)/?', FeedsEntryHandler),
('/api/feeds/(all|unread)/([0-9.]+)/?', FeedsEntryHandler),
('/api/feeds/import', FeedImportHandler),
('/api/feeds/update', FeedUpdateHandler),
('/api/feeds/(\d+)/?', FeedHandler),
('/api/feeds/(\d+)/(all|unread)/?', FeedEntryHandler),
('/api/feeds/(\d+)/(all|unread)/([0-9.]+)/?', FeedEntryHandler),
('/api/feeds/(\d+)/(entry-[a-z0-9]+)/?', EntryHandler),
('/api/feeds/(\d+)/(entry-[a-z0-9]+)/(read|unread)', EntryReadUnreadHandler),
('/api/feeds/v0.2.0/migration/status', MigrationStatusHandler),
('/api/feeds/v0.2.0/migration/execute/(\d+)/?', MigrationExecuteHandler),
('/api/feeds/v0.2.0/migration/execute/(\d+)/(entry-[a-z0-9]+)/?', MigrationExecuteHandler)
], debug=True)

def main():
run_wsgi_app(application)
Expand Down
6 changes: 3 additions & 3 deletions index.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,18 @@ indexes:

- kind: EntryModel
properties:
- name: read
- name: feed
- name: pagingKey
direction: desc

- kind: EntryModel
ancestor: yes
properties:
- name: feed
- name: read
- name: pagingKey
direction: desc

- kind: EntryModel
ancestor: yes
properties:
- name: read
- name: pagingKey
Expand Down
6 changes: 6 additions & 0 deletions queue.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
queue:
- name: low-priority-task
rate: 1/m
retry_parameters:
task_retry_limit: 1