Skip to content
This repository has been archived by the owner on Feb 8, 2018. It is now read-only.

move cached value computation into triggers #4045

Closed
wants to merge 12 commits into from
Closed

move cached value computation into triggers #4045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f34069f
Improve test coverage relevant to this PR (#4046)
chadwhitacre Jun 8, 2016
de839b0
Fix bug in test_pages where we ignored many pages
chadwhitacre Jun 15, 2016
862040a
Fixture for the next commit
chadwhitacre Jun 15, 2016
b3b8209
Clean up remaining tests
chadwhitacre Jun 15, 2016
2be11c3
Improve test coverage relevant to deslugging API
chadwhitacre Jun 15, 2016
ae5bcdb
Simplify some APIs to only accept Teams, not slugs
chadwhitacre Jun 15, 2016
a8485e2
Adapt to new deslugged APIs
chadwhitacre Jun 15, 2016
1833bf7
Add a new column and write to it alongside the old
chadwhitacre Jun 15, 2016
4976993
Failing tests for ids in payment_instructions
chadwhitacre Jun 16, 2016
4c6577e
Read {participant,team}_id instead of {}
chadwhitacre Jun 16, 2016
cfbdaf1
Drop payment_instructions.{participant,team}
chadwhitacre Jun 16, 2016
f0d2c1c
Stub out new PR for triggers
chadwhitacre Jun 2, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions defaults.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,6 @@ ASPEN_PROJECT_ROOT=.
ASPEN_SHOW_TRACEBACKS=yes
ASPEN_WWW_ROOT=www/

# https://github.com/benoitc/gunicorn/issues/186
GUNICORN_OPTS="--workers=1 --timeout=99999999"

# For testing Team review ticket posting
# Set your own username and an access token in local.env
TEAM_REVIEW_REPO=gratipay/test-gremlin
Expand Down
29 changes: 18 additions & 11 deletions gratipay/billing/payday.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,14 +399,21 @@ def update_stats(self):

-- Participants who have either received/given money

SELECT participant, team, amount, direction FROM payments WHERE payday = %(payday)s
SELECT p.id as participant_id
, t.id as team_id
, amount
, direction
FROM payments
JOIN participants p ON p.username = payments.participant
JOIN teams t ON t.slug = payments.team
WHERE payday = %(payday)s

UNION

-- Participants who weren't charged due to amount + due < MINIMUM_CHARGE

SELECT payload->>'participant' AS participant
, payload->>'team' AS team
SELECT (payload->>'participant_id')::bigint AS participant_id
, (payload->>'team_id')::bigint AS team_id
, '0' AS amount
, 'to-team' AS direction
FROM events
Expand All @@ -427,18 +434,18 @@ def update_stats(self):
SELECT COUNT(*)
FROM current_exchange_routes r
JOIN participants p ON p.id = r.participant
WHERE p.username = payload->>'participant'
WHERE p.id = (payload->>'participant_id')::bigint
AND network = 'braintree-cc'
AND error = ''
) > 0
)

UPDATE paydays p
SET nusers = (
SELECT COUNT(DISTINCT(participant)) FROM payments_and_dues
SELECT COUNT(DISTINCT(participant_id)) FROM payments_and_dues
)
, nteams = (
SELECT COUNT(DISTINCT(team)) FROM payments_and_dues
SELECT COUNT(DISTINCT(team_id)) FROM payments_and_dues
)
, volume = (
SELECT COALESCE(sum(amount), 0) FROM payments_and_dues WHERE direction='to-team'
Expand Down Expand Up @@ -480,17 +487,17 @@ def notify_participants(self):
p = e.participant
if p.notify_charge & i == 0:
continue
username = p.username
participant_id = p.id
nteams, top_team = self.db.one("""
WITH tippees AS (
SELECT t.slug, amount
FROM ( SELECT DISTINCT ON (team) team, amount
FROM ( SELECT DISTINCT ON (team_id) team_id, amount
FROM payment_instructions
WHERE mtime < %(ts_start)s
AND participant = %(username)s
ORDER BY team, mtime DESC
AND participant_id = %(participant_id)s
ORDER BY team_id, mtime DESC
) s
JOIN teams t ON s.team = t.slug
JOIN teams t ON s.team_id = t.id
JOIN participants p ON t.owner = p.username
WHERE s.amount > 0
AND t.is_approved IS true
Expand Down
1 change: 0 additions & 1 deletion gratipay/exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ def __str__(self):
class TooGreedy(Exception): pass
class NoSelfTipping(Exception): pass
class NoTippee(Exception): pass
class NoTeam(Exception): pass
class BadAmount(Exception): pass
class InvalidTeamName(Exception): pass

Expand Down
124 changes: 54 additions & 70 deletions gratipay/models/participant/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
UsernameContainsInvalidCharacters,
UsernameIsRestricted,
UsernameAlreadyTaken,
NoTeam,
BadAmount,
EmailAlreadyTaken,
CannotRemovePrimaryEmail,
Expand Down Expand Up @@ -125,6 +124,9 @@ def _from_thing(cls, thing, value):

""".format(thing), (value,))

def refresh(self):
self.__dict__.update(self.__class__.from_id(self.id).__dict__)


# Session Management
# ==================
Expand Down Expand Up @@ -323,15 +325,15 @@ def clear_payment_instructions(self, cursor):

SELECT ( SELECT teams.*::teams
FROM teams
WHERE slug=team
WHERE id=team_id
) AS team
FROM current_payment_instructions
WHERE participant = %s
WHERE participant_id = %s
AND amount > 0

""", (self.username,))
""", (self.id,))
for team in teams:
self.set_payment_instruction(team, '0.00', update_self=False, cursor=cursor)
self.set_payment_instruction(team, '0.00', cursor=cursor)


def clear_takes(self, cursor):
Expand Down Expand Up @@ -816,9 +818,8 @@ def update_avatar(self):
# Giving and Taking
# =================

def set_payment_instruction(self, team, amount, update_self=True, update_team=True,
cursor=None):
"""Given a Team or slug, and amount as str, returns a dict.
def set_payment_instruction(self, team, amount, cursor=None):
"""Given a Team instance, and amount as str, return a dict.

We INSERT instead of UPDATE, so that we have history to explore. The
COALESCE function returns the first of its arguments that is not NULL.
Expand All @@ -832,11 +833,6 @@ def set_payment_instruction(self, team, amount, update_self=True, update_team=Tr
"""
assert self.is_claimed # sanity check

if not isinstance(team, Team):
team, slug = Team.from_slug(team), team
if not team:
raise NoTeam(slug)

amount = Decimal(amount) # May raise InvalidOperation
if (amount < gratipay.MIN_PAYMENT) or (amount > gratipay.MAX_PAYMENT):
raise BadAmount
Expand All @@ -845,33 +841,32 @@ def set_payment_instruction(self, team, amount, update_self=True, update_team=Tr
NEW_PAYMENT_INSTRUCTION = """\

INSERT INTO payment_instructions
(ctime, participant, team, amount)
(ctime, participant_id, team_id, amount)
VALUES ( COALESCE (( SELECT ctime
FROM payment_instructions
WHERE (participant=%(participant)s AND team=%(team)s)
WHERE ( participant_id=%(participant_id)s
AND team_id=%(team_id)s
)
LIMIT 1
), CURRENT_TIMESTAMP)
, %(participant)s, %(team)s, %(amount)s
, %(participant_id)s, %(team_id)s, %(amount)s
)
RETURNING *

"""
args = dict(participant=self.username, team=team.slug, amount=amount)
args = dict(participant_id=self.id, team_id=team.id, amount=amount)
t = (cursor or self.db).one(NEW_PAYMENT_INSTRUCTION, args)
t_dict = t._asdict()

if amount > 0:
# Carry over any existing due
self._update_due(t_dict['team'], t_dict['id'], cursor)
self._update_due(t_dict['team_id'], t_dict['id'], cursor)
else:
self._reset_due(t_dict['team'], cursor=cursor)

if update_self:
# Update giving amount of participant
self.update_giving(cursor)
if update_team:
# Update receiving amount of team
team.update_receiving(cursor)
self._reset_due(t_dict['team_id'], cursor=cursor)

self.refresh()
team.refresh()

if team.slug == 'Gratipay':
# Update whether the participant is using Gratipay for free
self.update_is_free_rider(None if amount == 0 else False, cursor)
Expand All @@ -880,43 +875,32 @@ def set_payment_instruction(self, team, amount, update_self=True, update_team=Tr


def get_payment_instruction(self, team):
"""Given a slug, returns a dict.
"""Given a Team instance, return a dict.
"""

if not isinstance(team, Team):
team, slug = Team.from_slug(team), team
if not team:
raise NoTeam(slug)

default = dict(amount=Decimal('0.00'), is_funded=False)
return self.db.one("""\

SELECT *
FROM payment_instructions
WHERE participant=%s
AND team=%s
WHERE participant_id=%s
AND team_id=%s
ORDER BY mtime DESC
LIMIT 1

""", (self.username, team.slug), back_as=dict, default=default)
""", (self.id, team.id), back_as=dict, default=default)


def get_due(self, team):
"""Given a slug, return a Decimal.
"""Given a Team instance, return a Decimal.
"""
if not isinstance(team, Team):
team, slug = Team.from_slug(team), team
if not team:
raise NoTeam(slug)

return self.db.one("""\

SELECT due
FROM current_payment_instructions
WHERE participant = %s
AND team = %s
WHERE participant_id = %s
AND team_id = %s

""", (self.username, team.slug))
""", (self.id, team.id))


def get_giving_for_profile(self):
Expand All @@ -926,26 +910,26 @@ def get_giving_for_profile(self):
GIVING = """\

SELECT * FROM (
SELECT DISTINCT ON (pi.team)
pi.team AS team_slug
SELECT DISTINCT ON (pi.team_id)
t.slug AS team_slug
, pi.amount
, pi.due
, pi.ctime
, pi.mtime
, t.name AS team_name
FROM payment_instructions pi
JOIN teams t ON pi.team = t.slug
WHERE participant = %s
JOIN teams t ON pi.team_id = t.id
WHERE participant_id = %s
AND t.is_approved is true
AND t.is_closed is not true
ORDER BY pi.team
ORDER BY pi.team_id
, pi.mtime DESC
) AS foo
ORDER BY amount DESC
, team_slug

"""
giving = self.db.all(GIVING, (self.username,))
giving = self.db.all(GIVING, (self.id,))


# Compute the totals.
Expand Down Expand Up @@ -979,7 +963,7 @@ def update_giving_and_teams(self):
with self.db.get_cursor() as cursor:
updated_giving = self.update_giving(cursor)
for payment_instruction in updated_giving:
Team.from_slug(payment_instruction.team).update_receiving(cursor)
Team.from_id(payment_instruction.team_id).update_receiving(cursor)


def update_giving(self, cursor=None):
Expand All @@ -988,32 +972,32 @@ def update_giving(self, cursor=None):
updated = (cursor or self.db).all("""
UPDATE payment_instructions
SET is_funded = %(has_credit_card)s
WHERE participant = %(username)s
WHERE participant_id = %(participant_id)s
AND is_funded <> %(has_credit_card)s
RETURNING *
""", dict(username=self.username, has_credit_card=has_credit_card))
""", dict(participant_id=self.id, has_credit_card=has_credit_card))

r = (cursor or self.db).one("""
WITH pi AS (
SELECT amount
FROM current_payment_instructions cpi
JOIN teams t ON t.slug = cpi.team
WHERE participant = %(username)s
JOIN teams t ON t.id = cpi.team_id
WHERE participant_id = %(participant_id)s
AND amount > 0
AND is_funded
AND t.is_approved
)
UPDATE participants p
SET giving = COALESCE((SELECT sum(amount) FROM pi), 0)
, ngiving_to = COALESCE((SELECT count(amount) FROM pi), 0)
WHERE p.username=%(username)s
WHERE p.id=%(participant_id)s
RETURNING giving, ngiving_to
""", dict(username=self.username))
""", dict(participant_id=self.id))
self.set_attributes(giving=r.giving, ngiving_to=r.ngiving_to)

return updated

def _update_due(self, team, id, cursor=None):
def _update_due(self, team_id, id, cursor=None):
"""Transfer existing due value to newly inserted record
"""
# Copy due to new record
Expand All @@ -1022,33 +1006,33 @@ def _update_due(self, team, id, cursor=None):
SET due = COALESCE((
SELECT due
FROM payment_instructions s
WHERE participant=%(username)s
AND team = %(team)s
WHERE participant_id = %(participant_id)s
AND team_id = %(team_id)s
AND due > 0
), 0)
WHERE p.id = %(id)s
""", dict(username=self.username,team=team,id=id))
""", dict(participant_id=self.id, team_id=team_id, id=id))

# Reset older due values to 0
self._reset_due(team, except_for=id, cursor=cursor)
self._reset_due(team_id, except_for=id, cursor=cursor)
(cursor or self.db).run("""
UPDATE payment_instructions p
SET due = 0
WHERE participant = %(username)s
AND team = %(team)s
WHERE participant_id = %(participant_id)s
AND team_id = %(team_id)s
AND due > 0
AND p.id != %(id)s
""", dict(username=self.username,team=team,id=id))
""", dict(participant_id=self.id, team_id=team_id, id=id))

def _reset_due(self, team, except_for=-1, cursor=None):
def _reset_due(self, team_id, except_for=-1, cursor=None):
(cursor or self.db).run("""
UPDATE payment_instructions p
SET due = 0
WHERE participant = %(username)s
AND team = %(team)s
WHERE participant_id = %(participant_id)s
AND team_id = %(team_id)s
AND due > 0
AND p.id != %(id)s
""", dict(username=self.username,team=team,id=except_for))
""", dict(participant_id=self.id, team_id=team_id, id=except_for))

def update_taking(self, cursor=None):
(cursor or self.db).run("""
Expand Down
Loading