From 87d2788f95dd9e942c92f5b815729a533eaa1015 Mon Sep 17 00:00:00 2001 From: Ali Asadi Date: Wed, 2 Sep 2026 15:32:18 +0330 Subject: [PATCH] FEATURE(wallet): backfill command for company-side wallet history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several services (ipg, settlement, advertising, promotions) used to pass a user wallet (rial/reward) as their company-side wallet because they had no dedicated company account. They now each do. This adds a one-shot management command that repoints the company side of historical Transaction rows onto the new dedicated wallets and corrects the two affected Account.balance running totals. - apps/wallet/management/commands/backfill_company_wallets.py dry-run by default; --execute; --service ; --app- override. Idempotent (filters on the old account), single atomic + select_for_update. - docs/company_wallet_history_backfill.md — full write-up: model, mapping, balance-correction logic, consequences, run checklist, source commits. Not yet run against staging/prod. Co-Authored-By: Claude Sonnet 5 --- apps/wallet/management/__init__.py | 0 apps/wallet/management/commands/__init__.py | 0 .../commands/backfill_company_wallets.py | 234 ++++++++++++++++++ docs/company_wallet_history_backfill.md | 184 ++++++++++++++ 4 files changed, 418 insertions(+) create mode 100644 apps/wallet/management/__init__.py create mode 100644 apps/wallet/management/commands/__init__.py create mode 100644 apps/wallet/management/commands/backfill_company_wallets.py create mode 100644 docs/company_wallet_history_backfill.md diff --git a/apps/wallet/management/__init__.py b/apps/wallet/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/wallet/management/commands/__init__.py b/apps/wallet/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/wallet/management/commands/backfill_company_wallets.py b/apps/wallet/management/commands/backfill_company_wallets.py new file mode 100644 index 0000000..e4772dc --- /dev/null +++ b/apps/wallet/management/commands/backfill_company_wallets.py @@ -0,0 +1,234 @@ +"""Backfill historical Transaction rows onto the new dedicated company wallets. + +Several services used to pass a *user* wallet (rial / reward) as the company-side +account of their wallet calls, because they had no dedicated company account. +They now each have one. This command repoints the company side of every historical +``Transaction`` from the old (user) wallet to the new dedicated wallet, and fixes +the two denormalised ``Account.balance`` running totals so nothing is lost. + + python manage.py backfill_company_wallets # dry-run, all services + python manage.py backfill_company_wallets --service ipg # dry-run, one service + python manage.py backfill_company_wallets --execute # actually write + +Run it once, in the environment that owns the wallet DB. It is idempotent — a +second run finds nothing left to move (the filter is on the *old* account). + +App identities are resolved from the DB at runtime (by Application.name, with the +new dedicated wallet's existing account as a cross-check). Override with +--app- if resolution is ambiguous; --dry-run prints what it found. + +Full write-up: docs/company_wallet_history_backfill.md +""" + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction +from django.db.models import Q, Sum + +from apps.gooyal_oauth2.models import Application +from apps.wallet.constans import StateChoices, TypeChoices +from apps.wallet.models import Account, Transaction, Wallet + +# ───────────────────────────────────────────────────────────────────────────── +# Wallet UUIDs (unchanged user-side wallets + the new dedicated company wallets) +RIAL = "af7d967f-30c0-409b-9066-2549f2da5e5e" # WALLET_RIAL == WALLET_RIAL_DEPOSIT +REWARD = "e7c9d4d1-4d1f-43b2-96f7-4d4a168f480d" # WALLET_REWARD == WALLET_USER_BILLBOARD_VISIT_INCOME + +IPG_CREDIT = "5c693c93-6b13-476e-a720-e38f3798acae" +SETTLEMENT_TRANSIT = "939d9d70-3bda-4413-9f9e-756ef4e1525a" +SETTLEMENT_COMMISSION_INCOME = "ee8b050a-0ab7-48c3-a13c-01733de9bb1d" +ADVERTISING_TRANSIT = "052d38f0-d4de-40ff-85f6-9ee6e880b7e4" +PROMOTIONS_CREDIT = "f1f14c34-7e28-4d28-97c6-2bb8b2189ff3" # env still named WALLET_PROMOTIONS_TRANSIT + +# States in which the company-side balance effect is currently applied and must +# therefore travel with the rows when we repoint them: +# deposit flow → company is the PAYER, debited at submit() (PENDING) through SUCCESS +# withdraw flow → company is the PAYEE, credited only at verify() (SUCCESS) +PAYER_LIVE_STATES = (StateChoices.PENDING, StateChoices.SUCCESS, StateChoices.DELAYED, StateChoices.INCOMPLETE) +PAYEE_LIVE_STATES = (StateChoices.SUCCESS,) + +# Persian msgstr variants of the settlement leg descriptions (LANGUAGE_CODE is +# en-us, but a request-context call may have emitted the translated string). +COMMISSION_DESCRIPTIONS = ["settlement commission transaction", "کارمزد تسویه حساب"] + +# ───────────────────────────────────────────────────────────────────────────── +# service → what to move. Each "leg": +# side: 'payer' (deposit calls) or 'payee' (withdraw calls) — the company side +# old / new: wallet UUIDs +# desc_any: only rows whose details.description matches one of these (optional) +# desc_none: skip rows whose details.description matches one of these (optional) +SPECS = { + "ipg": { + "app_names": ["ipg", "ipg app", "gateway"], + "cross_check_wallet": IPG_CREDIT, + "legs": [ + {"side": "payer", "old": RIAL, "new": IPG_CREDIT}, + ], + }, + "settlement": { + "app_names": ["settlement"], + "cross_check_wallet": SETTLEMENT_TRANSIT, + "legs": [ + {"side": "payee", "old": RIAL, "new": SETTLEMENT_COMMISSION_INCOME, "desc_any": COMMISSION_DESCRIPTIONS}, + {"side": "payee", "old": REWARD, "new": SETTLEMENT_COMMISSION_INCOME, "desc_any": COMMISSION_DESCRIPTIONS}, + {"side": "payee", "old": RIAL, "new": SETTLEMENT_TRANSIT, "desc_none": COMMISSION_DESCRIPTIONS}, + {"side": "payee", "old": REWARD, "new": SETTLEMENT_TRANSIT, "desc_none": COMMISSION_DESCRIPTIONS}, + ], + }, + "advertising": { + "app_names": ["ad app", "advertising", "advertisement", "billboard"], + "cross_check_wallet": ADVERTISING_TRANSIT, + "legs": [ + # visit-reward payout, ad-balance refund, content/tip deposit to creator + {"side": "payer", "old": REWARD, "new": ADVERTISING_TRANSIT}, + # content/tip withdraw from the visitor + {"side": "payee", "old": REWARD, "new": ADVERTISING_TRANSIT}, + ], + }, + "promotions": { + "app_names": ["promotion", "promotions"], + "cross_check_wallet": PROMOTIONS_CREDIT, + "legs": [ + {"side": "payer", "old": REWARD, "new": PROMOTIONS_CREDIT}, + ], + }, +} + + +class Command(BaseCommand): + help = "Repoint historical Transaction company-side accounts onto the new dedicated wallets." + + def add_arguments(self, parser): + parser.add_argument("--execute", action="store_true", help="write changes (default: dry-run)") + parser.add_argument("--service", choices=sorted(SPECS), action="append", + help="limit to this service (repeatable); default: all") + for svc in SPECS: + parser.add_argument(f"--app-{svc}", dest=f"app_{svc}", metavar="UUID", + help=f"force the {svc} Application uuid instead of resolving by name") + + # ── app resolution ────────────────────────────────────────────────────── + def _resolve_app(self, svc, spec, options): + override = options.get(f"app_{svc}") + if override: + try: + return Application.objects.get(pk=override) + except Application.DoesNotExist: + raise CommandError(f"[{svc}] --app-{svc}={override} is not an Application") + + # 1) an existing APPLICATION account already sitting on the new dedicated wallet + acct = Account.objects.filter( + wallet_id=spec["cross_check_wallet"], owner_type=TypeChoices.APPLICATION + ).first() + if acct: + try: + return Application.objects.get(pk=acct.owner_uuid) + except Application.DoesNotExist: + pass + + # 2) by name + q = Q() + for name in spec["app_names"]: + q |= Q(name__iexact=name) | Q(name__icontains=name) + matches = list(Application.objects.filter(q)) + if len(matches) == 1: + return matches[0] + if not matches: + raise CommandError( + f"[{svc}] could not resolve the Application (tried names {spec['app_names']}). " + f"Re-run with --app-{svc} . Known apps: " + + ", ".join(f'{a.name}={a.pk}' for a in Application.objects.all()) + ) + raise CommandError( + f"[{svc}] ambiguous Application match: " + + ", ".join(f'{a.name}={a.pk}' for a in matches) + + f". Re-run with --app-{svc} ." + ) + + # ── one leg ───────────────────────────────────────────────────────────── + def _process_leg(self, svc, app, leg, execute): + side = leg["side"] + fk = f"{side}_account" + old_acct = Account.objects.filter( + owner_uuid=app.pk, owner_type=TypeChoices.APPLICATION, wallet_id=leg["old"] + ).first() + if not old_acct: + self.stdout.write(f" {svc}/{side} {leg['old']}→{leg['new']}: no old account, skip") + return + + qs = Transaction.objects.filter(application=app, **{fk: old_acct}) + if leg.get("desc_any"): + dq = Q() + for d in leg["desc_any"]: + dq |= Q(details__description__icontains=d) + qs = qs.filter(dq) + if leg.get("desc_none"): + for d in leg["desc_none"]: + qs = qs.exclude(details__description__icontains=d) + + total = qs.count() + live_states = PAYER_LIVE_STATES if side == "payer" else PAYEE_LIVE_STATES + delta = qs.filter(state__in=live_states).aggregate(s=Sum("amount"))["s"] or 0 + + # sign of the balance correction + # payer (deposit): these txns had debited old_acct by `delta` + # → give it back to old, take it from new + # payee (withdraw): these txns had credited old_acct by `delta` + # → remove from old, add to new + if side == "payer": + old_delta, new_delta = +delta, -delta + else: + old_delta, new_delta = -delta, +delta + + self.stdout.write( + f" {svc}/{side} {leg['old']}→{leg['new']}: " + f"{total} rows, Σ(live)={delta} " + f"balance: old {old_acct.balance}→{old_acct.balance + old_delta}, " + f"new →{new_delta:+d}" + ) + + if not execute or total == 0: + return + + new_acct, _ = Account.objects.select_for_update().get_or_create( + owner_uuid=app.pk, owner_type=TypeChoices.APPLICATION, wallet_id=leg["new"], + defaults={"balance": 0}, + ) + locked_old = Account.objects.select_for_update().get(pk=old_acct.pk) + moved = qs.update(**{fk: new_acct}) + Account.objects.filter(pk=locked_old.pk).update(balance=locked_old.balance + old_delta) + new_acct.refresh_from_db() + Account.objects.filter(pk=new_acct.pk).update(balance=new_acct.balance + new_delta) + self.stdout.write(self.style.SUCCESS(f" moved {moved} rows")) + + # ── entrypoint ────────────────────────────────────────────────────────── + def handle(self, *args, **options): + execute = options["execute"] + services = options.get("service") or list(SPECS) + + # fail early if a hard-coded wallet UUID is missing from this DB + for uid in {RIAL, REWARD, IPG_CREDIT, SETTLEMENT_TRANSIT, SETTLEMENT_COMMISSION_INCOME, + ADVERTISING_TRANSIT, PROMOTIONS_CREDIT}: + if not Wallet.objects.filter(pk=uid).exists(): + raise CommandError(f"Wallet {uid} not found in this database — wrong env?") + + self.stdout.write(self.style.WARNING("DRY RUN — no changes\n" if not execute else "EXECUTING\n")) + + ctx = transaction.atomic() if execute else _null_ctx() + with ctx: + for svc in services: + spec = SPECS[svc] + app = self._resolve_app(svc, spec, options) + self.stdout.write(f"{svc}: Application {app.name} ({app.pk})") + for leg in spec["legs"]: + self._process_leg(svc, app, leg, execute) + self.stdout.write("") + + if not execute: + self.stdout.write(self.style.WARNING("re-run with --execute to apply")) + + +class _null_ctx: + def __enter__(self): + return self + + def __exit__(self, *a): + return False diff --git a/docs/company_wallet_history_backfill.md b/docs/company_wallet_history_backfill.md new file mode 100644 index 0000000..b8cc526 --- /dev/null +++ b/docs/company_wallet_history_backfill.md @@ -0,0 +1,184 @@ +# Company-wallet history backfill + +**Status:** ready to run · dry-run first · not yet executed against staging/prod +**Command:** `python manage.py backfill_company_wallets` +**Scope:** wallet service DB only — `wallet_transaction`, `wallet_account` + +--- + +## 1. Background — why this is needed + +A wallet call has **two** wallet references: + +| Reference | Owner | Where it appears | +|---|---|---| +| **user-side** — `payer_wallet` / `payee_wallet` in the request body | the end user | resolved to `Account(owner=user, wallet=…)` | +| **company-side** — the first path segment of `/api/application//deposit\|withdraw/` | the calling application | resolved to `Account(owner=application, wallet=…)` | + +For a **deposit** call the company account is the transaction's **payer**; for a +**withdraw** call it is the **payee**. + +Historically several services had **no dedicated company account**, so they passed +a *user* wallet (the shared rial or reward wallet) as their own company-side +wallet. Their money therefore moved in and out of +`Account(owner=, wallet=)` — an account that structurally +looks like a user balance and does not reconcile against anything. + +Each of those services has since been given a real dedicated company wallet and +its code updated. This backfill rewrites the **historical** `Transaction` rows so +the company side points at the same dedicated wallet the current code uses, and +corrects the two affected `Account.balance` running totals. + +Nothing outside the wallet service stores a company wallet UUID on its own rows — +the other services only ever read `settings.WALLET_*` at call time — so this repo +is the only place with data to migrate. + +--- + +## 2. Data model recap + +``` +Transaction + ├─ payer_account ──FK(PROTECT)──▶ Account(owner_uuid, owner_type, wallet ─FK▶ Wallet, balance) + ├─ payee_account ──FK(PROTECT)──▶ Account(…) + ├─ application ───FK──▶ gooyal_oauth2.Application + ├─ amount, state, reference + └─ details (JSON: description, reference_id, payer_name, payee_name, …) +``` + +- Wallet identity is the `Wallet` row (its `uuid`); it is never denormalised onto + `Transaction`. The only lever is the `payer_account` / `payee_account` FK. +- `Account.balance` is a **denormalised running total**, mutated incrementally by + `Transaction.withdraw_from_payer_balance()` (at `submit()`, `CREATED→PENDING`) + and `deposit_to_payee_balance()` (at `verify()`, `PENDING→SUCCESS`), and undone + by `cancel()` / `rollback()`. Repointing an FK does **not** touch it — the + backfill fixes it explicitly. +- `StateChoices`: `CREATED 1 · DELAYED 2 · PENDING 3 · INCOMPLETE 4 · SUCCESS 5 · + FAILED 6 · EXPECTED_FAILURE 7 · ROLLED_BACK 8 · CANCELED 9`. +- `TypeChoices`: `USER 1 · APPLICATION 2`. + +--- + +## 3. What moves + +User-side wallets are **unchanged** — only their env-var names drifted: + +``` +WALLET_RIAL = WALLET_RIAL_DEPOSIT = af7d967f-30c0-409b-9066-2549f2da5e5e +WALLET_REWARD = WALLET_USER_BILLBOARD_VISIT_INCOME = e7c9d4d1-4d1f-43b2-96f7-4d4a168f480d +``` + +| Service | Flow(s) | Txn side | OLD company wallet | NEW company wallet | +|---|---|---|---|---| +| **ipg** | gateway top-up (`PaymentRequest.deposit_submit/verify`) | payer | `af7d967f…` rial | `WALLET_IPG_CREDIT` `5c693c93-6b13-476e-a720-e38f3798acae` | +| **settlement** | rial + reward payout legs | payee | `af7d967f…` / `e7c9d4d1…` | `WALLET_SETTLEMENT_TRANSIT` `939d9d70-3bda-4413-9f9e-756ef4e1525a` | +| **settlement** | commission legs¹ | payee | `af7d967f…` / `e7c9d4d1…` | `WALLET_SETTLEMENT_COMMISSION_INCOME` `ee8b050a-0ab7-48c3-a13c-01733de9bb1d` | +| **advertising** | billboard-visit reward payout, ad-balance refund, content/tip deposit to creator | payer | `e7c9d4d1…` reward | `WALLET_ADVERTISING_TRANSIT` `052d38f0-d4de-40ff-85f6-9ee6e880b7e4` | +| **advertising** | content/tip withdraw from visitor | payee | `e7c9d4d1…` reward | `WALLET_ADVERTISING_TRANSIT` `052d38f0…` | +| **promotions** | promotion payout (`Promotion.promote`) | payer | `e7c9d4d1…` reward (old `WALLET_REWARD`) | `WALLET_PROMOTIONS_CREDIT`² `f1f14c34-7e28-4d28-97c6-2bb8b2189ff3` | + +¹ Payout and commission legs land on the same old wallet and are told apart by +`details.description` ∈ {`"settlement commission transaction"`, `"کارمزد تسویه حساب"`}. +² The promotions code renamed `WALLET_PROMOTIONS_TRANSIT` → `WALLET_PROMOTIONS_CREDIT`; +some env files still use the old name. Same UUID. + +### Deliberately **not** touched + +| | Reason | +|---|---| +| advertising `AdPayment.submit()` charge | always withdrew into `WALLET_ADVERTISING_TRANSIT`, never a user wallet | +| advertising escrow (`EscrowWalletPayment`) | `WALLET_RIAL_ESCROW_PAYMENTS → WALLET_ADVERTISING_ESCROW` was a pure rename (same UUID); the commission split to `WALLET_ADVERTISING_ESCROW_INCOME` postdates any prod data (feature unreleased) | +| every user-side `payer` / `payee` | unchanged | +| `campaign`, `crm_backend`, `ipg` commission | `campaign`/`crm_backend` do balance reads only or have no dedicated account to move to; ipg has only the one flow | + +--- + +## 4. Balance correction + +For each `(service, leg)` the command: + +1. resolves `old_acct = Account(app, OLD_wallet)` and get-or-creates `new_acct = Account(app, NEW_wallet)`; +2. selects `rows = Transaction.filter(application=app, _account=old_acct[, description filter])`; +3. computes `Σ = sum(amount)` over `rows` **restricted to the states whose balance + effect is currently applied**: + - payer / deposit leg → `{PENDING, SUCCESS, DELAYED, INCOMPLETE}` + - payee / withdraw leg → `{SUCCESS}` +4. repoints **all** matched rows (any state) `rows.update(_account=new_acct)`; +5. shifts the running totals: + - payer leg: `old_acct.balance += Σ` , `new_acct.balance -= Σ` + *(the old account had been debited by Σ for these payouts — it gets it back; + the new account now carries the outflow)* + - payee leg: `old_acct.balance -= Σ` , `new_acct.balance += Σ` + *(the old account had been credited by Σ — it loses it; the new account gains it)* + +Net change across each pair is **zero**, so system-wide balance is conserved. + +Everything for one `--execute` run happens inside a single `transaction.atomic()` +with `select_for_update()` on every `Account` touched. + +### Consequences to accept before running + +- **The new dedicated accounts go strongly negative** — they are created today but + now carry months of historical outflow. Each service's application UUID must be + listed in `settings.ALLOWED_NEGATIVE_BALANCE_APPLICATIONS`; the balance itself + reading negative is expected and correct. +- **This rewrites historical financial records.** Any reconciliation or report + already produced from the old ledger will not reproduce afterwards. +- The old rial/reward application accounts are left at their residual (≈ 0 if every + historical call is accounted for). + +### Rejected alternative + +Leave every `Transaction` untouched and post one compensating transfer per leg +(old company account → new, for the live Σ). Keeps the ledger append-only, but +per-row history still shows the old wallet — which defeats the purpose. + +--- + +## 5. Running it + +```bash +# dry run — every service, no writes +python manage.py backfill_company_wallets + +# dry run — one service +python manage.py backfill_company_wallets --service settlement + +# apply +python manage.py backfill_company_wallets --execute + +# force an Application uuid if name resolution is ambiguous +python manage.py backfill_company_wallets --app-promotions --execute +``` + +**Application resolution** (printed in every run — verify it): + +1. an existing `owner_type=APPLICATION` account already on the service's new + dedicated wallet → its `owner_uuid`; +2. otherwise `Application.name` (`ipg` / `settlement` / `ad app` / `promotion…`); +3. ambiguous or missing → the command aborts and lists all known apps; pass + `--app- `. + +**Idempotent** — the row filter is on the *old* account, so a second run moves nothing. + +### Dry-run checklist + +- [ ] the resolved `Application` per service is correct +- [ ] settlement: commission-leg row count ≪ payout-leg row count (the + `details.description` split is working); investigate any settlement rows the + dry run leaves unclassified +- [ ] the printed balance deltas are plausible against current account balances +- [ ] each service's app UUID is in `ALLOWED_NEGATIVE_BALANCE_APPLICATIONS` + +--- + +## 6. How the mapping was derived + +Cross-repo git archaeology (commit refs current as of 2026-09-02): + +| Service | Commit(s) that introduced the dedicated wallet | +|---|---| +| ipg | `0083f1b` rename `WALLET_INCOME_FROM_IPG → WALLET_IPG_CREDIT`; `e4e13b2` "deposit user balance from IPG's own wallet, not the user rial wallet" | +| settlement | `cc2ffb9` "add settlement transit/commission wallets…"; `2037cd5` rename `WALLET_SETTLEMENT_TRANSIT → WALLET_SETTLEMENT_CREDIT` (code); env still `_TRANSIT` | +| advertising | `a8fb2a5` "pay billboard rewards and refunds out of WALLET_ADVERTISING_TRANSIT"; `cee2b1c` content/tip routing; `9df00d0` drop `WALLET_CONTENT_PAYMENT_TRANSIT` | +| promotions | `f59b881` add `WALLET_PROMOTIONS_TRANSIT` + per-recipient routing; `8725cdf` rename `→ WALLET_PROMOTIONS_CREDIT`; `ea070ad` `PromotionTransaction` ledger | -- 2.45.3