TNCSC_Odoo/scripts/migrate_wp_members.py

#!/usr/bin/env python3
"""Migrate TNCSC's real WordPress member export (users.csv) into Community
OS Membership.

This is deliberately separate from migrate_members.py, which expects a
clean, documented schema. users.csv is a real WordPress/WooCommerce export
with messy real-world data: duplicate "Country"/"State" columns (WordPress
exports the real profile ones AND a pair of empty leftover checkout-form
columns under the same header names - this script reads by column
*position*, not name, to get the real ones), free-text membership levels
that don't map 1:1 to our tier codes, ~229/493 rows with no legacy
membership ID at all, and several sensitive columns (date of birth,
citizenship/immigration-eligibility Yes/No answers) that have no field in
this system and are deliberately never read here.

Behaviour, confirmed with TNCSC before writing this:
  - Rows whose email is on the @abbuzz.com test/spam domain are skipped
    entirely (not real members).
  - "Level" maps to a tier code: Annual/Family -> FAM, Long-Term
    Membership -> LIFE, Student Annual Membership -> STU, Senior Annual
    Membership -> SEN. Blank Level, or a comma-separated multi-value Level
    (ambiguous - can't tell which one is current), is imported as a
    contact with NO tier set and flagged 'needs_review' in the report -
    never auto-guessed.
  - The legacy "TNCSC Membership ID" column (where present) is never used
    as the new member ID - the system generates its own via the
    configured Member ID Format. Instead it's preserved as an internal
    note (chatter message) on the partner, alongside the original raw
    Level and Join Date, so it's still there for reference/lookup.
  - Existing members are matched by email (case-insensitive) and updated;
    re-running the same file converges rather than duplicating.
  - State/province is intentionally NOT set - the source data for it is
    frequently garbage in this export (e.g. a city name duplicated into
    the state column, or blank), and guessing wrong is worse than leaving
    it blank for staff to fill in. Country is resolved by name lookup.

Usage:
    python scripts/migrate_wp_members.py \\
        --url https://your-odoo-host --db tncsc_site \\
        --username admin --password admin \\
        --csv users.csv \\
        --report data/raw/wp_members_migration_report.csv \\
        [--dry-run]
"""
import argparse
import csv
import sys
from pathlib import Path

from _migration_common import MigrationReport, authenticate, execute_kw, normalize_email, normalize_phone

EXPECTED_HEADER_PREFIX = ['Username', 'Email', 'First Name', 'Last Name']
JUNK_EMAIL_DOMAINS = {'abbuzz.com'}

# Column indices, verified against the real export - see module docstring
# for why these are read positionally rather than by (duplicated) name.
COL_EMAIL = 1
COL_FIRST_NAME = 2
COL_LAST_NAME = 3
COL_PHONE = 6
COL_ADDRESS1 = 7
COL_ADDRESS2 = 8
COL_ZIP = 9
COL_CITY = 10
COL_STATE = 11
COL_COUNTRY = 12
COL_LEGACY_MEMBER_ID = 27
COL_LEVEL = 28
COL_JOIN_DATE = 30

LEVEL_TO_TIER_CODE = {
    'Annual/Family': 'FAM',
    'Long-Term Membership': 'LIFE',
    'Student Annual Membership': 'STU',
    'Senior Annual Membership': 'SEN',
}


def read_rows(csv_path):
    with open(csv_path, newline='', encoding='utf-8-sig') as handle:
        reader = csv.reader(handle)
        header = next(reader)
        if header[:4] != EXPECTED_HEADER_PREFIX:
            raise RuntimeError(
                f"CSV header doesn't match the expected users.csv shape "
                f"(first 4 columns were {header[:4]!r}) - this script reads "
                f"columns by position, so a differently-shaped export would "
                f"silently misimport. Update the COL_* indices first."
            )
        for row in reader:
            yield row


def parse_join_date(value):
    """'2018-09-24 16:34:25' -> '2018-09-24'. Returns None if unparseable."""
    value = (value or '').strip()
    if not value:
        return None
    date_part = value.split(' ')[0]
    parts = date_part.split('-')
    if len(parts) == 3 and all(p.isdigit() for p in parts):
        return date_part
    return None


def resolve_tier(level_raw, tier_by_code):
    """Returns (tier_id_or_None, needs_review_reason_or_None)."""
    level = (level_raw or '').strip()
    if not level:
        return None, None
    if ',' in level:
        return None, f"ambiguous multi-value Level '{level}' - pick manually"
    code = LEVEL_TO_TIER_CODE.get(level)
    if not code:
        return None, f"unrecognized Level '{level}' - pick manually"
    tier_id = tier_by_code.get(code)
    if not tier_id:
        return None, f"tier code '{code}' (from Level '{level}') not found in this database"
    return tier_id, None


def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('--url', required=True)
    parser.add_argument('--db', required=True)
    parser.add_argument('--username', required=True)
    parser.add_argument('--password', required=True)
    parser.add_argument('--csv', required=True, type=Path)
    parser.add_argument('--report', required=True, type=Path)
    parser.add_argument('--dry-run', action='store_true')
    args = parser.parse_args()

    if not args.csv.exists():
        print(f"CSV file not found: {args.csv}", file=sys.stderr)
        return 1

    uid = authenticate(args.url, args.db, args.username, args.password)
    report = MigrationReport()

    def rpc(model, method, call_args, kwargs=None):
        return execute_kw(args.url, args.db, uid, args.password, model, method, call_args, kwargs)

    tier_by_code = {}
    for tier in rpc('community.membership.tier', 'search_read', [[]], {'fields': ['code']}):
        tier_by_code[tier['code']] = tier['id']

    countries_by_name = {}
    for country in rpc('res.country', 'search_read', [[]], {'fields': ['name']}):
        countries_by_name[country['name'].strip().lower()] = country['id']

    seen_emails = set()

    for row_number, row in enumerate(read_rows(args.csv), start=2):
        email = normalize_email(row[COL_EMAIL])
        key = email or f'row {row_number}'

        if not email:
            report.add(key, 'failed', 'missing email')
            continue
        if email.rsplit('@', 1)[-1] in JUNK_EMAIL_DOMAINS:
            report.add(key, 'skipped', 'test/spam domain')
            continue
        if email in seen_emails:
            report.add(key, 'failed', 'duplicate email in source file')
            continue
        seen_emails.add(email)

        first_name = (row[COL_FIRST_NAME] or '').strip()
        last_name = (row[COL_LAST_NAME] or '').strip()
        name = f'{first_name} {last_name}'.strip()
        if not name:
            report.add(key, 'failed', 'missing first and last name')
            continue

        tier_id, tier_review_reason = resolve_tier(row[COL_LEVEL], tier_by_code)

        vals = {
            'name': name,
            'email': email,
            'phone': normalize_phone(row[COL_PHONE]),
            'street': (row[COL_ADDRESS1] or '').strip() or False,
            'street2': (row[COL_ADDRESS2] or '').strip() or False,
            'zip': (row[COL_ZIP] or '').strip() or False,
            'city': (row[COL_CITY] or '').strip() or False,
        }
        country_name = (row[COL_COUNTRY] or '').strip().lower()
        if country_name in countries_by_name:
            vals['country_id'] = countries_by_name[country_name]

        if tier_id:
            vals['membership_tier_id'] = tier_id
        join_date = parse_join_date(row[COL_JOIN_DATE])
        if join_date:
            vals['membership_start'] = join_date

        existing = rpc('res.partner', 'search', [[('email', '=', email)]], {'limit': 1})

        if args.dry_run:
            action = 'would_update' if existing else 'would_create'
            report.add(key, 'needs_review' if tier_review_reason else action, tier_review_reason or '')
            continue

        if existing:
            rpc('res.partner', 'write', [existing, vals])
            partner_id = existing[0]
            action = 'updated'
        else:
            partner_id = rpc('res.partner', 'create', [vals])
            action = 'created'

        legacy_id = (row[COL_LEGACY_MEMBER_ID] or '').strip()
        raw_level = (row[COL_LEVEL] or '').strip()
        if legacy_id or raw_level:
            note_lines = ['Imported from TNCSC WordPress export (users.csv):']
            if legacy_id:
                note_lines.append(f'- Legacy TNCSC Membership ID: {legacy_id}')
            if raw_level:
                note_lines.append(f'- Original WordPress "Level": {raw_level}')
            rpc('res.partner', 'message_post', [[partner_id]], {
                'body': '<br/>'.join(note_lines),
            })

        if tier_id:
            partner_state = rpc('res.partner', 'read', [[partner_id], ['membership_state']])[0]['membership_state']
            if partner_state not in ('active', 'renewal_due'):
                rpc('res.partner', 'action_activate_membership', [[partner_id]])

        report.add(key, 'needs_review' if tier_review_reason else action, tier_review_reason or '')

    report.write(args.report)
    summary = report.summary()
    print(f"Done. {summary}")
    print(f"Report written to {args.report}")
    return 1 if summary.get('failed') else 0


if __name__ == '__main__':
    sys.exit(main())