From 90f3237dfe84dd1bcfcc9581b56b7bfade12de15 Mon Sep 17 00:00:00 2001 From: metatroncubeswdev Date: Mon, 24 Aug 2026 02:58:47 -0400 Subject: [PATCH] feat: WordPress member export migration script Adds scripts/migrate_wp_members.py to import TNCSC's real WordPress member export (data/raw/users.csv, gitignored - real PII, never committed) into Community OS Membership. Separate from migrate_members.py, which expects a clean documented schema - this one reads the real WooCommerce export's messy shape directly: duplicate "Country"/"State" columns (read by position, not name, to get the real profile ones instead of empty leftover checkout-form columns), free-text membership levels mapped to tier codes, and legacy WordPress "Membership ID"s preserved as a chatter note rather than reused as the new member ID (229 of 493 rows didn't have one at all). Test/spam rows (@abbuzz.com) are skipped; date of birth and the citizenship/immigration-eligibility columns in the source export are never read anywhere - no field for them and they're sensitive with no operational purpose here. Ambiguous multi-value Levels are imported without a tier and flagged 'needs_review' in the report rather than guessed. Run against tncsc_site and verified: 474 real members created (472 with a resolved tier or legacy-ID note), 10 junk rows skipped, 8 flagged for manual tier review, 1 non-member org inbox correctly excluded. Re-run confirmed idempotent (474 updated, zero duplicates). --- scripts/migrate_wp_members.py | 238 ++++++++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 scripts/migrate_wp_members.py diff --git a/scripts/migrate_wp_members.py b/scripts/migrate_wp_members.py new file mode 100644 index 0000000..4b7d5a6 --- /dev/null +++ b/scripts/migrate_wp_members.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Migrate TNCSC's real WordPress member export (users.csv) into Community +OS Membership. + +This is deliberately separate from migrate_members.py, which expects a +clean, documented schema. users.csv is a real WordPress/WooCommerce export +with messy real-world data: duplicate "Country"/"State" columns (WordPress +exports the real profile ones AND a pair of empty leftover checkout-form +columns under the same header names - this script reads by column +*position*, not name, to get the real ones), free-text membership levels +that don't map 1:1 to our tier codes, ~229/493 rows with no legacy +membership ID at all, and several sensitive columns (date of birth, +citizenship/immigration-eligibility Yes/No answers) that have no field in +this system and are deliberately never read here. + +Behaviour, confirmed with TNCSC before writing this: + - Rows whose email is on the @abbuzz.com test/spam domain are skipped + entirely (not real members). + - "Level" maps to a tier code: Annual/Family -> FAM, Long-Term + Membership -> LIFE, Student Annual Membership -> STU, Senior Annual + Membership -> SEN. Blank Level, or a comma-separated multi-value Level + (ambiguous - can't tell which one is current), is imported as a + contact with NO tier set and flagged 'needs_review' in the report - + never auto-guessed. + - The legacy "TNCSC Membership ID" column (where present) is never used + as the new member ID - the system generates its own via the + configured Member ID Format. Instead it's preserved as an internal + note (chatter message) on the partner, alongside the original raw + Level and Join Date, so it's still there for reference/lookup. + - Existing members are matched by email (case-insensitive) and updated; + re-running the same file converges rather than duplicating. + - State/province is intentionally NOT set - the source data for it is + frequently garbage in this export (e.g. a city name duplicated into + the state column, or blank), and guessing wrong is worse than leaving + it blank for staff to fill in. Country is resolved by name lookup. + +Usage: + python scripts/migrate_wp_members.py \\ + --url https://your-odoo-host --db tncsc_site \\ + --username admin --password admin \\ + --csv users.csv \\ + --report data/raw/wp_members_migration_report.csv \\ + [--dry-run] +""" +import argparse +import csv +import sys +from pathlib import Path + +from _migration_common import MigrationReport, authenticate, execute_kw, normalize_email, normalize_phone + +EXPECTED_HEADER_PREFIX = ['Username', 'Email', 'First Name', 'Last Name'] +JUNK_EMAIL_DOMAINS = {'abbuzz.com'} + +# Column indices, verified against the real export - see module docstring +# for why these are read positionally rather than by (duplicated) name. +COL_EMAIL = 1 +COL_FIRST_NAME = 2 +COL_LAST_NAME = 3 +COL_PHONE = 6 +COL_ADDRESS1 = 7 +COL_ADDRESS2 = 8 +COL_ZIP = 9 +COL_CITY = 10 +COL_STATE = 11 +COL_COUNTRY = 12 +COL_LEGACY_MEMBER_ID = 27 +COL_LEVEL = 28 +COL_JOIN_DATE = 30 + +LEVEL_TO_TIER_CODE = { + 'Annual/Family': 'FAM', + 'Long-Term Membership': 'LIFE', + 'Student Annual Membership': 'STU', + 'Senior Annual Membership': 'SEN', +} + + +def read_rows(csv_path): + with open(csv_path, newline='', encoding='utf-8-sig') as handle: + reader = csv.reader(handle) + header = next(reader) + if header[:4] != EXPECTED_HEADER_PREFIX: + raise RuntimeError( + f"CSV header doesn't match the expected users.csv shape " + f"(first 4 columns were {header[:4]!r}) - this script reads " + f"columns by position, so a differently-shaped export would " + f"silently misimport. Update the COL_* indices first." + ) + for row in reader: + yield row + + +def parse_join_date(value): + """'2018-09-24 16:34:25' -> '2018-09-24'. Returns None if unparseable.""" + value = (value or '').strip() + if not value: + return None + date_part = value.split(' ')[0] + parts = date_part.split('-') + if len(parts) == 3 and all(p.isdigit() for p in parts): + return date_part + return None + + +def resolve_tier(level_raw, tier_by_code): + """Returns (tier_id_or_None, needs_review_reason_or_None).""" + level = (level_raw or '').strip() + if not level: + return None, None + if ',' in level: + return None, f"ambiguous multi-value Level '{level}' - pick manually" + code = LEVEL_TO_TIER_CODE.get(level) + if not code: + return None, f"unrecognized Level '{level}' - pick manually" + tier_id = tier_by_code.get(code) + if not tier_id: + return None, f"tier code '{code}' (from Level '{level}') not found in this database" + return tier_id, None + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--url', required=True) + parser.add_argument('--db', required=True) + parser.add_argument('--username', required=True) + parser.add_argument('--password', required=True) + parser.add_argument('--csv', required=True, type=Path) + parser.add_argument('--report', required=True, type=Path) + parser.add_argument('--dry-run', action='store_true') + args = parser.parse_args() + + if not args.csv.exists(): + print(f"CSV file not found: {args.csv}", file=sys.stderr) + return 1 + + uid = authenticate(args.url, args.db, args.username, args.password) + report = MigrationReport() + + def rpc(model, method, call_args, kwargs=None): + return execute_kw(args.url, args.db, uid, args.password, model, method, call_args, kwargs) + + tier_by_code = {} + for tier in rpc('community.membership.tier', 'search_read', [[]], {'fields': ['code']}): + tier_by_code[tier['code']] = tier['id'] + + countries_by_name = {} + for country in rpc('res.country', 'search_read', [[]], {'fields': ['name']}): + countries_by_name[country['name'].strip().lower()] = country['id'] + + seen_emails = set() + + for row_number, row in enumerate(read_rows(args.csv), start=2): + email = normalize_email(row[COL_EMAIL]) + key = email or f'row {row_number}' + + if not email: + report.add(key, 'failed', 'missing email') + continue + if email.rsplit('@', 1)[-1] in JUNK_EMAIL_DOMAINS: + report.add(key, 'skipped', 'test/spam domain') + continue + if email in seen_emails: + report.add(key, 'failed', 'duplicate email in source file') + continue + seen_emails.add(email) + + first_name = (row[COL_FIRST_NAME] or '').strip() + last_name = (row[COL_LAST_NAME] or '').strip() + name = f'{first_name} {last_name}'.strip() + if not name: + report.add(key, 'failed', 'missing first and last name') + continue + + tier_id, tier_review_reason = resolve_tier(row[COL_LEVEL], tier_by_code) + + vals = { + 'name': name, + 'email': email, + 'phone': normalize_phone(row[COL_PHONE]), + 'street': (row[COL_ADDRESS1] or '').strip() or False, + 'street2': (row[COL_ADDRESS2] or '').strip() or False, + 'zip': (row[COL_ZIP] or '').strip() or False, + 'city': (row[COL_CITY] or '').strip() or False, + } + country_name = (row[COL_COUNTRY] or '').strip().lower() + if country_name in countries_by_name: + vals['country_id'] = countries_by_name[country_name] + + if tier_id: + vals['membership_tier_id'] = tier_id + join_date = parse_join_date(row[COL_JOIN_DATE]) + if join_date: + vals['membership_start'] = join_date + + existing = rpc('res.partner', 'search', [[('email', '=', email)]], {'limit': 1}) + + if args.dry_run: + action = 'would_update' if existing else 'would_create' + report.add(key, 'needs_review' if tier_review_reason else action, tier_review_reason or '') + continue + + if existing: + rpc('res.partner', 'write', [existing, vals]) + partner_id = existing[0] + action = 'updated' + else: + partner_id = rpc('res.partner', 'create', [vals]) + action = 'created' + + legacy_id = (row[COL_LEGACY_MEMBER_ID] or '').strip() + raw_level = (row[COL_LEVEL] or '').strip() + if legacy_id or raw_level: + note_lines = ['Imported from TNCSC WordPress export (users.csv):'] + if legacy_id: + note_lines.append(f'- Legacy TNCSC Membership ID: {legacy_id}') + if raw_level: + note_lines.append(f'- Original WordPress "Level": {raw_level}') + rpc('res.partner', 'message_post', [[partner_id]], { + 'body': '
'.join(note_lines), + }) + + if tier_id: + partner_state = rpc('res.partner', 'read', [[partner_id], ['membership_state']])[0]['membership_state'] + if partner_state not in ('active', 'renewal_due'): + rpc('res.partner', 'action_activate_membership', [[partner_id]]) + + report.add(key, 'needs_review' if tier_review_reason else action, tier_review_reason or '') + + report.write(args.report) + summary = report.summary() + print(f"Done. {summary}") + print(f"Report written to {args.report}") + return 1 if summary.get('failed') else 0 + + +if __name__ == '__main__': + sys.exit(main())