nkbmusic/tools/scan_songs.py

"""Scan the songs folder and write a manifest to review before import.

Reads tags, duration and bitrate, finds exact duplicates (same audio bytes),
and proposes a title, singer and album for every file. Open the CSV in Excel,
fix the columns, set `include` to no for anything to skip, then run
import_songs.py.

Usage:  python tools/scan_songs.py [songs_dir] [out_csv]
Needs:  pip install mutagen
"""
import csv
import hashlib
import re
import sys
from pathlib import Path

from mutagen import File as AudioFile

AUDIO_EXT = {".mp3", ".m4a", ".wma", ".wav", ".aac"}
# Formats the app streams directly; others must be converted to mp3 first.
PLAYABLE_EXT = {".mp3", ".m4a", ".aac"}

COLUMNS = [
    "include", "id", "file", "folder", "title", "singers", "album", "occasion",
    "year", "duration_s", "bitrate_kbps", "duplicate_of", "notes",
    "tag_title", "tag_artist", "tag_album",
]

JUNK_TITLE = re.compile(
    r"^(.*\btrack\s*\d+\w*.*|audio\s*t|\d+|.{1,3}|.*hits|<<<.*>>>|title|unknown.*|new( \(\d+\))?|(aud|wa)[-_ ]?\d.*|z \(\d+\))$", re.I)
# Artist tags that are placeholders, numbers or song titles rather than singers.
JUNK_ARTIST = re.compile(
    r"^(\d+|.*\bartists?|unknown.*|track\s*\d+|<<<.*>>>|latest|\d+[.\s_].*|zz.*|.*\bhits)$", re.I)
# Folders of film songs (not Badaga, owned by film labels): excluded by default.
FILM_FOLDERS = {"old hits"}


def first_tag(audio, *keys):
    tags = getattr(audio, "tags", None)
    if not tags:
        return ""
    for key in keys:
        value = tags.get(key)
        if value is None:
            continue
        text = value.text[0] if hasattr(value, "text") else (value[0] if isinstance(value, list) else value)
        text = str(text).strip()
        if text:
            return text
    return ""


def audio_digest(path):
    """Hash of the file minus the first 128 KB, so retagged copies still match."""
    h = hashlib.sha1()
    with open(path, "rb") as f:
        f.seek(131072)
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def clean_title(stem):
    t = re.sub(r"\.(mp3|m4a|wma)$", "", stem.strip(), flags=re.I)
    t = re.sub(r"\(\d+\)$", "", t).strip()             # "Song (3)" copies
    t = re.sub(r"^(zz|\d+[\s._-]*)", "", t, flags=re.I)  # leading track numbers
    t = re.sub(r"-?SS$", "", t)                        # uploader's tag on some files
    t = re.sub(r"\bAUDIO\s*T\b", "", t, flags=re.I)
    t = re.sub(r"[_]+", " ", t)
    t = re.sub(r"\s{2,}", " ", t).strip(" .-")
    return t.title() if t.isupper() or t.islower() else t


def main():
    root = Path(sys.argv[1] if len(sys.argv) > 1 else "songs")
    out = Path(sys.argv[2] if len(sys.argv) > 2 else "songs_manifest.csv")
    files = sorted(p for p in root.rglob("*") if p.suffix.lower() in AUDIO_EXT)

    seen = {}
    rows = []
    for n, path in enumerate(files, 1):
        notes = []
        try:
            audio = AudioFile(path)
        except Exception as e:  # unreadable file
            audio = None
            notes.append(f"unreadable: {e.__class__.__name__}")
        info = getattr(audio, "info", None)
        duration = round(info.length) if info and getattr(info, "length", 0) else 0
        bitrate = round(getattr(info, "bitrate", 0) / 1000) if info else 0

        tag_title = first_tag(audio, "TIT2", "\xa9nam", "Title", "title")
        tag_artist = first_tag(audio, "TPE1", "\xa9ART", "Author", "artist")
        tag_album = first_tag(audio, "TALB", "\xa9alb", "WM/AlbumTitle", "album")
        tag_year = first_tag(audio, "TDRC", "\xa9day", "WM/Year", "date")[:4]

        folder = path.parent.relative_to(root).as_posix()
        film = folder in FILM_FOLDERS
        title = clean_title(tag_title) if tag_title and not JUNK_TITLE.match(tag_title) else clean_title(path.stem)
        if re.search(r"\d{8,}", title):  # phone numbers in WhatsApp-forwarded file names
            title = ""
        if not title or JUNK_TITLE.match(title):
            title = ""
            notes.append("needs title")
        singers = "" if JUNK_ARTIST.match(tag_artist) else tag_artist
        if not singers:
            notes.append("needs singer")
        if film:
            notes.append("film song: not Badaga, needs label licence")
        if path.suffix.lower() not in PLAYABLE_EXT:
            notes.append("convert to mp3")
        if duration and duration < 60:
            notes.append("very short")
        if bitrate and bitrate < 96:
            notes.append("low quality")

        digest = audio_digest(path)
        duplicate_of = seen.get(digest, "")
        if duplicate_of:
            notes.append("duplicate")
        else:
            seen[digest] = f"s{n:04d}"

        rows.append({
            "include": "no" if film or duplicate_of or "unreadable" in " ".join(notes) else "yes",
            "id": f"s{n:04d}",
            "file": path.relative_to(root).as_posix(),
            "folder": folder,
            "title": title,
            "singers": singers,
            "album": tag_album if tag_album and not JUNK_TITLE.match(tag_album) else path.parent.name,
            "occasion": "folk",
            "year": tag_year if tag_year.isdigit() else "",
            "duration_s": duration,
            "bitrate_kbps": bitrate,
            "duplicate_of": duplicate_of,
            "notes": "; ".join(notes),
            "tag_title": tag_title,
            "tag_artist": tag_artist,
            "tag_album": tag_album,
        })

    # utf-8-sig so Excel shows Tamil text correctly.
    with open(out, "w", newline="", encoding="utf-8-sig") as f:
        w = csv.DictWriter(f, fieldnames=COLUMNS)
        w.writeheader()
        w.writerows(rows)

    dupes = sum(1 for r in rows if r["duplicate_of"])
    print(f"{len(rows)} files -> {out}")
    print(f"  duplicates: {dupes}")
    for flag in ("needs title", "needs singer", "convert to mp3", "low quality", "very short", "unreadable"):
        print(f"  {flag}: {sum(1 for r in rows if flag in r['notes'])}")
    print(f"  total hours: {sum(r['duration_s'] for r in rows) / 3600:.1f}")


if __name__ == "__main__":
    main()