#!/usr/bin/env python3
"""Reads what Naoki Matcha publishes about each tin it sells, from the seller's own pages. Used in article 16.

The question this answers is narrow: for each product, what is its name, what does the seller
disclose about it, where does the seller use the words that buyers search for, and what does a
gram cost.

Known failure modes in this kind of listing data, and the guard applied to each. Read this
before trusting any figure the script produces:
 1. **A product feed is not the page.** `products.json` returns `body_html`, but this shop puts
    its specification table — origin, grade, harvest, cultivar — in a theme block that never
    appears in the feed. Counting terms in `body_html` alone reports a word as absent when the
    page shows it. Every field here is read from the rendered product page.
 2. A Shopify variant's `grams` field is **shipping weight, not net weight**. At this shop a
    1.4 oz tin reports 45 g and a 3.5 oz tin reports 100 or 109 g for the same stated size. Net
    weight is taken from the size the seller states, and the source of each weight is recorded.
 3. Classification comes from **the seller's own categories** (`/collections/*`), not from words
    in the product name. A tin with "matcha" in its name may be a bundle, an accessory or a
    sweetened mix.
 4. Where a term appears matters. A word in a product's *name* is the seller naming the product;
    the same word in a category heading, a spec row or an image alt attribute is not. Each is
    counted in its own column, never summed into one "mentions" figure.
 5. Retry failed fetches. A page that fails to load is missing, never "the seller does not
    disclose this".
 6. Bundles and sets have no single net weight and no single grade. They are collected and
    marked, and excluded from per-gram statistics rather than silently dropped.
"""
import ssl, re, csv, json, time, html, urllib.request, collections

SHOP = 'naokimatcha.com'
UA = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120'}
CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE

# Guard 2: the seller prints "Net Wt. 1.4 oz (40g)" on the tin, so these are the seller's own
# roundings, not ounce-to-gram arithmetic. Anything not listed here is left blank, not guessed.
STATED_GRAMS = {'1.4': 40, '1.75': 50, '3.5': 100}

# The words this article is about. Counted per field, never summed (guard 4).
TERMS = ['ceremonial', 'superior', 'premium', 'culinary', 'first harvest', 'single cultivar']

# Cultivar names are checked against the whole rendered page, not just the spec table, so that
# "this listing names no cultivar" means the page names none anywhere. The list is published
# here rather than described, so the check can be re-run and disagreed with.
CULTIVARS = ['yabukita', 'okumidori', 'samidori', 'saemidori', 'asahi', 'tsuyuhikari',
             'kanayamidori', 'okuyutaka', 'asanoka', 'gokou', 'uji hikari', 'narino',
             'sayamakaori', 'yamakai', 'meiryoku', 'seimei', 'kirari', 'okuharuka']


def fetch(url, tries=4):
    for a in range(tries):
        try:
            r = urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=45, context=CTX)
            return r.read().decode('utf-8', 'ignore')
        except Exception as e:
            if a == tries - 1:
                print(f'    FETCH FAILED after {tries} tries: {url[:80]} {e}')
                return None
            time.sleep(2 * (a + 1))


def jget(url):
    t = fetch(url)
    try:
        return json.loads(t) if t else None
    except Exception:
        return None


def txt(s):
    return re.sub(r'\s+', ' ', html.unescape(re.sub(r'<[^>]+>', ' ', s or ''))).strip()


def accordions(page):
    """Every collapsible block on the product page, as {heading: body}. The spec table lives here
    and not in the product feed (guard 1)."""
    out = {}
    for m in re.finditer(r'<summary>.*?<span>(.*?)</span>.*?</summary>\s*'
                         r'<div class="accordion__content prose">(.*?)</div>\s*</details>', page, re.S):
        out[txt(m.group(1))] = txt(m.group(2))
    return out


def spec_fields(page):
    """The seller's specification rows, as Key -> Value, from whichever block carries them."""
    fields = {}
    for body_html_block in re.findall(r'<div class="accordion__content prose">(.*?)</div>', page, re.S):
        for p in re.findall(r'<p>(.*?)</p>', body_html_block, re.S):
            line = txt(p)
            m = re.match(r'([A-Za-z][A-Za-z /]{2,24}):\s*(.+)$', line)
            if m:
                fields.setdefault(m.group(1).strip(), m.group(2).strip())
    return fields


def count(text, term):
    return len(re.findall(r'(?<![a-z])' + re.escape(term) + r'(?![a-z])', (text or '').lower()))


if __name__ == '__main__':
    # Guard 3: the seller's own categories, fetched first so every product can be labelled with them.
    cols = (jget(f'https://{SHOP}/collections.json?limit=250') or {}).get('collections') or []
    member = collections.defaultdict(list)
    for c in cols:
        got = (jget(f'https://{SHOP}/collections/{c["handle"]}/products.json?limit=250') or {}).get('products')
        if got is None:
            print(f'  collection {c["handle"]}: fetch failed — recorded as unknown, not as empty')
            continue
        for p in got:
            member[p['handle']].append(c['title'])
        time.sleep(0.2)
    print(f"{len(cols)} collections read")

    feed = (jget(f'https://{SHOP}/products.json?limit=250') or {}).get('products') or []
    print(f"{len(feed)} products in the feed\n")

    rows = []
    for p in feed:
        handle, title = p['handle'], p['title']
        page = fetch(f'https://{SHOP}/products/{handle}')
        if page is None:
            rows.append({'product': title, 'handle': handle, 'note': 'product page fetch failed'})
            continue
        acc = accordions(page)
        spec = spec_fields(page)
        desc = txt(p.get('body_html'))
        cats = member.get(handle, [])
        alts = ' '.join(str((v.get('featured_image') or {}).get('alt') or '') for v in p['variants'])
        cat_blob = ' '.join(cats)
        # A bundle has no single net weight or grade (guard 6).
        is_bundle = bool(re.search(r'bundle|set|pack|trio|multipack|sampler', title, re.I))

        for v in p['variants']:
            vt = str(v.get('title') or '')
            oz = re.match(r'\s*([\d.]+)\s*(?:Ounce|oz)', vt, re.I)
            grams, wsrc = '', ''
            if oz:
                grams = STATED_GRAMS.get(oz.group(1), '')
                wsrc = f'seller-stated size {oz.group(1)} oz' if grams else f'{oz.group(1)} oz — no stated gram figure'
            try:
                price = float(v.get('price'))
            except (TypeError, ValueError):
                price = None
            row = {
                'product': title, 'handle': handle, 'variant': vt,
                'categories': '; '.join(cats), 'product_type': p.get('product_type') or '',
                'is_bundle': int(is_bundle),
                'price_usd': price, 'net_grams': grams, 'weight_source': wsrc,
                'per_gram': round(price / grams, 4) if (price and grams) else '',
                'shopify_grams_field': v.get('grams'),      # kept only to show it disagrees (guard 2)
                'sku': v.get('sku') or '', 'available': int(bool(v.get('available'))),
                'origin': spec.get('Origin', ''), 'grade': spec.get('Grade', ''),
                'harvest': spec.get('Harvest', ''), 'cultivar': spec.get('Cultivar', ''),
                'tasting_notes': spec.get('Tasting Notes', ''), 'serving': spec.get('Serving Size', ''),
                'spec_rows': len(spec), 'accordion_blocks': '; '.join(acc),
                'cultivars_named_on_page': '; '.join(c for c in CULTIVARS if count(page, c)),
                'description': desc, 'note': '',
            }
            for t in TERMS:                                   # guard 4: one column per place
                k = t.replace(' ', '_')
                row[f'name_{k}'] = count(title, t)
                row[f'spec_{k}'] = count(' '.join(spec.values()), t)
                row[f'desc_{k}'] = count(desc + ' ' + ' '.join(acc.values()), t)
                row[f'cat_{k}'] = count(cat_blob, t)
                row[f'alt_{k}'] = count(alts, t)
            rows.append(row)

        pg = f"${price/grams:.3f}/g" if (price and grams) else '—'
        print(f"  {title[:38]:<40}{vt[:22]:<24}{pg:<11} grade={spec.get('Grade','—')[:18]:<20}"
              f"origin={spec.get('Origin','—')[:16]:<18}cats={'; '.join(cats)[:34]}")
        time.sleep(0.3)

    cols_out = list(rows[0].keys())
    for r in rows:
        for k in cols_out:
            r.setdefault(k, '')
    path = 'data/naoki_listings_2026-08-26.csv'
    with open(path, 'w', newline='', encoding='utf-8') as f:
        w = csv.DictWriter(f, fieldnames=cols_out, extrasaction='ignore')
        w.writeheader(); w.writerows(rows)
    print(f"\n{len(rows)} rows -> {path}")

    print("\n=== Where the seller uses each term (counted per place, never summed) ===")
    uniq = {r['handle']: r for r in rows if r.get('product')}
    for t in TERMS:
        k = t.replace(' ', '_')
        n = sum(1 for r in uniq.values() if r.get(f'name_{k}'))
        s = sum(1 for r in uniq.values() if r.get(f'spec_{k}'))
        d = sum(1 for r in uniq.values() if r.get(f'desc_{k}'))
        c = sum(1 for r in uniq.values() if r.get(f'cat_{k}'))
        a = sum(1 for r in uniq.values() if r.get(f'alt_{k}'))
        print(f"  {t:<17} name {n:>2}/{len(uniq)}   spec row {s:>2}   description {d:>2}   "
              f"category {c:>2}   image alt {a:>2}")

    print("\n=== Cultivars named anywhere on the product page ===")
    for r in uniq.values():
        if r.get('product'):
            print(f"  {r['product'][:44]:<46}{r.get('cultivars_named_on_page') or '— none'}")