# Price and specification-disclosure database for the chasen, the bamboo matcha whisk.
import urllib.request, json, ssl, re, csv, html, time
UA={'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120 Safari/537.36'}
CTX=ssl.create_default_context(); CTX.check_hostname=False; CTX.verify_mode=ssl.CERT_NONE
SHOPS=[('rockys','https://www.rockysmatcha.com'),('jadeleaf','https://jadeleafmatcha.com'),
 ('matchaeologist','https://matchaeologist.com'),('ippodo_us','https://ippodotea.com'),('kettl','https://kettl.co'),
 ('rishi','https://rishi-tea.com'),('matchaful','https://matchaful.com'),('hojicha_co','https://hojichaco.com'),
 ('mizuba','https://mizubatea.com'),('tezumi','https://tezumi.com'),('nekohama','https://nekohama.co'),
 ('yunomi','https://yunomi.life')]
def cur_of(base):
    try:
        h=urllib.request.urlopen(urllib.request.Request(base+'/',headers=UA),timeout=20,context=CTX).read().decode('utf-8','ignore')
        m=re.search(r'Shopify\.currency\s*=\s*\{"active":"(\w{3})"',h)
        return m.group(1) if m else 'USD'
    except Exception: return 'USD'
def fetch(base):
    out,page=[],1
    while page<=14:
        try:
            r=urllib.request.urlopen(urllib.request.Request(f'{base}/products.json?limit=250&page={page}',headers=UA),timeout=30,context=CTX)
            ps=json.loads(r.read().decode('utf-8','ignore')).get('products',[])
        except Exception: break
        if not ps: break
        out+=ps; page+=1
    return out
# The whisk itself only. Sets, stands, holders and scoops are handled separately.
IS_WHISK=r'\bchasen\b|\bwhisk\b|\bwhisks\b'
EXCL=r'whisky|whiskers|whisker|holder|stand|kusenaoshi|rest\b|scoop|chashaku|spoon|bowl|chawan|sifter|strainer|cloth|towel|kit\b|set\b|bundle|starter|gift|electric|frother|milk'
# The Japanese characters inside the patterns below match Japanese-language product text:
# 本 is the counter for prongs, 高山 is Takayama, 真数穂 is the shin grade. They are matching
# rules, not prose, and must not be translated.
SPEC={
 'prongs': r'(\d{2,3})\s*(?:-|\s)?(?:prong|tine|bristle|tip|spike|本)',
 'bamboo_type': r'black bamboo|purple bamboo|smoked bamboo|susudake|white bamboo|kurotake|shichiku|hachiku',
 'takayama': r'takayama|nara\b|高山',
 'handmade': r'hand[- ]?carved|hand[- ]?crafted|hand[- ]?made|handmade|artisan|craftsman|master',
 'shin': r'\bshin\b|真数穂|shin chasen',
 'koicha': r'koicha|thick tea',
}
rows=[]
for shop,base in SHOPS:
    cur=cur_of(base); n=0
    for p in fetch(base):
        ti=p.get('title','')
        tl=ti.lower()
        if not re.search(IS_WHISK,tl): continue
        if re.search(EXCL,tl): continue
        body=re.sub(r'\s+',' ',html.unescape(re.sub(r'<[^>]+>',' ',(p.get('body_html') or '')))).lower()
        full=tl+' '+body[:900]
        r={'shop':shop,'cur':cur,'pid':p.get('id',''),'title':ti[:90]}
        m=re.search(SPEC['prongs'],full)
        r['prongs']=int(m.group(1)) if m and 40<=int(m.group(1))<=200 else ''
        for k in ['bamboo_type','takayama','handmade','shin','koicha']:
            r[k]=1 if re.search(SPEC[k],full) else 0
        for v in p.get('variants',[]):
            pr=v.get('price')
            if not pr: continue
            vt_l=(v.get('title') or '').lower()
            # Bulk variants such as "25 units", and sets, are not the price of a single whisk.
            if re.search(r'\b\d{2,}\s*(units?|pcs|pieces|sets?)\b|\bset\b|stand|holder|uru\b',vt_l): continue
            rr=dict(r); rr['variant']=(v.get('title') or '')[:40]; rr['price']=pr
            rows.append(rr); n+=1
    print(f"  {shop:<14} {n:>3} variants  ({cur})")
    time.sleep(0.4)
with open('data/chasen_prices_2026-08-25.csv','w',newline='') as f:
    w=csv.DictWriter(f,fieldnames=['shop','cur','pid','title','variant','price','prongs','bamboo_type','takayama','handmade','shin','koicha'])
    w.writeheader(); w.writerows(rows)
print(f"\nTotal {len(rows)} -> data/chasen_prices_2026-08-25.csv")