# -*- coding: utf-8 -*-
import re

t = open(r"C:\alrent\storage\temp\city9.html", encoding="utf-8", errors="replace").read()
print("=== homepage JSON-LD ===")
for m in re.finditer(r'<script type="application/ld\+json">(.*?)</script>', t, re.S):
    print(m.group(1)[:300])
    print("---")
print("yandex-verification:", t.count("yandex-verification"))
print("metrika:", t.count("metrika"))
print("mc.yandex:", t.count("mc.yandex"))
print("schema.org:", t.count("schema.org"))
print("ld+json:", t.count("ld+json"))

# ad card page
import urllib.request
import ssl
ctx = ssl._create_unverified_context()
# get a sample ad URL from sitemap
smap = urllib.request.urlopen("http://alrent.local/sitemap.xml", context=ctx, timeout=15).read().decode("utf-8", "replace")
ad_urls = re.findall(r"<loc>([^<]*-\d{4})</loc>", smap)[:3]
print("\nad urls:", ad_urls)
if ad_urls:
    url = ad_urls[0].replace("https://alrent.ru", "http://alrent.local")
    try:
        t2 = urllib.request.urlopen(url, context=ctx, timeout=15).read().decode("utf-8", "replace")
        print("\n=== ad page", url, "===")
        for m in re.finditer(r'<script type="application/ld\+json">(.*?)</script>', t2, re.S):
            print(m.group(1)[:400])
        print("ld+json on ad:", t2.count("ld+json"))
        print("itemtype:", t2.count("itemtype"))
        print("itemprop:", t2.count("itemprop"))
    except Exception as e:
        print("ad fetch err", e)