from datetime import datetime
from pathlib import Path
from urllib.parse import quote
import gzip

MAX_URLS_PER_SITEMAP = 50_000
MAX_UNCOMPRESSED_SIZE = 50 * 1024 * 1024 # 50 MB

def _quote_rel_path(p: Path, root: Path) -> str:
rel = p.relative_to(root).as_posix()
return "/".join(quote(part) for part in rel.split("/"))

def generate_sitemap(directory, base_url, include_pdf=True, recursive=True, output_path=None, compress=True):
root = Path(directory)
if not root.exists() or not root.is_dir():
raise ValueError(f"Directory not found: {directory}")

patterns = ['*.html']
if include_pdf:
patterns.append('*.pdf')

files = []
for pat in patterns:
if recursive:
files.extend([p for p in root.rglob(pat) if p.is_file()])
else:
files.extend([p for p in root.glob(pat) if p.is_file()])

files = sorted(files, key=lambda p: p.relative_to(root).as_posix())
print(f"Found {len(files)} files (html/pdf) in: {root}")

if not files:
print("No files found. No sitemap generated.")
return

if len(files) > MAX_URLS_PER_SITEMAP:
raise RuntimeError(f"URL count {len(files)} exceeds sitemap limit of {MAX_URLS_PER_SITEMAP}. Use multiple sitemaps.")

lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
]

for p in files:
lastmod = datetime.fromtimestamp(p.stat().st_mtime).date().isoformat()
loc = f"{base_url.rstrip('/')}/{_quote_rel_path(p, root)}"
lines.extend([
" <url>",
f" <loc>{loc}</loc>",
f" <lastmod>{lastmod}</lastmod>",
" <priority>0.7</priority>",
" </url>"
])

lines.append("</urlset>")
content = "\n".join(lines) + "\n"
content_bytes = content.encode("utf-8")

if len(content_bytes) > MAX_UNCOMPRESSED_SIZE:
raise RuntimeError("Generated sitemap exceeds 50MB uncompressed. Create multiple sitemaps / sitemap index.")

out_file = Path(output_path) if output_path else (root / "sitemap.xml")
out_file.parent.mkdir(parents=True, exist_ok=True)
out_file.write_bytes(content_bytes)
print(f"Wrote sitemap with {len(files)} entries to: {out_file.resolve()}")

if compress:
gz_path = out_file.with_suffix(out_file.suffix + ".gz")
with gzip.open(gz_path, "wb") as gz:
gz.write(content_bytes)
print(f"Wrote compressed sitemap to: {gz_path.resolve()}")

# Example call (adjust as needed)
if __name__ == "__main__":
generate_sitemap(r'C:\Users\User\Desktop\puresof\puresoft2006', 'https://www.puresoftwarecode.com', include_pdf=True, recursive=True)

import gzip
from datetime import datetime
from pathlib import Path
from urllib.parse import quote

MAX_URLS_PER_SITEMAP = 50_000
MAX_UNCOMPRESSED_SIZE = 50 * 1024 * 1024 # 50 MB

def _quote_rel_path(p: Path, root: Path) -> str:
rel = p.relative_to(root).as_posix()
return "/".join(quote(part) for part in rel.split("/"))
def write_sitemap_file(file_path, lines, compress=True):
"""Helper to write the XML content and its compressed GZ counterpart."""
content = "\n".join(lines) + "\n"
content_bytes = content.encode("utf-8")

if len(content_bytes) > MAX_UNCOMPRESSED_SIZE:
print(f"Warning: {file_path.name} exceeds 50MB uncompressed!")

# Save raw XML
file_path.write_bytes(content_bytes)

# Save compressed GZ
if compress:
gz_path = file_path.with_name(file_path.name + ".gz")
with gzip.open(gz_path, "wb") as gz:
gz.write(content_bytes)
def generate_sitemap(directory, base_url, include_pdf=True, recursive=True, output_dir=None, compress=True):
root = Path(directory)
if not root.exists() or not root.is_dir():
raise ValueError(f"Directory not found: {directory}")

# Establish target output folder
out_dir = Path(output_dir) if output_dir else root
out_dir.mkdir(parents=True, exist_ok=True)

patterns = ['*.html']
if include_pdf:
patterns.append('*.pdf')

files = []
for pat in patterns:
if recursive:
files.extend([p for p in root.rglob(pat) if p.is_file()])
else:
files.extend([p for p in root.glob(pat) if p.is_file()])

files = sorted(files, key=lambda p: p.relative_to(root).as_posix())
total_files = len(files)
print(f"Found {total_files} files (html/pdf) in: {root}")

if not files:
print("No files found. No sitemap generated.")
return

sitemap_files_created = []

# Chunk URLs into batches of 50,000
for chunk_idx in range(0, total_files, MAX_URLS_PER_SITEMAP):
chunk_files = files[chunk_idx : chunk_idx + MAX_URLS_PER_SITEMAP]
part_num = (chunk_idx // MAX_URLS_PER_SITEMAP) + 1

lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<urlset xmlns="http://sitemaps.org">'
]

for p in chunk_files:
lastmod = datetime.fromtimestamp(p.stat().st_mtime).date().isoformat()
loc = f"{base_url.rstrip('/')}/{_quote_rel_path(p, root)}"
if loc.endswith("/index.html"):
loc = loc[:-10]

lines.extend([
" <url>",
f" <loc>{loc}</loc>",
f" <lastmod>{lastmod}</lastmod>",
" <priority>0.7</priority>",
" </url>"
])

lines.append("</urlset>")

# Name the sub-sitemaps (e.g., sitemap-1.xml)
part_filename = f"sitemap-{part_num}.xml"
part_path = out_dir / part_filename
write_sitemap_file(part_path, lines, compress)
sitemap_files_created.append(part_filename)
print(f"Wrote part sitemap: {part_path.name} ({len(chunk_files)} entries)")

# Generate the main parent Sitemap Index file
index_lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<sitemapindex xmlns="http://sitemaps.org">'
]

current_date = datetime.now().date().isoformat()
for sitemap_file in sitemap_files_created:
# Use .xml.gz in index if compression is active, otherwise .xml
filename_to_register = sitemap_file + ".gz" if compress else sitemap_file
index_loc = f"{base_url.rstrip('/')}/{filename_to_register}"

index_lines.extend([
" <sitemap>",
f" <loc>{index_loc}</loc>",
f" <lastmod>{current_date}</lastmod>",
" </sitemap>"
])

index_lines.append("</sitemapindex>")

# Save the master index file as sitemap.xml
index_path = out_dir / "sitemap.xml"
write_sitemap_file(index_path, index_lines, compress)
print(f"\nSuccess! Wrote master index sitemap to: {index_path.resolve()}")
if __name__ == "__main__":
generate_sitemap(
directory=r'C:\Users\User\Desktop\puresof\puresoft2006', 
base_url='https://puresoftwarecode.com', 
include_pdf=True, 
recursive=True,
compress=True # Generates both regular and .gz files
)

from datetime import datetime
from pathlib import Path, PurePosixPath
from urllib.parse import quote, urljoin, urlparse, urldefrag
import gzip
from typing import List, Tuple, Set, Dict, Optional
import requests
from bs4 import BeautifulSoup
import urllib.robotparser
import email.utils
import fnmatch
import time

MAX_URLS_PER_SITEMAP = 50_000
MAX_UNCOMPRESSED_SIZE = 50 * 1024 * 1024 # 50 MB
CRAWL_TIMEOUT = 10.0 # seconds


def _quote_name(name: str) -> str:
return quote(name)


def _chunk_list(items: List, chunk_size: int):
for i in range(0, len(items), chunk_size):
yield items[i:i + chunk_size]


def _build_sitemap_content_from_items(items: List[Dict[str, str]]) -> bytes:
lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
]
for entry in items:
loc = entry["loc"]
lastmod = entry.get("lastmod")
lines.append(" <url>")
lines.append(f" <loc>{loc}</loc>")
if lastmod:
lines.append(f" <lastmod>{lastmod}</lastmod>")
lines.append(" <priority>0.7</priority>")
lines.append(" </url>")
lines.append("</urlset>")
return ("\n".join(lines) + "\n").encode("utf-8")


def _update_robots_txt(out_dir: Path, base_url: str, sitemap_urls: List[str], robots_name: str = "robots.txt"):
robots_path = out_dir / robots_name
sitemap_lines = [f"Sitemap: {u.rstrip('/')}" for u in sitemap_urls]

if robots_path.exists():
content = robots_path.read_text(encoding="utf-8").splitlines()
filtered = [ln for ln in content if not ln.strip().lower().startswith("sitemap:")]
if not any(line.strip() for line in filtered):
filtered = ["User-agent: *", "Disallow:"]
new_content = filtered + [""] + sitemap_lines
else:
new_content = ["User-agent: *", "Disallow:", ""] + sitemap_lines

robots_path.write_text("\n".join(new_content) + "\n", encoding="utf-8")
print(f"Updated robots file at: {robots_path.resolve()}")


def _parse_http_lastmod(header_value: Optional[str]) -> Optional[str]:
if not header_value:
return None
try:
dt = email.utils.parsedate_to_datetime(header_value)
return dt.date().isoformat()
except Exception:
return None


def _url_suffix(path: str) -> str:
# Use PurePosixPath to reliably extract suffix from a URL path
return PurePosixPath(path).suffix.lower()


def crawl_site(
root_url: str,
include_pdf: bool = True,
max_pages: int = 100000,
max_depth: int = 5,
allowed_prefixes: Optional[List[str]] = None,
exclude_query: bool = False,
exclude_patterns: Optional[List[str]] = None,
include_patterns: Optional[List[str]] = None,
respect_robots_crawl_delay: bool = True
) -> List[Dict[str, str]]:
parsed_root = urlparse(root_url)
base_origin = f"{parsed_root.scheme}://{parsed_root.netloc}"
rp = urllib.robotparser.RobotFileParser()
try:
rp.set_url(urljoin(base_origin, "/robots.txt"))
rp.read()
except Exception:
rp = None

session = requests.Session()
# Updated User-Agent to mimic a standard browser to prevent server blocks
session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"})

to_visit: List[Tuple[str, int]] = [(root_url, 0)]
visited: Set[str] = set()
results: List[Dict[str, str]] = []

exclude_patterns = exclude_patterns or []
include_patterns = include_patterns or []
allowed_prefixes = allowed_prefixes or []

# File extensions we want to skip downloading entirely (safety)
ignored_extensions = (
'.zip', '.rar', '.7z', '.tar', '.gz', '.mp3', '.mp4',
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp',
'.woff', '.woff2', '.ttf', '.eot', '.ico', '.css', '.js',
'.swf', '.exe', '.dmg', '.iso'
)

# Only these suffixes will be accepted into the sitemap
allowed_sitemap_suffixes = ('.html', '.htm', '.pdf', '')

crawl_delay = None
if respect_robots_crawl_delay and rp:
try:
crawl_delay = rp.crawl_delay("*")
except Exception:
crawl_delay = None

while to_visit and len(results) < max_pages:
url, depth = to_visit.pop(0)

if depth > max_depth:
continue

url, _ = urldefrag(url)
if exclude_query:
parsed_tmp = urlparse(url)
parsed_tmp = parsed_tmp._replace(query="")
url = parsed_tmp.geturl()

if url in visited:
continue
visited.add(url)

# --- PRE-FLIGHT VALIDATION CHECKS ---
if not url.lower().startswith(('http://', 'https://')):
continue

parsed_url = urlparse(url)
if f"{parsed_url.scheme}://{parsed_url.netloc}" != base_origin:
continue

# Quick skip for obviously unwanted extensions before requesting
if _url_suffix(parsed_url.path) in ignored_extensions:
continue

try:
if rp and not rp.can_fetch("*", url):
print(f"Blocked by robots.txt: {url}")
continue
except Exception:
pass

if crawl_delay:
time.sleep(crawl_delay)

try:
# stream=True allows header inspection without downloading body
resp = session.get(url, timeout=CRAWL_TIMEOUT, allow_redirects=True, stream=True)
except Exception as e:
print(f"Failed to GET {url}: {e}")
continue

if resp.status_code != 200:
resp.close()
continue

final_url = resp.url
final_url, _ = urldefrag(final_url)
if exclude_query:
pf = urlparse(final_url)
pf = pf._replace(query="")
final_url = pf.geturl()

parsed_final = urlparse(final_url)
if f"{parsed_final.scheme}://{parsed_final.netloc}" != base_origin:
resp.close()
continue

path = parsed_final.path or "/"
if allowed_prefixes:
matched_pref = any(path.startswith(pref) for pref in allowed_prefixes)
if not matched_pref:
resp.close()
continue

if exclude_patterns:
if any(fnmatch.fnmatch(path, pat) for pat in exclude_patterns):
resp.close()
continue
if include_patterns:
if not any(fnmatch.fnmatch(path, pat) for pat in include_patterns):
resp.close()
continue

lastmod = _parse_http_lastmod(resp.headers.get("Last-Modified"))

p_lower = parsed_final.path.lower()
suffix = _url_suffix(parsed_final.path)
content_type = resp.headers.get("Content-Type", "").lower()

# Determine types strictly: only accept HTML and PDF into sitemap
is_pdf = (suffix == ".pdf") or ("application/pdf" in content_type)
is_html = (suffix in (".html", ".htm", "")) or ("text/html" in content_type)

# Only store pdf and html pages in results
if is_pdf and include_pdf:
results.append({"loc": final_url, "lastmod": lastmod})
resp.close()
elif is_html:
# Read text content since it is HTML and we need to discover links
try:
html_text = resp.text
finally:
resp.close()
results.append({"loc": final_url, "lastmod": lastmod})
else:
# Not html or pdf -> skip
resp.close()
continue

# If this is HTML and within depth, parse links and queue only allowed suffixes
if "text/html" in content_type and depth < max_depth:
soup = BeautifulSoup(html_text, "html.parser")
for a in soup.find_all("a", href=True):
href = a["href"].strip()
if not href:
continue
abs_url = urljoin(final_url, href)
abs_url, _ = urldefrag(abs_url)
if exclude_query:
pf = urlparse(abs_url)
pf = pf._replace(query="")
abs_url = pf.geturl()

parsed_link = urlparse(abs_url)
if not abs_url.lower().startswith(('http://', 'https://')):
continue
if f"{parsed_link.scheme}://{parsed_link.netloc}" != base_origin:
continue

link_suffix = _url_suffix(parsed_link.path)
# Only follow/queue links that are HTML-like or PDFs (we allow PDFs to be queued so they can be HEAD-checked)
if link_suffix and link_suffix not in ('.html', '.htm', '.pdf'):
continue

to_visit.append((abs_url, depth + 1))

# Ensure the results returned contain only .html/.htm/.pdf locations (defensive)
filtered_results = []
for r in results:
parsed_r = urlparse(r["loc"])
if _url_suffix(parsed_r.path) in allowed_sitemap_suffixes or "text/html" in (session.head(r["loc"], allow_redirects=True, timeout=5).headers.get("Content-Type", "").lower() or ""):
filtered_results.append(r)
return filtered_results


if __name__ == "__main__":
target_url = "https://www.puresoftwarecode.com"
output_dir = r"C:\Users\User\Desktop\SitemapOut"

out_path = Path(output_dir)
out_path.mkdir(parents=True, exist_ok=True)

print(f"Crawling site: {target_url}...")
pages = crawl_site(target_url)
print(f"Found {len(pages)} pages.")

if pages:
xml_content = _build_sitemap_content_from_items(pages)

sitemap_xml_path = out_path / "sitemap.xml"
sitemap_xml_path.write_bytes(xml_content)
print(f"Saved: {sitemap_xml_path.resolve()}")

sitemap_gz_path = out_path / "sitemap.xml.gz"
with gzip.open(sitemap_gz_path, "wb") as f:
f.write(xml_content)
print(f"Saved: {sitemap_gz_path.resolve()}")

_update_robots_txt(out_path, target_url, [f"{target_url}/sitemap.xml"] )
else:
print("No pages found to build a sitemap.")