technology software

documentation technical

corporate

education learning

English

gdpr_consent

cookieconsent_status

consent

euconsent

_ga_3J9J1MDLV2

The domain name 'hub-ex8.pages.dev' uses the developer-focused generic top-level domain (.dev) with subdomain 'hub-ex8'. The core label 'pages' covers 5 characters split between 2 vowels and 3 consonants. Word splitting yields one word: pages. Average segment length settles at five characters. No strong language cues emerged from the frequency lists.

default-src 'self'; script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://aroralabs.org https://www.gstatic.com https://www.google.com https://apis.google.com https://www.googletagmanager.com https://pagead2.googlesyndication.com https://static.cloudflareinsights.com https://cdnjs.cloudflare.com https://unpkg.com https://cdn.jsdelivr.net https://esm.run; worker-src 'self' blob: https://cdnjs.cloudflare.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline' https://aroralabs.org https://fonts.googleapis.com https://cdnjs.cloudflare.com; img-src 'self' data: blob: https:; media-src 'self' blob: data:; font-src 'self' https://fonts.gstatic.com https://cdnjs.cloudflare.com data:; connect-src 'self' https://aroralabs.org https://us-central1-aroralabs-07.cloudfunctions.net https://identitytoolkit.googleapis.com https://securetoken.googleapis.com https://www.googleapis.com https://aroralabs-07.firebaseapp.com https://auth.aroralabs.org https://www.google-analytics.com https://analytics.google.com https://pagead2.googlesyndication.com https://cloudflareinsights.com https://ep1.adtrafficquality.google https://ep2.adtrafficquality.google https://firestore.googleapis.com https://firebaseappcheck.googleapis.com https://content-firebaseappcheck.googleapis.com https://www.google.com https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com https://esm.run https://staticimgly.com https://api.mymemory.translated.net https://api.qrserver.com https://ipwho.is https://ipapi.co https://freeipapi.com https://cloudflare-dns.com https://corsproxy.io https://thingproxy.freeboard.io https://is.gd; frame-src https://googleads.g.doubleclick.net https://tpc.googlesyndication.com https://aroralabs-07.firebaseapp.com https://auth.aroralabs.org https://accounts.google.com https://apis.google.com https://www.google.com; form-action 'self'; frame-ancestors 'none'

default-src 'self'; script-src 'self' https://pagead2.googlesyndication.com https://www.googletagmanager.com https://static.cloudflareinsights.com 'sha256-2dDj6qpConqfwvvfT3xCXLg5GUNXt6tyIejLWJi4yK0=' 'sha256-Baq5U4nUYC/4aSd4M8rQFBwM8+JfeJFih7dyLZFlhuo=' 'sha256-VnKwP9K4UhtLDLDt+CrdvBIttCnz7k+axn27Y+gfugg=' 'sha256-WInoAujV0rezRir63hEuvre3MIuxXxb62I2jZy8teOI=' 'sha256-a0lxgvGoqzNCLHNlRMMB/qhI9B9pV9vmigLMu73Y3CY=' 'sha256-fMfe2PyYPiOwWau7nUra2W9BeATEpAZVuDGVAnc6lDw=' 'sha256-l/1YMK1PBPxMsXiEXDejXinpYDRSIMvPPezykyd4z2s=' 'sha256-n4kBrdCPKGQdcR6eP5WqwHyxquRcHJz+QE3UetOA7/Y=' 'sha256-pT4C1IrDMKIQooE95+7ZzzKV5qerrwB8EE7bo9Lrtdo=' 'sha256-rgT3+Yc4OIWePWDHNqOmR5+eUfujFlTY2GiarDS7jk4=' 'sha256-z2izKzd/DK9TilvXt+4+NXMSlhmVqpVL6p5uO2kEiC8='; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data: blob: https:; font-src 'self' https://fonts.gstatic.com data:; connect-src 'self' https://firestore.googleapis.com https://api.web3forms.com https://www.google-analytics.com https://analytics.google.com https://pagead2.googlesyndication.com https://cloudflareinsights.com https://ep1.adtrafficquality.google https://ep2.adtrafficquality.google; frame-src 'self' https://googleads.g.doubleclick.net https://tpc.googlesyndication.com; form-action 'self' https://api.web3forms.com; frame-ancestors 'none'

require-trusted-types-for 'script'; report-uri https://csp.withgoogle.com/csp/apps-themes

aswift_0

google_esf

0c80c3278613dc643661a3e0696cac539949dfefec151b93110e82ecb084e5c5

f1d3db380a2d5b20aa2153e74cf00aa704063bac1beda2a2094d0ab5393741b3

df6da8caeca85d750f5819af198ef8e847f36cba53485ac5d358ae9e72f898b0

1b7c82c9eb49bf7fb2143bed9b7d375c691b3ef90cfef0317b432ee59ba957ab

c533ce4d89817df623f9c90df41deabc09e9db8e2309e9e0a6a26266b833870d

e1ac4849b9d4a498b68f8aca01e6ed99a11d94b35a2cec533b3916b8fb722fd6

5d6f2227b371cf1c3cc3a9b6465f16f99fd9a54c395da7e4b14d2f786033be99

6db71abcf515c8ea2bf5606a35f20b4af87866a4ec29117af013ce9b3eaecab5

6d8b5f609d5731c610a0c61f3ce74ba61aa856f17afd5e66b389dcc4af0bc029

c7d21628bda773e09e936f597b88828ef9cfcd238d45783798036be0acec2ced

610dd39d5b40286e6b1702cdd8096061a700960988ecc2eb13bfae744c52b435

6b0bafc15018b1e69b4ec79d15a5c9daa02d5fda65edef49406d57a8844cf157

6947761bfc4badfb949ebb629e562ac063bb2299f34627e675397454592b9285

3a6c59fbddf2fa066069a172f958a155426a03f0184b3481d8f453573f4a8d78

11695a6f72ab5db338b77bb1c7f28e69a89c23d258372b5ce4e5a83221573291

f30a9229a446745097ee05cece10ad725365936c7073475babfd048dabe09db7

🗺️
Sitemap Generator
Generate XML sitemaps

🏷️
Meta Tag Generator
Build SEO meta tags

🔗
OG Preview
Preview social share cards

🔑
Keyword Density
Analyse keyword focus

🔒
Privacy Policy
Generate a privacy policy

🧰
119 FREE TOOLS
from AroraLabs
One tab for every developer & business tool
Invoice generator, regex

viewport

https://hub.aroralabs.org/assets/og/robots.png

twitter:card

twitter:image

Free robots.txt generator with visual rule builder. Block AI scrapers (GPTBot, ChatGPT), protect private pages, and control Googlebot crawling. Download your robots.txt instantly.

description

robots.txt generator, robots file generator, block googlebot, block ai bots, crawl rules, disallow robots, seo robots

keywords

AlK2UR5SkAlj8jjdEc9p3F3xuFYlF6LYjAML3EOqw1g26eCwWPjdmecULvBH5MVPoqKYrOfPhYVL71xAXI1IBQoAAAB8eyJvcmlnaW4iOiJodHRwczovL2RvdWJsZWNsaWNrLm5ldDo0NDMiLCJmZWF0dXJlIjoiV2ViVmlld1hSZXF1ZXN0ZWRXaXRoRGVwcmVjYXRpb24iLCJleHBpcnkiOjE3NTgwNjcxOTksImlzU3ViZG9tYWluIjp0cnVlfQ==

Amm8/NmvvQfhwCib6I7ZsmUxiSCfOxWxHayJwyU1r3gRIItzr7bNQid6O8ZYaE1GSQTa69WwhPC9flq/oYkRBwsAAACCeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXN5bmRpY2F0aW9uLmNvbTo0NDMiLCJmZWF0dXJlIjoiV2ViVmlld1hSZXF1ZXN0ZWRXaXRoRGVwcmVjYXRpb24iLCJleHBpcnkiOjE3NTgwNjcxOTksImlzU3ViZG9tYWluIjp0cnVlfQ==

A9nrunKdU5m96PSN1XsSGr3qOP0lvPFUB2AiAylCDlN5DTl17uDFkpQuHj1AFtgWLxpLaiBZuhrtb2WOu7ofHwEAAACKeyJvcmlnaW4iOiJodHRwczovL2RvdWJsZWNsaWNrLm5ldDo0NDMiLCJmZWF0dXJlIjoiQUlQcm9tcHRBUElNdWx0aW1vZGFsSW5wdXQiLCJleHBpcnkiOjE3NzQzMTA0MDAsImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9

A93bovR+QVXNx2/38qDbmeYYf1wdte9EO37K9eMq3r+541qo0byhYU899BhPB7Cv9QqD7wIbR1B6OAc9kEfYCA4AAACQeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXN5bmRpY2F0aW9uLmNvbTo0NDMiLCJmZWF0dXJlIjoiQUlQcm9tcHRBUElNdWx0aW1vZGFsSW5wdXQiLCJleHBpcnkiOjE3NzQzMTA0MDAsImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9

A1S5fojrAunSDrFbD8OfGmFHdRFZymSM/1ss3G+NEttCLfHkXvlcF6LGLH8Mo5PakLO1sCASXU1/gQf6XGuTBgwAAACQeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXRhZ3NlcnZpY2VzLmNvbTo0NDMiLCJmZWF0dXJlIjoiQUlQcm9tcHRBUElNdWx0aW1vZGFsSW5wdXQiLCJleHBpcnkiOjE3NzQzMTA0MDAsImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9

Abuse Team

CloudFlare, Inc.

pages.dev

Abuse

Admin

Cloudflare-NOC

Cloudflare, Inc.

CLOUDFLARENET

Robots.txt Generator

AroraLabs

AroraLabs Hub

It's a plain-text file at the root of your domain that gives instructions to web crawlers about which URLs they may request. It's part of the voluntary Robots Exclusion Protocol — well-behaved bots like Googlebot and Bingbot read it before crawling and obey it, but it is advisory, not a security control. Anyone or any bot that ignores the protocol can still request the listed paths.

What does a robots.txt file actually do?

User-agent names the crawler a block of rules applies to (* matches all bots). Disallow lists a path prefix the bot should not request — Disallow: /admin/ blocks everything under /admin/, and Disallow: / blocks the whole site. Allow creates an exception, re-permitting a path inside an otherwise disallowed area, e.g. Disallow: /assets/ with Allow: /assets/public/.

What do User-agent, Disallow, and Allow mean?

It must live at the root of your domain so it loads at https://yourdomain.com/robots.txt — crawlers only check that exact location and ignore copies in subfolders. Upload it to your site's public root (often public/, www/, or public_html/). Each subdomain and protocol needs its own file. Platforms like Shopify and Squarespace manage it through their own settings instead.

Where do I place the robots.txt file?

The Sitemap directive points crawlers to your XML sitemap so they can discover your pages faster. It is independent of any User-agent block, must use a full absolute URL (e.g. Sitemap: https://yourdomain.com/sitemap.xml), and you can list several lines for multiple sitemaps. This generator places them at the end of the file, which is the conventional location.

How does the Sitemap directive work?

Crawl-delay asks a bot to wait the given number of seconds between requests, which can ease load on a small or shared server. Bing and Yandex honour it, but Google ignores it — set Googlebot's crawl rate in Google Search Console instead. Use it sparingly: a high delay slows how quickly your pages are crawled and updated in search results.

What is Crawl-delay and should I use it?

Only crawling. Disallow tells a bot not to fetch a page, but Google can still index a blocked URL — showing it with no description — if other sites link to it. To keep a page out of search results, let it be crawled and add a noindex meta tag or X-Robots-Tag HTTP header. Disallowing in robots.txt actually prevents Google from ever seeing that noindex instruction.

Does robots.txt block indexing or just crawling?

Partly. The Block AI Bots preset adds Disallow rules for GPTBot (OpenAI), ChatGPT-User, CCBot (Common Crawl), anthropic-ai, and Claude-Web — the main scrapers used to train large language models. Ethical operators honour these rules, but robots.txt is voluntary, so some scrapers ignore it. For content you must protect, combine it with server-side rate limiting or authentication.

Can I really stop AI companies from scraping my content?

window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag('consent','default',{analytics_storage:'denied',wait_for_update:2000});

gtag('js',new Date());gtag('config','G-3J9J1MDLV2');

window.kofiConfig={float:true,footer:true};


// ══════════════════════════════════════════════════
//  STATE
// ══════════════════════════════════════════════════
let state = {
  sitemaps: [],
  host: '',
  blocks: [
    { ua: '*', rules: [{ type: 'Allow', path: '/' }], crawlDelay: '' }
  ]
};

// ══════════════════════════════════════════════════
//  PRESETS
// ══════════════════════════════════════════════════
const PRESETS = {
  'allow-all': {
    host: '', sitemaps: [],
    blocks: [{ ua: '*', rules: [{ type: 'Allow', path: '/' }], crawlDelay: '' }]
  },
  'block-all': {
    host: '', sitemaps: [],
    blocks: [{ ua: '*', rules: [{ type: 'Disallow', path: '/' }], crawlDelay: '' }]
  },
  'block-ai': {
    host: '', sitemaps: [],
    blocks: [
      { ua: 'GPTBot',       rules: [{ type: 'Disallow', path: '/' }], crawlDelay: '' },
      { ua: 'ChatGPT-User', rules: [{ type: 'Disallow', path: '/' }], crawlDelay: '' },
      { ua: 'CCBot',        rules: [{ type: 'Disallow', path: '/' }], crawlDelay: '' },
      { ua: 'anthropic-ai', rules: [{ type: 'Disallow', path: '/' }], crawlDelay: '' },
      { ua: 'Claude-Web',   rules: [{ type: 'Disallow', path: '/' }], crawlDelay: '' },
      { ua: '*',            rules: [{ type: 'Allow', path: '/' }],    crawlDelay: '' }
    ]
  },
  'seo': {
    host: '', sitemaps: ['https://example.com/sitemap.xml'],
    blocks: [{
      ua: '*',
      rules: [
        { type: 'Allow',    path: '/' },
        { type: 'Disallow', path: '/admin/' },
        { type: 'Disallow', path: '/private/' },
        { type: 'Disallow', path: '/api/' },
        { type: 'Disallow', path: '/*.json$' },
        { type: 'Disallow', path: '/search' }
      ],
      crawlDelay: ''
    }]
  },
  'ecommerce': {
    host: '', sitemaps: ['https://example.com/sitemap.xml'],
    blocks: [{
      ua: '*',
      rules: [
        { type: 'Allow',    path: '/' },
        { type: 'Disallow', path: '/cart' },
        { type: 'Disallow', path: '/checkout' },
        { type: 'Disallow', path: '/account' },
        { type: 'Disallow', path: '/orders' },
        { type: 'Disallow', path: '/wishlist' },
        { type: 'Disallow', path: '/?s=' }
      ],
      crawlDelay: ''
    }]
  },
  'block-gimg': {
    host: '', sitemaps: [],
    blocks: [
      { ua: 'Googlebot-Image', rules: [{ type: 'Disallow', path: '/' }], crawlDelay: '' },
      { ua: '*',               rules: [{ type: 'Allow',    path: '/' }], crawlDelay: '' }
    ]
  }
};

function applyPreset(key) {
  const p = PRESETS[key];
  if (!p) return;
  state = JSON.parse(JSON.stringify(p));
  document.getElementById('hostInput').value = state.host;
  document.querySelectorAll('.preset-btn').forEach(b => {
    b.classList.toggle('active', b.dataset.preset === key);
  });
  hideValidation();
  renderAll();
}

function renderAll() { renderUABlocks(); renderSitemaps(); render(); }

// ══════════════════════════════════════════════════
//  UA BLOCKS DOM
// ══════════════════════════════════════════════════
const UA_SUGGESTIONS = ['*', 'Googlebot', 'Bingbot', 'GPTBot', 'ChatGPT-User',
  'CCBot', 'anthropic-ai', 'Claude-Web', 'Slurp', 'DuckDuckBot', 'Baiduspider',
  'Googlebot-Image', 'AdsBot-Google'];

function renderUABlocks() {
  const container = document.getElementById('uaBlocks');
  container.innerHTML = '';
  state.blocks.forEach((block, bi) => {
    const el = document.createElement('div');
    el.className = 'ua-block';
    el.dataset.bi = bi;

    const rulesHtml = block.rules.map((r, ri) => `
      <div class="rule-row" data-ri="${ri}">
        <select class="rule-type-select ${r.type.toLowerCase()}" onchange="updateRuleType(${bi},${ri},this.value)">
          <option value="Allow" ${r.type==='Allow'?'selected':''}>Allow</option>
          <option value="Disallow" ${r.type==='Disallow'?'selected':''}>Disallow</option>
        </select>
        <input type="text" class="rule-path-input" placeholder="/" value="${escHtml(r.path)}"
          oninput="updateRulePath(${bi},${ri},this.value)"
          onblur="validateRulePath(${bi},${ri},this)" />
        <button class="rule-del" title="Remove rule" onclick="removeRule(${bi},${ri})">
          <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>
        </button>
      </div>
    `).join('');

    el.innerHTML = `
      <div class="ua-block-head">
        <span class="ua-tag">User-agent</span>
        <input type="text" class="ua-name-input" list="ua-datalist"
          value="${escHtml(block.ua)}" placeholder="e.g. * or Googlebot"
          oninput="updateUAName(${bi},this.value)" />
        <button class="ua-block-del" title="Remove block" onclick="removeUABlock(${bi})">
          <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>
        </button>
      </div>
      <div class="ua-rules" id="rules-${bi}">${rulesHtml}</div>
      <div class="ua-footer">
        <button class="btn-add-rule" onclick="addRule(${bi})">
          <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5"><line x1="12" y1="5" x2="12" y2="19"/><line x1="5" y1="12" x2="19" y2="12"/></svg>
          Add Rule
        </button>
        <div class="crawl-row">
          <span class="crawl-label">Crawl-delay</span>
          <input type="number" class="crawl-input" placeholder="e.g. 10"
            min="0" value="${escHtml(block.crawlDelay)}"
            oninput="updateCrawlDelay(${bi},this.value)" />
        </div>
      </div>
    `;
    container.appendChild(el);
  });
}

function updateUAName(bi, val) { state.blocks[bi].ua = val.trim(); render(); }
function updateRuleType(bi, ri, val) {
  state.blocks[bi].rules[ri].type = val;
  const sel = document.querySelector(`.ua-block[data-bi="${bi}"] .rule-row[data-ri="${ri}"] .rule-type-select`);
  if (sel) { sel.className = 'rule-type-select ' + val.toLowerCase(); }
  render();
}
function updateRulePath(bi, ri, val) { state.blocks[bi].rules[ri].path = val; render(); }
function validateRulePath(bi, ri, input) {
  const v = input.value.trim();
  if (v && !v.startsWith('/') && !v.startsWith('*')) input.classList.add('invalid');
  else input.classList.remove('invalid');
}
function updateCrawlDelay(bi, val) { state.blocks[bi].crawlDelay = val; render(); }
function addRule(bi) {
  state.blocks[bi].rules.push({ type: 'Disallow', path: '' });
  renderUABlocks(); render();
  const block = document.querySelector(`.ua-block[data-bi="${bi}"]`);
  if (block) {
    const inputs = block.querySelectorAll('.rule-path-input');
    if (inputs.length) inputs[inputs.length - 1].focus();
  }
}
function removeRule(bi, ri) { state.blocks[bi].rules.splice(ri, 1); renderUABlocks(); render(); }
function addUABlock() {
  state.blocks.push({ ua: '*', rules: [{ type: 'Disallow', path: '' }], crawlDelay: '' });
  renderUABlocks(); render();
  const blocks = document.querySelectorAll('.ua-name-input');
  if (blocks.length) { blocks[blocks.length - 1].focus(); }
}
function removeUABlock(bi) { state.blocks.splice(bi, 1); renderUABlocks(); render(); }

// ══════════════════════════════════════════════════
//  SITEMAP DOM
// ══════════════════════════════════════════════════
function renderSitemaps() {
  const container = document.getElementById('sitemapList');
  container.innerHTML = '';
  state.sitemaps.forEach((url, i) => {
    const row = document.createElement('div');
    row.className = 'sitemap-row';
    row.innerHTML = `
      <input type="url" class="sitemap-input" placeholder="https://example.com/sitemap.xml"
        value="${escHtml(url)}" oninput="updateSitemap(${i},this.value)" />
      <button class="sitemap-del" onclick="removeSitemap(${i})" title="Remove">
        <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>
      </button>
    `;
    container.appendChild(row);
  });
}
function addSitemap() {
  state.sitemaps.push('');
  renderSitemaps();
  const inputs = document.querySelectorAll('.sitemap-input');
  if (inputs.length) inputs[inputs.length - 1].focus();
}
function removeSitemap(i) { state.sitemaps.splice(i, 1); renderSitemaps(); render(); }
function updateSitemap(i, val) { state.sitemaps[i] = val.trim(); render(); }

// ══════════════════════════════════════════════════
//  OUTPUT GENERATION
// ══════════════════════════════════════════════════
function buildRobots() {
  const lines = [];
  state.blocks.forEach(block => {
    lines.push(`User-agent: ${block.ua || '*'}`);
    block.rules.forEach(r => {
      if (r.path !== '' || r.type === 'Disallow') {
        lines.push(`${r.type}: ${r.path}`);
      }
    });
    if (block.crawlDelay) lines.push(`Crawl-delay: ${block.crawlDelay}`);
    lines.push('');
  });
  if (state.host) { lines.push(`Host: ${state.host}`, ''); }
  state.sitemaps.forEach(s => { if (s) lines.push(`Sitemap: ${s}`); });
  while (lines.length && lines[lines.length - 1] === '') lines.pop();
  return lines.join('\n');
}

// ══════════════════════════════════════════════════
//  SYNTAX HIGHLIGHTING
// ══════════════════════════════════════════════════
function highlight(text) {
  if (!text) return '<span class="syn-comment"># Your robots.txt will appear here…</span>';
  return text.split('\n').map(line => {
    if (line.startsWith('#')) return `<span class="syn-comment">${escHtml(line)}</span>`;
    const colonIdx = line.indexOf(':');
    if (colonIdx === -1) return escHtml(line) || ' ';
    const key = line.slice(0, colonIdx);
    const rest = line.slice(colonIdx);

    const keyLower = key.toLowerCase();
    let keyClass = 'syn-key';
    if (keyLower === 'user-agent') keyClass = 'syn-ua';
    else if (keyLower === 'allow')    keyClass = 'syn-allow';
    else if (keyLower === 'disallow') keyClass = 'syn-disallow';
    else if (keyLower === 'sitemap')  keyClass = 'syn-sitemap';

    const valStart = rest.indexOf(' ');
    const colon = valStart === -1 ? rest : rest.slice(0, valStart);
    const val   = valStart === -1 ? ''   : rest.slice(valStart);

    return `<span class="${keyClass}">${escHtml(key)}</span><span class="syn-colon">${escHtml(colon)}</span><span class="syn-val">${escHtml(val)}</span>`;
  }).join('\n');
}

function render() {
  const text = buildRobots();
  const preview = document.getElementById('previewCode');
  preview.innerHTML = highlight(text);
  const lines = text ? text.split('\n').length : 0;
  const bytes = new TextEncoder().encode(text).length;
  document.getElementById('charCount').textContent = `${lines} line${lines!==1?'s':''} · ${bytes} bytes`;
}

// ══════════════════════════════════════════════════
//  VALIDATE
// ══════════════════════════════════════════════════
function validate() {
  const errors = [];
  state.blocks.forEach((block, bi) => {
    if (!block.ua.trim()) errors.push(`Block #${bi+1}: User-agent name is empty.`);
    block.rules.forEach((r, ri) => {
      const p = r.path;
      if (p && !p.startsWith('/') && !p.startsWith('*')) {
        errors.push(`Block #${bi+1} rule #${ri+1}: Path "<strong>${escHtml(p)}</strong>" must start with <code>/</code>.`);
      }
    });
    if (block.crawlDelay && isNaN(Number(block.crawlDelay))) {
      errors.push(`Block #${bi+1}: Crawl-delay must be a number.`);
    }
  });
  state.sitemaps.forEach((s, i) => {
    if (s && !s.startsWith('http')) errors.push(`Sitemap #${i+1}: URL should start with http/https.`);
  });

  document.querySelectorAll('.rule-path-input').forEach(inp => {
    const v = inp.value.trim();
    if (v && !v.startsWith('/') && !v.startsWith('*')) inp.classList.add('invalid');
    else inp.classList.remove('invalid');
  });

  const alertEl = document.getElementById('validateAlert');
  const okEl = document.getElementById('validateOk');
  if (errors.length) {
    alertEl.innerHTML = '<strong>Issues found:</strong><ul style="margin-top:6px;padding-left:16px;display:flex;flex-direction:column;gap:3px">'
      + errors.map(e => `<li>${e}</li>`).join('') + '</ul>';
    alertEl.classList.add('show');
    okEl.classList.remove('show');
  } else {
    alertEl.classList.remove('show');
    okEl.innerHTML = '<strong>All good!</strong> Your robots.txt looks valid.';
    okEl.classList.add('show');
    showToast('Validation passed!', 'ok');
    setTimeout(() => okEl.classList.remove('show'), 4000);
  }
}

function hideValidation() {
  document.getElementById('validateAlert').classList.remove('show');
  document.getElementById('validateOk').classList.remove('show');
}

function resetState() {
  state = {
    sitemaps: [], host: '',
    blocks: [{ ua: '*', rules: [{ type: 'Allow', path: '/' }], crawlDelay: '' }]
  };
  document.getElementById('hostInput').value = '';
  document.querySelectorAll('.preset-btn').forEach(b => b.classList.remove('active'));
  hideValidation();
  renderAll();
  showToast('Reset to blank', 'ok');
}

async function copyOutput() {
  const text = buildRobots();
  if (!text) { showToast('Nothing to copy', 'err'); return; }
  try {
    await navigator.clipboard.writeText(text);
    showToast('Copied to clipboard!', 'ok');
  } catch {
    const ta = document.createElement('textarea');
    ta.value = text;
    ta.style.position = 'fixed';
    ta.style.opacity = '0';
    document.body.appendChild(ta);
    ta.select();
    document.execCommand('copy');
    document.body.removeChild(ta);
    showToast('Copied to clipboard!', 'ok');
  }
}

function downloadOutput() {
  const text = buildRobots();
  if (!text) { showToast('Nothing to download', 'err'); return; }
  const blob = new Blob([text], { type: 'text/plain' });
  const url = URL.createObjectURL(blob);
  const a = document.createElement('a');
  a.href = url;
  a.download = 'robots.txt';
  a.click();
  URL.revokeObjectURL(url);
  showToast('Downloaded robots.txt', 'ok');
}

let toastTimer;
function showToast(msg, type = '') {
  const el = document.getElementById('toast');
  clearTimeout(toastTimer);
  el.className = 'toast';
  el.textContent = msg;
  el.classList.add('show');
  toastTimer = setTimeout(() => el.classList.remove('show'), 2800);
}

function escHtml(s) {
  return String(s)
    .replace(/&/g,'&amp;')
    .replace(/</g,'&lt;')
    .replace(/>/g,'&gt;')
    .replace(/"/g,'&quot;');
}

// DATALIST for UA suggestions
const dl = document.createElement('datalist');
dl.id = 'ua-datalist';
UA_SUGGESTIONS.forEach(ua => {
  const opt = document.createElement('option');
  opt.value = ua;
  dl.appendChild(opt);
});
document.body.appendChild(dl);

// INIT
renderAll();


Requests	IP Address	Location	AS Autonomous System
9	142.250.154.154Google · CDN	United States	AS15169Google LLC
4	104.16.79.73Cloudflare · WAF	United States	AS13335Cloudflare, Inc.
4	188.114.97.3Cloudflare · WAF	United States	AS13335Cloudflare, Inc.
4	142.251.110.97Google · CDN	United States	AS15169Google LLC
4	172.66.47.186Cloudflare · WAF	United States	AS13335Cloudflare, Inc.
4	142.251.13.95Google · CDN	United States	AS15169Google LLC
29	6	-	-

Security Scan Report: hub-ex8.pages.dev

Summary

AI Security Verdict

Low Risk

Safety Factors

Details

Screenshot

Page Load Overview

Language Analysis

Primary Language

Detection Details

Website Classification

Primary Category

All Detected Categories

Detected Features

Domain & IP Information

Page Statistics

Detected Technologies7

Content Similarity HashesFor malware variant detection

TLSH (Trend Micro Locality Sensitive Hash)

ssdeep (Context Triggered Piecewise Hashing)

sdhash (Similarity Digest Hashing)

Image Hashes

Perceptual Hashes

Other Hashes

Scan History