#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /misc/*.css$
Allow: /misc/*.css?
Allow: /misc/*.js$
Allow: /misc/*.js?
Allow: /misc/*.gif
Allow: /misc/*.jpg
Allow: /misc/*.jpeg
Allow: /misc/*.png
Allow: /modules/*.css$
Allow: /modules/*.css?
Allow: /modules/*.js$
Allow: /modules/*.js?
Allow: /modules/*.gif
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.png
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /themes/*.css$
Allow: /themes/*.css?
Allow: /themes/*.js$
Allow: /themes/*.js?
Allow: /themes/*.gif
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.png
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /node/*/draft
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
Disallow: /?q=node/*/draft
# Printer-friendly and PDF generator URLs
Disallow: /printpdf/*
Disallow: /print/*
# Deprecated URLs.
Disallow: /node/*/share
# Filtered list pages.
Disallow: /whats-on?special*=
Disallow: /whats-on?*&special*=
Disallow: /whats-on?type=
Disallow: /whats-on?*&type=
Disallow: /whats-on?type*=
Disallow: /whats-on?*&type*=
Disallow: /whats-on?series=
Disallow: /whats-on?*&series=
Disallow: /whats-on?series*=
Disallow: /whats-on?*&series*=
Disallow: /whats-on?search=
Disallow: /whats-on?*&search=
Disallow: /stories?popular=
Disallow: /stories?*&popular=
Disallow: /stories?topic*=
Disallow: /stories?*&topic*=
Disallow: /stories?story_format*=
Disallow: /stories?*&story_format*=
Disallow: /about-library-publications/blogs?topic*=
Disallow: /about-library-publications/blogs?*&topic*=
Disallow: /about-library-publications/blogs?blog_name=
Disallow: /about-library-publications/blogs?*&blog_name=
Disallow: /about-library-publications/blogs?search=
Disallow: /about-library-publications/blogs?*&search=
Disallow: /blogs?topic*=
Disallow: /blogs?*&topic*=
Disallow: /blogs?blog_name=
Disallow: /blogs?*&blog_name=
Disallow: /blogs?search=
Disallow: /blogs?*&search=
Disallow: /learning/schools-and-teachers?stage=*
Disallow: /learning/schools-and-teachers?subject=*
Disallow: /learning/schools-and-teachers?content_format=*
Disallow: /learning/schools-and-teachers?*&stage=*
Disallow: /learning/schools-and-teachers?*&subject=*
Disallow: /learning/schools-and-teachers?*&content_format=*
Disallow: /learning/kids-and-families?special*=
Disallow: /learning/kids-and-families?*&special*=
Disallow: /learning/kids-and-families?type*=
Disallow: /learning/kids-and-families?*&type*=
Disallow: /learning/lifelong-learners?special*=
Disallow: /learning/lifelong-learners?*&special*=
Disallow: /learning/lifelong-learners?type*=
Disallow: /learning/lifelong-learners?*&type*=
Disallow: /about-library/media-centre?subject=*
Disallow: /about-library/media-centre?*&subject=*
# Filtered pathway pages on Find Legal Answers.
Disallow: /consumers-business-and-media-law?field*=
Disallow: /consumers-business-and-media-law?*&field*=
Disallow: /legal-information/consumers-business-and-media-law?field*=
Disallow: /legal-information/consumers-business-and-media-law?*&field*=
Disallow: /find-legal-answers/consumers-business-and-media-law?field*=
Disallow: /find-legal-answers/consumers-business-and-media-law?*&field*=
Disallow: /legal-information/family-relationships-and-children?field*=
Disallow: /legal-information/family-relationships-and-children?*&field*=
Disallow: /legal-information/neighbours-housing-and-environment?field*=
Disallow: /legal-information/neighbours-housing-and-environment?*&field*=
Disallow: /legal-information/wills-estates-and-planning-ahead?field*=
Disallow: /legal-information/wills-estates-and-planning-ahead?*&field*=
Disallow: /legal-information/police-and-crime?field*=
Disallow: /legal-information/police-and-crime?*&field*=
Disallow: /legal-information/fines-debt-and-money?field*=
Disallow: /legal-information/fines-debt-and-money?*&field*=
Disallow: /legal-information/courts-and-legal-system?field*=
Disallow: /legal-information/courts-and-legal-system?*&field*=
Disallow: /legal-information/work-and-employment?field*=
Disallow: /legal-information/work-and-employment?*&field*=
Disallow: /legal-information/cars-and-driving?field*=
Disallow: /legal-information/cars-and-driving?*&field*=
Disallow: /legal-information/government-and-rights?field*=
Disallow: /legal-information/government-and-rights?*&field*=
Disallow: /legal-information/accidents-compensation-and-insurance?field*=
Disallow: /legal-information/accidents-compensation-and-insurance?*&field*=
Disallow: /legal-information/health-and-ageing?field*=
Disallow: /legal-information/health-and-ageing?*&field*=

# Search results page.
Disallow: /search

# Tell bad bots to go away.
# @see CW-1180
# @see CW-1330
User-agent: MegaIndex.ru
Disallow: /
User-agent: megaindex.com
Disallow: /
User-agent: SiteImprove
Disallow: /
User-agent: semrush
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: DotBot/1.1
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: SISTRIX Crawler
Disallow: /
User-agent: disco_crawl
Disallow: /

# Tell some bots to settle down.
# @see CW-1180
User-agent: bingbot
Crawl-delay: 30
User-agent: msnbot
Crawl-delay: 30
User-agent: msnbot-media
Crawl-delay: 30
User-agent: YandexBot
Crawl-delay: 30
User-agent: YandexImages
Crawl-delay: 30
User-agent: Turnitinbot
Crawl-delay: 30
User-agent: Sogou web spider
Crawl-delay: 30
User-agent: Exabot
Crawl-delay: 30

User-agent: Funnelback
Disallow: /sitemap.xml
Disallow: /draft
Disallow: /node/*/draft
Disallow: /?q=node/*/draft

Sitemap: https://www.sl.nsw.gov.au/sitemap.xml