user-agent: *
allow: /
sitemap: http://www.agcscaffolding.co.uk/sitemap.xml
sitemap: http://www.agcscaffolding.co.uk/news/sitemap_index.xml
##### PAGES
Disallow: /privacy-cookies-policy/
##### WORDPRESS DIRECTORIES & FILES
Disallow: /news/wp-content/plugins/
Disallow: /news/wp-content/themes/
Disallow: /news/wp-content/upgrade/
Disallow: /news/wp-admin/
Disallow: /news/wp-includes/
Disallow: /news/tags/
 
 
#######################################
########### SCREAMING FROG ############
#######################################
 
User-agent: Screaming Frog SEO Spider
Allow: /
#Disallow: /*.gif$
#Disallow: /*.jpg$
#Disallow: /*.png$
#Disallow: /*.bmp$
#Disallow: /*.xml$
#Disallow: /*.css$
#Disallow: /*.js$
Disallow: /news/2014
Disallow: /news/2013
Disallow: /news/2012
Disallow: /news/2011
Disallow: /news/tag
Disallow: /news/author