user-agent: * disallow: /mysite/pgs/ user-agent: * disallow: /pgs/ # -------------------------------------------------------------------------------------------- # This is how FreeFind told me to write my FreeFind Exclusionary script user-agent: FreeFind disallow: /mysite/pgs/ # This is how I've changed the FreeFind Exclusionary script user-agent: FreeFind disallow: /pgs/ # -------------------------------------------------------------------------------------------- # Web bots: User-agent: AhrefsBot Disallow: / User-agent: AhrefsBot/5.2 Disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / # Above added M 12/4/17 user-agent: LinkWalker disallow: / user-agent: WhizBang disallow: / user-agent: MIIxpc disallow: / user-agent: MFC_Tear disallow: / user-agent: DIIbot disallow: / user-agent: ia_archiver disallow: / user-agent: WebZIP disallow: / user-agent: WebZIP/4.0 (http://www.spidersoft.com) disallow: / user-agent: WebZIP/4.1 (http://www.spidersoft.com) disallow: / user-agent: InternetSeer.com disallow: / user-agent: Teleport Pro/1.29.1590 disallow: / user-agent: Teleport Pro/1.29.1616 disallow: / user-agent: Teleport Pro/1.29.1718 disallow: / user-agent: Wget/1.6 disallow: / user-agent: Wget disallow: / user-agent: webbandit disallow: / user-agent: Webster Pro V2.9 Win32 disallow: / user-agent: MSProxy/2.0 disallow: / user-agent: fastlwspider disallow: / user-agent: SurfWalker disallow: / user-agent: GetWebPage disallow: / user-agent: lwp-trivial disallow: / user-agent: NICErsPRO disallow: / user-agent: Go!Zilla 3.3 (www.gozilla.com) disallow: / user-agent: Go!Zilla 3.5 (www.gozilla.com) disallow: / user-agent: Go!Zilla/4.0.0.26 disallow: / user-agent: crawler@alexa.com disallow: / user-agent: eCatch/3.0 disallow: / user-agent: Express WebPictures (www.express-soft.com) disallow: / user-agent: FileHound 1.2 disallow: / user-agent: FlashGet disallow: / user-agent: GetRight/3.3.3 disallow: / user-agent: GetRight/4.3 disallow: / user-agent: GetRight/4.5 disallow: / user-agent: GetRight/4.5b3 disallow: / user-agent: GetRight/4.5b7 disallow: / user-agent: ia_archiver/1.6 disallow: / user-agent: InstallShield DigitalWizard disallow: / user-agent: JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html) disallow: / user-agent: Mass Downloader 2.2 disallow: / user-agent: MetaProducts Download Express/1.0 disallow: / user-agent: Mister PiX version.dll disallow: / user-agent: Mozilla/4.0 (compatible; DnloadMage 1.0) disallow: / user-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AspTear 1.5) disallow: / user-agent: Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98) disallow: / user-agent: Mozilla/4.5 (compatible; HTTrack 3.16; Windows 98) disallow: / user-agent: NetAnts/1.23 disallow: / user-agent: Offline Explorer 1.* disallow: / user-agent: pavuk/0.9pl29b i686-pc-linux-gnu disallow: / user-agent: Python-urllib/2.0a1 disallow: / user-agent: Python-urllib/2.1.3 disallow: / user-agent: Python-urllib/2.2.1 disallow: / user-agent: RealDownload/4.0.0.42 disallow: / user-agent: SiteSnagger disallow: / user-agent: SQ Webscanner disallow: / user-agent: Stamina/1.4 disallow: / user-agent: UtilMind HTTPGet disallow: / user-agent: WebCopier v2.5 disallow: / user-agent: WebCopier v2.7a disallow: / user-agent: WebReaper v9 - www.webreaper.net disallow: / user-agent: WebReaper vx.x - www.webreaper.net disallow: / user-agent: WebReaper v9.8 - www.webreaper.net disallow: / user-agent: WWWOFFLE/2.x disallow: / # -------------------------------------------------------------------------------------------- # Email Bots: user-agent: EmailSiphon disallow: / user-agent: EmailWolf disallow: / user-agent: EmailCollector disallow: / user-agent: ExtractorPro disallow: / user-agent: EmailSiphon disallow: / user-agent: CherryPicker disallow: / user-agent: WebEMailExtractor disallow: / user-agent: Bullseye disallow: /