i need to programmatically detect when Google bot or any other search engine bot crawl our web site page. i have logic in Application_BeginRequest
of my global.asax that when any page is requested then i try to read two cookies value and if anyone is missing then i redirect user to a specific page from where user select country and from there few cookies are dropped in user pc.
protected void Application_BeginRequest(Object sender, EventArgs e)
{
string strCountryCookie = BBAreman.CountryCookie.GetCookieValue();
string strShippingCookie = BBAreman.CountryCookie.GetShippingCookieValue();
if (Request.Url.ToString().IndexOf(".asmx") == -1)
{
if (strCountryCookie.Trim() == "" || strShippingCookie.Trim() == "")
{
if (Request.Url.GetLeftPart(UriPartial.Authority).ToString() + "/index.aspx?ShowCountry=true" != HttpContext.Current.Request.Url.ToString())
{
Response.Redirect("~/index.aspx?ShowCountry=true");
}
}
}
}
now when Google bot access any of our page then bot is redirecting to a specific page and not being able to access any of page. as a result our web site is loosing ranking.
i never face this situation.so guys please guide me how to handle this situation. thanks
EDIT
i got a routine like
If(Request.Browser.Crawler) Then
'this is a web bot
Else
'this is a regular user
End If
i guess the above way i can detect that bot is crawling my page or any human? just tell me am i on right track or not ?
Solved
i solved it this way
public class Utility
{
public static bool IsCrawlByBot()
{
List<string> Crawlers = new List<string>()
{
"googlebot","bingbot","yandexbot","ahrefsbot","msnbot","linkedinbot","exabot","compspybot",
"yesupbot","paperlibot","tweetmemebot","semrushbot","gigabot","voilabot","adsbot-google",
"botlink","alkalinebot","araybot","undrip bot","borg-bot","boxseabot","yodaobot","admedia bot",
"ezooms.bot","confuzzledbot","coolbot","internet cruiser robot","yolinkbot","diibot","musobot",
"dragonbot","elfinbot","wikiobot","twitterbot","contextad bot","hambot","iajabot","news bot",
"irobot","socialradarbot","ko_yappo_robot","skimbot","psbot","rixbot","seznambot","careerbot",
"simbot","solbot","mail.ru_bot","spiderbot","blekkobot","bitlybot","techbot","void-bot",
"vwbot_k","diffbot","friendfeedbot","archive.org_bot","woriobot","crystalsemanticsbot","wepbot",
"spbot","tweetedtimes bot","mj12bot","who.is bot","psbot","robot","jbot","bbot","bot"
};
string ua = HttpContext.Current.Request.UserAgent.ToLower();
bool iscrawler = Crawlers.Exists(x => ua.Contains(x));
return iscrawler;
}
}
protected void Application_BeginRequest(Object sender, EventArgs e)
{
//if (!Request.Browser.Crawler)
if (!Utility.IsCrawlByBot())
{
string strCountryCookie = BBAreman.CountryCookie.GetCookieValue();
string strShippingCookie = BBAreman.CountryCookie.GetShippingCookieValue();
if (Request.Url.ToString().IndexOf(".asmx") == -1)
{
if (strCountryCookie.Trim() == "" || strShippingCookie.Trim() == "")
{
if (Request.Url.GetLeftPart(UriPartial.Authority).ToString() + "/index.aspx?ShowCountry=true" != HttpContext.Current.Request.Url.ToString())
{
Response.Redirect("~/index.aspx?ShowCountry=true");
}
}
}
}
}