Home Upgrade Search Memberlist Extras Hacker Tools Award Goals Help Wiki Contact

C# 

Simple Bing Scraper




Source:
Code
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Net;
using System.IO;

namespace BingScraper
{
   class Program
   {
       static void Main(string[] args)
       {
           Console.Title = "Simple Bing Scaper by w4zt3d";
          
           Console.Write("Scan for: ");
           int first = 1;
           int count = 100;
           string tofind = Console.ReadLine();
           if (tofind == "")
               return;
           string url = "https://www.bing.com/search?q=instreamset:url:\"/"+tofind+"\"&count=#count#&first=#first#";
           Regex regex = new Regex("href=\"(.*?)\" h=");
           WebClient webClient = new WebClient();
           List<string> alreadyFound = new List<string>();
           System.Threading.Thread.CurrentThread.Priority = System.Threading.ThreadPriority.Highest;
           File.WriteAllText("results.txt", "Bing Scraper by w4zt3d\n");
           int counter = 0;
           while (true)
           {
               try
               {
                   string code = webClient.DownloadString(url.Replace("#count#", count.ToString()).Replace("#first#", first.ToString()));
                   MatchCollection m = regex.Matches(code);
                   foreach(Match match in m)
                   {
                       string s = match.Groups[1].Value;
                          
                           if(s.EndsWith(tofind) & !alreadyFound.Contains(s))//<-- EDIT THIS PART FOR YOUR NEEDS!
                           {
                               alreadyFound.Add(s);
                               File.AppendAllText("results.txt", Environment.NewLine + s);
                               counter++;
                               Console.WriteLine("["+ counter +"] " + s);
                           }
                   }
               }
               catch
               {
                   Console.WriteLine("Request Failed!");
               }
               first += count;
           }
       }
   }
}
Hope someone can use it :)
Thanks, but who uses bing.. xd

Edit: Bing is actually nice XD
(May 21, 2017 10:24 AM)Finn. Wrote:  Thanks, but who uses bing.. xd
Its simpler and faster to scrape than google xdd
(May 21, 2017 10:24 AM)Finn. Wrote:  Thanks, but who uses bing.. xd

a lot of us in the research field of IR use bing as they provide data google does not.


Possibly Related Threads...
Thread / Author Replies Views Last Post
[.NET] by Who SoupDec 10, 2019 08:07 PM 29 1,397 Dec 10, 2019 08:07 PM
Last Post: webdev77
[Python] by NullApr 26, 2017 04:36 PM 5 286 Apr 26, 2017 04:36 PM
Last Post: ᏙictorҮ
[.NET] by SharoKurdiDec 7, 2016 12:39 PM 9 468 Dec 7, 2016 12:39 PM
Last Post: SharoKurdi