RedVodkaJelly Logo

PHP Google XML Sitemap Bot Class

POSTED AT 15:35 on 23rd May 2008

This bot will crawl any site and produce a valid XML Google sitemap used for Google sitemaps within the Webmaster tools section of their site.

Simply provide the domain name that you wish to crawl and this PHP class will crawl your site by finding all links and produce a valid XML sitemap.

If you would like a sitemap produced for you, but don’t know how to run this script then my company frozensheep offers this as a service - just get in touch with us.

Source Code

  1. <?php 
  2. /* 
  3. * 
  4. * Google Sitemap Bot 
  5. * 
  6. * Class to deal with creating an automatic google sitemap. 
  7. * Will create a valid google XML sitemap from any base URL 
  8. *  
  9. * Written By Jacob Wyke - jacob@redvodkajelly.com - www.redvodkajelly.com 
  10. *  
  11. * LICENSE 
  12. * ——- 
  13. * Feel free to use this as you wish, just give me credit where credits due and drop me an email telling me what your using it for so I can check out all the cool ways its been used. 
  14. * 
  15. * USAGE 
  16. * —– 
  17. * To use simply call the class with the desired URL. 
  18. * 
  19. *      $objBot = new RVJ_googleMapBot("http://www.redvodkajelly.com"); 
  20. * 
  21. *      $objBot->saveSiteMap("sitemap.xml"); 
  22. * 
  23. * Your sitemap will then be saved with any page on the domain found through crawling the site. 
  24. */ 
  25.  
  26. set_time_limit(0);

     

  27.  
  28. class RVJ_googleMapBot {
     
  29.  
  30.    var $strURL;
     
  31.    var $arrURLs = array();
     
  32.    var $arrCheckedURLs = array();
     
  33.    var $arrAllowedExtensions = array(’html’, ‘htm’, ‘php’, ‘php3′, ‘asp’, ‘asx’, ‘xml’, ‘cfc’); 
  34.    
     
  35.    /* 
  36.    * 
  37.    *   @Method:      RVJ_googleMapBot 
  38.    *   @Parameters:   1
     
  39.    *   @Param-1:      strURL - String - The URL yowant to crawl 
  40.    *   @Description:   Runs the bot to crawl the requested site 
  41.    * 
  42.    */

     

  43.    function RVJ_googleMapBot($strURL){
     
  44.       //store the passed details
     
  45.       $this->strURL = $strURL;
     
  46.       $this->arrURLs[] = $this->strURL;
     
  47.       $this->numLevelsDeep = $numLevelsDeep;
     
  48.       
     
  49.       $this->parseSite(); 
  50.    }
     
  51.  
  52.    /* 
  53.    * 
  54.    *   @Method:       
  55.    *   @Parameters:   0 
  56.    *   @Description:    
  57.    * 
  58.    */   

     

  59.    function parseSite(){
     
  60.       foreach($this->arrURLs as $strURL){
     
  61.          //check to make sure we havent already parsed this page
     
  62.          if(!in_array($strURL, $this->arrCheckedURLs)){
     
  63.             //ensure that the URL is for the same site
     
  64.             if(eregi($this->strURL, $strURL)){
     
  65.                //get the page content
     
  66.                $strPage = $this->getPage($strURL);
     
  67.                
     
  68.                //thanks to http://www.sphider.eu/ for the following regex 
  69.                preg_match_all("/(href)\s*=\s*[\'\"]?(([[a-z]{3,5}:\/\/(([.a-zA-Z0-9-])+(:[0-9]+)*))*([+:%\/\?~=&;\\\(\),._ a-zA-Z0-9-]*))(#[.a-zA-Z0-9-]*)?[\'\" ]?(\s*rel\s*=\s*[\'\"]?(nofollow)[\'\"]?)?/i", $strPage, $arrMatches); 
  70.                for($numX=0;$numX<count($arrMatches[0]);$numX++){

     

  71.                   $this->addURL($arrMatches[2][$numX]); 
  72.                } 
  73.             }
     
  74.              
  75.             //dump the page from memory or else PHP will complain its using too much memory 
  76.             unset($strPage); 
  77.              
  78.             //add this URL to the checked list 
  79.             $this->arrCheckedURLs[] = $strURL; 
  80.              
  81.             //remove any possible dupliate URLs 
  82.             $this->arrURLs = array_unique($this->arrURLs); 
  83.              
  84.             //run this method again 
  85.             $this->parseSite(); 
  86.          }

     

  87.       }
     
  88.    }
     
  89.  
  90.    /* 
  91.    * 
  92.    *   @Method:       
  93.    *   @Parameters:   0 
  94.    *   @Description:    
  95.    * 
  96.    */      
     
  97.    function getPage($strURL){
     
  98.       echo "Reading…$strURL<br/>\n";

     

  99.       if($strContent = @file_get_contents($strURL)){ 
  100.          return $strContent; 
  101.       }else{ 
  102.          //remove this URL from the list 
  103.          $numKey = array_search($strURL, $this->arrURLs); 
  104.          unset($this->arrURLs[$numKey]); 
  105.           
  106.       } 
  107.    }
     
  108.  
  109.    /* 
  110.    * 
  111.    *   @Method:       
  112.    *   @Parameters:   0 
  113.    *   @Description:    
  114.    * 
  115.    */       
  116.    function addURL($strURL){ 
  117.       //ensure that the link is to a full url 
  118.       if(substr($strURL, 0, 1) == ‘/’){ 
  119.          //its an internal link 
  120.          $strURL = $this->strURL.$strURL; 
  121.       } 
  122.       if(substr($strURL, 0, strlen($this->strURL))!=$this->strURL AND substr($strURL, 0, 4)!=’http’){ 
  123.          //its an internal link 
  124.          $strURL = $this->strURL."/".$strURL; 
  125.       } 
  126.  
  127.       //if its an internal link on the same URL add it to out list 
  128.       if(substr($strURL, 0, strlen($this->strURL))==$this->strURL){ 
  129.          if($this->checkURL($strURL)){ 
  130.             $this->arrURLs[] = $strURL; 
  131.          } 
  132.       } 
  133.    } 
  134.  
  135.    /* 
  136.    * 
  137.    *   @Method:       
  138.    *   @Parameters:   0 
  139.    *   @Description:    
  140.    * 
  141.    */       
  142.    function checkURL($strURL){ 
  143.       //get the file extension if there is one 
  144.       $arrExt = explode(".", $strURL); 
  145.       $arrExt = array_reverse($arrExt); 
  146.       $strExt = $arrExt[0]; 
  147.       if(strlen($strExt)<=4){ 
  148.          //check the string extension against allowed types 
  149.          if(!in_array($strExt, $this->arrAllowedExtensions)){ 
  150.             return 0; 
  151.          } 
  152.       } 
  153.        
  154.       return 1; 
  155.    } 
  156.  
  157.    /* 
  158.    * 
  159.    *   @Method:       
  160.    *   @Parameters:   0 
  161.    *   @Description:    
  162.    * 
  163.    */      

     

  164.    function displaySiteMap(){
     
  165.       $strSitemap = ‘<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.google.com/schemas/sitemap/0.84">’; 
  166.  
  167.       foreach($this->arrURLs as $strURL){ 
  168.          $strSitemap .= ‘ 
  169.           <url> 
  170.              <loc>’.$strURL.’</loc> 
  171.              <changefreq>weekly</changefreq> 
  172.           </url> 
  173.          ’; 
  174.       } 
  175.        
  176.       $strSitemap .= ‘</urlset>’;

     

  177.        
  178.       return $strSitemap; 
  179.    }
     
  180.  
  181.    /* 
  182.    * 
  183.    *   @Method:       
  184.    *   @Parameters:   0 
  185.    *   @Description:    
  186.    * 
  187.    */      
     
  188.    function saveSiteMap($strFile = "sitemap.xml"){

     

  189.       $resFP = fopen($strFile, ‘w+’);
     
  190.       if($resFP){ 
  191.          fwrite($resFP, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<urlset xmlns=\"http://www.google.com/schemas/sitemap/0.84\">\r\n"); 
  192.          foreach($this->arrURLs as $strURL){ 
  193.             fwrite($resFP, "<url>\r\n"); 
  194.             fwrite($resFP, "<loc>{$strURL}</loc>\r\n"); 
  195.             fwrite($resFP, "<changefreq>weekly</changefreq>\r\n"); 
  196.             fwrite($resFP, "</url>\r\n"); 
  197.          } 
  198.          fwrite($resFP, "</urlset>"); 
  199.           
  200.          fclose($resFP); 
  201.       } 
  202.    }

     

  203.    
     
  204. }
     
  205.  
  206. ?>