(.*?)<\/em>/"; // Build information about the site we're going to search. if($url = parse_url($start)) { if(isset($url['scheme'])) { $b_scheme = $url['scheme']; $b_url = $b_scheme."://"; } if(isset($url['host'])) { $b_host = $url['host']; $b_url = $b_url.$b_host; } if(isset($url['path'])) { $b_path = dirname($url['path']); $b_url = $b_url.$b_path; } } else { echo("\nError!\n"); echo("Description: Unable to parse starting URL. "); echo("Please enter a different URL to start from.\n"); echo("Starting URL: " .$start. "\n\n"); exit; } // Initialize our array of links. $links = array($start => "0"); // Initialize array of link levels $levels = array($start => "0"); // Starting link has level 0 $max_level = 2; // Set link limit for each level $link_limit = 10; // Key to follow external links or internal : both not supported $follow_external = true; // Initialize our array of search results. $gold = array(); // Initialize nesting level counter $nested = array(0 => 0, 1 => 0, 2 => 0); // Keep crawling until we run out of links. while($p_link = array_search("0", $links)) { // Mark this link as having been seen. $links[$p_link] = "1"; // Get the contents of the link we're currently looking at. // If we fail this, there's no point in going further. // We're going to surpress PHP's warning messages here as well. if(@ $contents = file_get_contents($p_link)) { // What link are we following? $indent_string = str_pad("", $levels[$p_link], ">")." "; echo("Following link: " .$indent_string.$p_link. "\n"); // Build information about the link we're currently looking at. if($url = parse_url($p_link)) { $p_url = $p_link; if(isset($url['scheme'])) { $p_scheme = $url['scheme']; $p_url = $p_scheme."://"; } if(isset($url['host'])) { $p_host = $url['host']; $p_url = $p_url.$p_host; } if(isset($url['path'])) { $p_path = dirname($url['path']); $p_url = $p_url.$p_path; } } // Extract all the search matches from the current page. preg_match_all($search, $contents, $search_results); // Put the search results into our pot of gold. for($i = 0; $i < count($search_results[1]); $i++) { $result = $search_results[1][$i]; if(array_search($result, $gold) === false) { $gold[] = $result; } } // Extract the links from the current page. preg_match_all("/href=\"(.*?)\"/i", $contents, $link_results); $this_page_links = array(); $this_page_levels = array(); // Loop through our extracted links and manipulate them. for($i = 0; $i < count($link_results[1]); $i++) { // Get an extracted link from out list and assume it's good. $c_link = $link_results[1][$i]; $c_valid = true; // Trim any whitespace that might be on our link. $c_link = trim($c_link); // Build information about our extracted link. // If we can't parse the URL, don't continue. // Surpress all PHP warnings here as well. if(@ $url = parse_url($c_link)) { if(isset($url['host'])) { $c_host = $url['host']; } if(isset($url['query'])) { $c_query = $url['query']; } if(isset($url['fragment'])) { $c_fragment = $url['fragment']; } } else { // If we won't be able to follow it, mark it as bad. $c_valid = false; } // Decide wether this link is internal or external. // We can follow either internal or external. if(preg_match("/^(http:|https:|ftp:|file:)/i", $c_link) && strpos($c_host, $b_host) === false) { $c_valid = $follow_external; } else { $c_valid = !$follow_external; } // Don't follow javascript or mailto links. if(preg_match("/^(mailto:|javascript:|news:)/i", $c_link)) { $c_valid = false; } // Don't follow pictures, zip files, etc. elseif(preg_match("/\.(jpg|gif|png|ico)$/i", $c_link)) { $c_valid = false; } elseif(preg_match("/\.(zip|rar|tar|gz)$/i", $c_link)) { $c_valid = false; } elseif(preg_match("/\.(c|pl|py|js|reg|orig)$/i", $c_link)) { $c_valid = false; } elseif(preg_match("/\.(exe|java|class)$/i", $c_link)) { $c_valid = false; } elseif(preg_match("/\.(css|xml|txt|doc|pdf|lit)$/i", $c_link)) { $c_valid = false; } elseif(preg_match("/\.(mp3|wav|ra|pm)$/i", $c_link)) { $c_valid = false; } // If our link's made it this far, it's good, so let's keep it. if($c_valid) { // Remove queries from the end of a link. if(isset($c_query)) { $c_link = preg_replace("/\?(.*?)$/", "", $c_link); } // Remove fragments from the end of a link. if(isset($c_fragment)) { $c_link = preg_replace("/#(.*?)$/", "", $c_link); } // Case 1: The URL is of the form: /directory/file if(preg_match("/^\//", $c_link)) { $c_link = $b_scheme."://".$b_host.$c_link; } // Case 2: The URL is of the form: ../directory/file if(preg_match("/^\.\.\//", $c_link)) { // How many directories will we have to backtrack into? preg_match_all("/\.\.\//", $c_link, $count); $count = count($count[0]); // Remove the relative bits from our link. $c_link = preg_replace("/\.\.\//", "", $c_link); // Remove leading and trailing slashes from our path. $p_path = preg_replace("/^\//", "", $p_path); $p_path = preg_replace("/\/$/", "", $p_path); // Backtrack the required number of directories. $path_array = explode("/", $p_path); $new_path = ""; for($j = $count; $j > 0; $j--) { array_pop($path_array); } for($j = 0; $j < count($path_array); $j++) { $new_path = $new_path.$path_array[$j]."/"; } // Tack our new path onto the begining of our link. $c_link = $p_scheme."://".$p_host."/".$new_path.$c_link; } // Case 3: The URL is of the form: ./directory/file $c_link = preg_replace("/^\.\//", "", $c_link); // Case 4: The URL is of the form: file if(!preg_match("/^http:/", $c_link)) { if(preg_match("/\/$/", $p_url)) { $c_link = $p_url.$c_link; } else { $c_link = $p_url."/".$c_link; } } // Remove any www. stuff from the start of our link. // $c_link = preg_replace("/^http:\/\/www\./", "http://", $c_link); // Add our extracted list to our list of links to look at. if(!array_key_exists($c_link, $links)) { $this_page_links[$c_link] = "0"; $this_page_levels[$c_link] = $levels[$p_link] + 1; } } // endif($c_valid) } // endfor Loop through our extracted links and manipulate them. if (count($this_page_links) > $link_limit) { $this_page_links = array_slice($this_page_links, 0, $link_limit); $this_page_levels = array_slice($this_page_levels, 0, $link_limit); } if ($levels[$p_link] < $max_level) { $insert_after = array_search($p_link, array_keys($links)); $links_tail = array_splice($links, $insert_after + 1); $levels_tail = array_splice($levels, $insert_after + 1); $links = $links + $this_page_links + $links_tail; $levels = $levels + $this_page_levels + $levels_tail; } } } // How many links did we end up following? echo("\nTotal number of links followed was ".count($links).".\n\n"); // What kind of search results did we get? echo("\nSearch results: \n\n"); for($i = 0; $i < count($gold); $i++) { echo($gold[$i]. "\n"); } echo("\nTotal number of search results found was ".count($gold).".\n\n"); ?>