Autoit.ES

Código: Seleccionar todo

#include <inet.au3>
#include <SQLite.au3>
#include <SQLite.dll.au3>

HotKeySet("{ESC}", "Parar")

$firsturl="http://"&"strelki.info" ; es nesesario el http
$dbname="nombre" & ".db" ; no poner algo mas que letras
$max_level=5 ; poner cero para sin limites de profundidad

Dim $ignorar[9] = [".jpg",".gif",".png",".bmp",".ico",".css","rss","rand=",".js"]

Local $info
Dim $szProtocol, $szDomain, $szPath, $szFile

_SQLite_Startup() ; buen dia base de SQlite
If NOT FileExists($dbname) Then ; chequeamos si existe la base de datos, si no existe se crea
$dbn=_SQLite_Open($dbname)
_SQLite_Exec($dbn,"CREATE TABLE urls (id INTEGER,url TEXT, level INTEGER, PRIMARY KEY (id), UNIQUE (url) ON CONFLICT IGNORE);")
_SQLite_Exec($dbn,"CREATE TABLE externos (id INTEGER,afuera TEXT, PRIMARY KEY (id), UNIQUE (afuera) ON CONFLICT IGNORE);")
_SQLite_Exec($dbn,"CREATE TABLE info (id INTEGER,lasturl TEXT, PRIMARY KEY (id));")
_SQLite_Exec($dbn,"INSERT INTO urls (url, level) VALUES ('"&$firsturl&"', 1);")
_SQLite_Exec($dbn,"INSERT INTO info (lasturl) VALUES ('1');")


Else
$dbn=_SQLite_Open($dbname)
EndIf

_SQLite_Exec($dbn,"PRAGMA cache_size=500000;")
_SQLite_Exec($dbn,"PRAGMA synchronous=OFF;")
_SQLite_Exec($dbn,"PRAGMA temp_store=2;")
_SQLite_Exec($dbn,"PRAGMA journal_mode=MEMORY;")

_SQLite_QuerySingleRow($dbn,"SELECT lasturl FROM info WHERE id=1",$info) ; busca la ultima pagina visitada

$o=$info[0]-1
$changes=1

While 1

	$o=$o+1
	Local $url_query
	Local $level_query
	if stringright($o,1)=0 Then _SQLite_Exec($dbn,"UPDATE info SET lasturl='"&$o&"';")
	_SQLite_QuerySingleRow($dbn,"SELECT url FROM urls WHERE id=" & $o,$url_query)
	_SQLite_QuerySingleRow($dbn,"SELECT level FROM urls WHERE id=" & $o,$level_query)
	$url=$url_query[0]
	$level=$level_query[0]
	;ConsoleWrite(@CR & $o & " Bajando url nivel:" & $level &" Direccion: " & $url)

	$source=_INetGetSource($url)
	$url_array = StringRegExp($source, '(?i)href=([^<>#\h]+)', 3)
	If @error = 0 Then
	$big_insert=""
		For $i = 0 To UBound($url_array) - 1
		 $found_url=$url_array[$i]

		 If $max_level<>0 Then
			 If $level>=$max_level then
				 ConsoleWrite(@CR & $o & " No se baja la url " & $found_url &" por haberse superado el limite")
				 ContinueLoop
			 EndIf
		 EndIf

         For $P=1 To $ignorar [0]
			 If StringInStr($found_url,$ignorar ($P),2) <> 0 then
			     ContinueLoop (2)
			 EndIf
		 Next

		 ; cambio respecto al original ahora se chequea sin la direccion sirve y luego se la limpia, no se si es por la base de datos que se la limpia
		 $found_url=StringReplace($found_url,chr(39),"")
		 $found_url=StringReplace($found_url,chr(34),"")
		 $found_url=StringReplace($found_url,"amp;","")
		 $found_url=StringReplace($found_url,"./","")

		 If StringInStr($found_url,"http://") = 0 then
		 $domain= _URLSplit($url, $szProtocol, $szDomain, $szPath, $szFile)
		 $found_url="http://" & $domain[2] & "/" & $found_url
		 Elseif StringInStr($found_url,$firsturl) = 0 then
		 Afuera ($found_url)
		 ContinueLoop
		 EndIf

		 $big_insert=$big_insert & "INSERT INTO urls (url, level) VALUES ('"&$found_url&"', "&$level+1&"); "

		 Next

	_SQLite_Exec($dbn,$big_insert)
    EndIf

	$changes=$changes-_SQLite_TotalChanges()
	ConsoleWrite(@CR & $o & " Bajando url nivel" & $level &" direccion: " & $url &" se encontraron " & $changes * -1 & " nuevas direcciones")
	$changes=_SQLite_TotalChanges()

WEnd

_SQLite_Shutdown ()

Func Afuera ($meter)
	consolewrite(@CR &"    ""Dirección externa encontrada: " & $meter )
	_SQLite_Exec($dbn,"INSERT INTO externos (afuera) VALUES ('"&$meter&"');")
EndFunc

Func Parar()
Exit 0
EndFunc


Func _URLSplit($szUrl, ByRef $szProtocol, ByRef $szDomain, ByRef $szPath, ByRef $szFile) ; no le he tocado una linea a esto anda muy bien
    Local $sSREPattern = '^(?s)(?i)(http|ftp|https|file)://(.*?/|.*$)(.*/){0,}(.*)$'
    Local $aUrlSRE = StringRegExp($szUrl, $sSREPattern, 2)
    If Not IsArray($aUrlSRE) Or UBound($aUrlSRE) - 1 <> 4 Then Return SetError(1, 0, 0)
    If StringRight($aUrlSRE[2], 1) = '/' Then
        $aUrlSRE[2] = StringTrimRight($aUrlSRE[2], 1)
        $aUrlSRE[3] = '/' & $aUrlSRE[3]
    EndIf
    $szProtocol = $aUrlSRE[1]
    $szDomain = $aUrlSRE[2]
    $szPath = $aUrlSRE[3]
    $szFile = $aUrlSRE[4]
    Return $aUrlSRE
EndFunc

Código: Seleccionar todo

For $P=1 To $ignorar [0]
If StringInStr($found_url,$ignorar ($P),2) <> 0 then
ContinueLoop (2)
EndIf
Next