Simple parsing modules

git-svn-id: file:///home/svn/framework3/trunk@9042 4d416f70-5f16-0410-b530-b9f4589650da

data/msfcrawler/basic.rb

@@ -18,60 +18,8 @@ class CrawlerSimple < BaseParser
 		
 		if hr and !hr.match(/^(\#|javascript\:)/) 
 			begin
-				uri = URI.parse(hr)
+				hreq = urltohash('GET',hr,request['uri'],nil)				
-			
-				tssl = false
-				if uri.scheme == "https"
-					tssl = true
-				else
-					tssl = false
-				end
-
-				if !uri.host or uri.host == nil
-					thost = request['rhost']
-					tssl = self.targetssl	
-				else
-					thost = uri.host	
-				end
-
-				if !uri.port or uri.port == nil
-					tport = request['rport']
-				else
-					tport = uri.port
-				end
-
-				if !uri.path or uri.path == nil
-					tpath = "/"
-				else
-					tpath = uri.path
-				end
 				
-				
-				
-				newp = Pathname.new(tpath)
-				oldp = Pathname.new(request['uri'])
-				if !newp.absolute?
-					if oldp.to_s[-1,1] == '/'
-						newp = oldp+newp
-					else
-						if !newp.to_s.empty?
-							newp = File.join(oldp.dirname,newp)
-						end
-					end		
-				end
-
-				hreq = {
-					'rhost'		=> thost,
-					'rport'		=> tport,
-					'uri'  		=> newp.to_s,
-					'method'   	=> 'GET',
-					'ctype'		=> 'text/plain',
-					'ssl'		=> tssl,
-					'query'		=> uri.query,
-					'data'		=> nil
-					
-				}
-
 				insertnewpath(hreq)
 					
 			rescue URI::InvalidURIError

data/msfcrawler/flash.rb

@@ -47,58 +47,8 @@ class CrawlerFlash < BaseParser
 				links.each do |link| 
 				
 					begin
-						uri = URI.parse(link[0])
+						hreq = urltohash('GET',link[0],request['uri'],nil)
-			
-						tssl = false
-						if uri.scheme == "https"
-							tssl = true
-						else
-							tssl = false
-						end
 
-						if !uri.host or uri.host == nil
-							thost = request['rhost']
-							tssl = self.targetssl	
-						else
-							thost = uri.host	
-						end
-
-						if !uri.port or uri.port == nil
-							tport = request['rport']
-						else
-							tport = uri.port
-						end
-
-						if !uri.path or uri.path == nil
-							tpath = "/"
-						else
-							tpath = uri.path
-						end
-				
-						newp = Pathname.new(tpath)
-						oldp = Pathname.new(request['uri'])
-						if !newp.absolute?
-							if oldp.to_s[-1,1] == '/'
-								newp = oldp+newp
-							else
-								if !newp.to_s.empty?
-									newp = File.join(oldp.dirname,newp)
-								end
-							end		
-						end
-
-						hreq = {
-							'rhost'		=> thost,
-							'rport'		=> tport,
-							'uri'  		=> newp.to_s,
-							'method'   	=> 'GET',
-							'ctype'		=> 'text/plain',
-							'ssl'		=> tssl,
-							'query'		=> uri.query,
-							'data'		=> nil
-						}
-				
-					
 						insertnewpath(hreq)
 					
 					rescue URI::InvalidURIError

data/msfcrawler/forms.rb

@@ -47,58 +47,10 @@ class CrawlerForms < BaseParser
 	
 		
 			begin
-				uri = URI.parse(hr)
+				hreq = urltohash(m,hr,request['uri'],data)
-			
-				tssl = false
-				if uri.scheme == "https"
-					tssl = true
-				else
-					tssl = false
-				end
-
-				if !uri.host or uri.host == nil
-					thost = request['rhost']
-					tssl = self.targetssl	
-				else
-					thost = uri.host	
-				end
-
-				if !uri.port or uri.port == nil
-					tport = request['rport']
-				else
-					tport = uri.port
-				end
-
-				if !uri.path or uri.path == nil
-					tpath = "/"
-				else
-					tpath = uri.path
-				end
 				
-				newp = Pathname.new(tpath)
+				hreq['ctype'] = 'application/x-www-form-urlencoded'
-				oldp = Pathname.new(request['uri'])
-				if !newp.absolute?
-					if oldp.to_s[-1,1] == '/'
-						newp = oldp+newp
-					else
-						if !newp.to_s.empty?
-							newp = File.join(oldp.dirname,newp)
-						end
-					end		
-				end
-				
-				hreq = {
-					'rhost'		=> thost,
-					'rport'		=> tport,
-					'uri'  		=> newp.to_s,
-					'method'   	=> m,
-					'ctype'		=> 'application/x-www-form-urlencoded',
-					'ssl'		=> tssl,
-					'query'		=> m == 'GET'? data : uri.query,
-					'data'		=> m == 'GET'? nil : data
 					
-				}
-				#puts hreq
 				insertnewpath(hreq)
 			
 					

data/msfcrawler/objects.rb

@@ -20,58 +20,8 @@ class CrawlerObjects < BaseParser
 			s = obj['src']
 
 			begin
-				uri = URI.parse(s)
+				hreq = urltohash('GET',s,request['uri'],nil)
-			
-				tssl = false
-				if uri.scheme == "https"
-					tssl = true
-				else
-					tssl = false
-				end
-
-				if !uri.host or uri.host == nil
-					thost = request['rhost']
-					tssl = self.targetssl	
-				else
-					thost = uri.host	
-				end
-
-				if !uri.port or uri.port == nil
-					tport = request['rport']
-				else
-					tport = uri.port
-				end
-
-				if !uri.path or uri.path == nil
-					tpath = "/"
-				else
-					tpath = uri.path
-				end
 				
-				newp = Pathname.new(tpath)
-				oldp = Pathname.new(request['uri'])
-				if !newp.absolute?
-					if oldp.to_s[-1,1] == '/'
-						newp = oldp+newp
-					else
-						if !newp.to_s.empty?
-							newp = File.join(oldp.dirname,newp)
-						end
-					end		
-				end
-				
-				hreq = {
-					'rhost'		=> thost,
-					'rport'		=> tport,
-					'uri'  		=> newp.to_s,
-					'method'   	=> 'GET',
-					'ctype'		=> 'text/html',
-					'ssl'		=> tssl,
-					'query'		=> uri.query,
-					'data'		=> nil
-					
-				}
-				#puts hreq
 				insertnewpath(hreq)
 			
 					

data/msfcrawler/scripts.rb

@@ -20,58 +20,8 @@ class CrawlerScripts < BaseParser
 			s = obj['src']
 
 			begin
-				uri = URI.parse(s)
+				hreq = urltohash('GET',s,request['uri'],nil)				
-			
-				tssl = false
-				if uri.scheme == "https"
-					tssl = true
-				else
-					tssl = false
-				end
-
-				if !uri.host or uri.host == nil
-					thost = request['rhost']
-					tssl = self.targetssl	
-				else
-					thost = uri.host	
-				end
-
-				if !uri.port or uri.port == nil
-					tport = request['rport']
-				else
-					tport = uri.port
-				end
-
-				if !uri.path or uri.path == nil
-					tpath = "/"
-				else
-					tpath = uri.path
-				end
 				
-				newp = Pathname.new(tpath)
-				oldp = Pathname.new(request['uri'])
-				if !newp.absolute?
-					if oldp.to_s[-1,1] == '/'
-						newp = oldp+newp
-					else
-						if !newp.to_s.empty?
-							newp = File.join(oldp.dirname,newp)
-						end
-					end		
-				end
-				
-				hreq = {
-					'rhost'		=> thost,
-					'rport'		=> tport,
-					'uri'  		=> newp.to_s,
-					'method'   	=> 'GET',
-					'ctype'		=> 'text/html',
-					'ssl'		=> tssl,
-					'query'		=> uri.query,
-					'data'		=> nil
-					
-				}
-				#puts hreq
 				insertnewpath(hreq)